From 723ead04e378cd858162a09abea56ff9226353ba Mon Sep 17 00:00:00 2001 From: Dane Schneider Date: Sun, 6 Sep 2026 22:39:55 -0700 Subject: [PATCH 1/5] feat(codex-security): allow Deep discovery model overrides --- plugins/codex-security/mcp-app/package.json | 2 +- plugins/codex-security/mcp-app/server.ts | 4 + .../mcp-app/src/deep-scan/executor.ts | 17 ++- .../mcp-app/src/deep-scan/store.ts | 18 +++ .../mcp-app/src/deep-scan/types.ts | 2 + .../mcp-app/tests/test_deep_scan_executor.mjs | 110 ++++++++++++++++++ .../tests/test_deep_scan_stdio_lifecycle.mjs | 19 ++- .../mcp-app/tests/test_deep_scan_store.mjs | 35 ++++++ .../references/config-preflight.md | 2 +- .../scripts/deep_scan_config.py | 21 +++- .../scripts/deep_scan_workbench.py | 20 +++- .../scripts/workbench_schema.py | 8 ++ .../tests/test_deep_scan_discovery_config.py | 108 +++++++++++++++++ .../codex-security/tests/test_workbench_db.py | 2 +- .../tests/test_workbench_deep_scan.py | 13 ++- .../test_workbench_setup_and_migrations.py | 66 ++++++++++- 16 files changed, 428 insertions(+), 19 deletions(-) create mode 100644 plugins/codex-security/tests/test_deep_scan_discovery_config.py diff --git a/plugins/codex-security/mcp-app/package.json b/plugins/codex-security/mcp-app/package.json index 6dfeb1cd70..a08f6242d6 100644 --- a/plugins/codex-security/mcp-app/package.json +++ b/plugins/codex-security/mcp-app/package.json @@ -1,6 +1,6 @@ { "name": "codex-security-mcp-app", - "version": "0.1.158", + "version": "0.1.159", "type": "module", "private": true, "scripts": { diff --git a/plugins/codex-security/mcp-app/server.ts b/plugins/codex-security/mcp-app/server.ts index 520f1a7740..6f074804fd 100644 --- a/plugins/codex-security/mcp-app/server.ts +++ b/plugins/codex-security/mcp-app/server.ts @@ -662,6 +662,10 @@ export function createCodexSecurityServer(): McpServer { store: deepScanStore, executor: new CodexSdkWorkerExecutor({ ...modelSettings, + discovery: { + model: begun.run.config.discoveryModel, + reasoningEffort: begun.run.config.discoveryReasoningEffort + }, parentSandbox, artifactContext: { pluginRoot: PLUGIN_ROOT, diff --git a/plugins/codex-security/mcp-app/src/deep-scan/executor.ts b/plugins/codex-security/mcp-app/src/deep-scan/executor.ts index 2d1753ebe2..1597b19837 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/executor.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/executor.ts @@ -23,6 +23,10 @@ import type { export interface CodexSdkWorkerModelSettings { model?: string; reasoningEffort?: string; + discovery?: { + model?: string; + reasoningEffort?: string; + }; artifactContext?: CodexSdkWorkerArtifactContext; parentSandbox?: DeepWorkerParentSandbox; } @@ -67,14 +71,21 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { env: childEnv, signal: request.signal }); + const modelSettings = request.kind === "discovery" + ? { + model: this.modelSettings.discovery?.model ?? this.modelSettings.model, + reasoningEffort: this.modelSettings.discovery?.reasoningEffort + ?? this.modelSettings.reasoningEffort + } + : this.modelSettings; const prompt = await fs.readFile(request.promptPath, "utf8"); const codex = new Codex({ codexPathOverride: executablePathForSpawn(codexPath), env: childEnv, config: { // The CLI can add effort levels before the pinned SDK widens ThreadOptions. - ...(this.modelSettings.reasoningEffort - ? { model_reasoning_effort: this.modelSettings.reasoningEffort } + ...(modelSettings.reasoningEffort + ? { model_reasoning_effort: modelSettings.reasoningEffort } : {}), mcp_servers: { // Discovery workers use the bundled skills and artifacts, not the parent workbench MCP. @@ -89,7 +100,7 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { configOverrides }); const threadOptions = { - ...(this.modelSettings.model ? { model: this.modelSettings.model } : {}), + ...(modelSettings.model ? { model: modelSettings.model } : {}), threadSource: "security_scan", approvalPolicy: "never", skipGitRepoCheck: true, diff --git a/plugins/codex-security/mcp-app/src/deep-scan/store.ts b/plugins/codex-security/mcp-app/src/deep-scan/store.ts index 8bf6b0bf7c..25943aa5db 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/store.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/store.ts @@ -562,6 +562,14 @@ export function parseDeepScan(result: JsonObject): DeepScanRunState { configValue.stopAfterNoNew, "deepScan.config.stopAfterNoNew" ); + const discoveryModel = optionalConfiguredString( + configValue.discoveryModel, + "deepScan.config.discoveryModel" + ); + const discoveryReasoningEffort = optionalConfiguredString( + configValue.discoveryReasoningEffort, + "deepScan.config.discoveryReasoningEffort" + ); const config: DeepScanConfig = { workers: positiveInteger(configValue.workers, "deepScan.config.workers"), subagents: nonNegativeInteger(configValue.subagents, "deepScan.config.subagents"), @@ -574,6 +582,8 @@ export function parseDeepScan(result: JsonObject): DeepScanRunState { configValue.maxDiscoveryRuns, "deepScan.config.maxDiscoveryRuns" ), + ...(discoveryModel === undefined ? {} : { discoveryModel }), + ...(discoveryReasoningEffort === undefined ? {} : { discoveryReasoningEffort }), ...(configValue.maxTimeHours === undefined ? {} : { @@ -746,6 +756,14 @@ function optionalString(value: unknown): string | undefined { return typeof value === "string" && value ? value : undefined; } +function optionalConfiguredString(value: unknown, field: string): string | undefined { + if (value === null || value === undefined) return undefined; + if (typeof value !== "string" || !value.trim()) { + throw new Error(`Codex Security workbench returned invalid ${field}.`); + } + return value; +} + function positiveInteger(value: unknown, field: string): number { if (!Number.isInteger(value) || Number(value) < 1) { throw new Error(`Codex Security workbench returned invalid ${field}.`); diff --git a/plugins/codex-security/mcp-app/src/deep-scan/types.ts b/plugins/codex-security/mcp-app/src/deep-scan/types.ts index dea2d9431c..83a6e585c1 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/types.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/types.ts @@ -28,6 +28,8 @@ export interface DeepScanConfig { stopAfterConsecutiveErrors: number; maxDiscoveryRuns: number; maxTimeHours?: number; + discoveryModel?: string; + discoveryReasoningEffort?: string; } export interface DeepScanCanonicalArtifacts { diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.mjs index b703cd2bb4..a07c6e89c7 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.mjs +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.mjs @@ -79,10 +79,13 @@ try { await testWorkerLaunchesWithoutGlobalCodex(); await testPreflightBindsExecutableAndHomeBeforeChangingCwd(); await testSdkInvocationAndThreadCapture(); + await testDiscoveryModelOverridesDoNotAffectReducer(); + await testPartialDiscoveryOverridesPreserveParentSettings(); await testBedrockCredentialsReachWorker(); await testArtifactServerUsesExtendedStartupTimeout(); await testZeroSubagentsPreservesHostRestrictions(); await testSdkResumesExistingThread(); + await testSdkResumesDiscoveryWithRoleOverride(); await testRetryNotificationDoesNotInterruptTurn(); await testSandboxNamespaceDiagnosticIsSanitized(); await testOwnedArtifactToolFailureDiagnosticIsSanitized(); @@ -671,6 +674,77 @@ async function testSdkInvocationAndThreadCapture() { } } +async function testDiscoveryModelOverridesDoNotAffectReducer() { + const fixture = await fakeCodexFixture(); + const previousPath = process.env.CODEX_CLI_PATH; + process.env.CODEX_CLI_PATH = fixture.executablePath; + try { + const promptPath = path.join(fixture.root, "prompt.md"); + const workingDirectory = path.join(fixture.root, "artifacts"); + await mkdir(workingDirectory); + await writeFile(promptPath, "fixture role-specific worker prompt\n"); + const executor = new CodexSdkWorkerExecutor({ + model: "gpt-6-astra", + reasoningEffort: "high", + discovery: { model: "gpt-5.6-luna", reasoningEffort: "xhigh" }, + parentSandbox: trustedParentSandbox + }); + + for (const [kind, model, effort] of [ + ["discovery", "gpt-5.6-luna", "xhigh"], + ["dedup", "gpt-6-astra", "high"] + ]) { + await executor.run({ + kind, + promptPath, + workingDirectory, + subagents: 0, + signal: new AbortController().signal + }); + const invocation = JSON.parse(await readFile(fixture.markerPath, "utf8")); + assertFlagPair(invocation.argv, "--model", model); + assert.equal(invocation.argv.includes(`model_reasoning_effort="${effort}"`), true); + assertReadOnlyWorkerPolicy(invocation.argv); + } + } finally { + restoreEnv("CODEX_CLI_PATH", previousPath); + } +} + +async function testPartialDiscoveryOverridesPreserveParentSettings() { + for (const [discovery, model, effort] of [ + [{ model: "gpt-5.6-luna" }, "gpt-5.6-luna", "high"], + [{ reasoningEffort: "xhigh" }, "gpt-6-astra", "xhigh"] + ]) { + const fixture = await fakeCodexFixture(); + const previousPath = process.env.CODEX_CLI_PATH; + process.env.CODEX_CLI_PATH = fixture.executablePath; + try { + const promptPath = path.join(fixture.root, "prompt.md"); + const workingDirectory = path.join(fixture.root, "artifacts"); + await mkdir(workingDirectory); + await writeFile(promptPath, "fixture partially overridden worker prompt\n"); + await new CodexSdkWorkerExecutor({ + model: "gpt-6-astra", + reasoningEffort: "high", + discovery, + parentSandbox: trustedParentSandbox + }).run({ + kind: "discovery", + promptPath, + workingDirectory, + subagents: 0, + signal: new AbortController().signal + }); + const invocation = JSON.parse(await readFile(fixture.markerPath, "utf8")); + assertFlagPair(invocation.argv, "--model", model); + assert.equal(invocation.argv.includes(`model_reasoning_effort="${effort}"`), true); + } finally { + restoreEnv("CODEX_CLI_PATH", previousPath); + } + } +} + async function testBedrockCredentialsReachWorker() { const fixture = await fakeCodexFixture(); const mcpConfig = JSON.parse( @@ -827,6 +901,42 @@ async function testSdkResumesExistingThread() { } } +async function testSdkResumesDiscoveryWithRoleOverride() { + const fixture = await fakeCodexFixture(); + const previousPath = process.env.CODEX_CLI_PATH; + process.env.CODEX_CLI_PATH = fixture.executablePath; + try { + const promptPath = path.join(fixture.root, "prompt.md"); + const workingDirectory = path.join(fixture.root, "artifacts"); + await mkdir(workingDirectory); + await writeFile(promptPath, "original discovery worker prompt\n"); + const result = await new CodexSdkWorkerExecutor({ + model: "gpt-6-astra", + reasoningEffort: "high", + discovery: { model: "gpt-5.6-luna", reasoningEffort: "xhigh" }, + parentSandbox: trustedReadOnlyParentSandbox + }).run({ + kind: "discovery", + promptPath, + workingDirectory, + subagents: 0, + signal: new AbortController().signal, + resumeThreadId: "fixture-existing-thread", + continuationPrompt: "continue the existing discovery worker\n" + }); + assert.equal(result.threadId, "fixture-existing-thread"); + const invocation = JSON.parse(await readFile(fixture.markerPath, "utf8")); + const resumeIndex = invocation.argv.indexOf("resume"); + assert.notEqual(resumeIndex, -1); + assert.equal(invocation.argv[resumeIndex + 1], "fixture-existing-thread"); + assertFlagPair(invocation.argv, "--model", "gpt-5.6-luna"); + assert.equal(invocation.argv.includes('model_reasoning_effort="xhigh"'), true); + assertReadOnlyWorkerPolicy(invocation.argv); + } finally { + restoreEnv("CODEX_CLI_PATH", previousPath); + } +} + async function testRetryNotificationDoesNotInterruptTurn() { const fixture = await fakeCodexFixture(); const previousPath = process.env.CODEX_CLI_PATH; diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.mjs index 0dd9ff2702..ce8aa5296b 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.mjs +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.mjs @@ -65,14 +65,17 @@ async function testDeepScanStdioLifecycle() { await mkdir(path.join(codexHome, "codex-security"), { recursive: true }); await writeFile(path.join(targetPath, "fixture.py"), "print('fixture')\n"); await writeFile(path.join(failedTargetPath, "fixture.py"), "print('failure fixture')\n"); + const deepScanConfigPath = path.join(codexHome, "codex-security", "config.toml"); await writeFile( - path.join(codexHome, "codex-security", "config.toml"), + deepScanConfigPath, [ "[deep_scan]", "workers = 1", "subagents = 0", "stop_after_no_new = 1", "max_discovery_runs = 2", + 'discovery_model = "gpt-5.6-luna"', + 'discovery_reasoning_effort = "high"', "" ].join("\n") ); @@ -175,8 +178,10 @@ async function testDeepScanStdioLifecycle() { "02_discovery", "in_scope_files.txt" )), { code: "ENOENT" }); - assertFlagPair(startedWorker.argv, "--model", "gpt-5.5"); - assert.equal(startedWorker.argv.includes('model_reasoning_effort="xhigh"'), true); + assert.equal(startedState.config.discoveryModel, "gpt-5.6-luna"); + assert.equal(startedState.config.discoveryReasoningEffort, "high"); + assertFlagPair(startedWorker.argv, "--model", "gpt-5.6-luna"); + assert.equal(startedWorker.argv.includes('model_reasoning_effort="high"'), true); assertReadOnlyWorkerInvocation(startedWorker.argv); // Discovery progress is admitted once the first complete Standard worker is active. @@ -310,6 +315,14 @@ async function testDeepScanStdioLifecycle() { assert.equal((await readJsonLines(startLogPath)).length, 1); const failureThreadId = "deep-scan-stdio-failure-thread"; + await writeFile(deepScanConfigPath, [ + "[deep_scan]", + "workers = 1", + "subagents = 0", + "stop_after_no_new = 1", + "max_discovery_runs = 2", + "" + ].join("\n")); server.sendRequest(20, "tools/call", toolCall( "start_codex_security_deep_scan", { targetPath: failedTargetPath, scope: ".", userContext: "stdio failure fixture" }, diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_store.mjs b/plugins/codex-security/mcp-app/tests/test_deep_scan_store.mjs index 09eb1f8840..d374b3fa0e 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_store.mjs +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_store.mjs @@ -22,6 +22,7 @@ await testTerminalProtocol(); testCanonicalNullAndPartialParsing(); testRunErrorParsing(); testConfiguredMaximumDurationParsing(); +testDiscoveryModelSettingsParsing(); await testWriteSerializationAndRecovery(); await testBeginUsesTheWriteQueue(); await testHeartbeatBypassesBlockedWriteQueue(); @@ -754,6 +755,25 @@ function idempotentPersistenceScenarios() { }]; } +function testDiscoveryModelSettingsParsing() { + const configured = parseDeepScan(stateResult(randomUUID(), { + deepScan: { + config: { + discoveryModel: "gpt-5.6-luna", + discoveryReasoningEffort: "xhigh" + } + } + })); + assert.equal(configured.config.discoveryModel, "gpt-5.6-luna"); + assert.equal(configured.config.discoveryReasoningEffort, "xhigh"); + + const legacy = parseDeepScan(stateResult(randomUUID(), { + deepScan: { config: { discoveryModel: null, discoveryReasoningEffort: null } } + })); + assert.equal(Object.hasOwn(legacy.config, "discoveryModel"), false); + assert.equal(Object.hasOwn(legacy.config, "discoveryReasoningEffort"), false); +} + function testInvalidPersistedConfig() { assert.throws( () => parseDeepScan(stateResult(randomUUID(), { @@ -782,6 +802,21 @@ function testInvalidPersistedConfig() { `invalid configured duration ${String(maxTimeHours)} must be rejected` ); } + for (const [field, value] of [ + ["discoveryModel", ""], + ["discoveryModel", " "], + ["discoveryModel", 5], + ["discoveryReasoningEffort", ""], + ["discoveryReasoningEffort", " "], + ["discoveryReasoningEffort", 5] + ]) { + assert.throws( + () => parseDeepScan(stateResult(randomUUID(), { + deepScan: { config: { [field]: value } } + })), + new RegExp(`invalid deepScan\\.config\\.${field}`) + ); + } } function testConfiguredMaximumDurationParsing() { diff --git a/plugins/codex-security/references/config-preflight.md b/plugins/codex-security/references/config-preflight.md index 6870b1925b..6299c9467c 100644 --- a/plugins/codex-security/references/config-preflight.md +++ b/plugins/codex-security/references/config-preflight.md @@ -60,7 +60,7 @@ When the profile includes remediation patches, present the concrete config delta Some remediation patches have `kind = "host_setting"`. Present those as host-level setup guidance, not as edits to persistent Codex config. -Deep Security Scan uses MCP-owned SDK sessions rather than the parent thread's worker pool. Its preflight does not require a particular parent delegation runtime, ownership, capacity, or depth. Discovery workers inherit the scan's model and use the reserved `codex_security_deep_scan_worker` permission profile with the parent's supported filesystem denials. The selected Codex executable must support permission-profile configuration and allowance checks. If that command fails to start or exits early, report its path and the tool's diagnostic; do not infer that its version is unsupported. If the tool identifies a missing API, ask the user to update the Codex installation at the reported path: the desktop app for an app-bundled executable, or the selected CLI otherwise. If Codex policy rejects the profile, report the tool's administrator guidance. Do not remove deny rules or select a broader sandbox to work around the error. +Deep Security Scan uses MCP-owned SDK sessions rather than the parent thread's worker pool. Its preflight does not require a particular parent delegation runtime, ownership, capacity, or depth. Discovery workers inherit the parent scan's model and reasoning effort unless explicitly overridden by the `[deep_scan]` settings `discovery_model` and `discovery_reasoning_effort`; reducer workers retain the parent settings. Workers use the reserved `codex_security_deep_scan_worker` permission profile with the parent's supported filesystem denials. The selected Codex executable must support permission-profile configuration and allowance checks. If that command fails to start or exits early, report its path and the tool's diagnostic; do not infer that its version is unsupported. If the tool identifies a missing API, ask the user to update the Codex installation at the reported path: the desktop app for an app-bundled executable, or the selected CLI otherwise. If Codex policy rejects the profile, report the tool's administrator guidance. Do not remove deny rules or select a broader sandbox to work around the error. Do not warn merely because a user's value differs from the profile's suggested patch. Warn or block only when the evaluated capability requirement is unmet. diff --git a/plugins/codex-security/scripts/deep_scan_config.py b/plugins/codex-security/scripts/deep_scan_config.py index 36130417d1..fbf4751090 100644 --- a/plugins/codex-security/scripts/deep_scan_config.py +++ b/plugins/codex-security/scripts/deep_scan_config.py @@ -27,6 +27,8 @@ "stop_after_consecutive_errors", "max_discovery_runs", "max_time_hours", + "discovery_model", + "discovery_reasoning_effort", } @@ -41,7 +43,7 @@ def config_path() -> Path: return codex_home() / "codex-security" / "config.toml" -def resolve_deep_scan_config(available_parallelism: int) -> dict[str, int | float]: +def resolve_deep_scan_config(available_parallelism: int) -> dict[str, int | float | str]: if isinstance(available_parallelism, bool) or available_parallelism < 1: raise SystemExit("Available parallelism must be a positive integer.") path = config_path() @@ -76,7 +78,7 @@ def resolve_deep_scan_config(available_parallelism: int) -> dict[str, int | floa "deep_scan.stop_after_no_new", minimum=1, ) - return { + resolved_config: dict[str, int | float | str] = { "workers": resolved_workers, "subagents": require_integer( configured.get("subagents", DEFAULT_SUBAGENTS), @@ -99,6 +101,15 @@ def resolve_deep_scan_config(available_parallelism: int) -> dict[str, int | floa "deep_scan.max_time_hours", ), } + if "discovery_model" in configured: + resolved_config["discoveryModel"] = require_nonempty_string( + configured["discovery_model"], "deep_scan.discovery_model" + ) + if "discovery_reasoning_effort" in configured: + resolved_config["discoveryReasoningEffort"] = require_nonempty_string( + configured["discovery_reasoning_effort"], "deep_scan.discovery_reasoning_effort" + ) + return resolved_config def require_integer(value: object, label: str, *, minimum: int) -> int: @@ -126,6 +137,12 @@ def require_positive_number(value: object, label: str) -> int | float: return value +def require_nonempty_string(value: object, label: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise SystemExit(f"{label} must be a non-empty string.") + return value.strip() + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--available-parallelism", type=int, required=True) diff --git a/plugins/codex-security/scripts/deep_scan_workbench.py b/plugins/codex-security/scripts/deep_scan_workbench.py index 853094a0af..1a229dffd3 100644 --- a/plugins/codex-security/scripts/deep_scan_workbench.py +++ b/plugins/codex-security/scripts/deep_scan_workbench.py @@ -467,6 +467,16 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A "stopAfterConsecutiveErrors": run["stop_after_consecutive_errors"], "maxDiscoveryRuns": run["max_discovery_runs"], "maxTimeHours": run["max_time_hours"], + **( + {"discoveryModel": run["discovery_model"]} + if run["discovery_model"] is not None + else {} + ), + **( + {"discoveryReasoningEffort": run["discovery_reasoning_effort"]} + if run["discovery_reasoning_effort"] is not None + else {} + ), }, "dispatchedCount": run["discovery_runs_dispatched"], "completionSequence": run["completion_sequence"], @@ -556,7 +566,7 @@ def deep_scan_result( return result -def effective_deep_scan_config(args: argparse.Namespace) -> dict[str, int | float]: +def effective_deep_scan_config(args: argparse.Namespace) -> dict[str, int | float | str]: available_parallelism = args.available_parallelism or os.cpu_count() or 1 return resolve_deep_scan_config(available_parallelism) @@ -564,7 +574,7 @@ def effective_deep_scan_config(args: argparse.Namespace) -> dict[str, int | floa def ensure_deep_scan_run( connection: sqlite3.Connection, scan: sqlite3.Row, - config: dict[str, int | float], + config: dict[str, int | float | str], workflow_version: str, timestamp: str, ) -> sqlite3.Row: @@ -582,9 +592,9 @@ def ensure_deep_scan_run( INSERT INTO deep_scan_runs ( scan_id, schema_version, workflow_version, status, phase, workers, subagents, stop_after_no_new, stop_after_consecutive_errors, - max_discovery_runs, max_time_hours, + max_discovery_runs, max_time_hours, discovery_model, discovery_reasoning_effort, created_at, updated_at - ) VALUES (?, 1, ?, 'running', 'setup', ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, 1, ?, 'running', 'setup', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( scan["id"], @@ -595,6 +605,8 @@ def ensure_deep_scan_run( config["stopAfterConsecutiveErrors"], config["maxDiscoveryRuns"], config["maxTimeHours"], + config.get("discoveryModel"), + config.get("discoveryReasoningEffort"), timestamp, timestamp, ), diff --git a/plugins/codex-security/scripts/workbench_schema.py b/plugins/codex-security/scripts/workbench_schema.py index 66baa2b4f8..1ff1e2f3cd 100644 --- a/plugins/codex-security/scripts/workbench_schema.py +++ b/plugins/codex-security/scripts/workbench_schema.py @@ -867,6 +867,14 @@ ); """, ), + ( + 42, + "persist deep scan discovery model settings", + """ + ALTER TABLE deep_scan_runs ADD COLUMN discovery_model TEXT; + ALTER TABLE deep_scan_runs ADD COLUMN discovery_reasoning_effort TEXT; + """, + ), ) diff --git a/plugins/codex-security/tests/test_deep_scan_discovery_config.py b/plugins/codex-security/tests/test_deep_scan_discovery_config.py new file mode 100644 index 0000000000..1259d3dee5 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_discovery_config.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest +from workbench_test_support import run_workbench + + +def deep_environment(codex_home: Path) -> dict[str, str]: + return {"CODEX_HOME": str(codex_home)} + + +def begin_target_scan( + state_dir: Path, codex_home: Path, target: Path, scan_root: Path +) -> dict[str, object]: + return run_workbench( + state_dir, + "begin-deep-scan", + "--thread-id", + "thread-deep-scan", + "--target-path", + str(target), + "--scope", + ".", + "--scan-root", + str(scan_root), + "--available-parallelism", + "16", + environment=deep_environment(codex_home), + ) + + +@pytest.mark.parametrize( + ("settings", "model", "effort"), + ( + ("", None, None), + ('discovery_model = "gpt-5.6-luna"\n', "gpt-5.6-luna", None), + ('discovery_reasoning_effort = "xhigh"\n', None, "xhigh"), + ( + 'discovery_model = "gpt-5.6-luna"\ndiscovery_reasoning_effort = "xhigh"\n', + "gpt-5.6-luna", + "xhigh", + ), + ), +) +def test_discovery_model_settings_are_independent_and_optional( + tmp_path: Path, settings: str, model: str | None, effort: str | None +) -> None: + state_dir = tmp_path / "state" + codex_home = tmp_path / "codex-home" + config_path = codex_home / "codex-security" / "config.toml" + config_path.parent.mkdir(parents=True) + config_path.write_text("[deep_scan]\n" + settings) + target = tmp_path / "target" + target.mkdir() + + deep_scan = begin_target_scan(state_dir, codex_home, target, tmp_path / "scans")["deepScan"] + + assert deep_scan["config"].get("discoveryModel") == model + assert deep_scan["config"].get("discoveryReasoningEffort") == effort + if model is None: + assert "discoveryModel" not in deep_scan["config"] + if effort is None: + assert "discoveryReasoningEffort" not in deep_scan["config"] + with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + assert connection.execute( + "SELECT discovery_model, discovery_reasoning_effort FROM deep_scan_runs" + ).fetchone() == (model, effort) + + +@pytest.mark.parametrize( + ("setting", "value"), + ( + ("discovery_model", '""'), + ("discovery_model", '" "'), + ("discovery_model", "false"), + ("discovery_reasoning_effort", '""'), + ("discovery_reasoning_effort", "12"), + ), +) +def test_invalid_discovery_model_setting_fails_before_scan_creation( + tmp_path: Path, setting: str, value: str +) -> None: + state_dir = tmp_path / "state" + codex_home = tmp_path / "codex-home" + config_path = codex_home / "codex-security" / "config.toml" + config_path.parent.mkdir(parents=True) + config_path.write_text(f"[deep_scan]\n{setting} = {value}\n") + target = tmp_path / "target" + target.mkdir() + + failed = run_workbench( + state_dir, + "begin-deep-scan", + "--thread-id", + "thread-deep-scan", + "--target-path", + str(target), + "--available-parallelism", + "8", + environment=deep_environment(codex_home), + check=False, + ) + + assert f"deep_scan.{setting} must be a non-empty string" in str(failed["stderr"]) + with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + assert connection.execute("SELECT COUNT(*) FROM scans").fetchone() == (0,) diff --git a/plugins/codex-security/tests/test_workbench_db.py b/plugins/codex-security/tests/test_workbench_db.py index f68ccdef94..b79de3fc90 100644 --- a/plugins/codex-security/tests/test_workbench_db.py +++ b/plugins/codex-security/tests/test_workbench_db.py @@ -1042,7 +1042,7 @@ def test_workbench_persists_progress_and_indexes_completed_findings(tmp_path: Pa ) } assert tables == EXPECTED_TABLES - assert connection.execute("SELECT COUNT(*) FROM schema_migrations").fetchone() == (41,) + assert connection.execute("SELECT COUNT(*) FROM schema_migrations").fetchone() == (42,) assert connection.execute("SELECT COUNT(*) FROM findings").fetchone() == (1,) assert connection.execute("SELECT COUNT(*) FROM finding_locations").fetchone() == (1,) diff --git a/plugins/codex-security/tests/test_workbench_deep_scan.py b/plugins/codex-security/tests/test_workbench_deep_scan.py index c58a274e27..dfaa486f28 100644 --- a/plugins/codex-security/tests/test_workbench_deep_scan.py +++ b/plugins/codex-security/tests/test_workbench_deep_scan.py @@ -279,7 +279,7 @@ def claim() -> dict[str, object]: return claim_deep_scan_coordinator(state_dir, codex_home, scan_id) with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: - assert connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone() == (41,) + assert connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone() == (42,) assert claim()["deepScan"]["coordinatorGeneration"] == 2 assert claim()["coordinatorDisposition"] == "observing" expire_deep_scan_coordinator(state_dir, scan_id) @@ -1341,6 +1341,8 @@ def test_target_begin_is_atomic_idempotent_and_snapshots_config(tmp_path: Path) "stop_after_no_new = 4\n" "max_discovery_runs = 12\n" "max_time_hours = 2.5\n" + 'discovery_model = "gpt-5.6-luna"\n' + 'discovery_reasoning_effort = "xhigh"\n' ) target = tmp_path / "target" target.mkdir() @@ -1359,6 +1361,8 @@ def test_target_begin_is_atomic_idempotent_and_snapshots_config(tmp_path: Path) "stopAfterConsecutiveErrors": 3, "maxDiscoveryRuns": 12, "maxTimeHours": 2.5, + "discoveryModel": "gpt-5.6-luna", + "discoveryReasoningEffort": "xhigh", } assert deep_scan["workers"] == [] scan_id = str(deep_scan["scanId"]) @@ -1368,7 +1372,9 @@ def test_target_begin_is_atomic_idempotent_and_snapshots_config(tmp_path: Path) assert scan["progress"]["coverage"]["filesTotal"] == 1 assert deep_scan["canonicalArtifacts"] is None - config_path.write_text("[deep_scan]\nworkers = 1\nmax_time_hours = 8\n") + config_path.write_text( + '[deep_scan]\nworkers = 1\nmax_time_hours = 8\ndiscovery_model = "gpt-5.6-sol"\n' + ) second = begin_target_scan(state_dir, codex_home, target, scan_root) assert second["startDisposition"] == "joined" assert second["deepScan"]["scanId"] == deep_scan["scanId"] @@ -1379,6 +1385,9 @@ def test_target_begin_is_atomic_idempotent_and_snapshots_config(tmp_path: Path) assert connection.execute("SELECT COUNT(*) FROM scans").fetchone() == (1,) assert connection.execute("SELECT COUNT(*) FROM deep_scan_runs").fetchone() == (1,) assert connection.execute("SELECT max_time_hours FROM deep_scan_runs").fetchone() == (2.5,) + assert connection.execute( + "SELECT discovery_model, discovery_reasoning_effort FROM deep_scan_runs" + ).fetchone() == ("gpt-5.6-luna", "xhigh") assert ( connection.execute( """ diff --git a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py index 679ee77c1d..06859d5a6d 100644 --- a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py +++ b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py @@ -405,7 +405,7 @@ def test_workbench_serializes_concurrent_first_run_migrations(tmp_path: Path) -> {"databasePath": str(state_dir / "workbench.sqlite3")}, ] with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: - assert connection.execute("SELECT COUNT(*) FROM schema_migrations").fetchone() == (41,) + assert connection.execute("SELECT COUNT(*) FROM schema_migrations").fetchone() == (42,) @pytest.mark.parametrize("previous_history", ["main", "comparison-preview"]) @@ -751,6 +751,62 @@ def test_deep_scan_time_limit_migration_backfills_and_repairs_existing_runs( ) +def test_discovery_model_migration_preserves_running_scan_settings() -> None: + namespace = runpy.run_path(str(SCRIPT), run_name="codex_security_workbench_db") + apply_migrations = namespace["apply_migrations"] + connection = sqlite3.connect(":memory:") + connection.row_factory = sqlite3.Row + previous_migrations = tuple( + migration for migration in namespace["MIGRATIONS"] if migration[0] < 42 + ) + with mock.patch.dict(apply_migrations.__globals__, {"MIGRATIONS": previous_migrations}): + apply_migrations(connection) + + timestamp = "2026-09-01T00:00:00Z" + connection.execute( + "INSERT INTO workspaces (id, created_at, updated_at) VALUES (?, ?, ?)", + ("legacy-workspace", timestamp, timestamp), + ) + connection.execute( + """ + INSERT INTO scans ( + id, workspace_id, target_path, target_revision, scope, mode, scan_dir, + model, reasoning_effort, status, phase, started_at, created_at, updated_at + ) VALUES (?, ?, '/legacy/target', 'legacy-revision', '.', 'deep', + '/legacy/scan', 'gpt-6-astra', 'xhigh', 'running', 'discovery', ?, ?, ?) + """, + ("legacy-scan", "legacy-workspace", timestamp, timestamp, timestamp), + ) + connection.execute( + """ + INSERT INTO deep_scan_runs ( + scan_id, schema_version, workflow_version, status, phase, + workers, subagents, stop_after_no_new, stop_after_consecutive_errors, + max_discovery_runs, created_at, updated_at + ) VALUES (?, 1, 'deep-scan-mcp/v1', 'running', 'discovery', + 4, 0, 4, 3, 40, ?, ?) + """, + ("legacy-scan", timestamp, timestamp), + ) + connection.commit() + + apply_migrations(connection) + + assert tuple( + connection.execute( + "SELECT discovery_model, discovery_reasoning_effort FROM deep_scan_runs" + ).fetchone() + ) == (None, None) + assert tuple(connection.execute("SELECT model, reasoning_effort FROM scans").fetchone()) == ( + "gpt-6-astra", + "xhigh", + ) + assert ( + connection.execute("SELECT version FROM schema_migrations WHERE version = 42").fetchone()[0] + == 42 + ) + + def test_workbench_reconciles_monorepo_migration_lineage() -> None: namespace = runpy.run_path(str(SCRIPT), run_name="codex_security_workbench_db") apply_migrations = namespace["apply_migrations"] @@ -867,6 +923,7 @@ def test_workbench_creates_single_final_schema(tmp_path: Path) -> None: (39, "store dedupe checkpoint bindings in columns"), (40, "index finding identity and comparison history"), (41, "checkpoint finding severity assessments"), + (42, "persist deep scan discovery model settings"), ] assert {row[1] for row in connection.execute("PRAGMA table_info(workspaces)")} >= { "diff_target_kind", @@ -924,6 +981,8 @@ def test_workbench_creates_single_final_schema(tmp_path: Path) -> None: "stop_after_consecutive_errors", "consecutive_errors", "max_time_hours", + "discovery_model", + "discovery_reasoning_effort", } assert { row[1] for row in connection.execute("PRAGMA table_info(finding_remediation_attempts)") @@ -969,7 +1028,7 @@ def test_workbench_upgrades_preexisting_database(tmp_path: Path) -> None: connection.execute("ALTER TABLE scans DROP COLUMN handoff_claim_token") run_workbench(state_dir, "database-info") with sqlite3.connect(database) as connection: - assert connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone() == (41,) + assert connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone() == (42,) assert {row[1] for row in connection.execute("PRAGMA table_info(scans)")} >= { "handoff_claimed_at", "handoff_claim_token", @@ -1996,6 +2055,7 @@ def test_workbench_upgrades_released_database_schema(tmp_path: Path) -> None: (39, "store dedupe checkpoint bindings in columns"), (40, "index finding identity and comparison history"), (41, "checkpoint finding severity assessments"), + (42, "persist deep scan discovery model settings"), ] assert "capability_preflight_json" in { row[1] for row in connection.execute("PRAGMA table_info(workspaces)") @@ -2079,6 +2139,7 @@ def test_workbench_upgrades_pre_release_phase_progress_migration(tmp_path: Path) (39, "store dedupe checkpoint bindings in columns"), (40, "index finding identity and comparison history"), (41, "checkpoint finding severity assessments"), + (42, "persist deep scan discovery model settings"), ] assert "continuation_thread_id" in { row[1] for row in connection.execute("PRAGMA table_info(scans)") @@ -2170,6 +2231,7 @@ def test_workbench_upgrades_pre_release_preflight_progress_migration(tmp_path: P (39, "store dedupe checkpoint bindings in columns"), (40, "index finding identity and comparison history"), (41, "checkpoint finding severity assessments"), + (42, "persist deep scan discovery model settings"), ] assert "continuation_thread_id" in { row[1] for row in connection.execute("PRAGMA table_info(scans)") From 3fee303b9c384d1cb69dbc193c5633530a74aae6 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 17 Oct 2026 03:34:33 +0000 Subject: [PATCH 2/5] fix(deep-scan): preserve discovery recipes and model-aware costs --- docs/project-configuration.md | 11 +- sdk/typescript/README.md | 9 + sdk/typescript/src/api.ts | 63 +++-- sdk/typescript/src/cli.ts | 4 +- sdk/typescript/src/cost.ts | 128 +++++++++- sdk/typescript/src/deep-config.ts | 81 ++++--- sdk/typescript/src/project-config.ts | 2 +- sdk/typescript/src/result.ts | 9 +- sdk/typescript/src/scan-settings.ts | 21 ++ sdk/typescript/tests-ts/api.test.ts | 43 +++- sdk/typescript/tests-ts/cost.test.ts | 221 ++++++++++++++++++ sdk/typescript/tests-ts/deep-config.test.ts | 36 +++ .../tests-ts/project-config.test.ts | 4 + sdk/typescript/tests-ts/result.test.ts | 31 +++ sdk/typescript/tests-ts/scan-recipe.test.ts | 28 +++ .../tests-ts/support/usage-rollout.ts | 5 + 16 files changed, 626 insertions(+), 70 deletions(-) diff --git a/docs/project-configuration.md b/docs/project-configuration.md index 0e55c15364..34ff3f0519 100644 --- a/docs/project-configuration.md +++ b/docs/project-configuration.md @@ -253,10 +253,13 @@ discovery_reasoning_effort = "high" Each omitted setting inherits the parent scan's choice. Setup and reducer workers keep the parent settings. Discovery choices are saved when the scan starts and -retained when workers resume. These settings have no new CLI flags or project-file -keys. Supplying all six orchestration settings forms a complete configuration that -does not read legacy TOML, including these optional choices; saved scan recipes -retain that behavior. +retained when workers resume or the scan is rerun. SDK callers can set +`discoveryModel` and `discoveryReasoningEffort` directly; each optional string +overrides its legacy TOML value. These settings have no new CLI flags or +project-file keys. Supplying all six orchestration settings forms a complete +configuration that does not read legacy TOML; include any desired discovery +overrides explicitly. Saved recipes preserve the original choices without reading +today's legacy configuration. A valid deep block can stay inactive in standard mode. Explicit deep CLI options still require deep mode. Deep diff scans and custom validation remain unsupported. diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index 606c65d479..d462aa6ce5 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -350,6 +350,15 @@ within each worker. Time and run limits apply to discovery. After discovery stops, the scan combines and returns completed findings. See [deep scan settings](docs/cli.md#configure-deep-scans) for defaults and saved settings. +Set the optional SDK fields `discoveryModel` and `discoveryReasoningEffort` to +override each discovery worker setting independently. They take precedence over +the legacy `[deep_scan]` TOML settings `discovery_model` and +`discovery_reasoning_effort`. If neither source sets a choice, it inherits the +parent scan's setting. Setup and reducer workers keep the parent settings. +Supplying all six orchestration settings skips legacy TOML, so include any desired discovery +overrides explicitly. Saved recipes preserve these choices for both resume and +rerun. + ### Progress and cost Use `onProgress` for scan progress and `onCost` for cost updates. diff --git a/sdk/typescript/src/api.ts b/sdk/typescript/src/api.ts index 005f227571..e27c4e737f 100644 --- a/sdk/typescript/src/api.ts +++ b/sdk/typescript/src/api.ts @@ -114,6 +114,7 @@ import { SCAN_AUTH_MODES, ScanSettingsSchema, type DeepScanOptions, + type ResolvedDeepScanOptions, type ScanAuthMode, type ScanSettings, type ScanPromptSettings, @@ -750,7 +751,7 @@ export class CodexSecurity { }); type ScanMetadata = Pick< ScanResultOptions, - "threadId" | "turnResult" | "sarifPath" | "repositoryFindings" + "threadId" | "turnResult" | "sarifPath" | "repositoryFindings" | "cost" >; if (state.scanId && state.scanDir) { await workflow.protectArtifacts(state.scanDir); @@ -765,6 +766,7 @@ export class CodexSecurity { metadata = { threadId: (scan["continuationThreadId"] as string) ?? "", turnResult: { status: "completed" }, + cost: scan["cost"] as ScanMetadata["cost"], }; } if (completed) { @@ -804,6 +806,7 @@ export class CodexSecurity { await workflow.complete("scan", { threadId: result.threadId, turnResult: result.turnResult, + cost: result.cost, sarifPath: result.sarifPath, repositoryFindings: result.repositoryFindings, } satisfies ScanMetadata); @@ -1143,7 +1146,11 @@ export class CodexSecurity { const configuration = await mergedCodexConfig(this.config); const model = scanModelConfiguration(configuration); const modelProvider = scanModelProvider(configuration); - validateScanCostLimit(options.maxCostUsd, model.model); + validateScanCostLimit( + options.maxCostUsd, + model.model, + inputs.deepScanConfiguration?.settings.discoveryModel, + ); const archiveDir = options.archiveExisting === true ? await planOutputArchive(inputs.outputDir) @@ -1884,7 +1891,11 @@ export class CodexSecurity { pluginVersion: runtime.plugin.version, }; const { model } = scanModelConfiguration(effectiveConfig); - validateScanCostLimit(options.maxCostUsd, model); + validateScanCostLimit( + options.maxCostUsd, + model, + deepScanConfiguration?.settings.discoveryModel, + ); if (mode === "deep" && options.maxCostUsd !== undefined) { budgetRecovery = { expectation, @@ -2632,17 +2643,20 @@ export class CodexSecurity { completedScan, "completed", ); - if (directDeepScan) { - result = new ScanResult({ - ...result, - turnResult: { - ...result.turnResult, - finalResponse: ( - await readScanFile(scanDir, "report.md", "report.md", signal) - ).toString("utf8"), - }, - }); - } + result = new ScanResult({ + ...result, + cost: completionCost, + ...(directDeepScan + ? { + turnResult: { + ...result.turnResult, + finalResponse: ( + await readScanFile(scanDir, "report.md", "report.md", signal) + ).toString("utf8"), + }, + } + : {}), + }); if (isRecord(completedScan) && Array.isArray(completedScan["warnings"])) { const targetWarnings = new Set([ ...preparedTargetWarnings, @@ -2904,7 +2918,10 @@ export class CodexSecurity { : undefined, ); } - return result; + return new ScanResult({ + ...result, + cost: snapshot?.cost ?? failure.cost, + }); } catch {} } if (activeScan !== null && scanTurnStatus !== null) { @@ -4807,7 +4824,7 @@ function scanRecipe({ failOnSeverity?: SeverityLevel; knowledgeBasePaths?: string[]; maxCostUsd?: number; - deepScan?: Required; + deepScan?: ResolvedDeepScanOptions; auth?: ScanAuthMode; cyberAccessProgram?: ScanSettings["cyberAccessProgram"]; }): JsonObject { @@ -4844,12 +4861,18 @@ async function prepareScanOutputDir( function validateScanCostLimit( maxCostUsd: number | undefined, model: string, + discoveryModel?: string, ): void { if (maxCostUsd === undefined) return; - if (estimateScanCost(model, { input_tokens: 0, output_tokens: 0 }) === null) { - throw new CodexSecurityError( - `A scan cost limit is not available for the configured model: ${model}.`, - ); + for (const selected of [model, discoveryModel]) { + if ( + selected !== undefined && + estimateScanCost(selected, { input_tokens: 0, output_tokens: 0 }) === null + ) { + throw new CodexSecurityError( + `A scan cost limit is not available for the configured model: ${selected}.`, + ); + } } } diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 31b820dd54..a49a922cd7 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -75,7 +75,6 @@ import { scanAuthentication, runtimeScanAuthentication, selectedScanEnvironment, - type DeepScanOptions, type ScanAuthMode, type ScanAuthentication, type ScanOptions, @@ -311,6 +310,7 @@ import { meetsSeverity, type FailureSeverity, type ResolvedScanSettings, + type ResolvedDeepScanOptions, } from "./scan-settings.js"; const PROGRESS_REFRESH_MILLISECONDS = 1_000; @@ -9354,7 +9354,7 @@ async function readDeepScanStop( dispatchedCount: number; completionSequence: number; noNewStreak: number; - config: Required; + config: ResolvedDeepScanOptions; createdAt: string; completedAt: string; } diff --git a/sdk/typescript/src/cost.ts b/sdk/typescript/src/cost.ts index a485b0e7bc..9704ae5354 100644 --- a/sdk/typescript/src/cost.ts +++ b/sdk/typescript/src/cost.ts @@ -87,6 +87,9 @@ function createSessionUsage() { startedAt: null as number | null, inheritedUsage: null as ScanTokenUsage | null, replaying: false, + model: null as string | null, + modelUsageOffset: null as ScanTokenUsage | null, + completedModelUsage: new Map(), usage: null as ScanTokenUsage | null, responseUsage: null as ScanTokenUsage | null, responseIds: new Set(), @@ -268,6 +271,7 @@ export class ScanCostTracker { } const usages = new Map(this.#receipts); + const sessions = new Map(); for (const [path, tracked] of this.#sessions) { const threadId = tracked.threadId; if (threadId === null || !included.has(threadId)) continue; @@ -282,6 +286,7 @@ export class ScanCostTracker { await readSessionUsage(path, session, repository); this.#sessions.set(path, session); } + sessions.set(threadId, session); let worker: number | undefined; if (threadId !== this.#threadId) { worker = this.#workers.get(threadId); @@ -319,18 +324,107 @@ export class ScanCostTracker { receipt.cache_write_input_tokens === 0)) ) { usages.set(threadId, session.usage); + } else if (receipt != null && session.usage !== null) { + // A final SDK receipt can advance the total while omitting cache-write + // and reasoning detail. Keep counters already observed in the rollout. + usages.set(threadId, { + ...receipt, + cache_write_input_tokens: Math.max( + receipt.cache_write_input_tokens, + session.usage.cache_write_input_tokens, + ), + reasoning_output_tokens: Math.max( + receipt.reasoning_output_tokens, + session.usage.reasoning_output_tokens, + ), + }); } } let usage: ScanTokenUsage | null = null; - for (const value of usages.values()) { + const modelUsages = new Map(); + for (const [threadId, value] of usages) { if (value === null) { this.#snapshot = { usage: null, cost: null }; return; } usage = addTokenUsage(usage, value); + const session = sessions.get(threadId); + for (const [model, completed] of session?.completedModelUsage ?? []) { + modelUsages.set( + model, + addTokenUsage(modelUsages.get(model) ?? null, completed), + ); + } + const current = + session?.modelUsageOffset == null + ? value + : subtractTokenUsage(value, session.modelUsageOffset); + if (current === null) { + throw new Error( + "Could not attribute scan token usage to its observed model.", + ); + } + const model = session?.model ?? this.#options.model; + modelUsages.set( + model, + addTokenUsage(modelUsages.get(model) ?? null, current), + ); } if (usage === null) return; - const cost = estimateScanCost(this.#options.model, usage); + // Zero-token control sessions do not contribute a model or price to a scan. + if (usage.total_tokens > 0) { + for (const [model, value] of modelUsages) { + if (value.total_tokens === 0) modelUsages.delete(model); + } + } + const costs: ScanCost[] = []; + for (const [model, value] of modelUsages) { + const cost = estimateScanCost(model, value); + if (cost === null) { + this.#snapshot = { usage, cost: null }; + this.#reportUsage(usage); + if (this.#options.maxCostUsd !== undefined) { + throw new Error( + `Could not verify the scan cost limit for observed model: ${model}.`, + ); + } + return; + } + costs.push(cost); + } + const cost = + costs.length === 1 + ? costs[0]! + : { + model: costs + .map(({ model }) => model) + .sort() + .join(", "), + inputTokens: usage.input_tokens, + cachedInputTokens: usage.cached_input_tokens, + cacheWriteInputTokens: usage.cache_write_input_tokens, + ...(usage.cache_write_input_tokens_reported === false + ? { cacheWriteInputTokensReported: false } + : {}), + outputTokens: usage.output_tokens, + estimatedUsd: costs.reduce( + (sum, value) => sum + value.estimatedUsd, + 0, + ), + estimatedUsdRange: { + min: costs.reduce( + (sum, value) => sum + value.estimatedUsdRange!.min, + 0, + ), + max: costs.some((value) => value.estimatedUsdRange!.max === null) + ? null + : costs.reduce( + (sum, value) => sum + value.estimatedUsdRange!.max!, + 0, + ), + context: "unknown" as const, + }, + }; this.#snapshot = { usage, cost }; this.#reportUsage(usage); this.#reportCost(cost); @@ -504,6 +598,36 @@ function readSessionEvent( return; } session.events?.push(event); + if ( + event["type"] === "turn_context" && + typeof payload["model"] === "string" + ) { + const model = payload["model"]; + // A resumed thread can change models. Keep its previous charges separate + // from the later increments in the thread's cumulative usage. + if ( + session.model !== null && + session.model !== model && + session.usage !== null + ) { + const completed = + session.modelUsageOffset === null + ? session.usage + : subtractTokenUsage(session.usage, session.modelUsageOffset); + if (completed !== null) { + session.completedModelUsage.set( + session.model, + addTokenUsage( + session.completedModelUsage.get(session.model) ?? null, + completed, + ), + ); + } + session.modelUsageOffset = session.usage; + } + session.model = model; + return; + } if (event["type"] === "token_usage_record") { const responseId = payload["response_id"]; if ( diff --git a/sdk/typescript/src/deep-config.ts b/sdk/typescript/src/deep-config.ts index 43af7fa239..6a9ed4942b 100644 --- a/sdk/typescript/src/deep-config.ts +++ b/sdk/typescript/src/deep-config.ts @@ -8,23 +8,29 @@ import { DEFAULT_DEEP_SCAN_SETTINGS } from "./deep-scan-defaults.js"; import { CodexSecurityError } from "./errors.js"; import { DEFAULT_SCAN_MODE, + DEEP_SCAN_DISCOVERY_SETTINGS, DEEP_SCAN_SETTINGS, DeepScanSettingsSchema, type DeepScanOptions, + type ResolvedDeepScanOptions, } from "./scan-settings.js"; import type { ScanMode } from "./scan-modes.js"; -const discoveryKeys = new Set([ - "discovery_model", - "discovery_reasoning_effort", -]); +const allSettings = [...DEEP_SCAN_SETTINGS, ...DEEP_SCAN_DISCOVERY_SETTINGS]; +const discoveryKeys = new Set( + DEEP_SCAN_DISCOVERY_SETTINGS.map(([, key]) => key), +); +type DeepScanSource = "default" | "legacy" | "override"; export type DeepScanSources = Record< - keyof DeepScanOptions, - "default" | "legacy" | "override" ->; + (typeof DEEP_SCAN_SETTINGS)[number][0], + DeepScanSource +> & + Partial< + Record<(typeof DEEP_SCAN_DISCOVERY_SETTINGS)[number][0], DeepScanSource> + >; export interface ResolvedDeepScanConfig { - settings: Required; + settings: ResolvedDeepScanOptions; sources: DeepScanSources; source: string; document: TomlTable; @@ -35,13 +41,15 @@ export function deepScanOptions( options: DeepScanOptions & { mode?: ScanMode }, ): DeepScanOptions { const selected: DeepScanOptions = {}; - for (const [name] of DEEP_SCAN_SETTINGS) { + for (const [name] of allSettings) { const value = options[name]; if (value === undefined) continue; if ((options.mode ?? DEFAULT_SCAN_MODE) !== "deep") { throw new CodexSecurityError("Deep scan settings require deep mode."); } - selected[name] = requireDeepScanValue(name, value, name); + Object.assign(selected, { + [name]: requireDeepScanValue(name, value, name), + }); } return selected; } @@ -50,7 +58,7 @@ function requireDeepScanValue( name: keyof DeepScanOptions, value: unknown, label: string, -): number { +): number | string { const parsed = DeepScanSettingsSchema.shape[name].unwrap().safeParse(value); if (!parsed.success) { throw new CodexSecurityError( @@ -82,29 +90,16 @@ export async function resolveDeepScanConfig( ); } const configured = (existing ?? {}) as TomlTable; - const keys = new Set([ - ...DEEP_SCAN_SETTINGS.map(([, key]) => key), - ...discoveryKeys, - ]); + const keys = new Set(allSettings.map(([, key]) => key)); const unknown = Object.keys(configured).filter((key) => !keys.has(key)); if (unknown.length > 0) { throw new CodexSecurityError( `Unknown Codex Security Deep Scan configuration ${unknown.join(", ")} in ${source}.`, ); } - for (const key of discoveryKeys) { - const value = configured[key]; - if (value === undefined) continue; - if (typeof value !== "string" || !value.trim()) { - throw new CodexSecurityError( - `Deep scan ${key} in ${source} must be a non-empty string.`, - ); - } - configured[key] = value.trim(); - } const settings = { ...DEFAULT_DEEP_SCAN_SETTINGS, - } as Required; + } as ResolvedDeepScanOptions; const sources = {} as DeepScanSources; for (const [name, key] of DEEP_SCAN_SETTINGS) { let value = Object.hasOwn(configured, key) @@ -119,11 +114,25 @@ export async function resolveDeepScanConfig( : Object.hasOwn(configured, key) ? "legacy" : "default"; - settings[name] = requireDeepScanValue( - name, - value, - sources[name] === "legacy" ? `${key} in ${source}` : name, - ); + Object.assign(settings, { + [name]: requireDeepScanValue( + name, + value, + sources[name] === "legacy" ? `${key} in ${source}` : name, + ), + }); + } + for (const [name, key] of DEEP_SCAN_DISCOVERY_SETTINGS) { + const value = explicit[name] ?? configured[key]; + if (value === undefined) continue; + sources[name] = explicit[name] === undefined ? "legacy" : "override"; + Object.assign(settings, { + [name]: requireDeepScanValue( + name, + value, + sources[name] === "legacy" ? `${key} in ${source}` : name, + ), + }); } return { settings, @@ -162,14 +171,14 @@ export async function writeDeepScanConfig( ...(retainAmbientSettings ? (document["deep_scan"] as TomlTable | undefined) : Object.fromEntries( - Object.entries(document["deep_scan"] ?? {}).filter(([key]) => - discoveryKeys.has(key), + Object.entries(document["deep_scan"] ?? {}).filter( + ([key]) => sameFile && discoveryKeys.has(key), ), )), ...Object.fromEntries( - DEEP_SCAN_SETTINGS.filter(([name]) => settings[name] !== undefined).map( - ([name, key]) => [key, settings[name]], - ), + allSettings + .filter(([name]) => settings[name] !== undefined) + .map(([name, key]) => [key, settings[name]]), ), }, } as JsonObject); diff --git a/sdk/typescript/src/project-config.ts b/sdk/typescript/src/project-config.ts index 97f3693141..1ae412e9ad 100644 --- a/sdk/typescript/src/project-config.ts +++ b/sdk/typescript/src/project-config.ts @@ -282,7 +282,7 @@ export function resolveScanSettings( overrides.target, ) ?? "repository"; const configuredDeep = file?.scan?.deep; - const deep: DeepScanOptions = {}; + const deep: DeepScanOptions = { ...explicitDeep }; if (mode === "deep") { for (const [name, , field] of DEEP_SCAN_SETTINGS) { const value = choose( diff --git a/sdk/typescript/src/result.ts b/sdk/typescript/src/result.ts index 4f1a937c9f..eeeb082aed 100644 --- a/sdk/typescript/src/result.ts +++ b/sdk/typescript/src/result.ts @@ -42,6 +42,7 @@ export interface ScanResultOptions { scanDir: string; threadId: string; turnResult: TurnResultMetadata; + cost?: Readonly | null; sarifPath?: string | null; threatModelPath?: string | null; repositoryFindings?: readonly RepositoryFinding[]; @@ -68,10 +69,10 @@ export class ScanResult { this.turnResult = options.turnResult; this.repositoryFindings = options.repositoryFindings; this.threatModelPath = options.threatModelPath ?? null; - this.cost = estimateScanCost( - options.turnResult.model, - options.turnResult.usage, - ); + this.cost = + options.cost === undefined + ? estimateScanCost(options.turnResult.model, options.turnResult.usage) + : options.cost; if (options.sarifPath !== undefined) { this.sarifPath = options.sarifPath; } else { diff --git a/sdk/typescript/src/scan-settings.ts b/sdk/typescript/src/scan-settings.ts index 7dd1036d3d..393f99a723 100644 --- a/sdk/typescript/src/scan-settings.ts +++ b/sdk/typescript/src/scan-settings.ts @@ -48,6 +48,11 @@ export const DEEP_SCAN_SETTINGS = [ ["maxTimeHours", "max_time_hours", "max_time_hours", "--max-time-hours"], ] as const; +export const DEEP_SCAN_DISCOVERY_SETTINGS = [ + ["discoveryModel", "discovery_model"], + ["discoveryReasoningEffort", "discovery_reasoning_effort"], +] as const; + const positiveIntegerError = "must be a positive integer"; const positiveInteger = z .number({ error: positiveIntegerError }) @@ -60,8 +65,20 @@ const nonnegativeInteger = z .nonnegative({ error: nonnegativeIntegerError }); const maximumHours = 96; const hoursError = `must be a positive number no greater than ${maximumHours}`; +const discoverySettingError = "must be a non-empty string"; +const discoverySetting = z + .string({ error: discoverySettingError }) + .trim() + .min(1, { error: discoverySettingError }) + .optional(); export const DeepScanSettingsSchema = z.strictObject({ + discoveryModel: discoverySetting.describe( + "Discovery worker model. Omit to inherit the parent model.", + ), + discoveryReasoningEffort: discoverySetting.describe( + "Discovery worker reasoning effort. Omit to inherit the parent effort.", + ), workers: positiveInteger.optional().meta({ default: DEFAULT_DEEP_SCAN_SETTINGS.workers, description: "Maximum concurrent deep-scan discovery workers.", @@ -95,6 +112,10 @@ export const DeepScanSettingsSchema = z.strictObject({ }); export type DeepScanOptions = z.infer; +export type ResolvedDeepScanOptions = Required< + Pick +> & + DeepScanOptions; const inputPath = z.string().min(1); diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.ts index 34eba77a21..5324ce2e20 100644 --- a/sdk/typescript/tests-ts/api.test.ts +++ b/sdk/typescript/tests-ts/api.test.ts @@ -155,6 +155,11 @@ test.each(["completed", "receipt-lost", "scan-interrupted", "prompt-files"])( if (scenario === "prompt-files") await writeFile(promptFile, scanPrompt); let modelCalls = 0; let completed = false; + let savedCost: JsonObject | null = null; + const usage = { input_tokens: 100, output_tokens: 10 }; + const expectedCostUsd = + estimateScanCost("gpt-5.6-sol", usage)!.estimatedUsd + + estimateScanCost("gpt-5.6-luna", usage)!.estimatedUsd; let loseReceipt = scenario === "receipt-lost"; const makeClient = async (attempt: number) => { const codexHome = join(root, `codex-home-${attempt}`); @@ -183,6 +188,7 @@ test.each(["completed", "receipt-lost", "scan-interrupted", "prompt-files"])( scan: { progress: { status: completed ? "complete" : "failed" }, continuationThreadId: "thread-1", + cost: savedCost, }, }; if (args[0] === "register-cli-scan") { @@ -194,7 +200,10 @@ test.each(["completed", "receipt-lost", "scan-interrupted", "prompt-files"])( }); return registration; } - if (args[0] === "complete-scan") completed = true; + if (args[0] === "complete-scan") { + completed = true; + savedCost = JSON.parse(args[args.indexOf("--cost-json") + 1]!); + } return mockWorkbench(args, input); }, createCodex: codexFactory(async (input: string) => { @@ -202,6 +211,13 @@ test.each(["completed", "receipt-lost", "scan-interrupted", "prompt-files"])( if (scenario === "prompt-files") expect(input).toContain(scanPrompt); if (scenario === "scan-interrupted" && modelCalls === 1) throw new Error("Synthetic interrupted scan"); + await writeUsageSession(codexHome, "thread-1", usage, { + model: "gpt-5.6-sol", + }); + await writeUsageSession(codexHome, "discovery-thread", usage, { + parent: "thread-1", + model: "gpt-5.6-luna", + }); await copyCompletedScan(root); return { events: completedEvents() }; }, "thread-1"), @@ -236,6 +252,10 @@ test.each(["completed", "receipt-lost", "scan-interrupted", "prompt-files"])( ...(scenario === "prompt-files" ? { scanPromptFile: replacement } : {}), }); expect(result.manifest.scan.id).toBe("scan_example_001"); + expect(result.cost?.estimatedUsd).toBe(expectedCostUsd); + expect(result.cost?.model).toContain("gpt-5.6-sol"); + expect(result.cost?.model).toContain("gpt-5.6-luna"); + expect(result.cost).toEqual(savedCost); expect(python).toHaveBeenCalledWith( expect.objectContaining({ protectedRoot: repository }), ); @@ -1225,6 +1245,21 @@ describe("CodexSecurity orchestration", () => { await expect( unpriced.preflight(repository, { maxCostUsd: 5 }), ).rejects.toThrow("cost limit is not available for the configured model"); + await expect( + client.preflight(repository, { + mode: "deep", + maxCostUsd: 5, + discoveryModel: "unknown-discovery-model", + }), + ).rejects.toThrow( + "cost limit is not available for the configured model: unknown-discovery-model", + ); + await expect( + client.preflight(repository, { + mode: "deep", + discoveryModel: "unknown-discovery-model", + }), + ).resolves.toMatchObject({ discoveryModel: "unknown-discovery-model" }); expect(prepareRuntime).not.toHaveBeenCalled(); await unpriced.close(); await client.close(); @@ -3403,6 +3438,8 @@ describe("CodexSecurity orchestration", () => { "stop_after_no_new = 7", "max_discovery_runs = 60", "max_time_hours = 48", + 'discovery_model = "synthetic-discovery"', + 'discovery_reasoning_effort = "high"', "[other]", "enabled = true", "", @@ -3459,6 +3496,8 @@ describe("CodexSecurity orchestration", () => { expect(configuration).toContain("stop_after_consecutive_errors = 2"); expect(configuration).toContain("max_discovery_runs = 10"); expect(configuration).toContain("max_time_hours = 1.5"); + expect(configuration).toContain('discovery_model = "synthetic-discovery"'); + expect(configuration).toContain('discovery_reasoning_effort = "high"'); expect(configuration).toContain("[other]"); expect(configuration).toContain("enabled = true"); expect(recipe).toMatchObject({ @@ -3472,6 +3511,8 @@ describe("CodexSecurity orchestration", () => { stopAfterConsecutiveErrors: 2, maxDiscoveryRuns: 10, maxTimeHours: 1.5, + discoveryModel: "synthetic-discovery", + discoveryReasoningEffort: "high", }, }); await client.close(); diff --git a/sdk/typescript/tests-ts/cost.test.ts b/sdk/typescript/tests-ts/cost.test.ts index 947cc610cb..f467508fbd 100644 --- a/sdk/typescript/tests-ts/cost.test.ts +++ b/sdk/typescript/tests-ts/cost.test.ts @@ -514,6 +514,227 @@ describe("scan cost", () => { }); describe("live scan cost tracking", () => { + test.each([ + ["gpt-5.6-luna", "gpt-5.6-sol", 4.0004, 8.0004, true], + ["gpt-5.6-sol", "gpt-5.6-luna", 0.208, 0.408, false], + ] as const)( + "prices %s parent and reducer separately from %s discovery, including resumed usage", + async ( + parentModel, + discoveryModel, + initialCost, + resumedCost, + exceedsBudget, + ) => { + const home = await codexHome(); + const parentUsage = { input_tokens: 1_000, output_tokens: 0 }; + await writeSession(home, "scan-thread", parentUsage, { + model: parentModel, + }); + await writeSession(home, "reducer-thread", parentUsage, { + parent: "scan-thread", + model: parentModel, + }); + const discoveryUsage = { input_tokens: 1_000_000, output_tokens: 0 }; + const worker = await writeSession( + home, + "discovery-thread", + discoveryUsage, + { + parent: "scan-thread", + model: discoveryModel, + }, + ); + const budget = new AbortController(); + const tracker = new ScanCostTracker({ + codexHome: home, + model: parentModel, + maxCostUsd: 1, + onCost: ({ estimatedUsd }) => { + if (estimatedUsd > 1) budget.abort(); + }, + }); + tracker.start("scan-thread"); + try { + const initial = await tracker.refresh(); + expect(initial.cost?.estimatedUsd).toBeCloseTo(initialCost, 10); + expect(initial.cost?.model).toBe("gpt-5.6-luna, gpt-5.6-sol"); + expect(initial.cost?.pricing).toBeUndefined(); + expect(budget.signal.aborted).toBe(exceedsBudget); + const resumedUsage = { input_tokens: 2_000_000, output_tokens: 0 }; + const response = nativeUsageRecord( + "discovery-thread", + "resumed-response", + discoveryUsage, + resumedUsage, + ); + await appendFile( + worker, + jsonLines([ + { type: "turn_context", payload: { model: discoveryModel } }, + response, + response, + legacyUsageRecord(discoveryUsage), + ]) + "\n", + ); + const resumed = await tracker.refresh(); + expect(resumed.cost?.estimatedUsd).toBeCloseTo(resumedCost, 10); + expect(resumed.usage).toMatchObject({ + input_tokens: 2_002_000, + output_tokens: 0, + }); + expect(budget.signal.aborted).toBe(exceedsBudget); + expect((await tracker.stop(parentUsage)).cost).toEqual(resumed.cost); + } finally { + await tracker.stop(); + } + }, + ); + + test("retains earlier model charges when a resumed session changes models", async () => { + const home = await codexHome(); + const usage = { input_tokens: 1_000_000, output_tokens: 0 }; + const path = await writeSession(home, "scan-thread", usage, { + model: "gpt-5.6-luna", + }); + const tracker = costTracker(home); + tracker.start("scan-thread"); + try { + expect((await tracker.refresh()).cost).toEqual( + estimateScanCost("gpt-5.6-luna", usage), + ); + await appendFile( + path, + jsonLines([ + { type: "turn_context", payload: { model: "gpt-5.6-sol" } }, + nativeUsageRecord("scan-thread", "second-turn", usage, { + input_tokens: 2_000_000, + output_tokens: 0, + }), + ]) + "\n", + ); + expect((await tracker.refresh()).cost?.estimatedUsd).toBe(4.2); + await appendFile( + path, + jsonLines([ + { type: "turn_context", payload: { model: "gpt-5.6-luna" } }, + legacyUsageRecord({ input_tokens: 3_000_000, output_tokens: 0 }), + ]) + "\n", + ); + const snapshot = await tracker.stop({ + input_tokens: 3_000_000, + output_tokens: 0, + }); + expect(snapshot.cost?.estimatedUsd).toBe(4.4); + expect(snapshot.cost?.inputTokens).toBe(3_000_000); + expect(snapshot.cost?.pricing).toBeUndefined(); + expect(snapshot.cost?.estimatedUsdRange?.min).toBe(4.4); + } finally { + await tracker.stop(); + } + }); + + test("keeps earlier-model detail when the final SDK receipt advances beyond its rollout", async () => { + const home = await codexHome(); + const original = { + input_tokens: 1_000, + cached_input_tokens: 20, + cache_write_input_tokens: 12, + output_tokens: 10, + reasoning_output_tokens: 5, + }; + const path = await writeSession(home, "scan-thread", original, { + model: "gpt-5.6-luna", + }); + await appendFile( + path, + jsonLines([ + { type: "turn_context", payload: { model: "gpt-5.6-sol" } }, + legacyUsageRecord({ + ...original, + input_tokens: 1_020, + output_tokens: 12, + }), + ]) + "\n", + ); + const tracker = costTracker(home, { maxCostUsd: 1 }); + tracker.start("scan-thread"); + const snapshot = await tracker.stop({ + input_tokens: 1_100, + cached_input_tokens: 20, + cache_write_input_tokens: 0, + output_tokens: 15, + }); + expect(snapshot.cost?.estimatedUsd).toBeCloseTo(0.000709, 12); + expect(snapshot.usage).toMatchObject({ + input_tokens: 1_100, + cached_input_tokens: 20, + cache_write_input_tokens: 12, + output_tokens: 15, + reasoning_output_tokens: 5, + }); + }); + + test("ignores inherited parent model context when pricing a fork", async () => { + const home = await codexHome(); + const path = await writeSession( + home, + childUuid7Thread, + {}, + { parent: scanThreadId }, + ); + const rollout = ownershipRollout([lowerUuid7Turn]); + rollout.splice(2, 0, { + type: "turn_context", + payload: { model: "gpt-5.6-sol" }, + }); + rollout.splice(rollout.length - 1, 0, { + type: "turn_context", + payload: { model: "gpt-5.6-luna" }, + }); + await writeFile(path, jsonLines(rollout) + "\n"); + const tracker = costTracker(home); + tracker.start(scanThreadId); + expect((await tracker.stop()).cost).toEqual( + estimateScanCost("gpt-5.6-luna", ownedSdkUsage), + ); + }); + + test.each([undefined, 1])( + "keeps unknown worker pricing unavailable with cost limit %p", + async (maxCostUsd) => { + const home = await codexHome(); + await writeSession( + home, + "worker-thread", + { input_tokens: 100, output_tokens: 10 }, + { + parent: "scan-thread", + model: "synthetic-unpriced-model", + }, + ); + const errors: unknown[] = []; + const usages: unknown[] = []; + const tracker = costTracker(home, { + maxCostUsd, + onUsage: (usage) => usages.push(usage), + onError: (error) => errors.push(error), + }); + tracker.start("scan-thread"); + if (maxCostUsd === undefined) { + expect((await tracker.stop()).cost).toBeNull(); + expect(errors).toEqual([]); + } else { + await waitFor(() => errors.length > 0); + expect(errors[0]).toBeInstanceOf(Error); + expect(String(errors[0])).toContain("synthetic-unpriced-model"); + await expect(tracker.stop()).rejects.toThrow("scan cost limit"); + } + expect(usages).toHaveLength(1); + expect(usages[0]).toMatchObject({ input_tokens: 100, output_tokens: 10 }); + }, + ); + test("keeps receipt cache-write detail when an equal legacy counter omits it", async () => { const home = await codexHome(); const usage = { diff --git a/sdk/typescript/tests-ts/deep-config.test.ts b/sdk/typescript/tests-ts/deep-config.test.ts index 4f4fb5b126..c46be139a9 100644 --- a/sdk/typescript/tests-ts/deep-config.test.ts +++ b/sdk/typescript/tests-ts/deep-config.test.ts @@ -123,9 +123,45 @@ test.each<{ ...discovery, }, ); + await writeFile(input.source, "invalid TOML ["); + const replay = await resolveDeepScanConfig(resolved.settings, input.source); + expect(replay.settings).toEqual(resolved.settings); + const replayDestination = join(input.root, "replay", "deep-scan.toml"); + await writeDeepScanConfig(replayDestination, replay); + expect( + parseToml(await readFile(replayDestination, "utf8"))["deep_scan"], + ).toEqual(parseToml(await readFile(destination, "utf8"))["deep_scan"]); }, ); +test("SDK discovery overrides merge independently with legacy settings", async () => { + const input = await fixture( + '[deep_scan]\ndiscovery_model = "legacy-model"\ndiscovery_reasoning_effort = "high"\n', + ); + const model = await resolveDeepScanConfig( + { discoveryModel: "sdk-model" }, + input.source, + ); + expect(model.settings).toMatchObject({ + discoveryModel: "sdk-model", + discoveryReasoningEffort: "high", + }); + const effort = await resolveDeepScanConfig( + { discoveryReasoningEffort: "low" }, + input.source, + ); + expect(effort.settings).toMatchObject({ + discoveryModel: "legacy-model", + discoveryReasoningEffort: "low", + }); + const complete = await resolveDeepScanConfig( + { ...DEFAULT_DEEP_SCAN_SETTINGS, discoveryModel: "sdk-model" }, + input.source, + ); + expect(complete.settings.discoveryModel).toBe("sdk-model"); + expect(complete.settings).not.toHaveProperty("discoveryReasoningEffort"); +}); + test("independent scans retain their own discovery configuration", async () => { const inputs = await Promise.all( ["first-model", "second-model"].map((model) => diff --git a/sdk/typescript/tests-ts/project-config.test.ts b/sdk/typescript/tests-ts/project-config.test.ts index bf1460a5dd..99a0640c50 100644 --- a/sdk/typescript/tests-ts/project-config.test.ts +++ b/sdk/typescript/tests-ts/project-config.test.ts @@ -361,6 +361,8 @@ describe("project configuration resolution", () => { project, { subagents: 0, + discoveryModel: "sdk-discovery", + discoveryReasoningEffort: "high", codexOverrides: { model: "gpt-5.6-sol", synthetic_setting: { enabled: false, itemCount: 0, names: [] }, @@ -370,6 +372,8 @@ describe("project configuration resolution", () => { ); expect(settings).toMatchObject({ subagents: 0, + discoveryModel: "sdk-discovery", + discoveryReasoningEffort: "high", workers: 8, }); expect(config).toEqual({ diff --git a/sdk/typescript/tests-ts/result.test.ts b/sdk/typescript/tests-ts/result.test.ts index 089bf6d9bb..44973f2437 100644 --- a/sdk/typescript/tests-ts/result.test.ts +++ b/sdk/typescript/tests-ts/result.test.ts @@ -52,6 +52,37 @@ const coverage = { } satisfies CoverageDocument; describe("ScanResult", () => { + test.each([ + null, + { + model: "gpt-5.6-luna, gpt-5.6-sol", + inputTokens: 1_100_000, + cachedInputTokens: 0, + cacheWriteInputTokens: 0, + outputTokens: 0, + estimatedUsd: 0.6, + }, + ])( + "preserves an observed multi-model cost, including unavailable pricing: %p", + (cost) => { + const result = new ScanResult({ + manifest, + findings, + coverage, + scanDir: "/scan", + threadId: "thread", + turnResult: { + model: "gpt-5.6-sol", + usage: { input_tokens: 1_100_000, output_tokens: 0 }, + }, + cost, + }); + expect(result.cost).toBe(cost); + expect(result.toJSON()["cost"]).toEqual(cost); + expect(new ScanResult({ ...result }).cost).toBe(cost); + }, + ); + test("rejects an unknown threshold with or without findings", () => { for (const levels of [[], ["high"]] satisfies SeverityLevel[][]) { const result = fakeResult(levels); diff --git a/sdk/typescript/tests-ts/scan-recipe.test.ts b/sdk/typescript/tests-ts/scan-recipe.test.ts index 41ff9c2fe7..daf5162a7b 100644 --- a/sdk/typescript/tests-ts/scan-recipe.test.ts +++ b/sdk/typescript/tests-ts/scan-recipe.test.ts @@ -34,6 +34,14 @@ test.each( const codexHome = join(root, "state", "codex-home"); await mkdir(repository); await mkdir(codexHome, { recursive: true }); + const legacyConfig = join(codexHome, "codex-security", "config.toml"); + if (mode === "deep") { + await mkdir(join(codexHome, "codex-security")); + await writeFile( + legacyConfig, + '[deep_scan]\ndiscovery_model = "synthetic-discovery"\ndiscovery_reasoning_effort = "high"\n', + ); + } await writeFile(join(repository, "source.py"), "# synthetic source\n"); const promptFile = join(root, "post-scan.md"); const postScanPrompt = @@ -112,6 +120,15 @@ test.each( scans[0]!.scanId, ]); expect(saved["recipe"]).toMatchObject({ mode, postScanPrompt }); + if (mode === "deep") { + expect(saved["recipe"]).toMatchObject({ + deepScan: { + discoveryModel: "synthetic-discovery", + discoveryReasoningEffort: "high", + }, + }); + await writeFile(legacyConfig, "invalid TOML ["); + } expect(JSON.stringify(saved)).not.toContain(promptFile); expect(JSON.stringify(saved)).not.toContain("synthetic-launch-key"); expect(JSON.stringify(saved)).not.toContain("synthetic-bedrock-key"); @@ -120,6 +137,17 @@ test.each( ...dependencies({ environment, currentDirectory: root, + onTurn: (_repository, options) => { + if (mode === "deep") + expect(options).toMatchObject({ + discoveryModel: "synthetic-discovery", + discoveryReasoningEffort: "high", + }); + else { + expect(options).not.toHaveProperty("discoveryModel"); + expect(options).not.toHaveProperty("discoveryReasoningEffort"); + } + }, onConfig: ({ codexOverrides }) => { expect(resolveCodexProfile(codexOverrides!)).toMatchObject( selected, diff --git a/sdk/typescript/tests-ts/support/usage-rollout.ts b/sdk/typescript/tests-ts/support/usage-rollout.ts index 87030c7365..8e80fc93c3 100644 --- a/sdk/typescript/tests-ts/support/usage-rollout.ts +++ b/sdk/typescript/tests-ts/support/usage-rollout.ts @@ -149,11 +149,13 @@ export async function writeSession( parent, cwd, timestamp, + model, parentField = "source", }: { parent?: string; cwd?: string; timestamp?: string; + model?: string; parentField?: SessionParentField; } = {}, ): Promise { @@ -172,6 +174,9 @@ export async function writeSession( ...(parent === undefined ? {} : parentMetadata(parent, parentField)), }, }), + ...(model === undefined + ? [] + : [JSON.stringify({ type: "turn_context", payload: { model } })]), JSON.stringify({ type: "event_msg", payload: { From 81e06750c7849d03aa30316181ce02fd4fd11e3e Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 17 Oct 2026 03:45:39 +0000 Subject: [PATCH 3/5] fix(deep-scan): restore discovery settings during schema repair --- .../mcp-app/src/workbench/migrations.ts | 2 +- .../mcp-app/tests/test_workbench_database.ts | 15 +++++++++++++-- .../codex-security/scripts/workbench_schema.py | 2 +- .../tests/test_workbench_setup_and_migrations.py | 14 ++++++++++++++ 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/plugins/codex-security/mcp-app/src/workbench/migrations.ts b/plugins/codex-security/mcp-app/src/workbench/migrations.ts index bc917ec1f2..8275c5d932 100644 --- a/plugins/codex-security/mcp-app/src/workbench/migrations.ts +++ b/plugins/codex-security/mcp-app/src/workbench/migrations.ts @@ -224,7 +224,7 @@ export function applyMigrations( ); } else if (item.version === 16) backfill = repairStableTargets(database); else if (applied.has(item.version)) { - if ([2, 12, 13, 26, 28, 31, 32, 47].includes(item.version)) + if ([2, 12, 13, 26, 28, 31, 32, 47, 50].includes(item.version)) repairAdditive(database, item.version); else if (item.version === 11) repairDeepScan(database); } else { diff --git a/plugins/codex-security/mcp-app/tests/test_workbench_database.ts b/plugins/codex-security/mcp-app/tests/test_workbench_database.ts index ff6dc51a7c..c375a3542b 100644 --- a/plugins/codex-security/mcp-app/tests/test_workbench_database.ts +++ b/plugins/codex-security/mcp-app/tests/test_workbench_database.ts @@ -511,7 +511,7 @@ test("recorded additive migrations restore missing columns and configured error database.exec(`INSERT INTO deep_scan_runs (scan_id, schema_version, workflow_version, status, phase, workers, subagents, stop_after_no_new, max_discovery_runs, created_at, updated_at) VALUES ('scan', 1, 'v1', 'running', 'discovery', 1, 0, 7, 10, 'created', 'updated')`); - for (const version of [27, 28, 31, 32, 47]) + for (const version of [27, 28, 31, 32, 47, 50]) database .prepare("INSERT INTO schema_migrations VALUES (?, ?, 'original')") .run(version, migrations[version - 1].name); @@ -528,8 +528,18 @@ test("recorded additive migrations restore missing columns and configured error .get()?.discovery_user_context_json, null, ); + const discoverySettings = () => + Object.values( + database + .prepare( + "SELECT discovery_model, discovery_reasoning_effort FROM deep_scan_runs", + ) + .get()!, + ); + assert.deepEqual(discoverySettings(), [null, null]); database.exec(`UPDATE deep_scan_runs SET stop_after_consecutive_errors = 2, - discovery_user_context_json = '"Original discovery context"'`); + discovery_user_context_json = '"Original discovery context"', + discovery_model = 'fixture-model', discovery_reasoning_effort = 'custom-effort'`); applyMigrations(database); assert.equal( database @@ -543,6 +553,7 @@ test("recorded additive migrations restore missing columns and configured error .get()?.discovery_user_context_json, '"Original discovery context"', ); + assert.deepEqual(discoverySettings(), ["fixture-model", "custom-effort"]); }); test("workflow and review checkpoints upgrade atomically with their preserved result data", (t) => { diff --git a/plugins/codex-security/scripts/workbench_schema.py b/plugins/codex-security/scripts/workbench_schema.py index 90271d5c1f..ccf1ddd695 100644 --- a/plugins/codex-security/scripts/workbench_schema.py +++ b/plugins/codex-security/scripts/workbench_schema.py @@ -75,7 +75,7 @@ def apply_migrations( elif version == 16: should_backfill_targets = repair_stable_targets_migration(connection) elif version in applied: - if version in (2, 12, 13, 26, 28, 31, 32, 47): + if version in (2, 12, 13, 26, 28, 31, 32, 47, 50): repair_additive_migration(connection, version) elif version == 11: repair_deep_scan_migration(connection) diff --git a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py index 6e27705d6d..34853a9cf6 100644 --- a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py +++ b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py @@ -1559,6 +1559,20 @@ def test_workbench_repairs_recorded_deep_scan_migration( assert "max_time_hours" in { row[1] for row in connection.execute("PRAGMA table_info(deep_scan_runs)") } + connection.execute( + """ + INSERT INTO deep_scan_runs ( + scan_id, schema_version, workflow_version, status, phase, + workers, subagents, stop_after_no_new, max_discovery_runs, + discovery_model, discovery_reasoning_effort, created_at, updated_at + ) VALUES (?, 1, 'deep-scan-mcp/v1', 'running', 'discovery', + 4, 0, 4, 40, 'fixture-model', 'custom-effort', 'created', 'updated') + """, + (scan_id,), + ) + assert connection.execute( + "SELECT discovery_model, discovery_reasoning_effort FROM deep_scan_runs" + ).fetchone() == ("fixture-model", "custom-effort") assert { row[0] for row in connection.execute( From 0db59926d78cd3ee1d87415c25b5b52a0936626c Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 17 Oct 2026 04:06:40 +0000 Subject: [PATCH 4/5] fix(sdk): preserve cost recomputation when copying scan results --- sdk/typescript/src/api.ts | 10 ++++---- sdk/typescript/src/result.ts | 6 ++--- sdk/typescript/tests-ts/result.test.ts | 32 ++++++++++++++++++++++++-- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/sdk/typescript/src/api.ts b/sdk/typescript/src/api.ts index e27c4e737f..42917dd81d 100644 --- a/sdk/typescript/src/api.ts +++ b/sdk/typescript/src/api.ts @@ -751,8 +751,8 @@ export class CodexSecurity { }); type ScanMetadata = Pick< ScanResultOptions, - "threadId" | "turnResult" | "sarifPath" | "repositoryFindings" | "cost" - >; + "threadId" | "turnResult" | "sarifPath" | "repositoryFindings" + > & { cost?: Readonly | null }; if (state.scanId && state.scanDir) { await workflow.protectArtifacts(state.scanDir); let metadata = state.stages.scan.result as ScanMetadata | undefined; @@ -782,6 +782,7 @@ export class CodexSecurity { ...contract, scanDir: state.scanDir, ...metadata, + observedCost: metadata.cost, threatModelPath: await readThreatModelPath(state.scanDir, { pythonPath: this.config.pythonPath, protectedRoot: local.protectedRoot, @@ -2645,7 +2646,7 @@ export class CodexSecurity { ); result = new ScanResult({ ...result, - cost: completionCost, + observedCost: completionCost, ...(directDeepScan ? { turnResult: { @@ -2727,6 +2728,7 @@ export class CodexSecurity { checkOpen(); result = new ScanResult({ ...result, + observedCost: result.cost, threatModelPath: isDeepStrictEqual( result.threatModel, followUpResult.threatModel, @@ -2920,7 +2922,7 @@ export class CodexSecurity { } return new ScanResult({ ...result, - cost: snapshot?.cost ?? failure.cost, + observedCost: snapshot?.cost ?? failure.cost, }); } catch {} } diff --git a/sdk/typescript/src/result.ts b/sdk/typescript/src/result.ts index eeeb082aed..2830e890e1 100644 --- a/sdk/typescript/src/result.ts +++ b/sdk/typescript/src/result.ts @@ -42,7 +42,7 @@ export interface ScanResultOptions { scanDir: string; threadId: string; turnResult: TurnResultMetadata; - cost?: Readonly | null; + observedCost?: Readonly | null; sarifPath?: string | null; threatModelPath?: string | null; repositoryFindings?: readonly RepositoryFinding[]; @@ -70,9 +70,9 @@ export class ScanResult { this.repositoryFindings = options.repositoryFindings; this.threatModelPath = options.threatModelPath ?? null; this.cost = - options.cost === undefined + options.observedCost === undefined ? estimateScanCost(options.turnResult.model, options.turnResult.usage) - : options.cost; + : options.observedCost; if (options.sarifPath !== undefined) { this.sarifPath = options.sarifPath; } else { diff --git a/sdk/typescript/tests-ts/result.test.ts b/sdk/typescript/tests-ts/result.test.ts index 44973f2437..c97590a848 100644 --- a/sdk/typescript/tests-ts/result.test.ts +++ b/sdk/typescript/tests-ts/result.test.ts @@ -75,14 +75,42 @@ describe("ScanResult", () => { model: "gpt-5.6-sol", usage: { input_tokens: 1_100_000, output_tokens: 0 }, }, - cost, + observedCost: cost, }); expect(result.cost).toBe(cost); expect(result.toJSON()["cost"]).toEqual(cost); - expect(new ScanResult({ ...result }).cost).toBe(cost); + expect( + new ScanResult({ ...result, observedCost: result.cost }).cost, + ).toBe(cost); }, ); + test("recomputes the cost when a copied result replaces its turn metadata", () => { + const original = new ScanResult({ + manifest, + findings, + coverage, + scanDir: "/scan", + threadId: "thread", + turnResult: { + model: "gpt-5.6-sol", + usage: { input_tokens: 1_000_000, output_tokens: 0 }, + }, + }); + expect(original.cost?.estimatedUsd).toBe(4); + const repriced = new ScanResult({ + ...original, + turnResult: { ...original.turnResult, model: "gpt-5.6-luna" }, + }); + expect(repriced.cost?.estimatedUsd).toBe(0.2); + const unpriced = new ScanResult({ + ...original, + turnResult: { ...original.turnResult, model: "synthetic-unpriced-model" }, + }); + expect(unpriced.cost).toBeNull(); + expect(original.cost?.estimatedUsd).toBe(4); + }); + test("rejects an unknown threshold with or without findings", () => { for (const levels of [[], ["high"]] satisfies SeverityLevel[][]) { const result = fakeResult(levels); From e55b1c278db98ef0fd2ce93448723c51f84b16ce Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 17 Oct 2026 04:22:55 +0000 Subject: [PATCH 5/5] fix(sdk): clear stale estimates when model pricing is unavailable --- sdk/typescript/src/cli.ts | 7 +- sdk/typescript/src/cost.ts | 24 +++--- sdk/typescript/src/scan-dashboard.ts | 10 ++- sdk/typescript/tests-ts/cli-show-cost.test.ts | 42 ++++++++++ sdk/typescript/tests-ts/cost.test.ts | 64 ++++++++++++++++ .../tests-ts/scan-dashboard.test.ts | 76 +++++++++++++++++++ 6 files changed, 210 insertions(+), 13 deletions(-) diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index a49a922cd7..88f40290cc 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -8212,6 +8212,7 @@ async function executeScan( let deepProgress: DeepScanProgress | null = null; let deepConsolidating = false; let runningCost: Readonly | null = null; + let costUpdates = 0; let runningUsage: Readonly | null = null; let maxCostUsd = arguments_.maxCostUsd; const showCost = arguments_.showCost === true || maxCostUsd !== undefined; @@ -8490,15 +8491,17 @@ async function executeScan( dashboard.setUsage(usage); return; } - const priorCost = runningCost; + const priorCostUpdates = costUpdates; // The tracker queues usage before cost; let the paired cost update render once. await Promise.resolve(); - if (runningCost !== priorCost) return; + if (costUpdates !== priorCostUpdates) return; + runningCost = null; progress?.stopTimer(); progress?.stage(`Tokens: ${formatTokenUsage(usage)}.`); progress?.startTimer(runningMessage()); }, onCost: (cost, limit = maxCostUsd) => { + costUpdates += 1; if (limit !== maxCostUsd && limit !== undefined) { dashboard?.note(`Total cost limit increased to ${formatUsd(limit)}.`); } diff --git a/sdk/typescript/src/cost.ts b/sdk/typescript/src/cost.ts index 9704ae5354..3ab10bde5e 100644 --- a/sdk/typescript/src/cost.ts +++ b/sdk/typescript/src/cost.ts @@ -198,8 +198,8 @@ export class ScanCostTracker { return this.#snapshot; const cost = estimateScanCost(this.#options.model, fallbackUsage); this.#snapshot = { usage: fallbackUsage ?? null, cost }; - this.#reportUsage(tokenUsage(fallbackUsage)); - this.#reportCost(cost); + const reportedUsage = this.#reportUsage(tokenUsage(fallbackUsage)); + this.#reportCost(cost, reportedUsage); return this.#snapshot; } @@ -382,6 +382,10 @@ export class ScanCostTracker { const cost = estimateScanCost(model, value); if (cost === null) { this.#snapshot = { usage, cost: null }; + if (this.#lastCost !== null) { + this.#lastCost = null; + this.#lastUsage = null; + } this.#reportUsage(usage); if (this.#options.maxCostUsd !== undefined) { throw new Error( @@ -426,8 +430,8 @@ export class ScanCostTracker { }, }; this.#snapshot = { usage, cost }; - this.#reportUsage(usage); - this.#reportCost(cost); + const reportedUsage = this.#reportUsage(usage); + this.#reportCost(cost, reportedUsage); } #reportWorkerProgress(session: SessionUsage): void { @@ -469,20 +473,22 @@ export class ScanCostTracker { } } - #reportCost(cost: ScanCost | null): void { + #reportCost(cost: ScanCost | null, reportedUsage = false): void { if (cost === null) return; const signature = JSON.stringify(cost); - if (signature === this.#lastCost) return; + // Usage without a paired cost update means its pricing is unavailable. + if (signature === this.#lastCost && !reportedUsage) return; this.#lastCost = signature; this.#options.onCost?.(cost); } - #reportUsage(usage: ScanTokenUsage | null): void { - if (usage === null) return; + #reportUsage(usage: ScanTokenUsage | null): boolean { + if (usage === null) return false; const signature = JSON.stringify(usage); - if (signature === this.#lastUsage) return; + if (signature === this.#lastUsage) return false; this.#lastUsage = signature; this.#options.onUsage?.(usage); + return this.#options.onUsage !== undefined; } } diff --git a/sdk/typescript/src/scan-dashboard.ts b/sdk/typescript/src/scan-dashboard.ts index e341b6f062..eabc9fc6ed 100644 --- a/sdk/typescript/src/scan-dashboard.ts +++ b/sdk/typescript/src/scan-dashboard.ts @@ -555,6 +555,7 @@ export class ScanDashboard { } public setUsage(usage: Readonly): void { + this.#cost = null; this.#tokens = formatTokenUsage(usage); this.#refresh(); } @@ -952,9 +953,12 @@ export class ScanDashboard { const costs = this.#components.flatMap(({ dashboard }) => dashboard.#cost === null ? [] : [dashboard.#cost], ); + const unavailable = this.#components.some( + ({ dashboard }) => dashboard.#tokens !== null && dashboard.#cost === null, + ); return wrapActivity( " COST ", - `${costs.length === 0 ? "waiting for usage" : formatScanCosts(costs)} · component scans only`, + `${unavailable ? "unavailable" : costs.length === 0 ? "waiting for usage" : formatScanCosts(costs)} · component scans only`, this.#width(), ); } @@ -1002,7 +1006,9 @@ export class ScanDashboard { const cost = this.#cost === null ? this.#options.maxCostUsd === undefined - ? "waiting for usage" + ? this.#tokens === null + ? "waiting for usage" + : "unavailable" : `— / ${formatUsd(this.#options.maxCostUsd)}` : `${formatScanCost(this.#cost)}${this.#options.maxCostUsd === undefined ? "" : `; short-context budget baseline: ${formatUsd(this.#cost.estimatedUsd)} / ${formatUsd(this.#options.maxCostUsd)} · ${budgetBar(this.#cost.estimatedUsd, this.#options.maxCostUsd)}`}`; return wrapActivity(" COST ", cost, this.#width()); diff --git a/sdk/typescript/tests-ts/cli-show-cost.test.ts b/sdk/typescript/tests-ts/cli-show-cost.test.ts index 457b5b3d08..df259e64f5 100644 --- a/sdk/typescript/tests-ts/cli-show-cost.test.ts +++ b/sdk/typescript/tests-ts/cli-show-cost.test.ts @@ -159,3 +159,45 @@ test.each( expect(outputs[1]).toBe(outputs[0]); expect(outputs[2]).toBe(outputs[0]); }); + +test.each(["direct", "deferred"])( + "clears stale prices and restores a paired update delivered %s", + async (delivery) => { + const { stderr, runCli } = createCliTest(main); + const deps = dependencies(); + const largerUsage = { + ...observedUsage, + input_tokens: 2_250, + total_tokens: 2_280, + }; + deps.createSecurity = () => + fakeSecurity(async (_repository, options = {}) => { + options.onScanStarted?.(); + options.onCost?.(result.cost!); + await options.onUsage?.(largerUsage); + options.onScanStarted?.(); + let running = stderr.text().trim().split("\n").at(-1)!; + expect(running).toContain("2,280 total"); + expect(running).not.toContain("$0.00488"); + for (let repeat = 0; repeat < 2; repeat += 1) { + // Reusing the same cost object must still count as a paired update. + if (delivery === "direct") { + const observed = options.onUsage?.(observedUsage); + options.onCost?.(result.cost!); + await observed; + } else { + await Promise.all([ + Promise.resolve().then(() => options.onUsage?.(observedUsage)), + Promise.resolve().then(() => options.onCost?.(result.cost!)), + ]); + } + options.onScanStarted?.(); + running = stderr.text().trim().split("\n").at(-1)!; + expect(running).toContain("1,280 total"); + expect(running).toContain("$0.00488"); + } + return result; + }); + expect(await runCli(["scan", ".", "--show-cost"], deps)).toBe(0); + }, +); diff --git a/sdk/typescript/tests-ts/cost.test.ts b/sdk/typescript/tests-ts/cost.test.ts index f467508fbd..bb712d317e 100644 --- a/sdk/typescript/tests-ts/cost.test.ts +++ b/sdk/typescript/tests-ts/cost.test.ts @@ -514,6 +514,70 @@ describe("scan cost", () => { }); describe("live scan cost tracking", () => { + test.each([1_000, 2_000])( + "invalidates a prior estimate when an unpriced rollout reports %p tokens", + async (inputTokens) => { + const home = await codexHome(); + const updates: string[] = []; + const tracker = costTracker(home, { + onUsage: () => updates.push("usage"), + onCost: () => updates.push("cost"), + }); + tracker.recordUsage( + { input_tokens: 1_000, output_tokens: 0 }, + "worker-thread", + ); + tracker.start("scan-thread"); + try { + expect((await tracker.refresh()).cost).not.toBeNull(); + expect(updates.splice(0)).toEqual(["usage", "cost"]); + await writeSession( + home, + "worker-thread", + { + input_tokens: inputTokens, + output_tokens: 0, + }, + { model: "synthetic-unpriced-model" }, + ); + expect((await tracker.refresh()).cost).toBeNull(); + expect(updates.splice(0)).toEqual(["usage"]); + await tracker.refresh(); + expect(updates).toEqual([]); + } finally { + await tracker.stop(); + } + }, + ); + + test("pairs changed usage with its still-valid unchanged price", async () => { + const home = await codexHome(); + const usage = { input_tokens: 100, output_tokens: 10 }; + const path = await writeSession(home, "scan-thread", usage); + const updates: string[] = []; + const tracker = costTracker(home, { + onUsage: () => updates.push("usage"), + onCost: () => updates.push("cost"), + }); + tracker.start("scan-thread"); + try { + const original = await tracker.refresh(); + expect(updates.splice(0)).toEqual(["usage", "cost"]); + await appendFile( + path, + jsonLines([ + legacyUsageRecord({ ...usage, reasoning_output_tokens: 5 }), + ]) + "\n", + ); + expect((await tracker.refresh()).cost).toEqual(original.cost); + expect(updates.splice(0)).toEqual(["usage", "cost"]); + await tracker.refresh(); + expect(updates).toEqual([]); + } finally { + await tracker.stop(); + } + }); + test.each([ ["gpt-5.6-luna", "gpt-5.6-sol", 4.0004, 8.0004, true], ["gpt-5.6-sol", "gpt-5.6-luna", 0.208, 0.408, false], diff --git a/sdk/typescript/tests-ts/scan-dashboard.test.ts b/sdk/typescript/tests-ts/scan-dashboard.test.ts index ea41aa7e11..d1d7b7e663 100644 --- a/sdk/typescript/tests-ts/scan-dashboard.test.ts +++ b/sdk/typescript/tests-ts/scan-dashboard.test.ts @@ -7,6 +7,7 @@ import { afterEach, describe, expect, test, mock, jest } from "bun:test"; import type { ComponentReceipt } from "../src/component-scan.js"; import { ScanDashboard } from "../src/scan-dashboard.js"; import { capture, fakeResult } from "./cli-fixtures.js"; +import { tokenUsage } from "../src/cost-model.js"; afterEach(() => jest.useRealTimers()); @@ -57,6 +58,81 @@ class DashboardTestInput extends EventEmitter { } describe("live scan dashboard", () => { + test("clears an unavailable price while retaining current tokens and restores a later estimate", () => { + const stderr = capture(true); + const dashboard = createDashboard(stderr.stream, { + model: { model: "gpt-5.6-sol", reasoningEffort: "xhigh" }, + showCost: true, + }); + const cost = fakeResult([], "complete", { + input_tokens: 1_000_000, + output_tokens: 0, + }).cost!; + dashboard.start(); + try { + dashboard.setCost(cost); + expect(lastFrame(stderr)).toContain("$4.00"); + dashboard.setUsage( + tokenUsage({ input_tokens: 3_000_000, output_tokens: 0 })!, + ); + expect(lastFrame(stderr)).toContain("3,000,000 total"); + expect(lastFrame(stderr)).toMatch(/COST\s+unavailable/); + expect(lastFrame(stderr)).not.toContain("$4.00"); + dashboard.setCost(cost); + expect(lastFrame(stderr)).toContain("$4.00"); + expect(lastFrame(stderr)).not.toMatch(/COST\s+unavailable/); + } finally { + dashboard.stop(); + } + }); + + test("does not present a known component subtotal as the complete cost", () => { + const stderr = capture(true); + const dashboard = createDashboard(stderr.stream, { + presentation: "components", + showCost: true, + }); + const receipts: ComponentReceipt[] = [1, 2].map((id) => ({ + id: `component-${id}`, + name: `Component ${id}`, + paths: [`src/${id}`], + status: "started", + outputDir: `/synthetic/results/${id}`, + })); + const cost = fakeResult([], "complete", { + input_tokens: 1_000_000, + output_tokens: 0, + }).cost!; + const frame = () => + stripVTControlCharacters(stderr.text().split("\u001B[H").at(-1)!); + dashboard.start(); + try { + dashboard.setComponents(receipts); + for (const receipt of receipts) { + dashboard.recordComponentEvent({ + componentId: receipt.id, + type: "cost", + value: cost, + }); + } + expect(frame().match(/COST\s+([^\n]+)/)?.[1]).toContain("$8.00"); + dashboard.recordComponentEvent({ + componentId: "component-2", + type: "usage", + value: tokenUsage({ input_tokens: 2_000_000, output_tokens: 0 })!, + }); + expect(frame()).toMatch(/COST\s+unavailable/); + dashboard.recordComponentEvent({ + componentId: "component-2", + type: "cost", + value: cost, + }); + expect(frame().match(/COST\s+([^\n]+)/)?.[1]).toContain("$8.00"); + } finally { + dashboard.stop(); + } + }); + test("flushes queued activity before stopping without rendering afterward", () => { const stderr = capture(true); const pending: (() => void)[] = [];