diff --git a/docs/project-configuration.md b/docs/project-configuration.md index 137de4c4e..34ff3f051 100644 --- a/docs/project-configuration.md +++ b/docs/project-configuration.md @@ -242,6 +242,25 @@ If a runtime aliases the ambient TOML file, only explicit overrides are written back; unrelated sections and inherited defaults remain untouched. Isolated runtime files still receive a complete snapshot. +Optional discovery model settings are available in the legacy user +`$CODEX_HOME/codex-security/config.toml` file (`~/.codex` when `CODEX_HOME` is unset): + +```toml +[deep_scan] +discovery_model = "your-discovery-model" +discovery_reasoning_effort = "high" +``` + +Each omitted setting inherits the parent scan's choice. Setup and reducer workers +keep the parent settings. Discovery choices are saved when the scan starts and +retained when workers resume or the scan is rerun. SDK callers can set +`discoveryModel` and `discoveryReasoningEffort` directly; each optional string +overrides its legacy TOML value. These settings have no new CLI flags or +project-file keys. Supplying all six orchestration settings forms a complete +configuration that does not read legacy TOML; include any desired discovery +overrides explicitly. Saved recipes preserve the original choices without reading +today's legacy configuration. + A valid deep block can stay inactive in standard mode. Explicit deep CLI options still require deep mode. Deep diff scans and custom validation remain unsupported. Counts retain their existing bounds; zero subagents is valid, and discovery time diff --git a/plugins/codex-security/mcp-app/src/deep-scan/engine.ts b/plugins/codex-security/mcp-app/src/deep-scan/engine.ts index 0316ab8bb..52ccc6679 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/engine.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/engine.ts @@ -34,6 +34,10 @@ export function startDeepScanEngine(options: { executor: new CodexSdkWorkerExecutor({ model: options.model, reasoningEffort: options.reasoningEffort, + discovery: { + model: run.config.discoveryModel, + reasoningEffort: run.config.discoveryReasoningEffort, + }, parentSandbox: options.parentSandbox, artifactContext: { pluginRoot, diff --git a/plugins/codex-security/mcp-app/src/deep-scan/executor.ts b/plugins/codex-security/mcp-app/src/deep-scan/executor.ts index 9e2d3a8d8..2c692f56b 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/executor.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/executor.ts @@ -54,6 +54,10 @@ import type { export interface CodexSdkWorkerModelSettings { model?: string; reasoningEffort?: string; + discovery?: { + model?: string; + reasoningEffort?: string; + }; artifactContext?: CodexSdkWorkerArtifactContext; parentSandbox?: DeepWorkerParentSandbox; } @@ -121,13 +125,16 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { // Keep host-bound inputs beside the prompt, outside the worker's output root. await writePrivateFile(includePathsFile, scopeInput); } + const discovery = + request.kind === "discovery" ? this.modelSettings.discovery : undefined; + const model = discovery?.model ?? this.modelSettings.model; + const reasoningEffort = + discovery?.reasoningEffort ?? this.modelSettings.reasoningEffort; // Keep one native configuration for the policy check and the worker turn. // Worker-owned tool and permission settings take precedence over inheritance. const configOverrides = profileConfigOverrides({ ...runtimeSettings.config, - ...(this.modelSettings.reasoningEffort - ? { model_reasoning_effort: this.modelSettings.reasoningEffort } - : {}), + ...(reasoningEffort ? { model_reasoning_effort: reasoningEffort } : {}), mcp_servers: { // A disabled server still needs a valid transport during native resolution. "codex-security": { command: "node", enabled: false }, @@ -187,9 +194,7 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { profileName: runtimeSettings.nativeProfile, }); const threadOptions = { - ...(this.modelSettings.model - ? { model: this.modelSettings.model } - : {}), + ...(model ? { model } : {}), threadSource: "security_scan", skipGitRepoCheck: true, workingDirectory: request.workingDirectory, diff --git a/plugins/codex-security/mcp-app/src/deep-scan/response.ts b/plugins/codex-security/mcp-app/src/deep-scan/response.ts index fbde85b76..954ba4512 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/response.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/response.ts @@ -16,6 +16,9 @@ const responseObject = (shape: T) => return value; }); const optionalString = optionalValue(requiredString); +const optionalConfiguredString = requiredString + .refine((value) => value.trim().length > 0) + .nullish(); export const workerSchema = responseObject({ id: requiredString, @@ -50,13 +53,25 @@ export const configSchema = z stopAfterConsecutiveErrors: positiveInteger.nullish(), maxDiscoveryRuns: positiveInteger, maxTimeHours: z.number().positive().max(96).optional(), + discoveryModel: optionalConfiguredString, + discoveryReasoningEffort: optionalConfiguredString, }) - .transform(({ stopAfterConsecutiveErrors, maxTimeHours, ...config }) => ({ - ...config, - stopAfterConsecutiveErrors: - stopAfterConsecutiveErrors ?? config.stopAfterNoNew, - ...(maxTimeHours === undefined ? {} : { maxTimeHours }), - })); + .transform( + ({ + stopAfterConsecutiveErrors, + maxTimeHours, + discoveryModel, + discoveryReasoningEffort, + ...config + }) => ({ + ...config, + stopAfterConsecutiveErrors: + stopAfterConsecutiveErrors ?? config.stopAfterNoNew, + ...(maxTimeHours === undefined ? {} : { maxTimeHours }), + ...(discoveryModel == null ? {} : { discoveryModel }), + ...(discoveryReasoningEffort == null ? {} : { discoveryReasoningEffort }), + }), + ); // Check the config object before status, then its fields before the run fields. export const runSchema = z diff --git a/plugins/codex-security/mcp-app/src/workbench/migrations.ts b/plugins/codex-security/mcp-app/src/workbench/migrations.ts index bc917ec1f..8275c5d93 100644 --- a/plugins/codex-security/mcp-app/src/workbench/migrations.ts +++ b/plugins/codex-security/mcp-app/src/workbench/migrations.ts @@ -224,7 +224,7 @@ export function applyMigrations( ); } else if (item.version === 16) backfill = repairStableTargets(database); else if (applied.has(item.version)) { - if ([2, 12, 13, 26, 28, 31, 32, 47].includes(item.version)) + if ([2, 12, 13, 26, 28, 31, 32, 47, 50].includes(item.version)) repairAdditive(database, item.version); else if (item.version === 11) repairDeepScan(database); } else { diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts index cbcc0d11f..b6f3ab7b6 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts @@ -1446,9 +1446,21 @@ wire_api = "responses" env_key = "SYNTHETIC_GATEWAY_KEY"`, ); const settings = [ - { model: "gpt-5.6-sol", reasoningEffort: "xhigh" }, - { model: "gpt-6-astra", reasoningEffort: "ultra" }, - { model: "gpt-6.1-sol", reasoningEffort: "max" }, + { + model: "gpt-5.6-sol", + reasoningEffort: "xhigh", + discovery: { model: "gpt-5.6-luna", reasoningEffort: "medium" }, + }, + { + model: "gpt-6-astra", + reasoningEffort: "ultra", + discovery: { model: "fixture-discovery-model" }, + }, + { + model: "gpt-6.1-sol", + reasoningEffort: "max", + discovery: { reasoningEffort: "fixture-future-effort" }, + }, { model: "gpt-6-sol", reasoningEffort: "high" }, { model: "fixture-future-model", reasoningEffort: "future-effort" }, // Omitted settings preserve the model and effort in the Codex home. @@ -1859,7 +1871,7 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`, }), ); // A running coordinator retains its settings if the source file changes. - for (const kind of ["discovery", "dedup"] as const) { + for (const kind of ["discovery", "dedup", "setup"] as const) { for (const resumeThreadId of [undefined, "fixture-resumed-thread"]) { launches.length = 0; await Promise.all( @@ -1922,14 +1934,16 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`, ({ args }) => args[0] === "exec", ); assert.equal(workerLaunches.length, settings.length); - for (const [ - index, - { model, reasoningEffort }, - ] of settings.entries()) { - const workerLaunch = workerLaunches.find(({ args }) => - model === undefined - ? !args.includes("--model") - : args[args.indexOf("--model") + 1] === model, + for (const [index, parent] of settings.entries()) { + const discovery = + kind === "discovery" ? parent.discovery : undefined; + const model = discovery?.model ?? parent.model; + const reasoningEffort = + discovery?.reasoningEffort ?? parent.reasoningEffort; + const workerLaunch = workerLaunches.find( + ({ environment }) => + environment!.CODEX_SECURITY_CONFIG_PATH === + workerConfigurations[index].path, ); assert.ok(workerLaunch, `missing worker launch for ${model}`); assert.equal( diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.ts index 3345f8474..910fdb10d 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_stdio_lifecycle.ts @@ -75,6 +75,13 @@ async function testDeepScanStdioLifecycle() { `.deep-scan-stdio-test-${randomUUID()}.cjs`, ); const threadId = "deep-scan-stdio-lifecycle-thread"; + const deepConfigPath = path.join(codexHome, "codex-security", "config.toml"); + const deepConfig = `[deep_scan] +workers = 1 +subagents = 0 +stop_after_no_new = 1 +max_discovery_runs = 2 +`; await mkdir(targetPath, { recursive: true }); await mkdir(failedTargetPath, { recursive: true }); @@ -111,12 +118,9 @@ async function testDeepScanStdioLifecycle() { "print('failure fixture')\n", ); await writeFile( - path.join(codexHome, "codex-security", "config.toml"), - `[deep_scan] -workers = 1 -subagents = 0 -stop_after_no_new = 1 -max_discovery_runs = 2 + deepConfigPath, + `${deepConfig}discovery_model = "gpt-5.6-luna" +discovery_reasoning_effort = "medium" `, ); await writeFakeCodex(fakeCodexPath); @@ -273,9 +277,11 @@ model_reasoning_summary = "none" ), { code: "ENOENT" }, ); - assertFlagPair(startedWorker.argv, "--model", "gpt-5.5"); + assert.equal(startedState.config.discoveryModel, "gpt-5.6-luna"); + assert.equal(startedState.config.discoveryReasoningEffort, "medium"); + assertFlagPair(startedWorker.argv, "--model", "gpt-5.6-luna"); assert.equal( - startedWorker.argv.includes('model_reasoning_effort="xhigh"'), + startedWorker.argv.includes('model_reasoning_effort="medium"'), true, ); assert.equal( @@ -486,6 +492,7 @@ model_reasoning_summary = "none" assertCanceled(lateJoin, scanId, startedState.scanDir); assert.equal((await readLogLines(startLogPath)).length, 1); + await writeFile(deepConfigPath, deepConfig); const failureThreadId = "deep-scan-stdio-failure-thread"; server.sendRequest( 20, @@ -662,6 +669,12 @@ model_reasoning_summary = "none" "the MCP server must remain responsive after canceling one scan", ); + await writeFile( + deepConfigPath, + `${deepConfig}discovery_model = "fixture-resumed-discovery-model" +discovery_reasoning_effort = "low" +`, + ); const resumedThreadId = "deep-scan-stdio-resumed-thread"; const opened = await server.request( 24, @@ -812,6 +825,12 @@ model_reasoning_summary = "none" resumedScanId, ]); await writeFile(restartControlPath, "after-restart"); + await writeFile( + deepConfigPath, + `${deepConfig}discovery_model = "fixture-unrelated-discovery-model" +discovery_reasoning_effort = "minimal" +`, + ); const resumedStartIndex = (await readJsonLines(startLogPath)).length; const restartedServer = startServer(serverBundlePath, environment); @@ -970,11 +989,15 @@ model_reasoning_summary = "none" assertFlagPair( execution.argv, "--model", - afterRestart ? "gpt-6.1-sol" : "gpt-5.6-sol", + context.workerLabel + ? "fixture-resumed-discovery-model" + : afterRestart + ? "gpt-6.1-sol" + : "gpt-5.6-sol", ); assert.ok( execution.argv.includes( - `model_reasoning_effort=${JSON.stringify(afterRestart ? "max" : "high")}`, + `model_reasoning_effort=${JSON.stringify(context.workerLabel ? "low" : afterRestart ? "max" : "high")}`, ), ); @@ -994,6 +1017,7 @@ model_reasoning_summary = "none" } finally { await restartedServer.stop(); } + await writeFile(deepConfigPath, deepConfig); await testCliDeepScanEngine({ fixtureRoot, environment, diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_store.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_store.ts index 478aea00b..b8f359950 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_store.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_store.ts @@ -19,6 +19,7 @@ await testTerminalProtocol(); testRunErrorParsing(); testLegacyResponseNormalization(); testConfiguredMaximumDurationParsing(); +testDiscoveryModelSettingsParsing(); await testWriteSerializationAndRecovery(); await testBeginUsesTheWriteQueue(); await testHeartbeatBypassesBlockedWriteQueue(); @@ -971,6 +972,33 @@ function testConfiguredMaximumDurationParsing() { assert.equal(Object.hasOwn(legacyState.config, "maxTimeHours"), false); } +function testDiscoveryModelSettingsParsing() { + for (const field of ["discoveryModel", "discoveryReasoningEffort"] as const) { + for (const value of [undefined, null, "custom-value", " custom-value "]) { + const state = parseDeepScan( + stateResult(randomUUID(), { + deepScan: { config: { [field]: value } }, + }), + ); + assert.equal(state.config[field], value ?? undefined); + assert.equal(Object.hasOwn(state.config, field), value != null); + } + for (const value of ["", " ", false, 12, [], {}]) { + assert.throws( + () => + parseDeepScan( + stateResult(randomUUID(), { + deepScan: { config: { [field]: value } }, + }), + ), + { + message: `Codex Security workbench returned invalid deepScan.config.${field}.`, + }, + ); + } + } +} + function testRunErrorParsing() { const createdAt = "2026-08-12T19:00:00Z"; const state = parseDeepScan( diff --git a/plugins/codex-security/mcp-app/tests/test_workbench_database.ts b/plugins/codex-security/mcp-app/tests/test_workbench_database.ts index ff6dc51a7..c375a3542 100644 --- a/plugins/codex-security/mcp-app/tests/test_workbench_database.ts +++ b/plugins/codex-security/mcp-app/tests/test_workbench_database.ts @@ -511,7 +511,7 @@ test("recorded additive migrations restore missing columns and configured error database.exec(`INSERT INTO deep_scan_runs (scan_id, schema_version, workflow_version, status, phase, workers, subagents, stop_after_no_new, max_discovery_runs, created_at, updated_at) VALUES ('scan', 1, 'v1', 'running', 'discovery', 1, 0, 7, 10, 'created', 'updated')`); - for (const version of [27, 28, 31, 32, 47]) + for (const version of [27, 28, 31, 32, 47, 50]) database .prepare("INSERT INTO schema_migrations VALUES (?, ?, 'original')") .run(version, migrations[version - 1].name); @@ -528,8 +528,18 @@ test("recorded additive migrations restore missing columns and configured error .get()?.discovery_user_context_json, null, ); + const discoverySettings = () => + Object.values( + database + .prepare( + "SELECT discovery_model, discovery_reasoning_effort FROM deep_scan_runs", + ) + .get()!, + ); + assert.deepEqual(discoverySettings(), [null, null]); database.exec(`UPDATE deep_scan_runs SET stop_after_consecutive_errors = 2, - discovery_user_context_json = '"Original discovery context"'`); + discovery_user_context_json = '"Original discovery context"', + discovery_model = 'fixture-model', discovery_reasoning_effort = 'custom-effort'`); applyMigrations(database); assert.equal( database @@ -543,6 +553,7 @@ test("recorded additive migrations restore missing columns and configured error .get()?.discovery_user_context_json, '"Original discovery context"', ); + assert.deepEqual(discoverySettings(), ["fixture-model", "custom-effort"]); }); test("workflow and review checkpoints upgrade atomically with their preserved result data", (t) => { diff --git a/plugins/codex-security/references/scan-prologue.md b/plugins/codex-security/references/scan-prologue.md index 62cd70825..4cd09498b 100644 --- a/plugins/codex-security/references/scan-prologue.md +++ b/plugins/codex-security/references/scan-prologue.md @@ -28,7 +28,7 @@ A top-level Standard scan reads `scan-runtime.md` once after its authoritative s ## Deep Worker Failures -Deep Security Scan uses MCP-owned SDK sessions rather than the parent thread's worker pool. Its preflight does not require a particular parent delegation runtime, ownership, capacity, or depth. Discovery workers inherit the scan's model and use the reserved `codex_security_deep_scan_worker` permission profile with the parent's supported filesystem denials. The selected Codex executable must support permission-profile configuration and allowance checks. If that command fails to start or exits early, report its path and the tool's diagnostic; do not infer that its version is unsupported. If the tool identifies a missing API, ask the user to update the Codex installation at the reported path: the desktop app for an app-bundled executable, or the selected CLI otherwise. If Codex policy rejects the profile, report the tool's administrator guidance. Do not remove deny rules or select a broader sandbox to work around the error. +Deep Security Scan uses MCP-owned SDK sessions rather than the parent thread's worker pool. Its preflight does not require a particular parent delegation runtime, ownership, capacity, or depth. Discovery workers inherit the scan's model and reasoning effort unless the user configures `discovery_model` or `discovery_reasoning_effort` in `[deep_scan]`; setup and reducer workers retain the parent settings. Workers use the reserved `codex_security_deep_scan_worker` permission profile with the parent's supported filesystem denials. The selected Codex executable must support permission-profile configuration and allowance checks. If that command fails to start or exits early, report its path and the tool's diagnostic; do not infer that its version is unsupported. If the tool identifies a missing API, ask the user to update the Codex installation at the reported path: the desktop app for an app-bundled executable, or the selected CLI otherwise. If Codex policy rejects the profile, report the tool's administrator guidance. Do not remove deny rules or select a broader sandbox to work around the error. Deep Scan workers use the selected Codex home's credentials and provider configuration. An existing `CODEX_API_KEY` remains selected. When the worker has no Codex account and its provider requires OpenAI authentication, it can use `OPENAI_API_KEY` through the SDK's API-key option. This does not replace a stored account, custom-provider authentication, or a forced ChatGPT login. The key must be present in the process that launched the plugin; setting it later in a separate shell does not update a running plugin. diff --git a/plugins/codex-security/scripts/deep_scan_config.py b/plugins/codex-security/scripts/deep_scan_config.py index d03e63b1d..d68b8289b 100644 --- a/plugins/codex-security/scripts/deep_scan_config.py +++ b/plugins/codex-security/scripts/deep_scan_config.py @@ -29,6 +29,8 @@ "stop_after_consecutive_errors", "max_discovery_runs", "max_time_hours", + "discovery_model", + "discovery_reasoning_effort", } @@ -45,7 +47,7 @@ def config_path() -> Path: return codex_home() / "codex-security" / "config.toml" -def resolve_deep_scan_config(available_parallelism: int) -> dict[str, int | float]: +def resolve_deep_scan_config(available_parallelism: int) -> dict[str, int | float | str]: if isinstance(available_parallelism, bool) or available_parallelism < 1: raise SystemExit("Available parallelism must be a positive integer.") path = config_path() @@ -80,7 +82,7 @@ def resolve_deep_scan_config(available_parallelism: int) -> dict[str, int | floa "deep_scan.stop_after_no_new", minimum=1, ) - return { + resolved_config: dict[str, int | float | str] = { "workers": resolved_workers, "subagents": require_integer( configured.get("subagents", DEFAULT_SUBAGENTS), @@ -103,6 +105,15 @@ def resolve_deep_scan_config(available_parallelism: int) -> dict[str, int | floa "deep_scan.max_time_hours", ), } + if "discovery_model" in configured: + resolved_config["discoveryModel"] = require_nonempty_string( + configured["discovery_model"], "deep_scan.discovery_model" + ) + if "discovery_reasoning_effort" in configured: + resolved_config["discoveryReasoningEffort"] = require_nonempty_string( + configured["discovery_reasoning_effort"], "deep_scan.discovery_reasoning_effort" + ) + return resolved_config def require_integer(value: object, label: str, *, minimum: int) -> int: @@ -128,6 +139,12 @@ def require_positive_number(value: object, label: str) -> int | float: return value +def require_nonempty_string(value: object, label: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise SystemExit(f"{label} must be a non-empty string.") + return value.strip() + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--available-parallelism", type=int, required=True) diff --git a/plugins/codex-security/scripts/deep_scan_workbench.py b/plugins/codex-security/scripts/deep_scan_workbench.py index cebf561ca..ae5570a6a 100644 --- a/plugins/codex-security/scripts/deep_scan_workbench.py +++ b/plugins/codex-security/scripts/deep_scan_workbench.py @@ -470,6 +470,16 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A "stopAfterConsecutiveErrors": run["stop_after_consecutive_errors"], "maxDiscoveryRuns": run["max_discovery_runs"], "maxTimeHours": run["max_time_hours"], + **( + {"discoveryModel": run["discovery_model"]} + if run["discovery_model"] is not None + else {} + ), + **( + {"discoveryReasoningEffort": run["discovery_reasoning_effort"]} + if run["discovery_reasoning_effort"] is not None + else {} + ), }, "dispatchedCount": run["discovery_runs_dispatched"], "completionSequence": run["completion_sequence"], @@ -559,7 +569,7 @@ def deep_scan_result( return result -def effective_deep_scan_config(args: argparse.Namespace) -> dict[str, int | float]: +def effective_deep_scan_config(args: argparse.Namespace) -> dict[str, int | float | str]: available_parallelism = args.available_parallelism or os.cpu_count() or 1 return resolve_deep_scan_config(available_parallelism) @@ -567,7 +577,7 @@ def effective_deep_scan_config(args: argparse.Namespace) -> dict[str, int | floa def ensure_deep_scan_run( connection: sqlite3.Connection, scan: sqlite3.Row, - config: dict[str, int | float], + config: dict[str, int | float | str], workflow_version: str, timestamp: str, ) -> sqlite3.Row: @@ -586,8 +596,9 @@ def ensure_deep_scan_run( scan_id, schema_version, workflow_version, status, phase, workers, subagents, stop_after_no_new, stop_after_consecutive_errors, max_discovery_runs, max_time_hours, discovery_user_context_json, + discovery_model, discovery_reasoning_effort, created_at, updated_at - ) VALUES (?, 1, ?, 'running', 'setup', ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, 1, ?, 'running', 'setup', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( scan["id"], @@ -599,6 +610,8 @@ def ensure_deep_scan_run( config["maxDiscoveryRuns"], config["maxTimeHours"], json.dumps(scan["user_context"]), + config.get("discoveryModel"), + config.get("discoveryReasoningEffort"), timestamp, timestamp, ), diff --git a/plugins/codex-security/scripts/workbench_schema.py b/plugins/codex-security/scripts/workbench_schema.py index 90271d5c1..ccf1ddd69 100644 --- a/plugins/codex-security/scripts/workbench_schema.py +++ b/plugins/codex-security/scripts/workbench_schema.py @@ -75,7 +75,7 @@ def apply_migrations( elif version == 16: should_backfill_targets = repair_stable_targets_migration(connection) elif version in applied: - if version in (2, 12, 13, 26, 28, 31, 32, 47): + if version in (2, 12, 13, 26, 28, 31, 32, 47, 50): repair_additive_migration(connection, version) elif version == 11: repair_deep_scan_migration(connection) diff --git a/plugins/codex-security/shared/workbench-migrations.json b/plugins/codex-security/shared/workbench-migrations.json index 11e368286..27bcee0b9 100644 --- a/plugins/codex-security/shared/workbench-migrations.json +++ b/plugins/codex-security/shared/workbench-migrations.json @@ -410,5 +410,13 @@ "statements": [ "CREATE INDEX finding_issue_receipts_by_destination_issue\nON finding_issue_receipts(destination_key, external_id, finding_id);" ] + }, + { + "version": 50, + "name": "persist deep scan discovery model settings", + "statements": [ + "ALTER TABLE deep_scan_runs ADD COLUMN discovery_model TEXT;", + "ALTER TABLE deep_scan_runs ADD COLUMN discovery_reasoning_effort TEXT;" + ] } ] diff --git a/plugins/codex-security/tests/test_deep_scan_discovery_config.py b/plugins/codex-security/tests/test_deep_scan_discovery_config.py new file mode 100644 index 000000000..1259d3dee --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_discovery_config.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest +from workbench_test_support import run_workbench + + +def deep_environment(codex_home: Path) -> dict[str, str]: + return {"CODEX_HOME": str(codex_home)} + + +def begin_target_scan( + state_dir: Path, codex_home: Path, target: Path, scan_root: Path +) -> dict[str, object]: + return run_workbench( + state_dir, + "begin-deep-scan", + "--thread-id", + "thread-deep-scan", + "--target-path", + str(target), + "--scope", + ".", + "--scan-root", + str(scan_root), + "--available-parallelism", + "16", + environment=deep_environment(codex_home), + ) + + +@pytest.mark.parametrize( + ("settings", "model", "effort"), + ( + ("", None, None), + ('discovery_model = "gpt-5.6-luna"\n', "gpt-5.6-luna", None), + ('discovery_reasoning_effort = "xhigh"\n', None, "xhigh"), + ( + 'discovery_model = "gpt-5.6-luna"\ndiscovery_reasoning_effort = "xhigh"\n', + "gpt-5.6-luna", + "xhigh", + ), + ), +) +def test_discovery_model_settings_are_independent_and_optional( + tmp_path: Path, settings: str, model: str | None, effort: str | None +) -> None: + state_dir = tmp_path / "state" + codex_home = tmp_path / "codex-home" + config_path = codex_home / "codex-security" / "config.toml" + config_path.parent.mkdir(parents=True) + config_path.write_text("[deep_scan]\n" + settings) + target = tmp_path / "target" + target.mkdir() + + deep_scan = begin_target_scan(state_dir, codex_home, target, tmp_path / "scans")["deepScan"] + + assert deep_scan["config"].get("discoveryModel") == model + assert deep_scan["config"].get("discoveryReasoningEffort") == effort + if model is None: + assert "discoveryModel" not in deep_scan["config"] + if effort is None: + assert "discoveryReasoningEffort" not in deep_scan["config"] + with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + assert connection.execute( + "SELECT discovery_model, discovery_reasoning_effort FROM deep_scan_runs" + ).fetchone() == (model, effort) + + +@pytest.mark.parametrize( + ("setting", "value"), + ( + ("discovery_model", '""'), + ("discovery_model", '" "'), + ("discovery_model", "false"), + ("discovery_reasoning_effort", '""'), + ("discovery_reasoning_effort", "12"), + ), +) +def test_invalid_discovery_model_setting_fails_before_scan_creation( + tmp_path: Path, setting: str, value: str +) -> None: + state_dir = tmp_path / "state" + codex_home = tmp_path / "codex-home" + config_path = codex_home / "codex-security" / "config.toml" + config_path.parent.mkdir(parents=True) + config_path.write_text(f"[deep_scan]\n{setting} = {value}\n") + target = tmp_path / "target" + target.mkdir() + + failed = run_workbench( + state_dir, + "begin-deep-scan", + "--thread-id", + "thread-deep-scan", + "--target-path", + str(target), + "--available-parallelism", + "8", + environment=deep_environment(codex_home), + check=False, + ) + + assert f"deep_scan.{setting} must be a non-empty string" in str(failed["stderr"]) + with sqlite3.connect(state_dir / "workbench.sqlite3") as connection: + assert connection.execute("SELECT COUNT(*) FROM scans").fetchone() == (0,) diff --git a/plugins/codex-security/tests/test_workbench_deep_scan.py b/plugins/codex-security/tests/test_workbench_deep_scan.py index d974a0468..6dd8b4617 100644 --- a/plugins/codex-security/tests/test_workbench_deep_scan.py +++ b/plugins/codex-security/tests/test_workbench_deep_scan.py @@ -1261,7 +1261,9 @@ def test_target_begin_is_atomic_idempotent_and_snapshots_config(tmp_path: Path) "subagents = 2\n" "stop_after_no_new = 4\n" "max_discovery_runs = 12\n" - "max_time_hours = 2.5\n", + "max_time_hours = 2.5\n" + 'discovery_model = "gpt-5.6-luna"\n' + 'discovery_reasoning_effort = "xhigh"\n', ) target = tmp_path / "target" target.mkdir() @@ -1280,6 +1282,8 @@ def test_target_begin_is_atomic_idempotent_and_snapshots_config(tmp_path: Path) "stopAfterConsecutiveErrors": 3, "maxDiscoveryRuns": 12, "maxTimeHours": 2.5, + "discoveryModel": "gpt-5.6-luna", + "discoveryReasoningEffort": "xhigh", } assert deep_scan["workers"] == [] scan_id = str(deep_scan["scanId"]) @@ -1289,7 +1293,9 @@ def test_target_begin_is_atomic_idempotent_and_snapshots_config(tmp_path: Path) assert scan["progress"]["coverage"]["filesTotal"] == 1 assert deep_scan["canonicalArtifacts"] is None - config_path.write_text("[deep_scan]\nworkers = 1\nmax_time_hours = 8\n") + config_path.write_text( + '[deep_scan]\nworkers = 1\nmax_time_hours = 8\ndiscovery_model = "gpt-5.6-sol"\n' + ) second = begin_target_scan(state_dir, codex_home, target, scan_root) assert second["startDisposition"] == "joined" assert second["deepScan"]["scanId"] == deep_scan["scanId"] @@ -1300,6 +1306,9 @@ def test_target_begin_is_atomic_idempotent_and_snapshots_config(tmp_path: Path) assert connection.execute("SELECT COUNT(*) FROM scans").fetchone() == (1,) assert connection.execute("SELECT COUNT(*) FROM deep_scan_runs").fetchone() == (1,) assert connection.execute("SELECT max_time_hours FROM deep_scan_runs").fetchone() == (2.5,) + assert connection.execute( + "SELECT discovery_model, discovery_reasoning_effort FROM deep_scan_runs" + ).fetchone() == ("gpt-5.6-luna", "xhigh") assert ( connection.execute( """ diff --git a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py index 20e22612f..34853a9cf 100644 --- a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py +++ b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py @@ -91,6 +91,7 @@ (47, "snapshot deep scan discovery context"), (48, "share finding issue receipts across trackers"), (49, "index finding issue receipt ownership"), + (50, "persist deep scan discovery model settings"), ] @@ -1193,6 +1194,54 @@ def test_deep_scan_time_limit_migration_backfills_and_repairs_existing_runs( ) +def test_discovery_model_migration_preserves_running_scan_settings() -> None: + connection, apply_migrations = create_historical_database(50) + with closing(connection): + timestamp = "2026-09-01T00:00:00Z" + connection.execute( + "INSERT INTO workspaces (id, created_at, updated_at) VALUES (?, ?, ?)", + ("legacy-workspace", timestamp, timestamp), + ) + connection.execute( + """ + INSERT INTO scans ( + id, workspace_id, target_path, target_revision, scope, mode, scan_dir, + model, reasoning_effort, status, phase, started_at, created_at, updated_at + ) VALUES (?, ?, '/legacy/target', 'legacy-revision', '.', 'deep', + '/legacy/scan', 'gpt-6-astra', 'xhigh', 'running', 'discovery', ?, ?, ?) + """, + ("legacy-scan", "legacy-workspace", timestamp, timestamp, timestamp), + ) + connection.execute( + """ + INSERT INTO deep_scan_runs ( + scan_id, schema_version, workflow_version, status, phase, + workers, subagents, stop_after_no_new, stop_after_consecutive_errors, + max_discovery_runs, discovery_user_context_json, created_at, updated_at + ) VALUES (?, 1, 'deep-scan-mcp/v1', 'running', 'discovery', + 4, 0, 4, 3, 40, '"Focus on parsing"', ?, ?) + """, + ("legacy-scan", timestamp, timestamp), + ) + connection.commit() + existing = dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) + + apply_migrations(connection) + apply_migrations(connection) + + assert dict(connection.execute("SELECT * FROM deep_scan_runs").fetchone()) == { + **existing, + "discovery_model": None, + "discovery_reasoning_effort": None, + } + assert tuple( + connection.execute("SELECT model, reasoning_effort FROM scans").fetchone() + ) == ( + "gpt-6-astra", + "xhigh", + ) + + def test_workbench_reconciles_monorepo_migration_lineage() -> None: connection, apply_migrations = create_historical_database(29) @@ -1380,6 +1429,8 @@ def test_workbench_creates_single_final_schema(tmp_path: Path) -> None: "stop_after_consecutive_errors", "consecutive_errors", "max_time_hours", + "discovery_model", + "discovery_reasoning_effort", } assert { row[1] for row in connection.execute("PRAGMA table_info(finding_remediation_attempts)") @@ -1508,6 +1559,20 @@ def test_workbench_repairs_recorded_deep_scan_migration( assert "max_time_hours" in { row[1] for row in connection.execute("PRAGMA table_info(deep_scan_runs)") } + connection.execute( + """ + INSERT INTO deep_scan_runs ( + scan_id, schema_version, workflow_version, status, phase, + workers, subagents, stop_after_no_new, max_discovery_runs, + discovery_model, discovery_reasoning_effort, created_at, updated_at + ) VALUES (?, 1, 'deep-scan-mcp/v1', 'running', 'discovery', + 4, 0, 4, 40, 'fixture-model', 'custom-effort', 'created', 'updated') + """, + (scan_id,), + ) + assert connection.execute( + "SELECT discovery_model, discovery_reasoning_effort FROM deep_scan_runs" + ).fetchone() == ("fixture-model", "custom-effort") assert { row[0] for row in connection.execute( diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index 606c65d47..d462aa6ce 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -350,6 +350,15 @@ within each worker. Time and run limits apply to discovery. After discovery stops, the scan combines and returns completed findings. See [deep scan settings](docs/cli.md#configure-deep-scans) for defaults and saved settings. +Set the optional SDK fields `discoveryModel` and `discoveryReasoningEffort` to +override each discovery worker setting independently. They take precedence over +the legacy `[deep_scan]` TOML settings `discovery_model` and +`discovery_reasoning_effort`. If neither source sets a choice, it inherits the +parent scan's setting. Setup and reducer workers keep the parent settings. +Supplying all six orchestration settings skips legacy TOML, so include any desired discovery +overrides explicitly. Saved recipes preserve these choices for both resume and +rerun. + ### Progress and cost Use `onProgress` for scan progress and `onCost` for cost updates. diff --git a/sdk/typescript/src/api.ts b/sdk/typescript/src/api.ts index 005f22757..42917dd81 100644 --- a/sdk/typescript/src/api.ts +++ b/sdk/typescript/src/api.ts @@ -114,6 +114,7 @@ import { SCAN_AUTH_MODES, ScanSettingsSchema, type DeepScanOptions, + type ResolvedDeepScanOptions, type ScanAuthMode, type ScanSettings, type ScanPromptSettings, @@ -751,7 +752,7 @@ export class CodexSecurity { type ScanMetadata = Pick< ScanResultOptions, "threadId" | "turnResult" | "sarifPath" | "repositoryFindings" - >; + > & { cost?: Readonly | null }; if (state.scanId && state.scanDir) { await workflow.protectArtifacts(state.scanDir); let metadata = state.stages.scan.result as ScanMetadata | undefined; @@ -765,6 +766,7 @@ export class CodexSecurity { metadata = { threadId: (scan["continuationThreadId"] as string) ?? "", turnResult: { status: "completed" }, + cost: scan["cost"] as ScanMetadata["cost"], }; } if (completed) { @@ -780,6 +782,7 @@ export class CodexSecurity { ...contract, scanDir: state.scanDir, ...metadata, + observedCost: metadata.cost, threatModelPath: await readThreatModelPath(state.scanDir, { pythonPath: this.config.pythonPath, protectedRoot: local.protectedRoot, @@ -804,6 +807,7 @@ export class CodexSecurity { await workflow.complete("scan", { threadId: result.threadId, turnResult: result.turnResult, + cost: result.cost, sarifPath: result.sarifPath, repositoryFindings: result.repositoryFindings, } satisfies ScanMetadata); @@ -1143,7 +1147,11 @@ export class CodexSecurity { const configuration = await mergedCodexConfig(this.config); const model = scanModelConfiguration(configuration); const modelProvider = scanModelProvider(configuration); - validateScanCostLimit(options.maxCostUsd, model.model); + validateScanCostLimit( + options.maxCostUsd, + model.model, + inputs.deepScanConfiguration?.settings.discoveryModel, + ); const archiveDir = options.archiveExisting === true ? await planOutputArchive(inputs.outputDir) @@ -1884,7 +1892,11 @@ export class CodexSecurity { pluginVersion: runtime.plugin.version, }; const { model } = scanModelConfiguration(effectiveConfig); - validateScanCostLimit(options.maxCostUsd, model); + validateScanCostLimit( + options.maxCostUsd, + model, + deepScanConfiguration?.settings.discoveryModel, + ); if (mode === "deep" && options.maxCostUsd !== undefined) { budgetRecovery = { expectation, @@ -2632,17 +2644,20 @@ export class CodexSecurity { completedScan, "completed", ); - if (directDeepScan) { - result = new ScanResult({ - ...result, - turnResult: { - ...result.turnResult, - finalResponse: ( - await readScanFile(scanDir, "report.md", "report.md", signal) - ).toString("utf8"), - }, - }); - } + result = new ScanResult({ + ...result, + observedCost: completionCost, + ...(directDeepScan + ? { + turnResult: { + ...result.turnResult, + finalResponse: ( + await readScanFile(scanDir, "report.md", "report.md", signal) + ).toString("utf8"), + }, + } + : {}), + }); if (isRecord(completedScan) && Array.isArray(completedScan["warnings"])) { const targetWarnings = new Set([ ...preparedTargetWarnings, @@ -2713,6 +2728,7 @@ export class CodexSecurity { checkOpen(); result = new ScanResult({ ...result, + observedCost: result.cost, threatModelPath: isDeepStrictEqual( result.threatModel, followUpResult.threatModel, @@ -2904,7 +2920,10 @@ export class CodexSecurity { : undefined, ); } - return result; + return new ScanResult({ + ...result, + observedCost: snapshot?.cost ?? failure.cost, + }); } catch {} } if (activeScan !== null && scanTurnStatus !== null) { @@ -4807,7 +4826,7 @@ function scanRecipe({ failOnSeverity?: SeverityLevel; knowledgeBasePaths?: string[]; maxCostUsd?: number; - deepScan?: Required; + deepScan?: ResolvedDeepScanOptions; auth?: ScanAuthMode; cyberAccessProgram?: ScanSettings["cyberAccessProgram"]; }): JsonObject { @@ -4844,12 +4863,18 @@ async function prepareScanOutputDir( function validateScanCostLimit( maxCostUsd: number | undefined, model: string, + discoveryModel?: string, ): void { if (maxCostUsd === undefined) return; - if (estimateScanCost(model, { input_tokens: 0, output_tokens: 0 }) === null) { - throw new CodexSecurityError( - `A scan cost limit is not available for the configured model: ${model}.`, - ); + for (const selected of [model, discoveryModel]) { + if ( + selected !== undefined && + estimateScanCost(selected, { input_tokens: 0, output_tokens: 0 }) === null + ) { + throw new CodexSecurityError( + `A scan cost limit is not available for the configured model: ${selected}.`, + ); + } } } diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 31b820dd5..88f40290c 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -75,7 +75,6 @@ import { scanAuthentication, runtimeScanAuthentication, selectedScanEnvironment, - type DeepScanOptions, type ScanAuthMode, type ScanAuthentication, type ScanOptions, @@ -311,6 +310,7 @@ import { meetsSeverity, type FailureSeverity, type ResolvedScanSettings, + type ResolvedDeepScanOptions, } from "./scan-settings.js"; const PROGRESS_REFRESH_MILLISECONDS = 1_000; @@ -8212,6 +8212,7 @@ async function executeScan( let deepProgress: DeepScanProgress | null = null; let deepConsolidating = false; let runningCost: Readonly | null = null; + let costUpdates = 0; let runningUsage: Readonly | null = null; let maxCostUsd = arguments_.maxCostUsd; const showCost = arguments_.showCost === true || maxCostUsd !== undefined; @@ -8490,15 +8491,17 @@ async function executeScan( dashboard.setUsage(usage); return; } - const priorCost = runningCost; + const priorCostUpdates = costUpdates; // The tracker queues usage before cost; let the paired cost update render once. await Promise.resolve(); - if (runningCost !== priorCost) return; + if (costUpdates !== priorCostUpdates) return; + runningCost = null; progress?.stopTimer(); progress?.stage(`Tokens: ${formatTokenUsage(usage)}.`); progress?.startTimer(runningMessage()); }, onCost: (cost, limit = maxCostUsd) => { + costUpdates += 1; if (limit !== maxCostUsd && limit !== undefined) { dashboard?.note(`Total cost limit increased to ${formatUsd(limit)}.`); } @@ -9354,7 +9357,7 @@ async function readDeepScanStop( dispatchedCount: number; completionSequence: number; noNewStreak: number; - config: Required; + config: ResolvedDeepScanOptions; createdAt: string; completedAt: string; } diff --git a/sdk/typescript/src/cost.ts b/sdk/typescript/src/cost.ts index a485b0e7b..3ab10bde5 100644 --- a/sdk/typescript/src/cost.ts +++ b/sdk/typescript/src/cost.ts @@ -87,6 +87,9 @@ function createSessionUsage() { startedAt: null as number | null, inheritedUsage: null as ScanTokenUsage | null, replaying: false, + model: null as string | null, + modelUsageOffset: null as ScanTokenUsage | null, + completedModelUsage: new Map(), usage: null as ScanTokenUsage | null, responseUsage: null as ScanTokenUsage | null, responseIds: new Set(), @@ -195,8 +198,8 @@ export class ScanCostTracker { return this.#snapshot; const cost = estimateScanCost(this.#options.model, fallbackUsage); this.#snapshot = { usage: fallbackUsage ?? null, cost }; - this.#reportUsage(tokenUsage(fallbackUsage)); - this.#reportCost(cost); + const reportedUsage = this.#reportUsage(tokenUsage(fallbackUsage)); + this.#reportCost(cost, reportedUsage); return this.#snapshot; } @@ -268,6 +271,7 @@ export class ScanCostTracker { } const usages = new Map(this.#receipts); + const sessions = new Map(); for (const [path, tracked] of this.#sessions) { const threadId = tracked.threadId; if (threadId === null || !included.has(threadId)) continue; @@ -282,6 +286,7 @@ export class ScanCostTracker { await readSessionUsage(path, session, repository); this.#sessions.set(path, session); } + sessions.set(threadId, session); let worker: number | undefined; if (threadId !== this.#threadId) { worker = this.#workers.get(threadId); @@ -319,21 +324,114 @@ export class ScanCostTracker { receipt.cache_write_input_tokens === 0)) ) { usages.set(threadId, session.usage); + } else if (receipt != null && session.usage !== null) { + // A final SDK receipt can advance the total while omitting cache-write + // and reasoning detail. Keep counters already observed in the rollout. + usages.set(threadId, { + ...receipt, + cache_write_input_tokens: Math.max( + receipt.cache_write_input_tokens, + session.usage.cache_write_input_tokens, + ), + reasoning_output_tokens: Math.max( + receipt.reasoning_output_tokens, + session.usage.reasoning_output_tokens, + ), + }); } } let usage: ScanTokenUsage | null = null; - for (const value of usages.values()) { + const modelUsages = new Map(); + for (const [threadId, value] of usages) { if (value === null) { this.#snapshot = { usage: null, cost: null }; return; } usage = addTokenUsage(usage, value); + const session = sessions.get(threadId); + for (const [model, completed] of session?.completedModelUsage ?? []) { + modelUsages.set( + model, + addTokenUsage(modelUsages.get(model) ?? null, completed), + ); + } + const current = + session?.modelUsageOffset == null + ? value + : subtractTokenUsage(value, session.modelUsageOffset); + if (current === null) { + throw new Error( + "Could not attribute scan token usage to its observed model.", + ); + } + const model = session?.model ?? this.#options.model; + modelUsages.set( + model, + addTokenUsage(modelUsages.get(model) ?? null, current), + ); } if (usage === null) return; - const cost = estimateScanCost(this.#options.model, usage); + // Zero-token control sessions do not contribute a model or price to a scan. + if (usage.total_tokens > 0) { + for (const [model, value] of modelUsages) { + if (value.total_tokens === 0) modelUsages.delete(model); + } + } + const costs: ScanCost[] = []; + for (const [model, value] of modelUsages) { + const cost = estimateScanCost(model, value); + if (cost === null) { + this.#snapshot = { usage, cost: null }; + if (this.#lastCost !== null) { + this.#lastCost = null; + this.#lastUsage = null; + } + this.#reportUsage(usage); + if (this.#options.maxCostUsd !== undefined) { + throw new Error( + `Could not verify the scan cost limit for observed model: ${model}.`, + ); + } + return; + } + costs.push(cost); + } + const cost = + costs.length === 1 + ? costs[0]! + : { + model: costs + .map(({ model }) => model) + .sort() + .join(", "), + inputTokens: usage.input_tokens, + cachedInputTokens: usage.cached_input_tokens, + cacheWriteInputTokens: usage.cache_write_input_tokens, + ...(usage.cache_write_input_tokens_reported === false + ? { cacheWriteInputTokensReported: false } + : {}), + outputTokens: usage.output_tokens, + estimatedUsd: costs.reduce( + (sum, value) => sum + value.estimatedUsd, + 0, + ), + estimatedUsdRange: { + min: costs.reduce( + (sum, value) => sum + value.estimatedUsdRange!.min, + 0, + ), + max: costs.some((value) => value.estimatedUsdRange!.max === null) + ? null + : costs.reduce( + (sum, value) => sum + value.estimatedUsdRange!.max!, + 0, + ), + context: "unknown" as const, + }, + }; this.#snapshot = { usage, cost }; - this.#reportUsage(usage); - this.#reportCost(cost); + const reportedUsage = this.#reportUsage(usage); + this.#reportCost(cost, reportedUsage); } #reportWorkerProgress(session: SessionUsage): void { @@ -375,20 +473,22 @@ export class ScanCostTracker { } } - #reportCost(cost: ScanCost | null): void { + #reportCost(cost: ScanCost | null, reportedUsage = false): void { if (cost === null) return; const signature = JSON.stringify(cost); - if (signature === this.#lastCost) return; + // Usage without a paired cost update means its pricing is unavailable. + if (signature === this.#lastCost && !reportedUsage) return; this.#lastCost = signature; this.#options.onCost?.(cost); } - #reportUsage(usage: ScanTokenUsage | null): void { - if (usage === null) return; + #reportUsage(usage: ScanTokenUsage | null): boolean { + if (usage === null) return false; const signature = JSON.stringify(usage); - if (signature === this.#lastUsage) return; + if (signature === this.#lastUsage) return false; this.#lastUsage = signature; this.#options.onUsage?.(usage); + return this.#options.onUsage !== undefined; } } @@ -504,6 +604,36 @@ function readSessionEvent( return; } session.events?.push(event); + if ( + event["type"] === "turn_context" && + typeof payload["model"] === "string" + ) { + const model = payload["model"]; + // A resumed thread can change models. Keep its previous charges separate + // from the later increments in the thread's cumulative usage. + if ( + session.model !== null && + session.model !== model && + session.usage !== null + ) { + const completed = + session.modelUsageOffset === null + ? session.usage + : subtractTokenUsage(session.usage, session.modelUsageOffset); + if (completed !== null) { + session.completedModelUsage.set( + session.model, + addTokenUsage( + session.completedModelUsage.get(session.model) ?? null, + completed, + ), + ); + } + session.modelUsageOffset = session.usage; + } + session.model = model; + return; + } if (event["type"] === "token_usage_record") { const responseId = payload["response_id"]; if ( diff --git a/sdk/typescript/src/deep-config.ts b/sdk/typescript/src/deep-config.ts index 5937cdc49..6a9ed4942 100644 --- a/sdk/typescript/src/deep-config.ts +++ b/sdk/typescript/src/deep-config.ts @@ -8,18 +8,29 @@ import { DEFAULT_DEEP_SCAN_SETTINGS } from "./deep-scan-defaults.js"; import { CodexSecurityError } from "./errors.js"; import { DEFAULT_SCAN_MODE, + DEEP_SCAN_DISCOVERY_SETTINGS, DEEP_SCAN_SETTINGS, DeepScanSettingsSchema, type DeepScanOptions, + type ResolvedDeepScanOptions, } from "./scan-settings.js"; import type { ScanMode } from "./scan-modes.js"; +const allSettings = [...DEEP_SCAN_SETTINGS, ...DEEP_SCAN_DISCOVERY_SETTINGS]; +const discoveryKeys = new Set( + DEEP_SCAN_DISCOVERY_SETTINGS.map(([, key]) => key), +); + +type DeepScanSource = "default" | "legacy" | "override"; export type DeepScanSources = Record< - keyof DeepScanOptions, - "default" | "legacy" | "override" ->; + (typeof DEEP_SCAN_SETTINGS)[number][0], + DeepScanSource +> & + Partial< + Record<(typeof DEEP_SCAN_DISCOVERY_SETTINGS)[number][0], DeepScanSource> + >; export interface ResolvedDeepScanConfig { - settings: Required; + settings: ResolvedDeepScanOptions; sources: DeepScanSources; source: string; document: TomlTable; @@ -30,13 +41,15 @@ export function deepScanOptions( options: DeepScanOptions & { mode?: ScanMode }, ): DeepScanOptions { const selected: DeepScanOptions = {}; - for (const [name] of DEEP_SCAN_SETTINGS) { + for (const [name] of allSettings) { const value = options[name]; if (value === undefined) continue; if ((options.mode ?? DEFAULT_SCAN_MODE) !== "deep") { throw new CodexSecurityError("Deep scan settings require deep mode."); } - selected[name] = requireDeepScanValue(name, value, name); + Object.assign(selected, { + [name]: requireDeepScanValue(name, value, name), + }); } return selected; } @@ -45,7 +58,7 @@ function requireDeepScanValue( name: keyof DeepScanOptions, value: unknown, label: string, -): number { +): number | string { const parsed = DeepScanSettingsSchema.shape[name].unwrap().safeParse(value); if (!parsed.success) { throw new CodexSecurityError( @@ -77,7 +90,7 @@ export async function resolveDeepScanConfig( ); } const configured = (existing ?? {}) as TomlTable; - const keys = new Set(DEEP_SCAN_SETTINGS.map(([, key]) => key)); + const keys = new Set(allSettings.map(([, key]) => key)); const unknown = Object.keys(configured).filter((key) => !keys.has(key)); if (unknown.length > 0) { throw new CodexSecurityError( @@ -86,7 +99,7 @@ export async function resolveDeepScanConfig( } const settings = { ...DEFAULT_DEEP_SCAN_SETTINGS, - } as Required; + } as ResolvedDeepScanOptions; const sources = {} as DeepScanSources; for (const [name, key] of DEEP_SCAN_SETTINGS) { let value = Object.hasOwn(configured, key) @@ -101,11 +114,25 @@ export async function resolveDeepScanConfig( : Object.hasOwn(configured, key) ? "legacy" : "default"; - settings[name] = requireDeepScanValue( - name, - value, - sources[name] === "legacy" ? `${key} in ${source}` : name, - ); + Object.assign(settings, { + [name]: requireDeepScanValue( + name, + value, + sources[name] === "legacy" ? `${key} in ${source}` : name, + ), + }); + } + for (const [name, key] of DEEP_SCAN_DISCOVERY_SETTINGS) { + const value = explicit[name] ?? configured[key]; + if (value === undefined) continue; + sources[name] = explicit[name] === undefined ? "legacy" : "override"; + Object.assign(settings, { + [name]: requireDeepScanValue( + name, + value, + sources[name] === "legacy" ? `${key} in ${source}` : name, + ), + }); } return { settings, @@ -143,11 +170,15 @@ export async function writeDeepScanConfig( deep_scan: { ...(retainAmbientSettings ? (document["deep_scan"] as TomlTable | undefined) - : {}), + : Object.fromEntries( + Object.entries(document["deep_scan"] ?? {}).filter( + ([key]) => sameFile && discoveryKeys.has(key), + ), + )), ...Object.fromEntries( - DEEP_SCAN_SETTINGS.filter(([name]) => settings[name] !== undefined).map( - ([name, key]) => [key, settings[name]], - ), + allSettings + .filter(([name]) => settings[name] !== undefined) + .map(([name, key]) => [key, settings[name]]), ), }, } as JsonObject); diff --git a/sdk/typescript/src/project-config.ts b/sdk/typescript/src/project-config.ts index 97f369314..1ae412e9a 100644 --- a/sdk/typescript/src/project-config.ts +++ b/sdk/typescript/src/project-config.ts @@ -282,7 +282,7 @@ export function resolveScanSettings( overrides.target, ) ?? "repository"; const configuredDeep = file?.scan?.deep; - const deep: DeepScanOptions = {}; + const deep: DeepScanOptions = { ...explicitDeep }; if (mode === "deep") { for (const [name, , field] of DEEP_SCAN_SETTINGS) { const value = choose( diff --git a/sdk/typescript/src/result.ts b/sdk/typescript/src/result.ts index 4f1a937c9..2830e890e 100644 --- a/sdk/typescript/src/result.ts +++ b/sdk/typescript/src/result.ts @@ -42,6 +42,7 @@ export interface ScanResultOptions { scanDir: string; threadId: string; turnResult: TurnResultMetadata; + observedCost?: Readonly | null; sarifPath?: string | null; threatModelPath?: string | null; repositoryFindings?: readonly RepositoryFinding[]; @@ -68,10 +69,10 @@ export class ScanResult { this.turnResult = options.turnResult; this.repositoryFindings = options.repositoryFindings; this.threatModelPath = options.threatModelPath ?? null; - this.cost = estimateScanCost( - options.turnResult.model, - options.turnResult.usage, - ); + this.cost = + options.observedCost === undefined + ? estimateScanCost(options.turnResult.model, options.turnResult.usage) + : options.observedCost; if (options.sarifPath !== undefined) { this.sarifPath = options.sarifPath; } else { diff --git a/sdk/typescript/src/scan-dashboard.ts b/sdk/typescript/src/scan-dashboard.ts index e341b6f06..eabc9fc6e 100644 --- a/sdk/typescript/src/scan-dashboard.ts +++ b/sdk/typescript/src/scan-dashboard.ts @@ -555,6 +555,7 @@ export class ScanDashboard { } public setUsage(usage: Readonly): void { + this.#cost = null; this.#tokens = formatTokenUsage(usage); this.#refresh(); } @@ -952,9 +953,12 @@ export class ScanDashboard { const costs = this.#components.flatMap(({ dashboard }) => dashboard.#cost === null ? [] : [dashboard.#cost], ); + const unavailable = this.#components.some( + ({ dashboard }) => dashboard.#tokens !== null && dashboard.#cost === null, + ); return wrapActivity( " COST ", - `${costs.length === 0 ? "waiting for usage" : formatScanCosts(costs)} · component scans only`, + `${unavailable ? "unavailable" : costs.length === 0 ? "waiting for usage" : formatScanCosts(costs)} · component scans only`, this.#width(), ); } @@ -1002,7 +1006,9 @@ export class ScanDashboard { const cost = this.#cost === null ? this.#options.maxCostUsd === undefined - ? "waiting for usage" + ? this.#tokens === null + ? "waiting for usage" + : "unavailable" : `— / ${formatUsd(this.#options.maxCostUsd)}` : `${formatScanCost(this.#cost)}${this.#options.maxCostUsd === undefined ? "" : `; short-context budget baseline: ${formatUsd(this.#cost.estimatedUsd)} / ${formatUsd(this.#options.maxCostUsd)} · ${budgetBar(this.#cost.estimatedUsd, this.#options.maxCostUsd)}`}`; return wrapActivity(" COST ", cost, this.#width()); diff --git a/sdk/typescript/src/scan-settings.ts b/sdk/typescript/src/scan-settings.ts index 7dd1036d3..393f99a72 100644 --- a/sdk/typescript/src/scan-settings.ts +++ b/sdk/typescript/src/scan-settings.ts @@ -48,6 +48,11 @@ export const DEEP_SCAN_SETTINGS = [ ["maxTimeHours", "max_time_hours", "max_time_hours", "--max-time-hours"], ] as const; +export const DEEP_SCAN_DISCOVERY_SETTINGS = [ + ["discoveryModel", "discovery_model"], + ["discoveryReasoningEffort", "discovery_reasoning_effort"], +] as const; + const positiveIntegerError = "must be a positive integer"; const positiveInteger = z .number({ error: positiveIntegerError }) @@ -60,8 +65,20 @@ const nonnegativeInteger = z .nonnegative({ error: nonnegativeIntegerError }); const maximumHours = 96; const hoursError = `must be a positive number no greater than ${maximumHours}`; +const discoverySettingError = "must be a non-empty string"; +const discoverySetting = z + .string({ error: discoverySettingError }) + .trim() + .min(1, { error: discoverySettingError }) + .optional(); export const DeepScanSettingsSchema = z.strictObject({ + discoveryModel: discoverySetting.describe( + "Discovery worker model. Omit to inherit the parent model.", + ), + discoveryReasoningEffort: discoverySetting.describe( + "Discovery worker reasoning effort. Omit to inherit the parent effort.", + ), workers: positiveInteger.optional().meta({ default: DEFAULT_DEEP_SCAN_SETTINGS.workers, description: "Maximum concurrent deep-scan discovery workers.", @@ -95,6 +112,10 @@ export const DeepScanSettingsSchema = z.strictObject({ }); export type DeepScanOptions = z.infer; +export type ResolvedDeepScanOptions = Required< + Pick +> & + DeepScanOptions; const inputPath = z.string().min(1); diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.ts index 34eba77a2..5324ce2e2 100644 --- a/sdk/typescript/tests-ts/api.test.ts +++ b/sdk/typescript/tests-ts/api.test.ts @@ -155,6 +155,11 @@ test.each(["completed", "receipt-lost", "scan-interrupted", "prompt-files"])( if (scenario === "prompt-files") await writeFile(promptFile, scanPrompt); let modelCalls = 0; let completed = false; + let savedCost: JsonObject | null = null; + const usage = { input_tokens: 100, output_tokens: 10 }; + const expectedCostUsd = + estimateScanCost("gpt-5.6-sol", usage)!.estimatedUsd + + estimateScanCost("gpt-5.6-luna", usage)!.estimatedUsd; let loseReceipt = scenario === "receipt-lost"; const makeClient = async (attempt: number) => { const codexHome = join(root, `codex-home-${attempt}`); @@ -183,6 +188,7 @@ test.each(["completed", "receipt-lost", "scan-interrupted", "prompt-files"])( scan: { progress: { status: completed ? "complete" : "failed" }, continuationThreadId: "thread-1", + cost: savedCost, }, }; if (args[0] === "register-cli-scan") { @@ -194,7 +200,10 @@ test.each(["completed", "receipt-lost", "scan-interrupted", "prompt-files"])( }); return registration; } - if (args[0] === "complete-scan") completed = true; + if (args[0] === "complete-scan") { + completed = true; + savedCost = JSON.parse(args[args.indexOf("--cost-json") + 1]!); + } return mockWorkbench(args, input); }, createCodex: codexFactory(async (input: string) => { @@ -202,6 +211,13 @@ test.each(["completed", "receipt-lost", "scan-interrupted", "prompt-files"])( if (scenario === "prompt-files") expect(input).toContain(scanPrompt); if (scenario === "scan-interrupted" && modelCalls === 1) throw new Error("Synthetic interrupted scan"); + await writeUsageSession(codexHome, "thread-1", usage, { + model: "gpt-5.6-sol", + }); + await writeUsageSession(codexHome, "discovery-thread", usage, { + parent: "thread-1", + model: "gpt-5.6-luna", + }); await copyCompletedScan(root); return { events: completedEvents() }; }, "thread-1"), @@ -236,6 +252,10 @@ test.each(["completed", "receipt-lost", "scan-interrupted", "prompt-files"])( ...(scenario === "prompt-files" ? { scanPromptFile: replacement } : {}), }); expect(result.manifest.scan.id).toBe("scan_example_001"); + expect(result.cost?.estimatedUsd).toBe(expectedCostUsd); + expect(result.cost?.model).toContain("gpt-5.6-sol"); + expect(result.cost?.model).toContain("gpt-5.6-luna"); + expect(result.cost).toEqual(savedCost); expect(python).toHaveBeenCalledWith( expect.objectContaining({ protectedRoot: repository }), ); @@ -1225,6 +1245,21 @@ describe("CodexSecurity orchestration", () => { await expect( unpriced.preflight(repository, { maxCostUsd: 5 }), ).rejects.toThrow("cost limit is not available for the configured model"); + await expect( + client.preflight(repository, { + mode: "deep", + maxCostUsd: 5, + discoveryModel: "unknown-discovery-model", + }), + ).rejects.toThrow( + "cost limit is not available for the configured model: unknown-discovery-model", + ); + await expect( + client.preflight(repository, { + mode: "deep", + discoveryModel: "unknown-discovery-model", + }), + ).resolves.toMatchObject({ discoveryModel: "unknown-discovery-model" }); expect(prepareRuntime).not.toHaveBeenCalled(); await unpriced.close(); await client.close(); @@ -3403,6 +3438,8 @@ describe("CodexSecurity orchestration", () => { "stop_after_no_new = 7", "max_discovery_runs = 60", "max_time_hours = 48", + 'discovery_model = "synthetic-discovery"', + 'discovery_reasoning_effort = "high"', "[other]", "enabled = true", "", @@ -3459,6 +3496,8 @@ describe("CodexSecurity orchestration", () => { expect(configuration).toContain("stop_after_consecutive_errors = 2"); expect(configuration).toContain("max_discovery_runs = 10"); expect(configuration).toContain("max_time_hours = 1.5"); + expect(configuration).toContain('discovery_model = "synthetic-discovery"'); + expect(configuration).toContain('discovery_reasoning_effort = "high"'); expect(configuration).toContain("[other]"); expect(configuration).toContain("enabled = true"); expect(recipe).toMatchObject({ @@ -3472,6 +3511,8 @@ describe("CodexSecurity orchestration", () => { stopAfterConsecutiveErrors: 2, maxDiscoveryRuns: 10, maxTimeHours: 1.5, + discoveryModel: "synthetic-discovery", + discoveryReasoningEffort: "high", }, }); await client.close(); diff --git a/sdk/typescript/tests-ts/cli-show-cost.test.ts b/sdk/typescript/tests-ts/cli-show-cost.test.ts index 457b5b3d0..df259e64f 100644 --- a/sdk/typescript/tests-ts/cli-show-cost.test.ts +++ b/sdk/typescript/tests-ts/cli-show-cost.test.ts @@ -159,3 +159,45 @@ test.each( expect(outputs[1]).toBe(outputs[0]); expect(outputs[2]).toBe(outputs[0]); }); + +test.each(["direct", "deferred"])( + "clears stale prices and restores a paired update delivered %s", + async (delivery) => { + const { stderr, runCli } = createCliTest(main); + const deps = dependencies(); + const largerUsage = { + ...observedUsage, + input_tokens: 2_250, + total_tokens: 2_280, + }; + deps.createSecurity = () => + fakeSecurity(async (_repository, options = {}) => { + options.onScanStarted?.(); + options.onCost?.(result.cost!); + await options.onUsage?.(largerUsage); + options.onScanStarted?.(); + let running = stderr.text().trim().split("\n").at(-1)!; + expect(running).toContain("2,280 total"); + expect(running).not.toContain("$0.00488"); + for (let repeat = 0; repeat < 2; repeat += 1) { + // Reusing the same cost object must still count as a paired update. + if (delivery === "direct") { + const observed = options.onUsage?.(observedUsage); + options.onCost?.(result.cost!); + await observed; + } else { + await Promise.all([ + Promise.resolve().then(() => options.onUsage?.(observedUsage)), + Promise.resolve().then(() => options.onCost?.(result.cost!)), + ]); + } + options.onScanStarted?.(); + running = stderr.text().trim().split("\n").at(-1)!; + expect(running).toContain("1,280 total"); + expect(running).toContain("$0.00488"); + } + return result; + }); + expect(await runCli(["scan", ".", "--show-cost"], deps)).toBe(0); + }, +); diff --git a/sdk/typescript/tests-ts/cost.test.ts b/sdk/typescript/tests-ts/cost.test.ts index 947cc610c..bb712d317 100644 --- a/sdk/typescript/tests-ts/cost.test.ts +++ b/sdk/typescript/tests-ts/cost.test.ts @@ -514,6 +514,291 @@ describe("scan cost", () => { }); describe("live scan cost tracking", () => { + test.each([1_000, 2_000])( + "invalidates a prior estimate when an unpriced rollout reports %p tokens", + async (inputTokens) => { + const home = await codexHome(); + const updates: string[] = []; + const tracker = costTracker(home, { + onUsage: () => updates.push("usage"), + onCost: () => updates.push("cost"), + }); + tracker.recordUsage( + { input_tokens: 1_000, output_tokens: 0 }, + "worker-thread", + ); + tracker.start("scan-thread"); + try { + expect((await tracker.refresh()).cost).not.toBeNull(); + expect(updates.splice(0)).toEqual(["usage", "cost"]); + await writeSession( + home, + "worker-thread", + { + input_tokens: inputTokens, + output_tokens: 0, + }, + { model: "synthetic-unpriced-model" }, + ); + expect((await tracker.refresh()).cost).toBeNull(); + expect(updates.splice(0)).toEqual(["usage"]); + await tracker.refresh(); + expect(updates).toEqual([]); + } finally { + await tracker.stop(); + } + }, + ); + + test("pairs changed usage with its still-valid unchanged price", async () => { + const home = await codexHome(); + const usage = { input_tokens: 100, output_tokens: 10 }; + const path = await writeSession(home, "scan-thread", usage); + const updates: string[] = []; + const tracker = costTracker(home, { + onUsage: () => updates.push("usage"), + onCost: () => updates.push("cost"), + }); + tracker.start("scan-thread"); + try { + const original = await tracker.refresh(); + expect(updates.splice(0)).toEqual(["usage", "cost"]); + await appendFile( + path, + jsonLines([ + legacyUsageRecord({ ...usage, reasoning_output_tokens: 5 }), + ]) + "\n", + ); + expect((await tracker.refresh()).cost).toEqual(original.cost); + expect(updates.splice(0)).toEqual(["usage", "cost"]); + await tracker.refresh(); + expect(updates).toEqual([]); + } finally { + await tracker.stop(); + } + }); + + test.each([ + ["gpt-5.6-luna", "gpt-5.6-sol", 4.0004, 8.0004, true], + ["gpt-5.6-sol", "gpt-5.6-luna", 0.208, 0.408, false], + ] as const)( + "prices %s parent and reducer separately from %s discovery, including resumed usage", + async ( + parentModel, + discoveryModel, + initialCost, + resumedCost, + exceedsBudget, + ) => { + const home = await codexHome(); + const parentUsage = { input_tokens: 1_000, output_tokens: 0 }; + await writeSession(home, "scan-thread", parentUsage, { + model: parentModel, + }); + await writeSession(home, "reducer-thread", parentUsage, { + parent: "scan-thread", + model: parentModel, + }); + const discoveryUsage = { input_tokens: 1_000_000, output_tokens: 0 }; + const worker = await writeSession( + home, + "discovery-thread", + discoveryUsage, + { + parent: "scan-thread", + model: discoveryModel, + }, + ); + const budget = new AbortController(); + const tracker = new ScanCostTracker({ + codexHome: home, + model: parentModel, + maxCostUsd: 1, + onCost: ({ estimatedUsd }) => { + if (estimatedUsd > 1) budget.abort(); + }, + }); + tracker.start("scan-thread"); + try { + const initial = await tracker.refresh(); + expect(initial.cost?.estimatedUsd).toBeCloseTo(initialCost, 10); + expect(initial.cost?.model).toBe("gpt-5.6-luna, gpt-5.6-sol"); + expect(initial.cost?.pricing).toBeUndefined(); + expect(budget.signal.aborted).toBe(exceedsBudget); + const resumedUsage = { input_tokens: 2_000_000, output_tokens: 0 }; + const response = nativeUsageRecord( + "discovery-thread", + "resumed-response", + discoveryUsage, + resumedUsage, + ); + await appendFile( + worker, + jsonLines([ + { type: "turn_context", payload: { model: discoveryModel } }, + response, + response, + legacyUsageRecord(discoveryUsage), + ]) + "\n", + ); + const resumed = await tracker.refresh(); + expect(resumed.cost?.estimatedUsd).toBeCloseTo(resumedCost, 10); + expect(resumed.usage).toMatchObject({ + input_tokens: 2_002_000, + output_tokens: 0, + }); + expect(budget.signal.aborted).toBe(exceedsBudget); + expect((await tracker.stop(parentUsage)).cost).toEqual(resumed.cost); + } finally { + await tracker.stop(); + } + }, + ); + + test("retains earlier model charges when a resumed session changes models", async () => { + const home = await codexHome(); + const usage = { input_tokens: 1_000_000, output_tokens: 0 }; + const path = await writeSession(home, "scan-thread", usage, { + model: "gpt-5.6-luna", + }); + const tracker = costTracker(home); + tracker.start("scan-thread"); + try { + expect((await tracker.refresh()).cost).toEqual( + estimateScanCost("gpt-5.6-luna", usage), + ); + await appendFile( + path, + jsonLines([ + { type: "turn_context", payload: { model: "gpt-5.6-sol" } }, + nativeUsageRecord("scan-thread", "second-turn", usage, { + input_tokens: 2_000_000, + output_tokens: 0, + }), + ]) + "\n", + ); + expect((await tracker.refresh()).cost?.estimatedUsd).toBe(4.2); + await appendFile( + path, + jsonLines([ + { type: "turn_context", payload: { model: "gpt-5.6-luna" } }, + legacyUsageRecord({ input_tokens: 3_000_000, output_tokens: 0 }), + ]) + "\n", + ); + const snapshot = await tracker.stop({ + input_tokens: 3_000_000, + output_tokens: 0, + }); + expect(snapshot.cost?.estimatedUsd).toBe(4.4); + expect(snapshot.cost?.inputTokens).toBe(3_000_000); + expect(snapshot.cost?.pricing).toBeUndefined(); + expect(snapshot.cost?.estimatedUsdRange?.min).toBe(4.4); + } finally { + await tracker.stop(); + } + }); + + test("keeps earlier-model detail when the final SDK receipt advances beyond its rollout", async () => { + const home = await codexHome(); + const original = { + input_tokens: 1_000, + cached_input_tokens: 20, + cache_write_input_tokens: 12, + output_tokens: 10, + reasoning_output_tokens: 5, + }; + const path = await writeSession(home, "scan-thread", original, { + model: "gpt-5.6-luna", + }); + await appendFile( + path, + jsonLines([ + { type: "turn_context", payload: { model: "gpt-5.6-sol" } }, + legacyUsageRecord({ + ...original, + input_tokens: 1_020, + output_tokens: 12, + }), + ]) + "\n", + ); + const tracker = costTracker(home, { maxCostUsd: 1 }); + tracker.start("scan-thread"); + const snapshot = await tracker.stop({ + input_tokens: 1_100, + cached_input_tokens: 20, + cache_write_input_tokens: 0, + output_tokens: 15, + }); + expect(snapshot.cost?.estimatedUsd).toBeCloseTo(0.000709, 12); + expect(snapshot.usage).toMatchObject({ + input_tokens: 1_100, + cached_input_tokens: 20, + cache_write_input_tokens: 12, + output_tokens: 15, + reasoning_output_tokens: 5, + }); + }); + + test("ignores inherited parent model context when pricing a fork", async () => { + const home = await codexHome(); + const path = await writeSession( + home, + childUuid7Thread, + {}, + { parent: scanThreadId }, + ); + const rollout = ownershipRollout([lowerUuid7Turn]); + rollout.splice(2, 0, { + type: "turn_context", + payload: { model: "gpt-5.6-sol" }, + }); + rollout.splice(rollout.length - 1, 0, { + type: "turn_context", + payload: { model: "gpt-5.6-luna" }, + }); + await writeFile(path, jsonLines(rollout) + "\n"); + const tracker = costTracker(home); + tracker.start(scanThreadId); + expect((await tracker.stop()).cost).toEqual( + estimateScanCost("gpt-5.6-luna", ownedSdkUsage), + ); + }); + + test.each([undefined, 1])( + "keeps unknown worker pricing unavailable with cost limit %p", + async (maxCostUsd) => { + const home = await codexHome(); + await writeSession( + home, + "worker-thread", + { input_tokens: 100, output_tokens: 10 }, + { + parent: "scan-thread", + model: "synthetic-unpriced-model", + }, + ); + const errors: unknown[] = []; + const usages: unknown[] = []; + const tracker = costTracker(home, { + maxCostUsd, + onUsage: (usage) => usages.push(usage), + onError: (error) => errors.push(error), + }); + tracker.start("scan-thread"); + if (maxCostUsd === undefined) { + expect((await tracker.stop()).cost).toBeNull(); + expect(errors).toEqual([]); + } else { + await waitFor(() => errors.length > 0); + expect(errors[0]).toBeInstanceOf(Error); + expect(String(errors[0])).toContain("synthetic-unpriced-model"); + await expect(tracker.stop()).rejects.toThrow("scan cost limit"); + } + expect(usages).toHaveLength(1); + expect(usages[0]).toMatchObject({ input_tokens: 100, output_tokens: 10 }); + }, + ); + test("keeps receipt cache-write detail when an equal legacy counter omits it", async () => { const home = await codexHome(); const usage = { diff --git a/sdk/typescript/tests-ts/deep-config.test.ts b/sdk/typescript/tests-ts/deep-config.test.ts index 486cc9ca8..c46be139a 100644 --- a/sdk/typescript/tests-ts/deep-config.test.ts +++ b/sdk/typescript/tests-ts/deep-config.test.ts @@ -76,6 +76,112 @@ test("normalizes legacy auto and preserves explicit zero while merging", async ( }); }); +test.each<{ + name: string; + contents: string; + discovery: Record; +}>([ + { name: "neither", contents: "", discovery: {} }, + { + name: "model only", + contents: 'discovery_model = " fixture-discovery "\n', + discovery: { discovery_model: "fixture-discovery" }, + }, + { + name: "effort only", + contents: 'discovery_reasoning_effort = " future-effort "\n', + discovery: { discovery_reasoning_effort: "future-effort" }, + }, + { + name: "both", + contents: + 'discovery_model = "fixture-discovery"\ndiscovery_reasoning_effort = "high"\n', + discovery: { + discovery_model: "fixture-discovery", + discovery_reasoning_effort: "high", + }, + }, +])( + "snapshots optional discovery settings: $name", + async ({ contents, discovery }) => { + const input = await fixture(`[deep_scan]\nworkers = 2\n${contents}`); + const resolved = await resolveDeepScanConfig({}, input.source); + await writeFile( + input.source, + '[deep_scan]\ndiscovery_model = "later-model"\n', + ); + const destination = join(input.root, "runtime", "deep-scan.toml"); + await writeDeepScanConfig(destination, resolved); + expect(parseToml(await readFile(destination, "utf8"))["deep_scan"]).toEqual( + { + workers: 2, + subagents: 3, + stop_after_no_new: 4, + stop_after_consecutive_errors: 3, + max_discovery_runs: 40, + max_time_hours: 96, + ...discovery, + }, + ); + await writeFile(input.source, "invalid TOML ["); + const replay = await resolveDeepScanConfig(resolved.settings, input.source); + expect(replay.settings).toEqual(resolved.settings); + const replayDestination = join(input.root, "replay", "deep-scan.toml"); + await writeDeepScanConfig(replayDestination, replay); + expect( + parseToml(await readFile(replayDestination, "utf8"))["deep_scan"], + ).toEqual(parseToml(await readFile(destination, "utf8"))["deep_scan"]); + }, +); + +test("SDK discovery overrides merge independently with legacy settings", async () => { + const input = await fixture( + '[deep_scan]\ndiscovery_model = "legacy-model"\ndiscovery_reasoning_effort = "high"\n', + ); + const model = await resolveDeepScanConfig( + { discoveryModel: "sdk-model" }, + input.source, + ); + expect(model.settings).toMatchObject({ + discoveryModel: "sdk-model", + discoveryReasoningEffort: "high", + }); + const effort = await resolveDeepScanConfig( + { discoveryReasoningEffort: "low" }, + input.source, + ); + expect(effort.settings).toMatchObject({ + discoveryModel: "legacy-model", + discoveryReasoningEffort: "low", + }); + const complete = await resolveDeepScanConfig( + { ...DEFAULT_DEEP_SCAN_SETTINGS, discoveryModel: "sdk-model" }, + input.source, + ); + expect(complete.settings.discoveryModel).toBe("sdk-model"); + expect(complete.settings).not.toHaveProperty("discoveryReasoningEffort"); +}); + +test("independent scans retain their own discovery configuration", async () => { + const inputs = await Promise.all( + ["first-model", "second-model"].map((model) => + fixture(`[deep_scan]\ndiscovery_model = "${model}"\n`), + ), + ); + const snapshots = await Promise.all( + inputs.map(async (input) => { + const resolved = await resolveDeepScanConfig({}, input.source); + const destination = join(input.root, "runtime", "deep-scan.toml"); + await writeDeepScanConfig(destination, resolved); + return parseToml(await readFile(destination, "utf8"))["deep_scan"]; + }), + ); + expect(snapshots).toMatchObject([ + { discovery_model: "first-model" }, + { discovery_model: "second-model" }, + ]); +}); + test("validates legacy values after matching explicit overrides", async () => { const input = await fixture('[deep_scan]\nworkers = "invalid"\n'); await expect(resolveDeepScanConfig({}, input.source)).rejects.toThrow( @@ -98,7 +204,7 @@ test.each(["same path", "directory link"])( "updating ambient overrides through the %s does not pin inherited defaults", async (kind) => { const input = await fixture( - "[deep_scan]\nworkers = 7\n[other]\nkeep = true\n", + '[deep_scan]\nworkers = 7\ndiscovery_model = "fixture-discovery"\n[other]\nkeep = true\n', ); let destination = input.source; if (kind === "directory link") { @@ -111,7 +217,7 @@ test.each(["same path", "directory link"])( await resolveDeepScanConfig({ workers: 8 }, input.source), ); expect(parseToml(await readFile(input.source, "utf8"))).toEqual({ - deep_scan: { workers: 8 }, + deep_scan: { workers: 8, discovery_model: "fixture-discovery" }, other: { keep: true }, }); }, @@ -139,6 +245,12 @@ test.each([ ["[deep_scan]\nstop_after_consecutive_errors = 0\n", "integer"], ["[deep_scan]\nmax_time_hours = 97\n", "no greater than 96"], ["[deep_scan]\nworkres = 2\n", "Unknown"], + ['[deep_scan]\ndiscovery_model = ""\n', "non-empty string"], + ['[deep_scan]\ndiscovery_model = " "\n', "non-empty string"], + ["[deep_scan]\ndiscovery_model = false\n", "non-empty string"], + ['[deep_scan]\ndiscovery_reasoning_effort = ""\n', "non-empty string"], + ['[deep_scan]\ndiscovery_reasoning_effort = " "\n', "non-empty string"], + ["[deep_scan]\ndiscovery_reasoning_effort = 12\n", "non-empty string"], ["[deep_scan\n", "Cannot read"], ])("rejects invalid ambient settings: %s", async (contents, message) => { const input = await fixture(contents); @@ -168,6 +280,20 @@ test("complete saved settings do not read a changed or invalid legacy file", asy expect(await readFile(input.source, "utf8")).toBe("not valid TOML ["); }); +test("complete saved settings do not adopt new ambient discovery overrides", async () => { + const input = await fixture( + '[deep_scan]\ndiscovery_model = "later-model"\ndiscovery_reasoning_effort = "high"\n', + ); + const destination = join(input.root, "runtime", "deep-scan.toml"); + await writeDeepScanConfig( + destination, + await resolveDeepScanConfig(DEFAULT_DEEP_SCAN_SETTINGS, input.source), + ); + const snapshot = parseToml(await readFile(destination, "utf8"))["deep_scan"]; + expect(snapshot).not.toHaveProperty("discovery_model"); + expect(snapshot).not.toHaveProperty("discovery_reasoning_effort"); +}); + test.each(["same path", "directory link"])( "complete settings preserve ambient sections through the %s", async (destinationKind) => { @@ -178,7 +304,7 @@ test.each(["same path", "directory link"])( ); await writeFile( input.source, - "[deep_scan]\nworkers = 9\n[other]\nenabled = false\n", + '[deep_scan]\nworkers = 9\ndiscovery_model = "fixture-discovery"\n[other]\nenabled = false\n', ); let destination = input.source; if (destinationKind === "directory link") { @@ -195,6 +321,7 @@ test.each(["same path", "directory link"])( stop_after_consecutive_errors: 3, max_discovery_runs: 40, max_time_hours: 96, + discovery_model: "fixture-discovery", }, other: { enabled: false }, }); diff --git a/sdk/typescript/tests-ts/project-config.test.ts b/sdk/typescript/tests-ts/project-config.test.ts index bf1460a5d..99a0640c5 100644 --- a/sdk/typescript/tests-ts/project-config.test.ts +++ b/sdk/typescript/tests-ts/project-config.test.ts @@ -361,6 +361,8 @@ describe("project configuration resolution", () => { project, { subagents: 0, + discoveryModel: "sdk-discovery", + discoveryReasoningEffort: "high", codexOverrides: { model: "gpt-5.6-sol", synthetic_setting: { enabled: false, itemCount: 0, names: [] }, @@ -370,6 +372,8 @@ describe("project configuration resolution", () => { ); expect(settings).toMatchObject({ subagents: 0, + discoveryModel: "sdk-discovery", + discoveryReasoningEffort: "high", workers: 8, }); expect(config).toEqual({ diff --git a/sdk/typescript/tests-ts/result.test.ts b/sdk/typescript/tests-ts/result.test.ts index 089bf6d9b..c97590a84 100644 --- a/sdk/typescript/tests-ts/result.test.ts +++ b/sdk/typescript/tests-ts/result.test.ts @@ -52,6 +52,65 @@ const coverage = { } satisfies CoverageDocument; describe("ScanResult", () => { + test.each([ + null, + { + model: "gpt-5.6-luna, gpt-5.6-sol", + inputTokens: 1_100_000, + cachedInputTokens: 0, + cacheWriteInputTokens: 0, + outputTokens: 0, + estimatedUsd: 0.6, + }, + ])( + "preserves an observed multi-model cost, including unavailable pricing: %p", + (cost) => { + const result = new ScanResult({ + manifest, + findings, + coverage, + scanDir: "/scan", + threadId: "thread", + turnResult: { + model: "gpt-5.6-sol", + usage: { input_tokens: 1_100_000, output_tokens: 0 }, + }, + observedCost: cost, + }); + expect(result.cost).toBe(cost); + expect(result.toJSON()["cost"]).toEqual(cost); + expect( + new ScanResult({ ...result, observedCost: result.cost }).cost, + ).toBe(cost); + }, + ); + + test("recomputes the cost when a copied result replaces its turn metadata", () => { + const original = new ScanResult({ + manifest, + findings, + coverage, + scanDir: "/scan", + threadId: "thread", + turnResult: { + model: "gpt-5.6-sol", + usage: { input_tokens: 1_000_000, output_tokens: 0 }, + }, + }); + expect(original.cost?.estimatedUsd).toBe(4); + const repriced = new ScanResult({ + ...original, + turnResult: { ...original.turnResult, model: "gpt-5.6-luna" }, + }); + expect(repriced.cost?.estimatedUsd).toBe(0.2); + const unpriced = new ScanResult({ + ...original, + turnResult: { ...original.turnResult, model: "synthetic-unpriced-model" }, + }); + expect(unpriced.cost).toBeNull(); + expect(original.cost?.estimatedUsd).toBe(4); + }); + test("rejects an unknown threshold with or without findings", () => { for (const levels of [[], ["high"]] satisfies SeverityLevel[][]) { const result = fakeResult(levels); diff --git a/sdk/typescript/tests-ts/scan-dashboard.test.ts b/sdk/typescript/tests-ts/scan-dashboard.test.ts index ea41aa7e1..d1d7b7e66 100644 --- a/sdk/typescript/tests-ts/scan-dashboard.test.ts +++ b/sdk/typescript/tests-ts/scan-dashboard.test.ts @@ -7,6 +7,7 @@ import { afterEach, describe, expect, test, mock, jest } from "bun:test"; import type { ComponentReceipt } from "../src/component-scan.js"; import { ScanDashboard } from "../src/scan-dashboard.js"; import { capture, fakeResult } from "./cli-fixtures.js"; +import { tokenUsage } from "../src/cost-model.js"; afterEach(() => jest.useRealTimers()); @@ -57,6 +58,81 @@ class DashboardTestInput extends EventEmitter { } describe("live scan dashboard", () => { + test("clears an unavailable price while retaining current tokens and restores a later estimate", () => { + const stderr = capture(true); + const dashboard = createDashboard(stderr.stream, { + model: { model: "gpt-5.6-sol", reasoningEffort: "xhigh" }, + showCost: true, + }); + const cost = fakeResult([], "complete", { + input_tokens: 1_000_000, + output_tokens: 0, + }).cost!; + dashboard.start(); + try { + dashboard.setCost(cost); + expect(lastFrame(stderr)).toContain("$4.00"); + dashboard.setUsage( + tokenUsage({ input_tokens: 3_000_000, output_tokens: 0 })!, + ); + expect(lastFrame(stderr)).toContain("3,000,000 total"); + expect(lastFrame(stderr)).toMatch(/COST\s+unavailable/); + expect(lastFrame(stderr)).not.toContain("$4.00"); + dashboard.setCost(cost); + expect(lastFrame(stderr)).toContain("$4.00"); + expect(lastFrame(stderr)).not.toMatch(/COST\s+unavailable/); + } finally { + dashboard.stop(); + } + }); + + test("does not present a known component subtotal as the complete cost", () => { + const stderr = capture(true); + const dashboard = createDashboard(stderr.stream, { + presentation: "components", + showCost: true, + }); + const receipts: ComponentReceipt[] = [1, 2].map((id) => ({ + id: `component-${id}`, + name: `Component ${id}`, + paths: [`src/${id}`], + status: "started", + outputDir: `/synthetic/results/${id}`, + })); + const cost = fakeResult([], "complete", { + input_tokens: 1_000_000, + output_tokens: 0, + }).cost!; + const frame = () => + stripVTControlCharacters(stderr.text().split("\u001B[H").at(-1)!); + dashboard.start(); + try { + dashboard.setComponents(receipts); + for (const receipt of receipts) { + dashboard.recordComponentEvent({ + componentId: receipt.id, + type: "cost", + value: cost, + }); + } + expect(frame().match(/COST\s+([^\n]+)/)?.[1]).toContain("$8.00"); + dashboard.recordComponentEvent({ + componentId: "component-2", + type: "usage", + value: tokenUsage({ input_tokens: 2_000_000, output_tokens: 0 })!, + }); + expect(frame()).toMatch(/COST\s+unavailable/); + dashboard.recordComponentEvent({ + componentId: "component-2", + type: "cost", + value: cost, + }); + expect(frame().match(/COST\s+([^\n]+)/)?.[1]).toContain("$8.00"); + } finally { + dashboard.stop(); + } + }); + test("flushes queued activity before stopping without rendering afterward", () => { const stderr = capture(true); const pending: (() => void)[] = []; diff --git a/sdk/typescript/tests-ts/scan-recipe.test.ts b/sdk/typescript/tests-ts/scan-recipe.test.ts index 41ff9c2fe..daf5162a7 100644 --- a/sdk/typescript/tests-ts/scan-recipe.test.ts +++ b/sdk/typescript/tests-ts/scan-recipe.test.ts @@ -34,6 +34,14 @@ test.each( const codexHome = join(root, "state", "codex-home"); await mkdir(repository); await mkdir(codexHome, { recursive: true }); + const legacyConfig = join(codexHome, "codex-security", "config.toml"); + if (mode === "deep") { + await mkdir(join(codexHome, "codex-security")); + await writeFile( + legacyConfig, + '[deep_scan]\ndiscovery_model = "synthetic-discovery"\ndiscovery_reasoning_effort = "high"\n', + ); + } await writeFile(join(repository, "source.py"), "# synthetic source\n"); const promptFile = join(root, "post-scan.md"); const postScanPrompt = @@ -112,6 +120,15 @@ test.each( scans[0]!.scanId, ]); expect(saved["recipe"]).toMatchObject({ mode, postScanPrompt }); + if (mode === "deep") { + expect(saved["recipe"]).toMatchObject({ + deepScan: { + discoveryModel: "synthetic-discovery", + discoveryReasoningEffort: "high", + }, + }); + await writeFile(legacyConfig, "invalid TOML ["); + } expect(JSON.stringify(saved)).not.toContain(promptFile); expect(JSON.stringify(saved)).not.toContain("synthetic-launch-key"); expect(JSON.stringify(saved)).not.toContain("synthetic-bedrock-key"); @@ -120,6 +137,17 @@ test.each( ...dependencies({ environment, currentDirectory: root, + onTurn: (_repository, options) => { + if (mode === "deep") + expect(options).toMatchObject({ + discoveryModel: "synthetic-discovery", + discoveryReasoningEffort: "high", + }); + else { + expect(options).not.toHaveProperty("discoveryModel"); + expect(options).not.toHaveProperty("discoveryReasoningEffort"); + } + }, onConfig: ({ codexOverrides }) => { expect(resolveCodexProfile(codexOverrides!)).toMatchObject( selected, diff --git a/sdk/typescript/tests-ts/support/usage-rollout.ts b/sdk/typescript/tests-ts/support/usage-rollout.ts index 87030c736..8e80fc93c 100644 --- a/sdk/typescript/tests-ts/support/usage-rollout.ts +++ b/sdk/typescript/tests-ts/support/usage-rollout.ts @@ -149,11 +149,13 @@ export async function writeSession( parent, cwd, timestamp, + model, parentField = "source", }: { parent?: string; cwd?: string; timestamp?: string; + model?: string; parentField?: SessionParentField; } = {}, ): Promise { @@ -172,6 +174,9 @@ export async function writeSession( ...(parent === undefined ? {} : parentMetadata(parent, parentField)), }, }), + ...(model === undefined + ? [] + : [JSON.stringify({ type: "turn_context", payload: { model } })]), JSON.stringify({ type: "event_msg", payload: {