diff --git a/github-action/README.md b/github-action/README.md index 8586adccc..5e277e4ff 100644 --- a/github-action/README.md +++ b/github-action/README.md @@ -387,6 +387,21 @@ The Action uses the CLI’s validated JSON result for findings and scan status. If the CLI exits without a usable result, the Action reports failure and leaves report paths empty; it does not recover unvalidated files from disk. +### Request attribution + +The Action adds `codex_security_integration=github_action` and +`codex_security_provider` (the configured provider family) through the CLI's existing +`responses_api_metadata` setting. These accompany the CLI's request attribution +on model requests supported by the provider. This attribution contains no +repository, workflow, run, actor, credential, or finding data. +Native managed configuration may override the provider selection before inference. + +The Action continues to set `analytics.enabled=false`. Request metadata is +separate from native analytics and does not create an additional telemetry +request or publish scan results to Codex Security Cloud. Requests sent to other +providers do not establish OpenAI-side usage reporting. Summaries, annotations, +and artifact uploads retain their existing controls. + ## Troubleshooting - **Checkout or history errors:** use the triggering revision, `fetch-depth: 0` diff --git a/github-action/dist/index.cjs b/github-action/dist/index.cjs index 2494315f4..6718a13c0 100644 --- a/github-action/dist/index.cjs +++ b/github-action/dist/index.cjs @@ -55631,7 +55631,14 @@ function scanArguments(inputs, target, resultsDirectory) { "--format", "json" ]; - args.push("--codex", "analytics.enabled=false"); + args.push( + "--codex", + "analytics.enabled=false", + "--codex", + 'responses_api_metadata.codex_security_integration="github_action"', + "--codex", + `responses_api_metadata.codex_security_provider=${JSON.stringify(inputs.provider)}` + ); for (const path6 of inputs.paths) args.push(`--path=./${path6}`); const options = [ ["--cyber-access-program", inputs.cyberAccessProgram], diff --git a/github-action/src/inputs.ts b/github-action/src/inputs.ts index afd9ac8d4..7791ba579 100644 --- a/github-action/src/inputs.ts +++ b/github-action/src/inputs.ts @@ -67,7 +67,9 @@ export function scanArguments(inputs: Inputs, target: {repository: string; diffB '--output-dir', resultsDirectory, '--format', 'json']; // Preserve the pinned CLI's sandbox and automatic approval-review defaults. // Forcing approval_policy="never" prevents recovery from hosted Linux sandbox errors. - args.push('--codex', 'analytics.enabled=false'); + args.push('--codex', 'analytics.enabled=false', + '--codex', 'responses_api_metadata.codex_security_integration="github_action"', + '--codex', `responses_api_metadata.codex_security_provider=${JSON.stringify(inputs.provider)}`); // Bind the option value and keep it explicitly relative so the CLI treats // leading '-' and '~' as literal repository filenames. for (const path of inputs.paths) args.push(`--path=./${path}`); diff --git a/github-action/tests/main.test.ts b/github-action/tests/main.test.ts index c30f3d074..7c87a96dc 100644 --- a/github-action/tests/main.test.ts +++ b/github-action/tests/main.test.ts @@ -16,6 +16,14 @@ import { copyCompletedScan } from './fixtures.js'; type Scenario = 'schedule' | 'pr' | 'policy-pr'; const inputKey = (name: string): string => `INPUT_${name.toUpperCase()}`; +function assertAttribution(args: readonly string[], provider: string): void { + assert.deepEqual(args.flatMap((arg, index) => arg === '--codex' ? [args[index + 1]] : []), [ + 'analytics.enabled=false', + 'responses_api_metadata.codex_security_integration="github_action"', + `responses_api_metadata.codex_security_provider="${provider}"`, + ]); +} + function outputValues(text: string): Record { const values: Record = {}; const lines = text.split(/\r?\n/u); @@ -186,6 +194,7 @@ test('PR severity failure retains SARIF outputs despite unrelated scope notes', test('scheduled complete report-only scan succeeds', async (t) => { const app = await harness(t); const result = await app.run(); + assertAttribution(result.args, 'openai'); assert.equal(result.exitCode, 0); assert.equal(result.outputs['scan-status'], 'completed'); assert.equal(result.outputs['policy-status'], 'passed'); assert.equal(result.outputs['sarif-upload-ready'], 'true'); assert.equal(result.outputs['analysis-ref'], 'refs/heads/main'); @@ -212,6 +221,7 @@ test('Deep scans forward their budget and publish completed reports with cleanup const app = await harness(t); app.setInput('mode', 'deep'); app.setInput('max-time-hours', '1.5'); const result = await app.run(); + assertAttribution(result.args, 'openai'); assert.equal(result.args[result.args.indexOf('--mode') + 1], 'deep'); assert.equal(result.args[result.args.indexOf('--max-time-hours') + 1], '1.5'); assert.equal(result.exitCode, 0); assert.equal(result.cleanups, 1); @@ -254,6 +264,7 @@ for (const mode of ['standard', 'deep']) { assert.equal(result.exitCode, 0); assert.equal(result.outputs['scan-status'], 'completed'); assert.equal(result.outputs['sarif-upload-ready'], 'true'); assert.equal(result.args[result.args.indexOf('--provider') + 1], 'amazon-bedrock'); + assertAttribution(result.args, 'amazon-bedrock'); assert.equal(result.args[result.args.indexOf('--auth') + 1], 'auto'); assert.equal(result.environment.AWS_ACCESS_KEY_ID, 'synthetic-access-key'); assert.equal(result.environment.AWS_SECRET_ACCESS_KEY, 'synthetic-secret-key'); @@ -274,6 +285,7 @@ for (const [provider, key] of [['openrouter', 'OPENROUTER_API_KEY'], ['fireworks const result = await app.run(); assert.equal(result.exitCode, 0); assert.equal(result.outputs['scan-status'], 'completed'); assert.equal(result.args[result.args.indexOf('--provider') + 1], provider); + assertAttribution(result.args, provider); assert.equal(result.args[result.args.indexOf('--auth') + 1], 'api-key'); assert.equal(result.environment[key], expectedKey); for (const other of ['OPENAI_API_KEY', 'CODEX_API_KEY', 'OPENROUTER_API_KEY', 'FIREWORKS_API_KEY', 'AWS_SESSION_TOKEN', 'GITHUB_TOKEN']) diff --git a/plugins/codex-security/mcp-app/src/deep-scan/executor.ts b/plugins/codex-security/mcp-app/src/deep-scan/executor.ts index 9e2d3a8d8..84568f83c 100644 --- a/plugins/codex-security/mcp-app/src/deep-scan/executor.ts +++ b/plugins/codex-security/mcp-app/src/deep-scan/executor.ts @@ -125,6 +125,17 @@ export class CodexSdkWorkerExecutor implements CodexWorkerExecutor { // Worker-owned tool and permission settings take precedence over inheritance. const configOverrides = profileConfigOverrides({ ...runtimeSettings.config, + responses_api_metadata: { + codex_security_surface: "plugin", + codex_security_command: "scan", + codex_security_scan_mode: "deep", + // Keep the host's integration/provider context as this coordinator + // launches or resumes workers with a different role. + ...(isRecord(runtimeSettings.config.responses_api_metadata) + ? runtimeSettings.config.responses_api_metadata + : {}), + codex_security_worker_role: request.kind, + }, ...(this.modelSettings.reasoningEffort ? { model_reasoning_effort: this.modelSettings.reasoningEffort } : {}), @@ -516,6 +527,27 @@ function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; } +function mergeConfigTables( + base: Record, + override: Record, +): Record { + return Object.fromEntries([ + ...Object.entries(base), + ...Object.entries(override).map(([key, value]) => { + const inherited = base[key]; + return [ + key, + isRecord(inherited) && + !Array.isArray(inherited) && + isRecord(value) && + !Array.isArray(value) + ? mergeConfigTables(inherited, value) + : value, + ]; + }), + ]); +} + async function workerRuntimeSettings( environment: Record, ): Promise { @@ -531,10 +563,13 @@ async function workerRuntimeSettings( typeof config.profile === "string" && isRecord(profiles) ? profiles[config.profile] : undefined; - const selected = { ...config, ...(isRecord(profile) ? profile : {}) }; - for (const key of ["analytics", "responses_api_metadata"]) { + const selected: Record = { + ...config, + ...(isRecord(profile) ? profile : {}), + }; + for (const key of ["analytics", "otel", "responses_api_metadata"]) { if (isRecord(config[key]) && isRecord(profile) && isRecord(profile[key])) { - selected[key] = { ...config[key], ...profile[key] }; + selected[key] = mergeConfigTables(config[key], profile[key]); } } const inherited = Object.fromEntries( @@ -542,6 +577,7 @@ async function workerRuntimeSettings( "model_reasoning_summary", "service_tier", "analytics", + "otel", "responses_api_metadata", "skills", ].map((key) => [key, selected[key]]), diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_worker_telemetry_fixture.ts b/plugins/codex-security/mcp-app/tests/deep_scan_worker_telemetry_fixture.ts new file mode 100644 index 000000000..be07b5a81 --- /dev/null +++ b/plugins/codex-security/mcp-app/tests/deep_scan_worker_telemetry_fixture.ts @@ -0,0 +1,81 @@ +/** Root, selected-profile, and SDK snapshot telemetry used by launch tests. */ +export function workerTelemetryFixture( + index: number, + hasConfiguration: boolean, +) { + // Cover missing settings, native config inheritance, and snapshot overrides. + const mergeProfileTelemetry = index === 0 && hasConfiguration; + const telemetry = + index === 0 && !hasConfiguration + ? {} + : { + analytics: { enabled: index % 2 === 0 }, + otel: { + environment: `synthetic-scan-${index}`, + exporter: { + "otlp-http": { + endpoint: `http://localhost:${4318 + index}/v1/logs`, + headers: { "X-Scan": `synthetic-scan-${index}` }, + }, + }, + log_user_prompt: false, + }, + responses_api_metadata: { + codex_security_surface: index % 2 === 0 ? "cli" : "sdk", + codex_security_command: "scan", + codex_security_package_version: `0.2.${index}`, + codex_security_plugin_version: `0.1.${index}`, + codex_security_integration: "github_action", + codex_security_provider: + index % 2 === 0 ? "openai" : "amazon-bedrock", + custom_tag: `synthetic-scan-${index}`, + "team.tag": `literal-key-${index}`, + ...(mergeProfileTelemetry + ? { + root_only: "native-root", + profile_only: "selected-profile", + } + : {}), + }, + }; + const overrideTelemetry = index === 1 || index === 2; + const inheritedTelemetry = + overrideTelemetry || index === 4 + ? { + analytics: { enabled: index % 2 !== 0 }, + otel: { environment: "preflight-default", exporter: "none" }, + responses_api_metadata: { custom_tag: "preflight-default" }, + } + : mergeProfileTelemetry + ? { + analytics: { enabled: true }, + otel: { + environment: "native-root", + exporter: { + "otlp-http": { + endpoint: "http://localhost:4317/v1/logs", + headers: { "X-Scan": "synthetic-scan-0" }, + }, + }, + log_user_prompt: false, + }, + responses_api_metadata: { + codex_security_surface: "cli", + codex_security_command: "scan", + codex_security_package_version: "0.2.0", + codex_security_plugin_version: "0.1.0", + codex_security_integration: "github_action", + codex_security_provider: "openai", + custom_tag: "preflight-default", + "team.tag": "literal-key-0", + root_only: "native-root", + }, + } + : telemetry; + return { + mergeProfileTelemetry, + telemetry, + inheritedTelemetry, + workerTelemetry: overrideTelemetry ? telemetry : {}, + }; +} diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts index cbcc0d11f..748c82c87 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_executor.ts @@ -36,6 +36,7 @@ import type { DeepScanWorkerKind, } from "../src/deep-scan/types.js"; import { testDeepScanContextSnapshots } from "./deep_scan_context_snapshot_cases.ts"; +import { workerTelemetryFixture } from "./deep_scan_worker_telemetry_fixture.ts"; const executorSource = new URL("../src/deep-scan/executor.ts", import.meta.url); const { bundledCodexSdkEnvironment } = await importModule({ @@ -1515,47 +1516,12 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`, }, ], }; - // Cover missing settings, native config inheritance, and snapshot overrides. - const mergeProfileTelemetry = index === 0 && configuration !== ""; - const telemetry = - index === 0 && configuration === "" - ? {} - : { - analytics: { enabled: index % 2 === 0 }, - responses_api_metadata: { - codex_security_surface: index % 2 === 0 ? "cli" : "sdk", - codex_security_command: "scan", - codex_security_package_version: `0.2.${index}`, - codex_security_plugin_version: `0.1.${index}`, - custom_tag: `synthetic-scan-${index}`, - ...(mergeProfileTelemetry - ? { - root_only: "native-root", - profile_only: "selected-profile", - } - : {}), - }, - }; - const overrideTelemetry = index === 1 || index === 2; - const inheritedTelemetry = - overrideTelemetry || index === 4 - ? { - analytics: { enabled: index % 2 !== 0 }, - responses_api_metadata: { custom_tag: "preflight-default" }, - } - : mergeProfileTelemetry - ? { - analytics: { enabled: true }, - responses_api_metadata: { - codex_security_surface: "cli", - codex_security_command: "scan", - codex_security_package_version: "0.2.0", - codex_security_plugin_version: "0.1.0", - custom_tag: "preflight-default", - root_only: "native-root", - }, - } - : telemetry; + const { + mergeProfileTelemetry, + telemetry, + inheritedTelemetry, + workerTelemetry, + } = workerTelemetryFixture(index, configuration !== ""); const features = index === 0 ? undefined @@ -1674,7 +1640,7 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`, skills, features, telemetry, - workerTelemetry: overrideTelemetry ? telemetry : {}, + workerTelemetry, configuration: { ...parsedConfiguration, ...inheritedTelemetry, @@ -1718,6 +1684,14 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`, ...(mergeProfileTelemetry ? { analytics: {}, + otel: { + environment: "synthetic-scan-0", + exporter: { + "otlp-http": { + endpoint: "http://localhost:4318/v1/logs", + }, + }, + }, responses_api_metadata: { custom_tag: "synthetic-scan-0", profile_only: "selected-profile", @@ -2054,7 +2028,7 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`, parseToml( nativeConfigOverrides(invocation.argv) .filter((override) => - /^(analytics|responses_api_metadata)[.=]/.test( + /^(analytics|otel|responses_api_metadata)[.=]/.test( override, ), ) @@ -2062,7 +2036,17 @@ env_key = "SYNTHETIC_GATEWAY_KEY"`, ), ), ), - workerConfigurations[index].telemetry, + { + ...workerConfigurations[index].telemetry, + responses_api_metadata: { + codex_security_surface: "plugin", + codex_security_command: "scan", + codex_security_scan_mode: "deep", + ...workerConfigurations[index].telemetry + .responses_api_metadata, + codex_security_worker_role: kind, + }, + }, ); assert.equal( invocation.cacheDirectory, diff --git a/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.ts b/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.ts index 9ab0a7eff..44f10a6fb 100644 --- a/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.ts +++ b/plugins/codex-security/mcp-app/tests/test_deep_scan_store_integration.ts @@ -39,6 +39,7 @@ export { recordCodexSecurityScanDraftViaWorkbench, saveScanDraftCheckpoint } fro await testCanonicalParentCheckpointsSurviveSuccessiveWrites(); await testFreeformFailureMessagesAgainstRealWorkbench(); +await testWorkerRetryThreadsRemainInScanUsage(); await testReducerCommitAndFinishAgainstRealWorkbench(); await testReducerCommitAndFinishAgainstRealWorkbench(true); await testExpiredDeadlineWithoutCompletedDiscoveryAgainstRealWorkbench(); @@ -228,6 +229,58 @@ function createWorkbenchRunner(environment: NodeJS.ProcessEnv, bounded = true) { }; } +async function testWorkerRetryThreadsRemainInScanUsage() { + const { fixtureRoot, targetPath, environment } = await createWorkbenchFixture( + "deep-scan-retry-usage-", + ); + const runWorkbench = createWorkbenchRunner(environment); + const store = new WorkbenchDeepScanStore(runWorkbench); + try { + const threadId = "retry-owner"; + const run = await store.begin({ + targetPath, + threadId, + scanRoot: path.join(fixtureRoot, "scans"), + }); + await store.claimCoordinator({ scanId: run.scanId, threadId }); + const worker = await createWorkerFixture(run, "retry", "discovery"); + await store.updateWorker({ ...worker, status: "queued" }); + await store.updateWorker({ + ...worker, + status: "running", + threadId: "first-attempt", + }); + const retried = { + ...worker, + attempt: 2, + status: "running" as const, + threadId: "retry-attempt", + }; + await store.updateWorker(retried); + await store.updateWorker(retried); + + const resumed = await new WorkbenchDeepScanStore(runWorkbench).get( + run.scanId, + threadId, + ); + assert.equal(resumed.persistedWorkers.length, 1); + assert.equal(resumed.persistedWorkers[0].threadId, "retry-attempt"); + assert.equal(resumed.persistedWorkers[0].attempt, 2); + const { scan } = await runWorkbench(["get-scan", "--scan-id", run.scanId]); + assert.deepEqual(scan.executionThreadIds, [ + "first-attempt", + "retry-attempt", + ]); + assert.deepEqual(scan.threadIds, [ + threadId, + "first-attempt", + "retry-attempt", + ]); + } finally { + await rm(fixtureRoot, { recursive: true, force: true }); + } +} + async function testFreeformFailureMessagesAgainstRealWorkbench() { const { fixtureRoot, targetPath, environment } = await createWorkbenchFixture( "deep-scan-error-transport-", diff --git a/plugins/codex-security/mcp-app/tests/test_workbench_database.ts b/plugins/codex-security/mcp-app/tests/test_workbench_database.ts index ff6dc51a7..1eeb51f91 100644 --- a/plugins/codex-security/mcp-app/tests/test_workbench_database.ts +++ b/plugins/codex-security/mcp-app/tests/test_workbench_database.ts @@ -168,6 +168,43 @@ test("every released schema upgrades to the same current schema and remains idem } }); +test("worker thread history survives retry, replay, and worker retirement", (t) => { + const database = memory(t, 49); + insertScan(database); + database.exec(` + INSERT INTO deep_scan_runs ( + scan_id, schema_version, workflow_version, status, phase, workers, + subagents, stop_after_no_new, max_discovery_runs, created_at, updated_at + ) VALUES ('scan', 1, 'fixture', 'running', 'discovery', 1, 0, 3, 10, 'created', 'updated'); + INSERT INTO deep_scan_workers ( + id, scan_id, kind, status, prompt_path, artifact_dir, attempt, + sdk_thread_id, created_at, updated_at + ) VALUES ('worker', 'scan', 'discovery', 'running', '/synthetic/prompt.md', + '/synthetic/output', 1, 'first-thread', 'created', 'updated'); + `); + applyMigrations(database); + database.exec(` + UPDATE deep_scan_workers SET sdk_thread_id = 'retry-thread', attempt = 2; + UPDATE deep_scan_workers SET sdk_thread_id = 'retry-thread', attempt = 2; + DELETE FROM deep_scan_workers; + `); + applyMigrations(database); + assert.deepEqual( + database + .prepare( + "SELECT sdk_thread_id FROM deep_scan_worker_threads ORDER BY sdk_thread_id", + ) + .all() + .map((row) => row.sdk_thread_id), + ["first-thread", "retry-thread"], + ); + database.exec("DELETE FROM scans"); + assert.deepEqual( + database.prepare("SELECT * FROM deep_scan_worker_threads").all(), + [], + ); +}); + test("configured state paths use native parent traversal semantics", async () => { const directory = await temporary.create("workbench-symlink-"); const actual = join(directory, "actual"); diff --git a/plugins/codex-security/scripts/workbench_scan_usage.py b/plugins/codex-security/scripts/workbench_scan_usage.py index e46a52cac..0f9854123 100644 --- a/plugins/codex-security/scripts/workbench_scan_usage.py +++ b/plugins/codex-security/scripts/workbench_scan_usage.py @@ -120,6 +120,7 @@ def collect_scan_usage( total = _empty_token_usage() accepted_thread_ids: set[str] = set() + measured_thread_ids: set[str] = set() excluded_thread_ids: set[str] = set() for session in sessions: if session.parent_thread_id in excluded_thread_ids: @@ -138,6 +139,10 @@ def collect_scan_usage( started_at=started_at, completed_at=stopped_at, completed_turn_id=completed_turn_id if session.thread_id == thread_id else None, + is_scan_owner=( + scan["mode"] == "deep" + and session.thread_id == scan["deep_scan_owner_thread_id"] + ), ) except (OSError, UnicodeError, ValueError): missing_thread_ids.add(session.thread_id) @@ -153,16 +158,22 @@ def collect_scan_usage( missing_thread_ids.add(session.thread_id) continue accepted_thread_ids.add(session.thread_id) + if "token_usage_unavailable" in session_warnings: + missing_thread_ids.add(session.thread_id) + continue + measured_thread_ids.add(session.thread_id) _add_token_usage(total, session_usage) if not accepted_thread_ids: return _unavailable_usage("scan_thread_unavailable", warnings=warnings) + if not measured_thread_ids: + return _unavailable_usage("token_usage_unavailable", warnings=warnings) result: dict[str, Any] = { "coverage": "partial" if missing_thread_ids or warnings else "complete", "source": "codex_rollout", **total, - "threadCount": len(accepted_thread_ids), + "threadCount": len(measured_thread_ids), } if missing_thread_ids: result["missingThreadCount"] = len(missing_thread_ids) @@ -196,8 +207,8 @@ def _scan_root_thread_ids( for row in connection.execute( """ SELECT DISTINCT sdk_thread_id - FROM deep_scan_workers - WHERE scan_id = ? AND sdk_thread_id IS NOT NULL + FROM deep_scan_worker_threads + WHERE scan_id = ? ORDER BY sdk_thread_id """, (scan["id"],), @@ -440,6 +451,7 @@ def _read_rollout_usage( started_at: datetime, completed_at: datetime | None, completed_turn_id: str | None = None, + is_scan_owner: bool = False, ) -> tuple[dict[str, int], set[str]]: total = _empty_token_usage() warnings: set[str] = set() @@ -451,6 +463,8 @@ def _read_rollout_usage( response_ids: set[str] = set() current_turn_id: str | None = None completed_turn_measured = False + usage_observed = False + direct_control_session = False with session.path.open("rb") as source: for line_number, raw_line in enumerate(source, start=1): @@ -488,6 +502,13 @@ def _read_rollout_usage( and not recorded_parent and not payload.get("forked_from_id") ) + # prepareCliDeepScanSession creates this owner through app-server + # without a model turn, so its initial token usage is known zero. + direct_control_session = ( + boundary_reached + and is_scan_owner + and payload.get("originator") == "codex_security_deep_scan" + ) continue if not isinstance(payload, dict): @@ -513,6 +534,19 @@ def _read_rollout_usage( previous = inherited_usage continue if event.get("type") == "event_msg" and payload.get("type") == "task_started": + # The same owner can resume as a model-backed coordinator. Only + # a provably post-scan turn leaves the original control zero intact. + task_started_at = _timestamp(event.get("timestamp")) + if ( + completed_at is None + or task_started_at is None + or task_started_at <= completed_at + or ( + completed_turn_id is not None + and payload.get("turn_id") == completed_turn_id + ) + ): + direct_control_session = False if ( completed_turn_id is not None and current_turn_id == completed_turn_id @@ -585,6 +619,7 @@ def _read_rollout_usage( if turn_usage is not None: observed_response_usage = turn_usage continue + usage_observed = True if is_completion_turn: completed_turn_measured = True if turn_usage is not None and any( @@ -614,6 +649,7 @@ def _read_rollout_usage( ) if completed_at is not None and timestamp > completed_at and not is_completion_turn: continue + usage_observed = True if is_completion_turn: completed_turn_measured = True if delta["totalTokens"] <= 0: @@ -623,6 +659,8 @@ def _read_rollout_usage( _add_token_usage(total, response_usage if response_usage is not None else legacy_usage) if not boundary_reached: warnings.add("thread_ownership_unavailable") + elif not usage_observed and not direct_control_session: + warnings.add("token_usage_unavailable") if completed_turn_id is not None and not completed_turn_measured: warnings.add("completion_turn_unavailable") return total, warnings diff --git a/plugins/codex-security/shared/workbench-migrations.json b/plugins/codex-security/shared/workbench-migrations.json index 11e368286..e55fe7034 100644 --- a/plugins/codex-security/shared/workbench-migrations.json +++ b/plugins/codex-security/shared/workbench-migrations.json @@ -410,5 +410,15 @@ "statements": [ "CREATE INDEX finding_issue_receipts_by_destination_issue\nON finding_issue_receipts(destination_key, external_id, finding_id);" ] + }, + { + "version": 50, + "name": "preserve deep scan worker thread history", + "statements": [ + "CREATE TABLE deep_scan_worker_threads (\n scan_id TEXT NOT NULL REFERENCES scans(id) ON DELETE CASCADE,\n worker_id TEXT NOT NULL,\n sdk_thread_id TEXT NOT NULL,\n PRIMARY KEY (scan_id, worker_id, sdk_thread_id)\n);", + "INSERT INTO deep_scan_worker_threads (scan_id, worker_id, sdk_thread_id)\nSELECT scan_id, id, sdk_thread_id FROM deep_scan_workers\nWHERE sdk_thread_id IS NOT NULL;", + "CREATE TRIGGER record_deep_scan_worker_thread_insert\nAFTER INSERT ON deep_scan_workers\nWHEN NEW.sdk_thread_id IS NOT NULL\nBEGIN\n INSERT OR IGNORE INTO deep_scan_worker_threads (scan_id, worker_id, sdk_thread_id)\n VALUES (NEW.scan_id, NEW.id, NEW.sdk_thread_id);\nEND;", + "CREATE TRIGGER record_deep_scan_worker_thread_update\nAFTER UPDATE OF sdk_thread_id ON deep_scan_workers\nWHEN NEW.sdk_thread_id IS NOT NULL\nBEGIN\n INSERT OR IGNORE INTO deep_scan_worker_threads (scan_id, worker_id, sdk_thread_id)\n VALUES (NEW.scan_id, NEW.id, NEW.sdk_thread_id);\nEND;" + ] } ] diff --git a/plugins/codex-security/tests/test_workbench_db.py b/plugins/codex-security/tests/test_workbench_db.py index 47af92867..35a27d849 100644 --- a/plugins/codex-security/tests/test_workbench_db.py +++ b/plugins/codex-security/tests/test_workbench_db.py @@ -70,6 +70,7 @@ EXPECTED_TABLES = { "deep_scan_dedup_inputs", "deep_scan_runs", + "deep_scan_worker_threads", "deep_scan_workers", "finding_decisions", "finding_dedupe_group_members", diff --git a/plugins/codex-security/tests/test_workbench_scan_usage.py b/plugins/codex-security/tests/test_workbench_scan_usage.py index 11126db43..81379eae5 100644 --- a/plugins/codex-security/tests/test_workbench_scan_usage.py +++ b/plugins/codex-security/tests/test_workbench_scan_usage.py @@ -169,6 +169,7 @@ def _rollout( task_started_at: datetime | None = None, session_started_at: datetime | None = None, copied_events: list[dict[str, Any]] | None = None, + originator: str | None = None, ) -> Path: directory = root / "rollouts" directory.mkdir(exist_ok=True) @@ -183,6 +184,8 @@ def _rollout( }, *(copied_events or []), ] + if originator is not None: + records[0]["payload"]["originator"] = originator if session_started_at is not None: records[0]["payload"]["timestamp"] = session_started_at.isoformat().replace("+00:00", "Z") if parent_thread_id is not None and include_task_start: @@ -917,6 +920,76 @@ def test_completion_reports_unavailable_without_fabricating_zero(tmp_path: Path) assert "totalTokens" not in usage +@pytest.mark.parametrize("observation", ["missing", "null", "before", "legacy_zero", "native_zero"]) +def test_completion_distinguishes_missing_usage_from_observed_zero( + tmp_path: Path, observation: str +) -> None: + fixture = _start_scan(tmp_path) + counted = fixture.started_at + timedelta(microseconds=1) + events = [] + if observation == "null": + events = [_event(counted, "event_msg", {"type": "token_count", "info": None})] + elif observation == "before": + events = [_token_event(fixture.started_at - timedelta(seconds=1), 10, 1)] + elif observation == "legacy_zero": + events = [_token_event(counted, 0, 0)] + elif observation == "native_zero": + events = [_response_event(counted, "scan-parent", "zero-response", 0, 0)] + _state_graph( + fixture.environment, + {"scan-parent": _rollout(tmp_path, "scan-parent", events)}, + [], + ) + + usage = _complete_scan(fixture)["scan"]["usage"] + + if observation.endswith("_zero"): + assert usage == { + "coverage": "complete", + "source": "codex_rollout", + **_counts(0, 0, 0), + "threadCount": 1, + } + else: + assert usage == { + "coverage": "unavailable", + "source": "codex_rollout", + "threadCount": 0, + "warnings": ["token_usage_unavailable"], + } + + +@pytest.mark.parametrize("missing_thread", ["scan-parent", "scan-worker"]) +def test_missing_usage_preserves_measured_parent_or_descendant( + tmp_path: Path, missing_thread: str +) -> None: + fixture = _start_scan(tmp_path) + counted = fixture.started_at + timedelta(microseconds=1) + _state_graph( + fixture.environment, + { + thread: _rollout( + tmp_path, + thread, + [] if thread == missing_thread else [_token_event(counted, 10, 2)], + parent_thread_id="scan-parent" if thread == "scan-worker" else None, + task_started_at=counted, + ) + for thread in ["scan-parent", "scan-worker"] + }, + [("scan-parent", "scan-worker")], + ) + + assert _complete_scan(fixture)["scan"]["usage"] == { + "coverage": "partial", + "source": "codex_rollout", + **_counts(10, 0, 2), + "threadCount": 1, + "missingThreadCount": 1, + "warnings": ["token_usage_unavailable"], + } + + @pytest.mark.skipif(sys.platform != "darwin", reason="macOS system path aliases") @pytest.mark.parametrize("temporary_root", [tempfile.gettempdir(), "/tmp"], ids=["var", "tmp"]) def test_completion_accepts_macos_system_rollout_alias( @@ -952,25 +1025,29 @@ def test_completion_rejects_non_system_rollout_symlink(tmp_path: Path) -> None: } -def test_completion_counts_deep_sdk_workers_and_descendants(tmp_path: Path) -> None: +def test_completion_counts_deep_worker_retries_and_descendants(tmp_path: Path) -> None: fixture = _start_scan(tmp_path, mode="deep") counted = fixture.started_at + timedelta(microseconds=1) artifact = fixture.scan_dir / "artifacts" / "usage-worker" artifact.mkdir(parents=True) prompt = artifact / "prompt.md" prompt.write_text("Review the fixture target.\n", encoding="utf-8") - upsert_deep_worker( - fixture.state_dir, - fixture.scan_id, - str(uuid.uuid4()), - "discovery", - "running", - str(prompt), - str(artifact), - "--sdk-thread-id", - "sdk-worker", - environment=fixture.environment, - ) + worker_id = str(uuid.uuid4()) + for attempt, thread in [(1, "sdk-worker"), (2, "sdk-retry"), (2, "sdk-retry")]: + upsert_deep_worker( + fixture.state_dir, + fixture.scan_id, + worker_id, + "discovery", + "running", + str(prompt), + str(artifact), + "--attempt", + str(attempt), + "--sdk-thread-id", + thread, + environment=fixture.environment, + ) _state_graph( fixture.environment, { @@ -982,15 +1059,15 @@ def test_completion_counts_deep_sdk_workers_and_descendants(tmp_path: Path) -> N [_token_event(counted, 7, 2)], parent_thread_id="sdk-worker", ), + "sdk-retry": _rollout(tmp_path, "sdk-retry", [_token_event(counted, 30, 3)]), }, [("sdk-worker", "sdk-child")], ) - usage = _complete_scan(fixture)["scan"]["usage"] - assert usage == { + assert _complete_scan(fixture)["scan"]["usage"] == { "coverage": "complete", "source": "codex_rollout", - **_counts(37, 0, 10), - "threadCount": 3, + **_counts(67, 0, 13), + "threadCount": 4, } @@ -1085,6 +1162,146 @@ def test_completion_preserves_explicit_legacy_cost(tmp_path: Path) -> None: assert "usage" not in completed +@pytest.mark.parametrize( + "worker_state", + ["absent", "measured", "missing_graph_child", "missing_registered_worker", "unmeasured"], +) +def test_completion_persists_direct_control_zero_without_hiding_missing_workers( + tmp_path: Path, worker_state: str +) -> None: + fixture = _start_scan(tmp_path, mode="deep") + counted = fixture.started_at + timedelta(microseconds=1) + threads = { + # The app-server client persists this originator while injecting context, + # without starting a model turn or producing a native token receipt. + "scan-parent": _rollout( + tmp_path, + "scan-parent", + [ + _event(counted, "turn_context", {}), + _event(counted, "response_item", {"type": "message"}), + ], + originator="codex_security_deep_scan", + ), + } + edges = [] + measured_worker = worker_state not in {"absent", "unmeasured"} + if worker_state != "absent": + threads["scan-worker"] = _rollout( + tmp_path, + "scan-worker", + [_token_event(counted, 100, 10)] if measured_worker else [], + parent_thread_id="scan-parent", + task_started_at=counted, + originator="codex_security_deep_scan", + ) + edges.append(("scan-parent", "scan-worker")) + if worker_state == "missing_graph_child": + threads["missing-worker"] = tmp_path / "missing-worker.jsonl" + edges.append(("scan-parent", "missing-worker")) + elif worker_state == "missing_registered_worker": + with sqlite3.connect(fixture.state_dir / "workbench.sqlite3") as connection: + connection.execute( + "INSERT INTO deep_scan_worker_threads (scan_id, worker_id, sdk_thread_id) " + "VALUES (?, ?, ?)", + (fixture.scan_id, "earlier-attempt", "missing-worker"), + ) + _state_graph(fixture.environment, threads, edges) + missing_warning = { + "missing_graph_child": "rollout_unavailable", + "missing_registered_worker": "scan_root_unavailable", + "unmeasured": "token_usage_unavailable", + }.get(worker_state) + usage = { + "coverage": "complete" if missing_warning is None else "partial", + "source": "codex_rollout", + **_counts(100 if measured_worker else 0, 0, 10 if measured_worker else 0), + "threadCount": 2 if measured_worker else 1, + **({"missingThreadCount": 1, "warnings": [missing_warning]} if missing_warning else {}), + } + + completed = _complete_scan(fixture)["scan"] + + assert completed["usage"] == usage + assert "cost" not in completed + repeated = scan_command( + fixture.state_dir, "complete-scan", fixture.scan_id, environment=fixture.environment + )["scan"] + assert repeated["usage"] == usage + assert "cost" not in repeated + with sqlite3.connect(fixture.state_dir / "workbench.sqlite3") as connection: + stored = connection.execute( + "SELECT cost_json FROM scans WHERE id = ?", (fixture.scan_id,) + ).fetchone()[0] + assert json.loads(stored) == {"usage": usage} + + +@pytest.mark.parametrize("turn_start", ["before_scan", "during_scan", "untimestamped", "post_scan"]) +def test_direct_control_resumed_model_turn_requires_a_usage_receipt( + tmp_path: Path, turn_start: str +) -> None: + fixture = _start_scan(tmp_path, mode="deep") + counted = fixture.started_at + timedelta(microseconds=1) + timestamp = ( + fixture.started_at - timedelta(seconds=1) + if turn_start == "before_scan" + else counted + timedelta(days=1) + if turn_start == "post_scan" + else counted + ) + event = _event(timestamp, "event_msg", {"type": "task_started", "turn_id": "resumed-turn"}) + if turn_start == "untimestamped": + del event["timestamp"] + _state_graph( + fixture.environment, + { + "scan-parent": _rollout( + tmp_path, "scan-parent", [event], originator="codex_security_deep_scan" + ), + "scan-worker": _rollout( + tmp_path, + "scan-worker", + [_token_event(counted, 100, 10)], + parent_thread_id="scan-parent", + ), + }, + [("scan-parent", "scan-worker")], + ) + + usage = _complete_scan(fixture)["scan"]["usage"] + + assert usage == { + "coverage": "complete" if turn_start == "post_scan" else "partial", + "source": "codex_rollout", + **_counts(100, 0, 10), + "threadCount": 2 if turn_start == "post_scan" else 1, + **( + {} + if turn_start == "post_scan" + else {"missingThreadCount": 1, "warnings": ["token_usage_unavailable"]} + ), + } + + +@pytest.mark.parametrize("originator", ["codex_sdk_ts", "synthetic_override", None]) +def test_metadata_only_deep_owner_requires_direct_control_originator( + tmp_path: Path, originator: str | None +) -> None: + fixture = _start_scan(tmp_path, mode="deep") + _state_graph( + fixture.environment, + {"scan-parent": _rollout(tmp_path, "scan-parent", [], originator=originator)}, + [], + ) + + assert _complete_scan(fixture)["scan"]["usage"] == { + "coverage": "unavailable", + "source": "codex_rollout", + "threadCount": 0, + "warnings": ["token_usage_unavailable"], + } + + def test_usage_is_returned_by_completion_without_an_extra_command(tmp_path: Path) -> None: fixture = _start_scan(tmp_path) counted = fixture.started_at + timedelta(microseconds=1) diff --git a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py index 20e22612f..5086a0ff3 100644 --- a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py +++ b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py @@ -91,6 +91,7 @@ (47, "snapshot deep scan discovery context"), (48, "share finding issue receipts across trackers"), (49, "index finding issue receipt ownership"), + (50, "preserve deep scan worker thread history"), ] diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md index b92fdff53..3984099a5 100644 --- a/sdk/typescript/docs/cli.md +++ b/sdk/typescript/docs/cli.md @@ -651,14 +651,28 @@ that command. Model, effort, and provider choices carry into patch-risk assessme Model requests include Codex Security attribution in `responses_api_metadata`: `codex_security_surface` (`cli` or `sdk`), `codex_security_command`, `codex_security_package_version`, and `codex_security_plugin_version` when the -selected plugin declares a version. These fields identify the workflow and -versions producing a model request; they do not record command starts, -completions, or failures. +selected plugin declares a version. The shared context adds +`codex_security_provider` for the configured provider family when the wrapper +selects a provider, and +`codex_security_scan_mode` for scans. Custom provider names use `custom`. +Provider attribution reflects the wrapper's selected profile or default; +native managed configuration may select a different provider before inference. +The GitHub Action supplies `codex_security_integration=github_action`; +Deep Scan workers retain the context and add `codex_security_worker_role`. +Caller metadata, including literal dotted keys, is preserved. Connected-app +publication uses the same request attribution; embedding requests identify +the package and version in their `User-Agent`. + +These fields provide request attribution for supported inference providers, including +Bedrock. They do not create a separate telemetry collector, record command +starts or outcomes, or make another provider's traffic visible to OpenAI. Disable Codex usage analytics and built-in metrics with `--codex 'analytics.enabled=false'`. The setting carries into `scan --patch`, patch-risk assessment, and Deep Scan discovery and reducer workers, including -resumed workers. Deep Scan workers also preserve the scan's request metadata. +resumed workers. Selected-profile settings take precedence over root settings. +Explicit OpenTelemetry settings also carry into scan patches and Deep Scan +workers, including exporter choices and prompt-logging controls. Model-request attribution metadata remains attached when analytics are disabled. The setting does not control configured OpenTelemetry exporters, integrations, authentication, or CLI update checks. Validation uses isolated configuration; diff --git a/sdk/typescript/src/api.ts b/sdk/typescript/src/api.ts index 005f22757..1424b49be 100644 --- a/sdk/typescript/src/api.ts +++ b/sdk/typescript/src/api.ts @@ -100,7 +100,7 @@ import { type DeepScanProgress, } from "./deep-progress.js"; import { findScanSession } from "./scan-logs.js"; -import { runDeepScan } from "./deep-scan.js"; +import { DIRECT_DEEP_SCAN_CONTROL_USAGE, runDeepScan } from "./deep-scan.js"; import { deepScanOptions, resolveDeepScanConfig, @@ -195,7 +195,7 @@ import { type ScanWorkerStatus, } from "./worker-progress.js"; import { CODEX_SECURITY_THREAD_SOURCES } from "./thread-source.js"; -import { codexSecurityRequestMetadata } from "./request-metadata.js"; +import { codexSecurityRequestContext } from "./request-metadata.js"; import { CODEX_EXECUTABLE_VERSION, CODEX_SDK_VERSION } from "./version.js"; import { bundledCodexSdkEnvironment } from "./codex-sdk-environment.js"; import { @@ -887,7 +887,7 @@ export class CodexSecurity { session.sessionConfig["approval_policy"] = "never"; const { codex } = await this.#createSessionCodex( session, - "validate", + { command: "validate" }, { CODEX_SECURITY_REPOSITORY: inputs.repository, CODEX_SECURITY_PLUGIN_ROOT: runtime.plugin.pluginRoot, @@ -998,7 +998,7 @@ export class CodexSecurity { }; const { codex } = await this.#createSessionCodex( session, - "patch", + { command: "patch" }, { CODEX_SECURITY_REPOSITORY: repository, CODEX_SECURITY_PLUGIN_ROOT: session.runtime.plugin.pluginRoot, @@ -1028,7 +1028,9 @@ export class CodexSecurity { onCost: options.onCost === undefined ? undefined - : notifyObserver(options, "onCost"), + : (cost, complete) => { + if (complete) notifyObserver(options, "onCost")(cost); + }, onError: reportTrackingError, }); const thread = codex.startThread({ @@ -1365,7 +1367,7 @@ export class CodexSecurity { ); const { codex } = await this.#createSessionCodex( session, - "policy", + { command: "policy" }, { CODEX_SECURITY_REPOSITORY: target.repository, CODEX_SECURITY_PLUGIN_ROOT: runtime.plugin.pluginRoot, @@ -1383,9 +1385,12 @@ export class CodexSecurity { `permissions.${POLICY_PERMISSION_PROFILE}.filesystem=${inlineToml(policyFilesystem)}`, ], ); - const reportCost = (current: Readonly): void => { + const reportCost = ( + current: Readonly, + complete = true, + ): void => { const total = addScanCosts(accumulatedCost, current); - if (completeCost) notifyObserver(options, "onCost")(total); + if (completeCost && complete) notifyObserver(options, "onCost")(total); if ( options.maxCostUsd !== undefined && total.estimatedUsd > options.maxCostUsd @@ -1557,6 +1562,7 @@ export class CodexSecurity { ]); let maxCostUsd = options.maxCostUsd; let latestCost: Readonly | null = null; + let latestCostComplete = false; let notifiedLimit: number | undefined; let scanDir = ""; let targetPathsFile: string | null = null; @@ -1798,16 +1804,12 @@ export class CodexSecurity { ); const workerSnapshot: JsonObject = { ...workerRuntimeConfig, - responses_api_metadata: { - ...(isRecord(workerRuntimeConfig["responses_api_metadata"]) - ? workerRuntimeConfig["responses_api_metadata"] - : {}), - ...codexSecurityRequestMetadata( - this.#surface, - "scan", - runtime.plugin.version, - ), - }, + responses_api_metadata: codexSecurityRequestContext(effectiveConfig, { + surface: this.#surface, + command: "scan", + pluginVersion: runtime.plugin.version, + mode, + }).metadata, ...(workerEnvironment === undefined ? {} : { environment: workerEnvironment }), @@ -1959,9 +1961,11 @@ export class CodexSecurity { onCost: options.onCost === undefined && options.maxCostUsd === undefined ? undefined - : (cost) => { + : (cost, complete) => { latestCost = cost; - notifyObserver(options, "onCost")(cost, maxCostUsd); + latestCostComplete = complete; + if (complete) + notifyObserver(options, "onCost")(cost, maxCostUsd); if ( maxCostUsd !== undefined && cost.estimatedUsd > maxCostUsd @@ -2013,7 +2017,11 @@ export class CodexSecurity { ); if (budgetSignal.aborted) return; maxCostUsd = next; - notifyObserver(options, "onCost")(latestCost!, maxCostUsd); + if (latestCostComplete) + notifyObserver(options, "onCost")( + latestCost!, + maxCostUsd, + ); }) .catch((error: unknown) => { if (!budgetSignal.aborted) { @@ -2375,7 +2383,7 @@ export class CodexSecurity { const { codex, environment, codexOptions } = await this.#createSessionCodex( session, - "scan", + { command: "scan", mode }, runtimePaths, options.auth, git, @@ -2475,6 +2483,8 @@ export class CodexSecurity { return; } if (budgetRecovery !== null) budgetRecovery.threadId = threadId; + if (directDeepScan) + tracker.recordUsage(DIRECT_DEEP_SCAN_CONTROL_USAGE, threadId); tracker.start(threadId); try { await workbench( @@ -2765,7 +2775,10 @@ export class CodexSecurity { "all", ); if (previousFindings !== undefined) { - const matchingRuntimeConfig = structuredClone(workerRuntimeConfig); + const matchingRuntimeConfig: JsonObject = { + ...structuredClone(workerRuntimeConfig), + responses_api_metadata: workerSnapshot["responses_api_metadata"]!, + }; const matchingFeatures = matchingRuntimeConfig["features"]; if (isRecord(matchingFeatures)) delete matchingFeatures["plugins"]; await matchCompletedScan({ @@ -3206,7 +3219,7 @@ export class CodexSecurity { async #createSessionCodex( session: PreparedSession, - command: string, + { command, mode }: { command: string; mode?: ScanMode }, runtimePaths: Record, auth: ScanAuthMode = "auto", git?: InspectedExecutable, @@ -3355,11 +3368,16 @@ export class CodexSecurity { ...configOverrides, ]; } - const configuredResponsesMetadata = isRecord( - sdkCodexConfig["responses_api_metadata"], - ) - ? sdkCodexConfig["responses_api_metadata"] - : {}; + const requestContext = codexSecurityRequestContext( + config ?? sessionConfig, + { + surface: this.#surface, + command, + pluginVersion: runtime.plugin.version, + mode, + }, + ); + delete sdkCodexConfig["responses_api_metadata"]; const configuredCodexPath = environmentValue( this.#dependencies.environment, "CODEX_CLI_PATH", @@ -3388,23 +3406,9 @@ export class CodexSecurity { ...(runtime.providerProfile === undefined ? {} : { nativeProfile: runtime.providerProfile.name }), - ...(configOverrides.length > 0 - ? { - configOverrides, - } - : {}), + configOverrides: [...configOverrides, requestContext.configOverride], env: sdkEnvironment, - config: { - ...(sdkCodexConfig as NonNullable), - responses_api_metadata: { - ...configuredResponsesMetadata, - ...codexSecurityRequestMetadata( - this.#surface, - command, - runtime.plugin.version, - ), - }, - }, + config: sdkCodexConfig as NonNullable, }; const codex = await this.#dependencies.createCodex(codexOptions); return { codex, environment, codexOptions }; @@ -4462,7 +4466,8 @@ export async function runScanEvents( } // Managed scans reconcile saved checkpoints before admitting canonical artifacts. if (options.workbenchValidated && options.onFinalize !== undefined) { - usage = (await options.onFinalize(usage)) ?? usage; + const finalizedUsage = await options.onFinalize(usage); + if (finalizedUsage !== undefined) usage = finalizedUsage; } options.signal.throwIfAborted(); await requireScanArtifacts( @@ -4498,7 +4503,8 @@ export async function runScanEvents( ); } if (!options.workbenchValidated && options.onFinalize !== undefined) { - usage = (await options.onFinalize(usage)) ?? usage; + const finalizedUsage = await options.onFinalize(usage); + if (finalizedUsage !== undefined) usage = finalizedUsage; } const result = await collectResult( { @@ -5594,6 +5600,7 @@ function selectedWorkerRuntimeConfig( [ ...CODEX_AUTH_CONFIG_KEYS, "analytics", + "otel", "responses_api_metadata", "openai_base_url", "features", diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 31b820dd5..fc6cefe66 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -201,6 +201,7 @@ import { checkScanPublication, forceTerminatePublicationProcesses as terminatePublishers, publishScan, + publishScanInternal, type CheckScanPublicationOptions, type PublishScanProgress, type PublishScanResult, @@ -237,7 +238,7 @@ import { type ScanMatchingBatch, } from "./scan-comparison.js"; import { scanActivitiesFromEvent } from "./scan-activity.js"; -import { codexSecurityRequestMetadata } from "./request-metadata.js"; +import { codexSecurityRequestContext } from "./request-metadata.js"; import { CODEX_SECURITY_THREAD_SOURCES, type CodexSecurityThreadSource, @@ -1081,6 +1082,8 @@ interface SkillRunOptions { provider?: string; providerConfiguration?: JsonObject; environment?: NodeJS.ProcessEnv; + requestMetadata?: JsonObject; + otel?: JsonObject; patchArtifact?: { path: string; repository: string; @@ -3155,27 +3158,28 @@ export async function main( } let result; try { - result = await (dependencies.publishScan ?? publishScan)( - resolveCliPath(currentDirectory, scanDir), - { - ...destination!, - ...(options.findingId.length === 0 - ? {} - : { findingIds: options.findingId }), - ...(selectedScans[0]?.scanId === undefined - ? {} - : { expectedScanId: selectedScans[0].scanId }), - dryRun: options.dryRun, - signal: controller.signal, - ...(options.skipExisting ? { skipExisting: true } : {}), - ...(options.dryRun - ? {} - : { - onProgress: (event: PublishScanProgress) => - progress.observe(event), - }), - }, - ); + result = await ( + dependencies.publishScan ?? + ((directory, options) => + publishScanInternal(directory, options, { surface: "cli" })) + )(resolveCliPath(currentDirectory, scanDir), { + ...destination!, + ...(options.findingId.length === 0 + ? {} + : { findingIds: options.findingId }), + ...(selectedScans[0]?.scanId === undefined + ? {} + : { expectedScanId: selectedScans[0].scanId }), + dryRun: options.dryRun, + signal: controller.signal, + ...(options.skipExisting ? { skipExisting: true } : {}), + ...(options.dryRun + ? {} + : { + onProgress: (event: PublishScanProgress) => + progress.observe(event), + }), + }); } finally { progress.stop(); } @@ -7655,6 +7659,26 @@ async function runSkill( const threadSource = patch ? CODEX_SECURITY_THREAD_SOURCES.remediation : CODEX_SECURITY_THREAD_SOURCES.validation; + const requestContext = codexSecurityRequestContext( + { + ...effectiveOverrides, + ...(options.requestMetadata === undefined + ? {} + : { responses_api_metadata: options.requestMetadata }), + }, + { + surface: "cli", + command: assess + ? "assess-patch-risk" + : verify + ? "verify-fix" + : patch + ? "patch" + : "validate", + pluginVersion, + providerResolved: provider !== undefined || validate, + }, + ); return dependencies.runCodex( [ ...(appServer @@ -7676,6 +7700,9 @@ async function runSkill( value.startsWith("analytics.") || value.startsWith("analytics="), ) .flatMap((value) => ["--config", value]), + ...(options.otel === undefined + ? [] + : ["--config", `otel=${inlineToml(options.otel)}`]), ...(provider === undefined ? [] : ["--config", `model_provider=${JSON.stringify(provider)}`]), @@ -7690,22 +7717,8 @@ async function runSkill( ...(verify || assess ? ["--config", 'approvals_reviewer="auto_review"'] : []), - ...Object.entries( - codexSecurityRequestMetadata( - "cli", - assess - ? "assess-patch-risk" - : verify - ? "verify-fix" - : patch - ? "patch" - : "validate", - pluginVersion, - ), - ).flatMap(([key, value]) => [ - "--config", - `responses_api_metadata.${key}=${JSON.stringify(value)}`, - ]), + "--config", + requestContext.configOverride, ...(options.safetyIdentifier === undefined ? [] : [ @@ -8301,6 +8314,8 @@ async function executeScan( let auth: ScanAuthMode | undefined = arguments_.auth; let patchAnalyticsOverride: string | undefined; let patchServiceTier: JsonValue | undefined; + let patchRequestMetadata: JsonObject | undefined; + let patchOtel: JsonObject | undefined; let selectedAuthentication: ScanAuthentication | null = null; let repository = ""; let failed = false; @@ -8326,13 +8341,22 @@ async function executeScan( }; ({ model: effectiveModel, reasoningEffort: effectiveReasoningEffort } = scanModelConfiguration(effectiveConfiguration)); - patchServiceTier = - resolveCodexProfile(effectiveConfiguration)["service_tier"] ?? "default"; + const resolvedConfiguration = resolveCodexProfile(effectiveConfiguration); + patchServiceTier = resolvedConfiguration["service_tier"] ?? "default"; const provider = scanModelProvider(effectiveConfiguration); - const analytics = effectiveConfiguration["analytics"]; + const analytics = resolvedConfiguration["analytics"]; if (isJsonObject(analytics) && analytics["enabled"] !== undefined) { patchAnalyticsOverride = `analytics.enabled=${JSON.stringify(analytics["enabled"])}`; } + patchRequestMetadata = { + ...codexSecurityRequestContext(resolvedConfiguration, { + surface: "cli", + command: "scan", + mode: arguments_.mode, + }).metadata, + }; + const otel = resolvedConfiguration["otel"]; + if (isJsonObject(otel)) patchOtel = otel; auth = !arguments_.dryRun && !arguments_.mock && interactive ? await chooseInteractiveAuthentication( @@ -9078,6 +9102,8 @@ async function executeScan( { ...providerOptions, serviceTier: patchServiceTier, + requestMetadata: patchRequestMetadata, + otel: patchOtel, safetyIdentifier: arguments_.safetyIdentifier, auth, findingInstructions: patchSelection?.instructions, diff --git a/sdk/typescript/src/cost.ts b/sdk/typescript/src/cost.ts index a485b0e7b..5faa6723b 100644 --- a/sdk/typescript/src/cost.ts +++ b/sdk/typescript/src/cost.ts @@ -58,7 +58,8 @@ interface ScanCostTrackerOptions { scanDirectory?: string; maxCostUsd?: number; expectedFilesTotal?: number; - onCost?: (cost: Readonly) => void; + /** Incomplete costs are measured subtotals, usable for spending limits only. */ + onCost?: (cost: Readonly, complete: boolean) => void; onUsage?: (usage: Readonly) => void; onActivity?: (activity: ScanActivity) => void; onProgress?: (progress: ScanProgress) => void; @@ -311,6 +312,9 @@ export class ScanCostTracker { this.#reportWorkerProgress(session); } const receipt = usages.get(threadId); + // A discovered thread without a receipt has unknown usage, not zero. + // Retain it so a parent-only subtotal cannot appear to be complete. + if (receipt === undefined) usages.set(threadId, null); if ( session.usage !== null && (session.usage.total_tokens > (receipt?.total_tokens ?? -1) || @@ -322,18 +326,22 @@ export class ScanCostTracker { } } let usage: ScanTokenUsage | null = null; + let complete = true; for (const value of usages.values()) { if (value === null) { - this.#snapshot = { usage: null, cost: null }; - return; + complete = false; + continue; } usage = addTokenUsage(usage, value); } + if (!complete) this.#snapshot = { usage: null, cost: null }; if (usage === null) return; const cost = estimateScanCost(this.#options.model, usage); - this.#snapshot = { usage, cost }; - this.#reportUsage(usage); - this.#reportCost(cost); + if (complete) { + this.#snapshot = { usage, cost }; + this.#reportUsage(usage); + } + this.#reportCost(cost, complete); } #reportWorkerProgress(session: SessionUsage): void { @@ -375,12 +383,12 @@ export class ScanCostTracker { } } - #reportCost(cost: ScanCost | null): void { + #reportCost(cost: ScanCost | null, complete = true): void { if (cost === null) return; - const signature = JSON.stringify(cost); + const signature = JSON.stringify([cost, complete]); if (signature === this.#lastCost) return; this.#lastCost = signature; - this.#options.onCost?.(cost); + this.#options.onCost?.(cost, complete); } #reportUsage(usage: ScanTokenUsage | null): void { diff --git a/sdk/typescript/src/deduplication/codex-review.ts b/sdk/typescript/src/deduplication/codex-review.ts index 857d029e3..5508c842c 100644 --- a/sdk/typescript/src/deduplication/codex-review.ts +++ b/sdk/typescript/src/deduplication/codex-review.ts @@ -22,7 +22,7 @@ import { import { CODEX_SECURITY_THREAD_SOURCES } from "../thread-source.js"; import { VERSION } from "../version.js"; import type { CodexSecuritySurface } from "../api.js"; -import { codexSecurityRequestMetadata } from "../request-metadata.js"; +import { codexSecurityRequestContext } from "../request-metadata.js"; import { DeduplicationReviewError, type DeduplicationReviewFailureCategory, @@ -388,11 +388,14 @@ export class CodexReviewRunner { update_plan: { enabled: false }, experimental_request_user_input: { enabled: false }, }, - responses_api_metadata: { - ...(executionConfig["responses_api_metadata"] as - Record | undefined), - ...codexSecurityRequestMetadata(this.surface, "dedupe"), - }, + responses_api_metadata: codexSecurityRequestContext( + executionConfig, + { + surface: this.surface, + command: "dedupe", + providerResolved: false, + }, + ).metadata, features: { code_mode: { direct_only_tool_namespaces: ["review_validator"], diff --git a/sdk/typescript/src/deep-scan.ts b/sdk/typescript/src/deep-scan.ts index 17ba6bb8c..e81f1079a 100644 --- a/sdk/typescript/src/deep-scan.ts +++ b/sdk/typescript/src/deep-scan.ts @@ -19,6 +19,14 @@ export interface DirectDeepScanOptions { signal: AbortSignal; } +// The direct coordinator records context but never calls a model itself. +export const DIRECT_DEEP_SCAN_CONTROL_USAGE = { + input_tokens: 0, + cached_input_tokens: 0, + cache_write_input_tokens: 0, + output_tokens: 0, +} as const; + /** Run the same coordinator as the plugin, without a parent model turn. */ export async function* runDeepScan( options: DirectDeepScanOptions, @@ -95,12 +103,7 @@ export async function* runDeepScan( // existing scan tracker from the native sessions below this scan directory. yield { type: "turn.completed", - usage: { - input_tokens: 0, - cached_input_tokens: 0, - cache_write_input_tokens: 0, - output_tokens: 0, - }, + usage: { ...DIRECT_DEEP_SCAN_CONTROL_USAGE }, }; } diff --git a/sdk/typescript/src/publish.ts b/sdk/typescript/src/publish.ts index b2cac0e2a..b87ad0b6f 100644 --- a/sdk/typescript/src/publish.ts +++ b/sdk/typescript/src/publish.ts @@ -1,4 +1,6 @@ import { findingEntry, notify } from "./value.js"; +import type { CodexSecuritySurface } from "./api.js"; +import { codexSecurityRequestContext } from "./request-metadata.js"; import { spawn, spawnSync, @@ -173,6 +175,7 @@ export interface PublicationCodexResult { } export interface PublishScanDependencies { + surface?: CodexSecuritySurface; environment?: NodeJS.ProcessEnv; linearClient?: LinearClientFactory<"users" | "createIssue">; prepare?: typeof prepareScanPublication; @@ -331,6 +334,15 @@ export async function publishScanInternal( options.signal, ); } else { + const requestContext = codexSecurityRequestContext( + {}, + { + surface: dependencies.surface ?? "sdk", + command: "publish", + // Native Codex resolves provider and metadata from all its config layers. + providerResolved: false, + }, + ); invocation = await (dependencies.runCodex ?? runPublicationCodex)( command!, [ @@ -339,6 +351,8 @@ export async function publishScanInternal( "gpt-5.6-luna", "-c", 'model_reasoning_effort="low"', + "-c", + requestContext.configOverride, "--ephemeral", "--json", "--sandbox", diff --git a/sdk/typescript/src/request-metadata.ts b/sdk/typescript/src/request-metadata.ts index 0460d51ef..b8f198044 100644 --- a/sdk/typescript/src/request-metadata.ts +++ b/sdk/typescript/src/request-metadata.ts @@ -1,18 +1,62 @@ import type { CodexSecuritySurface } from "./api.js"; +import { inlineToml, resolveCodexProfile, type JsonObject } from "./config.js"; +import { isRecord } from "./record.js"; +import type { ScanMode } from "./scan-modes.js"; import { VERSION } from "./version.js"; -/** Attribution for model requests, independent of native analytics events. */ -export function codexSecurityRequestMetadata( - surface: CodexSecuritySurface, - command: string, - pluginVersion?: string, +interface RequestContextOptions { + surface: CodexSecuritySurface; + command: string; + pluginVersion?: string; + mode?: ScanMode; + /** False when this caller leaves provider selection to native config layers. */ + providerResolved?: boolean; +} + +/** Per-operation attribution, independent of native analytics and OTel settings. + * Provider family reflects wrapper configuration; native managed settings can override it. */ +export function codexSecurityRequestContext( + config: Readonly, + context: RequestContextOptions, ) { - return { - codex_security_surface: surface, - codex_security_command: command, + const resolved = resolveCodexProfile(config); + const configured = resolved["responses_api_metadata"]; + const metadata = Object.freeze({ + ...(isRecord(configured) ? configured : {}), + codex_security_surface: context.surface, + codex_security_command: context.command, codex_security_package_version: VERSION, - ...(pluginVersion === undefined + ...(context.pluginVersion === undefined + ? {} + : { codex_security_plugin_version: context.pluginVersion }), + ...(context.providerResolved === false ? {} - : { codex_security_plugin_version: pluginVersion }), - }; + : { + codex_security_provider: providerFamily(resolved["model_provider"]), + }), + ...(context.mode === undefined + ? {} + : { codex_security_scan_mode: context.mode }), + }) as Readonly>; + return Object.freeze({ + metadata, + // The SDK's structured config flattens literal dotted keys into nested tables. + configOverride: `responses_api_metadata=${inlineToml(metadata)}`, + }); +} + +function providerFamily(provider: unknown): string { + if (typeof provider !== "string") + return provider === undefined ? "openai" : "custom"; + switch (provider) { + case "openai": + case "amazon-bedrock": + case "openrouter": + case "fireworks": + case "ollama": + case "lmstudio": + return provider; + default: + return "custom"; + } } diff --git a/sdk/typescript/src/scan-comparison.ts b/sdk/typescript/src/scan-comparison.ts index 43769ea6a..7a2908e61 100644 --- a/sdk/typescript/src/scan-comparison.ts +++ b/sdk/typescript/src/scan-comparison.ts @@ -25,7 +25,6 @@ import { DEFAULT_CODEX_CONFIG, deepMerge, hasCommandAuth, - inlineToml, mergedCodexConfig, normalizeLegacyWindowsSandboxOverride, resolveCodexProfile, @@ -38,7 +37,7 @@ import { type JsonObject, } from "./config.js"; import { CodexSecurityError, ConfigurationError } from "./errors.js"; -import { codexSecurityRequestMetadata } from "./request-metadata.js"; +import { codexSecurityRequestContext } from "./request-metadata.js"; import { createProfileCodex, createProviderProfile, @@ -570,15 +569,21 @@ async function startReadOnlyCodexThread( ); } const sdkConfig = structuredCodexConfig(config); - const requestMetadata = { - ...(homeExecutionConfig["responses_api_metadata"] as - JsonObject | undefined), - ...(sdkConfig["responses_api_metadata"] as JsonObject | undefined), - ...codexSecurityRequestMetadata( - runtimeOptions.surface, - runtimeOptions.command, - ), - }; + const requestContext = codexSecurityRequestContext( + { + ...sdkConfig, + responses_api_metadata: { + ...(homeExecutionConfig["responses_api_metadata"] as + JsonObject | undefined), + ...(sdkConfig["responses_api_metadata"] as JsonObject | undefined), + }, + }, + { + surface: runtimeOptions.surface, + command: runtimeOptions.command, + providerResolved: sdkConfig["model_provider"] !== undefined, + }, + ); delete sdkConfig["responses_api_metadata"]; delete sdkConfig["default_permissions"]; const providerSettings = commandAuth ? providerConfig : (config ?? {}); @@ -594,8 +599,7 @@ async function startReadOnlyCodexThread( ); const command = resolveCodexCommand(environment); const codexOptions: CodexOptions = { - // A single table preserves literal keys that the SDK would split on dots. - configOverrides: [`responses_api_metadata=${inlineToml(requestMetadata)}`], + configOverrides: [requestContext.configOverride], codexPathOverride: executablePathForSpawn(command.command), env: environment, // The SDK forwards its apiKey option as CODEX_API_KEY for Codex exec. diff --git a/sdk/typescript/src/server/embeddings.ts b/sdk/typescript/src/server/embeddings.ts index 4f68a9ef4..b06ff843b 100644 --- a/sdk/typescript/src/server/embeddings.ts +++ b/sdk/typescript/src/server/embeddings.ts @@ -2,6 +2,7 @@ import { Tiktoken } from "js-tiktoken/lite"; import cl100kBase from "js-tiktoken/ranks/cl100k_base"; import { errorMessage } from "../errors.js"; import type { Finding } from "../models.js"; +import { VERSION } from "../version.js"; import { FindingsError } from "./errors.js"; import type { FindingEmbedding } from "./storage.js"; @@ -93,6 +94,7 @@ export class OpenAiFindingEmbedder implements FindingEmbedder { headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json", + "User-Agent": `codex-security/${VERSION}`, }, body: JSON.stringify({ model: EMBEDDING_MODEL, diff --git a/sdk/typescript/tests-ts/api-attribution-concurrency.test.ts b/sdk/typescript/tests-ts/api-attribution-concurrency.test.ts index dc9f5e414..15678abb1 100644 --- a/sdk/typescript/tests-ts/api-attribution-concurrency.test.ts +++ b/sdk/typescript/tests-ts/api-attribution-concurrency.test.ts @@ -11,6 +11,7 @@ import { PLUGIN_ROOT } from "./plugin-root.js"; import { createApiTestFixtures } from "./support/temporary-directories.js"; import { InternalSecurity } from "./support/internal-security.js"; import { VERSION } from "../src/version.js"; +import { requestMetadata } from "./support/request-metadata.js"; const fixtures = createApiTestFixtures(); @@ -94,8 +95,11 @@ describe("delegated scan attribution", () => { pluginPath: PLUGIN_ROOT, codexOverrides: { analytics: { enabled: surface === "sdk" }, + otel: { environment: `synthetic-${surface}` }, responses_api_metadata: { custom_attribution: surface, + "team.tag": `synthetic-${surface}`, + codex_security_integration: "github_action", codex_security_surface: "spoofed", codex_security_command: "spoofed", codex_security_package_version: "spoofed", @@ -203,12 +207,22 @@ describe("delegated scan attribution", () => { features, ...(model === undefined ? {} : { model }), analytics: { enabled: surface === "sdk" }, - responses_api_metadata: { - custom_attribution: surface, - codex_security_surface: surface, - codex_security_command: "scan", - codex_security_package_version: VERSION, - }, + otel: { environment: `synthetic-${surface}` }, + }); + expect(options.config).not.toHaveProperty( + "responses_api_metadata", + ); + expect( + requestMetadata(options.configOverrides), + ).toMatchObject({ + custom_attribution: surface, + "team.tag": `synthetic-${surface}`, + codex_security_integration: "github_action", + codex_security_surface: surface, + codex_security_command: "scan", + codex_security_package_version: VERSION, + codex_security_provider: "openai", + codex_security_scan_mode: mode, }); const deepConfigPath = options.env?.["CODEX_SECURITY_DEEP_SCAN_CONFIG_PATH"]; @@ -221,8 +235,10 @@ describe("delegated scan attribution", () => { parseToml(deepConfig)["worker_runtime"], ).toMatchObject({ analytics: { enabled: surface === "sdk" }, - responses_api_metadata: - options.config?.["responses_api_metadata"], + otel: { environment: `synthetic-${surface}` }, + responses_api_metadata: requestMetadata( + options.configOverrides, + ), }); } expect(threadOptions.threadSource).toBe("security_scan"); @@ -286,6 +302,7 @@ describe("delegated scan attribution", () => { expect(sharedConfig).not.toHaveProperty( "responses_api_metadata", ); + expect(sharedConfig).not.toHaveProperty("otel"); expect(sharedConfig).not.toHaveProperty("model"); expect(sharedConfig).not.toHaveProperty("codex_security"); expect(sharedConfig).not.toHaveProperty("analytics"); diff --git a/sdk/typescript/tests-ts/api-credentials.test.ts b/sdk/typescript/tests-ts/api-credentials.test.ts index 0b50f8ca0..f88854d47 100644 --- a/sdk/typescript/tests-ts/api-credentials.test.ts +++ b/sdk/typescript/tests-ts/api-credentials.test.ts @@ -1,3 +1,4 @@ +import { requestMetadata } from "./support/request-metadata.js"; import { nodeCommand } from "./support/shell.js"; import { existsSync } from "node:fs"; import { chmod, mkdir, readFile, stat, writeFile } from "node:fs/promises"; @@ -342,10 +343,10 @@ describe("CodexSecurity orchestration", () => { shell_environment_policy: { set: { PRIVATE_TOKEN: "RUNTIME_SHELL_SECRET" }, }, - responses_api_metadata: { - request_trace: "preserve-configured-metadata", - codex_security_surface: "sdk", - }, + }); + expect(requestMetadata(options.configOverrides)).toMatchObject({ + request_trace: "preserve-configured-metadata", + codex_security_surface: "sdk", }); if (process.platform !== "win32") { expect((await stat(configPath!)).mode & 0o777).toBe(0o600); diff --git a/sdk/typescript/tests-ts/api-direct-deep-scan.test.ts b/sdk/typescript/tests-ts/api-direct-deep-scan.test.ts index caf3da15f..5076368d2 100644 --- a/sdk/typescript/tests-ts/api-direct-deep-scan.test.ts +++ b/sdk/typescript/tests-ts/api-direct-deep-scan.test.ts @@ -14,12 +14,13 @@ import { } from "./support/api-events.js"; import { createApiTestFixtures } from "./support/temporary-directories.js"; import { runTestInSubprocess } from "./support/test-subprocess.js"; +import { writeSession } from "./support/usage-rollout.js"; const { temporaryDirectory, cleanup } = createApiTestFixtures(); afterEach(cleanup); test.each([undefined, "Summarize the completed scan."])( - "Deep Scan uses the finalized report with post-scan prompt %p", + "Deep Scan preserves worker usage and the finalized report with post-scan prompt %p", async (postScanPrompt) => { const root = await temporaryDirectory(); const repository = join(root, "repository"); @@ -31,6 +32,9 @@ test.each([undefined, "Summarize the completed scan."])( const finalReport = "# Final scan report\n\nFinalized synthetic findings.\n"; const commands: string[] = []; + let completionReceipt: string | undefined; + const { promise: runningCost, resolve: recordCost } = + Promise.withResolvers(); const parentTurn = mock(async () => { throw new Error("A parent model turn must not execute the Deep Scan."); }); @@ -42,70 +46,108 @@ test.each([undefined, "Summarize the completed scan."])( runStreamed: postScanTurn, })); const started = mock(); - const client = TestClient.withDependencies({ - ...scanRuntimeDependencies(codexHome, scanDir), - canRunDirectDeepScan: () => true, - environment: { - CODEX_SECURITY_STATE_DIR: join(root, "state"), - CODEX_API_KEY: "synthetic-direct-scan-key", - }, - createCodex: () => ({ - startThread: () => ({ id: null, runStreamed: parentTurn }), - resumeThread, - }), - runDeepScan: async function* (options) { - expect(options.scanId).toBe("scan_example_001"); - expect(options.codexOptions.env).toMatchObject({ - CODEX_SECURITY_SCAN_ID: options.scanId, - CODEX_SECURITY_SCAN_DIR: scanDir, - CODEX_SECURITY_REPOSITORY: repository, - }); - await copyCompletedScan(root); - const coveragePath = join(scanDir, "coverage.json"); - const coverage = JSON.parse(await readFile(coveragePath, "utf8")); - coverage.mode = "deep_repository"; - const coverageBytes = JSON.stringify(coverage); - await writeFile(coveragePath, coverageBytes); - const manifestPath = join(scanDir, "scan-manifest.json"); - const manifest = JSON.parse(await readFile(manifestPath, "utf8")); - const artifact = manifest.scan.artifacts.find( - (entry: { path: string }) => entry.path === "coverage.json", - ); - artifact.sha256 = hash("sha256", coverageBytes); - await writeFile(manifestPath, JSON.stringify(manifest)); - yield { type: "thread.started", thread_id: "engine-session" }; - yield { - type: "turn.completed", - usage: { - input_tokens: 0, - cached_input_tokens: 0, - cache_write_input_tokens: 0, - output_tokens: 0, - }, - }; - }, - runWorkbench: async (_options, args, input) => { - commands.push(args[0]!); - if (args[0] === "prepare-scan-completion") { - await writeFile( - join(scanDir, "report.md"), - "# Prepared scan report\n", + const client = new TestClient( + { codexOverrides: { model: "gpt-5.6-sol" } }, + { + ...scanRuntimeDependencies(codexHome, scanDir), + canRunDirectDeepScan: () => true, + environment: { + CODEX_SECURITY_STATE_DIR: join(root, "state"), + CODEX_API_KEY: "synthetic-direct-scan-key", + }, + createCodex: () => ({ + startThread: () => ({ id: null, runStreamed: parentTurn }), + resumeThread, + }), + runDeepScan: async function* (options) { + expect(options.scanId).toBe("scan_example_001"); + expect(options.codexOptions.env).toMatchObject({ + CODEX_SECURITY_SCAN_ID: options.scanId, + CODEX_SECURITY_SCAN_DIR: scanDir, + CODEX_SECURITY_REPOSITORY: repository, + }); + await copyCompletedScan(root); + const coveragePath = join(scanDir, "coverage.json"); + const coverage = JSON.parse(await readFile(coveragePath, "utf8")); + coverage.mode = "deep_repository"; + const coverageBytes = JSON.stringify(coverage); + await writeFile(coveragePath, coverageBytes); + const manifestPath = join(scanDir, "scan-manifest.json"); + const manifest = JSON.parse(await readFile(manifestPath, "utf8")); + const artifact = manifest.scan.artifacts.find( + (entry: { path: string }) => entry.path === "coverage.json", ); - } else if (args[0] === "complete-scan") { - await writeFile(join(scanDir, "report.md"), finalReport); - } - return mockWorkbench(args, input); + artifact.sha256 = hash("sha256", coverageBytes); + await writeFile(manifestPath, JSON.stringify(manifest)); + await writeSession( + codexHome, + "engine-session", + {}, + { + cwd: scanDir, + timestamp: "2026-07-26T12:00:00Z", + }, + ); + await writeSession( + codexHome, + "discovery-session", + { input_tokens: 100, output_tokens: 10 }, + { + cwd: join( + scanDir, + "artifacts", + "deep_discovery", + "workers", + "one", + "output", + ), + timestamp: "2026-07-26T12:00:01Z", + }, + ); + yield { type: "thread.started", thread_id: "engine-session" }; + // The host must count workers while the no-inference coordinator runs. + expect(await runningCost).toBe(100); + yield { + type: "turn.completed", + usage: { + input_tokens: 0, + cached_input_tokens: 0, + cache_write_input_tokens: 0, + output_tokens: 0, + }, + }; + }, + runWorkbench: async (_options, args, input) => { + commands.push(args[0]!); + if (args[0] === "prepare-scan-completion") { + await writeFile( + join(scanDir, "report.md"), + "# Prepared scan report\n", + ); + } else if (args[0] === "complete-scan") { + const receiptIndex = args.indexOf("--cost-json"); + completionReceipt = + receiptIndex < 0 ? undefined : args[receiptIndex + 1]; + await writeFile(join(scanDir, "report.md"), finalReport); + } + return mockWorkbench(args, input); + }, }, - }); + ); try { const result = await client.run(repository, { mode: "deep", postScanPrompt, onScanStarted: started, + onCost: (cost) => recordCost(cost.inputTokens), }); expect(result.threadId).toBe("engine-session"); expect(result.reportPath).toBe(join(scanDir, "report.md")); expect(result.turnResult.finalResponse).toBe(finalReport); + expect(result.cost?.inputTokens).toBe(100); + expect(JSON.parse(completionReceipt!)).toMatchObject({ + inputTokens: 100, + }); expect(result.toJSON()["turn"]).toMatchObject({ finalResponse: finalReport, }); diff --git a/sdk/typescript/tests-ts/api-patch.test.ts b/sdk/typescript/tests-ts/api-patch.test.ts index 775c36817..64b7ba713 100644 --- a/sdk/typescript/tests-ts/api-patch.test.ts +++ b/sdk/typescript/tests-ts/api-patch.test.ts @@ -1,3 +1,4 @@ +import { requestMetadata } from "./support/request-metadata.js"; import { mkdir, readFile, realpath, writeFile } from "node:fs/promises"; import { join } from "node:path"; import type { @@ -300,9 +301,11 @@ describe("CodexSecurity headless patching", () => { model_reasoning_effort: "medium", features: { plugins: false }, default_permissions: "codex_security_scan", - responses_api_metadata: { codex_security_surface: "sdk" }, }, }); + expect(requestMetadata(captured.codex?.configOverrides)).toMatchObject({ + codex_security_surface: "sdk", + }); expect(captured.thread).not.toHaveProperty("sandboxMode"); expect(captured.thread).not.toHaveProperty("networkAccessEnabled"); expect(captured.codex?.env?.["OPENAI_API_KEY"]).toBeUndefined(); @@ -360,7 +363,9 @@ describe("CodexSecurity headless patching", () => { const { client, options, captured, codexHome } = await patchClient(); await using security = client; await security.patch(options); - expect(parseToml(captured.codex!.configOverrides!.join("\n"))).toEqual({ + expect( + parseToml(captured.codex!.configOverrides!.join("\n")), + ).toMatchObject({ permissions: { codex_security_scan: { filesystem: { @@ -417,7 +422,9 @@ describe("CodexSecurity headless patching", () => { ).toEqual({ model_providers: { "synthetic.provider": provider }, }); - expect(parseToml(captured.codex!.configOverrides!.join("\n"))).toEqual({ + expect( + parseToml(captured.codex!.configOverrides!.join("\n")), + ).toMatchObject({ permissions: { codex_security_scan: { filesystem: { diff --git a/sdk/typescript/tests-ts/api-policy.test.ts b/sdk/typescript/tests-ts/api-policy.test.ts index 21879f23b..dc5e18166 100644 --- a/sdk/typescript/tests-ts/api-policy.test.ts +++ b/sdk/typescript/tests-ts/api-policy.test.ts @@ -1,4 +1,6 @@ +import { requestMetadata } from "./support/request-metadata.js"; import { execFileSync } from "node:child_process"; +import { once } from "node:events"; import { link, mkdir, @@ -43,6 +45,7 @@ import { stageResult, } from "./support/security-policy.js"; import { rejecting, throwing } from "./support/errors.js"; +import { writeSession } from "./support/usage-rollout.js"; const { fixture, cleanup } = createPolicyTestFixtures(); afterEach(cleanup); @@ -272,24 +275,22 @@ describe("CodexSecurity policy API", () => { expect(f.configuration()!.config!["default_permissions"]).toBe( "codex_security_policy", ); - expect(parseToml(overrides.join("\n"))).toEqual({ - permissions: { - codex_security_scan: { - filesystem: { - ":root": "read", - ":workspace_roots": "write", - [await realpath(f.runtime.codexHome)]: { ".": "deny" }, - [f.runtime.deepScanConfigPath!]: { ".": "deny" }, - }, + expect(parseToml(overrides.join("\n"))["permissions"]).toEqual({ + codex_security_scan: { + filesystem: { + ":root": "read", + ":workspace_roots": "write", + [await realpath(f.runtime.codexHome)]: { ".": "deny" }, + [f.runtime.deepScanConfigPath!]: { ".": "deny" }, }, - codex_security_policy: { - filesystem: { - ":minimal": "read", - ":workspace_roots": "read", - ...Object.fromEntries( - readPaths.map((path) => [path, { ".": "read" }]), - ), - }, + }, + codex_security_policy: { + filesystem: { + ":minimal": "read", + ":workspace_roots": "read", + ...Object.fromEntries( + readPaths.map((path) => [path, { ".": "read" }]), + ), }, }, }); @@ -1367,9 +1368,10 @@ describe("CodexSecurity policy API", () => { web_search: "disabled", sandbox_workspace_write: { network_access: false }, }); - expect(f.configuration()?.config?.["responses_api_metadata"]).toMatchObject( - { codex_security_surface: "cli", codex_security_command: "policy" }, - ); + expect(requestMetadata(f.configuration()?.configOverrides)).toMatchObject({ + codex_security_surface: "cli", + codex_security_command: "policy", + }); expect(f.configuration()?.env?.["CODEX_SECURITY_REPOSITORY"]).toBe( f.repository, ); @@ -1705,6 +1707,55 @@ describe("CodexSecurity policy API", () => { await f.security.close(); }); + test("stops a policy stage at its measured budget while worker usage is missing", async () => { + let interruptedStage: SecurityPolicyStage | undefined; + const f = await setup({ + stream: async function* (stage, signal) { + yield { type: "thread.started", thread_id: `policy-${stage}` }; + yield { type: "turn.started" }; + if (!signal.aborted) await once(signal, "abort"); + interruptedStage = stage; + throw signal.reason; + }, + }); + await writeSession(f.runtime.codexHome, "policy-architecture", { + input_tokens: 3_000, + output_tokens: 0, + }); + await writeSession( + f.runtime.codexHome, + "policy-worker", + {}, + { parent: "policy-architecture" }, + ); + const costs = mock(); + const cancellation = new AbortController(); + const timeout = setTimeout( + () => cancellation.abort(new Error("Policy budget was not enforced.")), + 5_000, + ); + try { + await expect( + f.security.generatePolicy(f.repository, { + outputDir: f.outputDir, + maxCostUsd: 0.004, + signal: cancellation.signal, + onCost: costs, + }), + ).rejects.toThrow("exceeded its $0.004 cost limit"); + expect(interruptedStage).toBe("architecture"); + expect(f.threads).toHaveLength(1); + expect(costs).not.toHaveBeenCalled(); + await expect( + readFile(join(f.outputDir, "project-spec.md")), + ).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readdir(f.repository)).toEqual([]); + } finally { + clearTimeout(timeout); + await f.security.close(); + } + }); + test("optional observer failures do not stop policy generation", async () => { const f = await setup(); const errors: string[] = []; diff --git a/sdk/typescript/tests-ts/api-usage.test.ts b/sdk/typescript/tests-ts/api-usage.test.ts new file mode 100644 index 000000000..f2c5f15e5 --- /dev/null +++ b/sdk/typescript/tests-ts/api-usage.test.ts @@ -0,0 +1,154 @@ +import { afterEach, expect, mock, test } from "bun:test"; +import { once } from "node:events"; +import { appendFile, mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import { setImmediate } from "node:timers/promises"; +import type { ThreadEvent } from "@openai/codex-sdk"; +import { ScanCostTracker } from "../src/cost.js"; +import { ScanCostLimitExceededError } from "../src/errors.js"; +import { copyCompletedScan } from "./plugin-root.js"; +import { + completedEvents, + runEvents, + scanRuntimeDependencies, +} from "./support/api-events.js"; +import { mockWorkbench, TestClient } from "./support/api-client.js"; +import { createApiTestFixtures } from "./support/temporary-directories.js"; +import { writeSession } from "./support/usage-rollout.js"; + +const { temporaryDirectory, cleanup } = createApiTestFixtures(); +afterEach(cleanup); + +test.each([ + [false, null], + [true, null], + [false, undefined], + [true, undefined], +] as const)( + "preserves finalizer results (workbenchValidated=%p, result=%p)", + async (workbenchValidated, finalizedUsage) => { + const scanDir = await copyCompletedScan(await temporaryDirectory()); + const codexHome = await temporaryDirectory(); + await writeSession( + codexHome, + "worker-thread", + {}, + { + parent: "thread-1", + }, + ); + const tracker = new ScanCostTracker({ + codexHome, + model: "gpt-5.6-sol", + }); + const result = await runEvents(scanDir, completedEvents(), { + workbenchValidated, + onThreadStarted: (threadId) => tracker.start(threadId), + onFinalize: async (usage) => { + const snapshot = await tracker.stop(usage); + return finalizedUsage === null ? snapshot.usage : undefined; + }, + }); + + expect(result.toJSON()).toMatchObject({ + cost: + finalizedUsage === null ? null : { inputTokens: 10, outputTokens: 3 }, + turn: { + usage: + finalizedUsage === null + ? null + : { input_tokens: 10, output_tokens: 3 }, + status: "completed", + }, + }); + }, +); + +test.each([false, true])( + "enforces measured spending with a missing worker after raising the limit: %p", + async (raiseLimit) => { + const root = await temporaryDirectory(); + const repository = join(root, "repository"); + const codexHome = join(root, "codex-home"); + const scanDir = join(root, "scan"); + await mkdir(repository); + await mkdir(scanDir, { mode: 0o700 }); + await writeSession( + codexHome, + "pending-worker", + {}, + { + parent: "scan-thread", + }, + ); + const parent = await writeSession(codexHome, "scan-thread", { + input_tokens: raiseLimit ? 900 : 3_000, + output_tokens: 0, + }); + const limitSaved = Promise.withResolvers(); + const costs = mock(); + const usages = mock(); + const approaching = mock(() => 0.008); + const client = TestClient.withDependencies({ + ...scanRuntimeDependencies(codexHome, scanDir), + runWorkbench: async (_options, args, input) => { + if (args[0] === "set-scan-cost-limit") limitSaved.resolve(); + return mockWorkbench(args, input); + }, + createCodex: () => ({ + startThread: () => ({ + id: null, + runStreamed: async (_input, { signal }) => ({ + events: (async function* (): AsyncGenerator { + yield { type: "thread.started", thread_id: "scan-thread" }; + if (raiseLimit) { + await limitSaved.promise; + // Resume after the caller adopts the saved limit. + await setImmediate(); + await appendFile( + parent, + JSON.stringify({ + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: 3_000, + output_tokens: 0, + }, + }, + }, + }) + "\n", + ); + } + if (!signal?.aborted) await once(signal!, "abort"); + throw new DOMException("aborted", "AbortError"); + })(), + }), + }), + }), + }); + const keepAlive = setTimeout(() => {}, 10_000); + try { + await expect( + client.run(repository, { + maxCostUsd: 0.004, + signal: AbortSignal.timeout(5_000), + onCost: costs, + onUsage: usages, + ...(raiseLimit ? { onBudgetApproaching: approaching } : {}), + }), + ).rejects.toMatchObject({ + name: ScanCostLimitExceededError.name, + maxCostUsd: raiseLimit ? 0.008 : 0.004, + cost: { inputTokens: 3_000, estimatedUsd: 0.012 }, + }); + expect(costs).not.toHaveBeenCalled(); + expect(usages).not.toHaveBeenCalled(); + expect(approaching).toHaveBeenCalledTimes(raiseLimit ? 1 : 0); + } finally { + clearTimeout(keepAlive); + await client.close(); + } + }, +); diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.ts index 34eba77a2..ace9f1295 100644 --- a/sdk/typescript/tests-ts/api.test.ts +++ b/sdk/typescript/tests-ts/api.test.ts @@ -1,3 +1,4 @@ +import { requestMetadata } from "./support/request-metadata.js"; import { once } from "node:events"; import { createServer, type Socket } from "node:net"; import { @@ -560,22 +561,26 @@ describe("CodexSecurity finding validation", () => { web_search: "disabled", approval_policy: "never", analytics: { enabled: false }, - responses_api_metadata: { - codex_security_surface: "sdk", - codex_security_command: "validate", - codex_security_package_version: VERSION, - codex_security_plugin_version: - preparedRuntime("unused").plugin.version, - }, }, }); + expect(requestMetadata(captured.codex?.configOverrides)).toMatchObject({ + codex_security_surface: "sdk", + codex_security_command: "validate", + codex_security_package_version: VERSION, + codex_security_plugin_version: preparedRuntime("unused").plugin.version, + codex_security_provider: "openai", + }); const credentialHome = captured.codex!.env!["CODEX_HOME"]!; const credentialDenies = { [credentialHome]: { ".": "deny" }, [join(credentialHome, "deep-scan.toml")]: { ".": "deny" }, }; expect( - captured.codex?.configOverrides?.map((override) => parseToml(override)), + captured.codex?.configOverrides + ?.filter( + (override) => !override.startsWith("responses_api_metadata="), + ) + .map((override) => parseToml(override)), ).toEqual([ { permissions: { @@ -632,6 +637,12 @@ describe("CodexSecurity finding validation", () => { review: { web_search: "cached", approval_policy: "on-request", + analytics: { enabled: false }, + otel: { exporter: "none" }, + responses_api_metadata: { + "team.tag": "synthetic-validation", + codex_security_integration: "github_action", + }, }, }, }, @@ -666,8 +677,19 @@ describe("CodexSecurity finding validation", () => { expect(config).toMatchObject({ web_search: "disabled", approval_policy: "never", + analytics: { enabled: false }, + otel: { exporter: "none" }, default_permissions: "codex_security_validation", + responses_api_metadata: { + "team.tag": "synthetic-validation", + codex_security_integration: "github_action", + codex_security_command: "validate", + codex_security_provider: "openai", + }, }); + expect( + overrides.filter((value) => value.startsWith("responses_api_metadata")), + ).toHaveLength(1); expect(config).not.toHaveProperty("profile"); expect(config).not.toHaveProperty("profiles"); }); @@ -1968,13 +1990,18 @@ describe("CodexSecurity orchestration", () => { { profile: "cloud", analytics: { enabled: true }, + otel: { environment: "synthetic-root" }, responses_api_metadata: { custom_attribution: "root" }, model_context_window: 64_000, model_auto_compact_token_limit: 48_000, profiles: { cloud: { analytics: { enabled: false }, - responses_api_metadata: { custom_attribution: "selected" }, + otel: { environment: "synthetic-selected" }, + responses_api_metadata: { + custom_attribution: "selected", + codex_security_integration: "github_action", + }, model_reasoning_summary: "concise", service_tier: "fast", model_context_window: 96_000, @@ -2098,12 +2125,15 @@ describe("CodexSecurity orchestration", () => { expect(workerConfig["analytics"]).toEqual( selected["analytics"], ); + expect(workerConfig["otel"]).toEqual(selected["otel"]); expect(workerConfig["responses_api_metadata"]).toMatchObject({ ...(selected["responses_api_metadata"] as JsonObject | undefined), codex_security_surface: "sdk", codex_security_command: "scan", codex_security_package_version: VERSION, + codex_security_provider: "amazon-bedrock", + codex_security_scan_mode: "deep", }); const windows = (resolveCodexProfile(overrides)["windows"] ?? DEFAULT_CODEX_CONFIG["windows"]) as { sandbox: string }; @@ -4616,6 +4646,12 @@ describe("CodexSecurity orchestration", () => { { codexOverrides: { ...providerConfig, + analytics: { enabled: false }, + otel: { environment: "synthetic-matching" }, + responses_api_metadata: { + "team.tag": "synthetic-matching", + codex_security_integration: "github_action", + }, ...(failure === "budget-context" ? { model: "gpt-6.1-sol", @@ -4698,6 +4734,17 @@ describe("CodexSecurity orchestration", () => { }, async matchFindings(input, options, runtimeOptions) { expect(options?.cyberAccessProgram).toBe("daybreak_blue"); + expect(options?.config?.codexOverrides).toMatchObject({ + analytics: { enabled: false }, + otel: { environment: "synthetic-matching" }, + responses_api_metadata: { + "team.tag": "synthetic-matching", + codex_security_integration: "github_action", + codex_security_command: "scan", + codex_security_scan_mode: "standard", + codex_security_provider: "custom", + }, + }); if (failure !== "managed") expect(options?.config?.codexOverrides).toMatchObject({ ...providerConfig, diff --git a/sdk/typescript/tests-ts/cli-patch.test.ts b/sdk/typescript/tests-ts/cli-patch.test.ts index 8159999c6..ba66e3c00 100644 --- a/sdk/typescript/tests-ts/cli-patch.test.ts +++ b/sdk/typescript/tests-ts/cli-patch.test.ts @@ -1,3 +1,4 @@ +import { requestMetadata } from "./support/request-metadata.js"; import { gitText } from "./support/shell.js"; import { emptyPage } from "./support/linear-pagination.js"; import { resolving } from "./support/promises.js"; @@ -1117,6 +1118,69 @@ describe("scan and patch workflow", () => { }, ); + test.each([ + [undefined, false], + [true, false], + [false, true], + ] as const)( + "passes selected-profile telemetry to scan patches (root=%p, profile=%p)", + async (rootAnalytics, profileAnalytics) => { + const result = resultWithFindings(["high"]); + let childArgs: readonly string[] = []; + const outcome = await runWorkflow( + [ + "scan", + "--patch", + "--mode", + "standard", + "--json", + ...(rootAnalytics === undefined + ? [] + : ["--codex", `analytics.enabled=${rootAnalytics}`]), + "--codex", + 'profile="quiet"', + "--codex", + `profiles.quiet.analytics.enabled=${profileAnalytics}`, + "--codex", + 'profiles.quiet.model_provider="amazon-bedrock"', + "--codex", + 'responses_api_metadata={ "team.tag"="root", caller="preserved" }', + "--codex", + 'profiles.quiet.responses_api_metadata={ "team.tag"="selected", codex_security_integration="github_action" }', + "--codex", + "otel.log_user_prompt=false", + "--codex", + 'profiles.quiet.otel.metrics_exporter="none"', + ], + { + result, + onCodex: (args, output) => { + childArgs = args; + completePatches(args, output); + return 0; + }, + }, + ); + + expect(outcome.exitCode, outcome.stderr).toBe(0); + expect(childArgs).toContain(`analytics.enabled=${profileAnalytics}`); + expect(requestMetadata(childArgs)).toMatchObject({ + "team.tag": "selected", + caller: "preserved", + codex_security_surface: "cli", + codex_security_command: "patch", + codex_security_integration: "github_action", + codex_security_provider: "amazon-bedrock", + codex_security_scan_mode: "standard", + }); + expect( + parseToml(childArgs.find((value) => value.startsWith("otel="))!)[ + "otel" + ], + ).toEqual({ log_user_prompt: false, metrics_exporter: "none" }); + }, + ); + test("continues with separate patch tasks when one finding fails", async () => { const result = resultWithFindings(["critical", "high", "medium"]); const tasks: string[] = []; @@ -1365,9 +1429,9 @@ describe("scan and patch workflow", () => { ) { expect(output.command).toBe("patch"); expect(output.appServer?.sandbox).toBe("read-only"); - expect(args).toContain( - 'responses_api_metadata.codex_security_command="assess-patch-risk"', - ); + expect(requestMetadata(args)).toMatchObject({ + codex_security_command: "assess-patch-risk", + }); expect(output.appServer?.prompt).toContain( "", ); diff --git a/sdk/typescript/tests-ts/cli-project-config.test.ts b/sdk/typescript/tests-ts/cli-project-config.test.ts index f004016dd..cde3b9bf1 100644 --- a/sdk/typescript/tests-ts/cli-project-config.test.ts +++ b/sdk/typescript/tests-ts/cli-project-config.test.ts @@ -1225,7 +1225,9 @@ test.each([ codexHome: input.root, model, onUsage: options?.onUsage, - onCost: options?.onCost, + onCost: (cost, complete) => { + if (complete) options?.onCost?.(cost); + }, }); tracker.start("synthetic-scan-thread"); await tracker.stop(usage); diff --git a/sdk/typescript/tests-ts/cli-skills.test.ts b/sdk/typescript/tests-ts/cli-skills.test.ts index b696cda29..ca4de5a4a 100644 --- a/sdk/typescript/tests-ts/cli-skills.test.ts +++ b/sdk/typescript/tests-ts/cli-skills.test.ts @@ -1,3 +1,4 @@ +import { requestMetadata } from "./support/request-metadata.js"; import { execFileSync, spawn } from "node:child_process"; import { mkdir, rm, symlink, writeFile } from "node:fs/promises"; import * as filesystem from "node:fs/promises"; @@ -292,13 +293,7 @@ describe("CLI skill commands", () => { "--config", 'approval_policy="never"', "--config", - 'responses_api_metadata.codex_security_surface="cli"', - "--config", - `responses_api_metadata.codex_security_command=${JSON.stringify(command)}`, - "--config", - `responses_api_metadata.codex_security_package_version=${JSON.stringify(VERSION)}`, - "--config", - `responses_api_metadata.codex_security_plugin_version=${JSON.stringify((await pluginMetadata(PLUGIN_ROOT)).version)}`, + expect.stringMatching(/^responses_api_metadata=\{/u), ...(command === "patch" ? [] : [ @@ -312,6 +307,16 @@ describe("CLI skill commands", () => { "-", ]), ]); + expect(requestMetadata(invocation!)).toEqual({ + codex_security_surface: "cli", + codex_security_command: command, + codex_security_package_version: VERSION, + codex_security_plugin_version: (await pluginMetadata(PLUGIN_ROOT)) + .version, + ...(command === "validate" + ? { codex_security_provider: "openai" } + : {}), + }); if (command === "validate") { expect(prompt).toContain( await filesystem.readFile( diff --git a/sdk/typescript/tests-ts/cli-verify-fix.test.ts b/sdk/typescript/tests-ts/cli-verify-fix.test.ts index 96696e3a0..5b3b25f8d 100644 --- a/sdk/typescript/tests-ts/cli-verify-fix.test.ts +++ b/sdk/typescript/tests-ts/cli-verify-fix.test.ts @@ -1,3 +1,4 @@ +import { requestMetadata } from "./support/request-metadata.js"; import { describe, expect, test, mock } from "bun:test"; import { readFile } from "node:fs/promises"; import { join } from "node:path"; @@ -49,12 +50,10 @@ describe("read-only finding verification", () => { }, linearClient, onCodex: (args, output, processEnvironment, input) => { - expect(args).toContain( - 'responses_api_metadata.codex_security_command="verify-fix"', - ); - expect(args).toContain( - `responses_api_metadata.codex_security_package_version=${JSON.stringify(VERSION)}`, - ); + expect(requestMetadata(args)).toMatchObject({ + codex_security_command: "verify-fix", + codex_security_package_version: VERSION, + }); expect(args[0]).toBe("app-server"); expect(input).toBeUndefined(); expect(args).toContain('approval_policy="on-request"'); diff --git a/sdk/typescript/tests-ts/cost.test.ts b/sdk/typescript/tests-ts/cost.test.ts index 947cc610c..58424e060 100644 --- a/sdk/typescript/tests-ts/cost.test.ts +++ b/sdk/typescript/tests-ts/cost.test.ts @@ -2606,6 +2606,99 @@ describe("live scan cost tracking", () => { }); }); + test("reports budget subtotals until missing worker usage becomes measured", async () => { + const home = await codexHome(); + const parentUsage = { input_tokens: 1_000_000, output_tokens: 100_000 }; + await writeSession(home, "scan-thread", parentUsage); + const costs: Array<[number, boolean]> = []; + const usages: unknown[] = []; + const tracker = costTracker(home, { + onCost: (cost, complete) => costs.push([cost.estimatedUsd, complete]), + onUsage: (usage) => usages.push(usage), + }); + tracker.start("scan-thread"); + expect((await tracker.refresh()).cost?.estimatedUsd).toBe(6); + + const worker = await writeSession( + home, + "worker-thread", + {}, + { + parent: "scan-thread", + }, + ); + expect(await tracker.refresh()).toEqual({ usage: null, cost: null }); + expect(costs.at(-1)).toEqual([6, false]); + expect(usages).toHaveLength(1); + + await appendFile( + worker, + jsonLines([legacyUsageRecord({ input_tokens: 0, output_tokens: 0 })]) + + "\n", + ); + expect((await tracker.refresh()).usage).toMatchObject({ + ...parentUsage, + total_tokens: 1_100_000, + }); + expect(costs.at(-1)).toEqual([6, true]); + expect(usages).toHaveLength(1); + + await appendFile( + worker, + jsonLines([ + legacyUsageRecord({ input_tokens: 500_000, output_tokens: 50_000 }), + ]) + "\n", + ); + expect((await tracker.stop(parentUsage)).usage).toMatchObject({ + input_tokens: 1_500_000, + output_tokens: 150_000, + total_tokens: 1_650_000, + }); + expect(costs).toEqual([ + [6, true], + [6, false], + [6, true], + [9, true], + ]); + expect(usages).toHaveLength(2); + }); + + test("keeps independent Deep worker usage unknown until a receipt supplies it", async () => { + const home = await codexHome(); + const scanDirectory = join(home, "scans", "current"); + const parentUsage = { input_tokens: 100, output_tokens: 10 }; + await writeSession(home, "scan-thread", parentUsage, { + cwd: scanDirectory, + timestamp: "2026-07-26T12:00:00Z", + }); + await writeSession( + home, + "deep-worker", + {}, + { + cwd: join( + scanDirectory, + "artifacts", + "deep_discovery", + "workers", + "one", + "output", + ), + timestamp: "2026-07-26T12:01:00Z", + }, + ); + const tracker = costTracker(home, { scanDirectory }); + tracker.start("scan-thread"); + expect(await tracker.refresh()).toEqual({ usage: null, cost: null }); + + tracker.recordUsage({ input_tokens: 50, output_tokens: 5 }, "deep-worker"); + expect((await tracker.stop(parentUsage)).usage).toMatchObject({ + input_tokens: 150, + output_tokens: 15, + total_tokens: 165, + }); + }); + test.each(["receipt", "receipt-and-log", "unknown"] as const)( "accounts for a separate validation turn with %s usage", async (source) => { diff --git a/sdk/typescript/tests-ts/direct-deep-scan.test.ts b/sdk/typescript/tests-ts/direct-deep-scan.test.ts index 919f438cc..25437155a 100644 --- a/sdk/typescript/tests-ts/direct-deep-scan.test.ts +++ b/sdk/typescript/tests-ts/direct-deep-scan.test.ts @@ -9,7 +9,10 @@ import { import { join } from "node:path"; import { afterEach, test } from "node:test"; import { setTimeout } from "node:timers/promises"; -import { runDeepScan } from "../src/deep-scan.js"; +import { + DIRECT_DEEP_SCAN_CONTROL_USAGE, + runDeepScan, +} from "../src/deep-scan.js"; import { ScanCostTracker } from "../src/cost.js"; import { PLUGIN_ROOT } from "./plugin-root.js"; import { createApiTestFixtures } from "./support/temporary-directories.js"; @@ -150,6 +153,11 @@ test("isolates direct engine credentials and runtime snapshots across concurrent run.map((event) => event.type), ["thread.started", "turn.completed"], ); + const firstUsage = events[0]![1]!["usage"] as Record; + const secondUsage = events[1]![1]!["usage"] as Record; + firstUsage["input_tokens"] = 100; + assert.equal(secondUsage["input_tokens"], 0); + assert.equal(DIRECT_DEEP_SCAN_CONTROL_USAGE.input_tokens, 0); }); test("preserves complete worker usage and cost after direct engine completion", async () => { @@ -230,8 +238,27 @@ test("preserves complete worker usage and cost after direct engine completion", scanDirectory: scanDir, model: "gpt-6.1-sol", }); + tracker.recordUsage(DIRECT_DEEP_SCAN_CONTROL_USAGE, "engine-session"); tracker.start("engine-session"); const running = await tracker.refresh(); + await writeSession( + codexHome, + "waiting-session", + {}, + { + parent: "engine-session", + }, + ); + assert.deepEqual(await tracker.refresh(), { usage: null, cost: null }); + tracker.recordUsage( + { + input_tokens: 0, + cached_input_tokens: 0, + cache_write_input_tokens: 0, + output_tokens: 0, + }, + "waiting-session", + ); const completed = (await events.next()).value; assert.equal(completed?.type, "turn.completed"); const final = await tracker.stop(completed?.["usage"]); diff --git a/sdk/typescript/tests-ts/finding-embeddings.test.ts b/sdk/typescript/tests-ts/finding-embeddings.test.ts index acfef9fb1..b9bebc196 100644 --- a/sdk/typescript/tests-ts/finding-embeddings.test.ts +++ b/sdk/typescript/tests-ts/finding-embeddings.test.ts @@ -11,6 +11,7 @@ import { OpenAiFindingEmbedder, } from "../src/server/embeddings.js"; import { PLUGIN_ROOT } from "./plugin-root.js"; +import { VERSION } from "../src/version.js"; const example = ( await readJson( @@ -63,6 +64,7 @@ test("uses the configured embedding model and preserves response indexes", async expect(url).toBe("https://api.openai.com/v1/embeddings"); expect(init.headers).toMatchObject({ Authorization: "Bearer synthetic-key", + "User-Agent": `codex-security/${VERSION}`, }); const request = JSON.parse(String(init.body)); expect(request).toMatchObject({ diff --git a/sdk/typescript/tests-ts/publish.test.ts b/sdk/typescript/tests-ts/publish.test.ts index 3815b6def..a6989ee90 100644 --- a/sdk/typescript/tests-ts/publish.test.ts +++ b/sdk/typescript/tests-ts/publish.test.ts @@ -23,6 +23,8 @@ import { UnknownLinearError, } from "@linear/sdk"; import { afterEach, describe, expect, spyOn, test, mock } from "bun:test"; +import { requestMetadata } from "./support/request-metadata.js"; +import { VERSION } from "../src/version.js"; import { forceTerminatePublicationProcesses, publishScanInternal, @@ -1224,7 +1226,7 @@ describe("connected Linear publication", () => { expect(result.counts).toEqual({ findings: 2, created: 2, failed: 0 }); }); - test("reuses ambient Codex configuration and loads exact issue data from a private file", async () => { + test("attributes publication while preserving ambient Codex configuration and private issue data", async () => { const publication = preparedPublication(); const stateDirectory = await temporaryDirectory(); const environment = { @@ -1275,6 +1277,8 @@ describe("connected Linear publication", () => { "gpt-5.6-luna", "-c", 'model_reasoning_effort="low"', + "-c", + expect.stringMatching(/^responses_api_metadata=\{/u), "--ephemeral", "--json", "--sandbox", @@ -1286,6 +1290,11 @@ describe("connected Linear publication", () => { ]); expect(args).not.toContain("--ignore-user-config"); expect(args).not.toContain("--disable"); + expect(requestMetadata(args)).toEqual({ + codex_security_surface: "sdk", + codex_security_command: "publish", + codex_security_package_version: VERSION, + }); expect(inheritedEnvironment).toBe(environment); expect(input).toContain("already-connected hosted Linear application"); expect(input).toContain("untrusted inert data"); diff --git a/sdk/typescript/tests-ts/request-metadata.test.ts b/sdk/typescript/tests-ts/request-metadata.test.ts new file mode 100644 index 000000000..cf24773d8 --- /dev/null +++ b/sdk/typescript/tests-ts/request-metadata.test.ts @@ -0,0 +1,131 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { afterEach, expect, test } from "bun:test"; +import { parse } from "smol-toml"; +import { codexSecurityRequestContext } from "../src/request-metadata.js"; +import { VERSION } from "../src/version.js"; +import { resolveCodexCommand } from "../src/runtime.js"; +import { PLUGIN_ROOT } from "./plugin-root.js"; +import { createApiTestFixtures } from "./support/temporary-directories.js"; + +const { temporaryDirectory, cleanup } = createApiTestFixtures( + "codex-security-request-metadata-", +); +afterEach(cleanup); + +test.each([ + [undefined, "openai"], + ["openai", "openai"], + ["amazon-bedrock", "amazon-bedrock"], + ["openrouter", "openrouter"], + ["fireworks", "fireworks"], + ["ollama", "ollama"], + ["lmstudio", "lmstudio"], + ["synthetic.private-provider", "custom"], +] as const)("attributes provider %p as %s", (provider, expected) => { + const context = codexSecurityRequestContext( + provider === undefined ? {} : { model_provider: provider }, + { surface: "sdk", command: "scan" }, + ); + expect(context.metadata["codex_security_provider"]).toBe(expected); +}); + +test("selected profile metadata and provider win without changing telemetry settings", () => { + const config = { + model_provider: "openai", + analytics: { enabled: true }, + otel: { exporter: "none" }, + responses_api_metadata: { "root.tag": "retained", "team.tag": "root" }, + profile: "selected.profile", + profiles: { + "selected.profile": { + model_provider: "amazon-bedrock", + analytics: { enabled: false }, + otel: { + exporter: { "otlp-http": { endpoint: "http://localhost:4318" } }, + }, + responses_api_metadata: { + "team.tag": "selected", + codex_security_integration: "github_action", + codex_security_surface: "spoofed", + codex_security_command: "spoofed", + codex_security_package_version: "spoofed", + codex_security_plugin_version: "spoofed", + codex_security_provider: "spoofed", + codex_security_scan_mode: "spoofed", + }, + }, + unused: { model_provider: "ollama" }, + }, + }; + const original = structuredClone(config); + const context = codexSecurityRequestContext(config, { + surface: "cli", + command: "scan", + pluginVersion: "1.2.3", + mode: "deep", + }); + expect(context.metadata).toEqual({ + "root.tag": "retained", + "team.tag": "selected", + codex_security_integration: "github_action", + codex_security_surface: "cli", + codex_security_command: "scan", + codex_security_package_version: VERSION, + codex_security_plugin_version: "1.2.3", + codex_security_provider: "amazon-bedrock", + codex_security_scan_mode: "deep", + }); + expect(parse(context.configOverride)).toEqual({ + responses_api_metadata: context.metadata, + }); + expect(config).toEqual(original); +}); + +test("leaves provider attribution to native configuration when unresolved", () => { + const context = codexSecurityRequestContext( + {}, + { surface: "cli", command: "patch", providerResolved: false }, + ); + expect(context.metadata).not.toHaveProperty("codex_security_provider"); + expect(context.metadata).not.toHaveProperty("codex_security_scan_mode"); + const inherited = codexSecurityRequestContext( + { responses_api_metadata: { codex_security_provider: "amazon-bedrock" } }, + { surface: "cli", command: "patch", providerResolved: false }, + ); + expect(inherited.metadata["codex_security_provider"]).toBe("amazon-bedrock"); +}); + +test("native table overrides preserve inherited literal metadata and opt-outs", async () => { + const home = await temporaryDirectory(); + await writeFile( + join(home, "config.toml"), + '[analytics]\nenabled=false\n[responses_api_metadata]\n"home.tag"="preserved"\ncodex_security_integration="github_action"\n', + ); + const context = codexSecurityRequestContext( + { responses_api_metadata: { "caller.tag": "synthetic" } }, + { surface: "cli", command: "publish", providerResolved: false }, + ); + const { readDeepScanRuntimeConfig } = await import( + pathToFileURL(join(PLUGIN_ROOT, "mcp", "permission-profile-preflight.mjs")) + .href + ); + const config = await readDeepScanRuntimeConfig({ + codexPath: resolveCodexCommand({}).command, + cwd: home, + configOverrides: [context.configOverride], + env: { + PATH: process.env["PATH"], + SystemRoot: process.env["SystemRoot"], + CODEX_HOME: home, + }, + signal: new AbortController().signal, + }); + expect(config.responses_api_metadata).toEqual({ + "home.tag": "preserved", + codex_security_integration: "github_action", + ...context.metadata, + }); + expect(config.analytics).toEqual({ enabled: false }); +}); diff --git a/sdk/typescript/tests-ts/scan-comparison.test.ts b/sdk/typescript/tests-ts/scan-comparison.test.ts index ae56318d7..98570c36d 100644 --- a/sdk/typescript/tests-ts/scan-comparison.test.ts +++ b/sdk/typescript/tests-ts/scan-comparison.test.ts @@ -306,6 +306,7 @@ describe("semantic scan comparison", () => { codex_security_surface: "sdk", codex_security_command: "compare", codex_security_package_version: VERSION, + ...(withProfile ? { codex_security_provider: "custom" } : {}), }, }); const parsed = await runCodexCommand( @@ -470,6 +471,11 @@ describe("semantic scan comparison", () => { codex_security_surface: "sdk", codex_security_command: "compare", codex_security_package_version: VERSION, + codex_security_provider: ["fireworks", "amazon-bedrock"].includes( + selection, + ) + ? selection + : "custom", }, default_permissions: "codex_security_deep_scan_worker", permissions: { @@ -1224,6 +1230,7 @@ describe("semantic scan comparison", () => { codex_security_surface: "sdk", codex_security_command: "compare", codex_security_package_version: VERSION, + codex_security_provider: "openai", }, }); expect(profileClient.profiles).toHaveLength(0); diff --git a/sdk/typescript/tests-ts/support/request-metadata.ts b/sdk/typescript/tests-ts/support/request-metadata.ts new file mode 100644 index 000000000..31d3b50bc --- /dev/null +++ b/sdk/typescript/tests-ts/support/request-metadata.ts @@ -0,0 +1,10 @@ +import { parse } from "smol-toml"; + +export function requestMetadata(args: readonly string[] | undefined) { + const override = args?.find((value) => + value.startsWith("responses_api_metadata="), + ); + return override === undefined + ? undefined + : parse(override)["responses_api_metadata"]; +}