test(evals): compare production Standard scan time budgets

.github/workflows/node-ci.yml

@@ -466,6 +466,14 @@ jobs:
       - name: Test secret discovery eval contracts
         if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
         run: node --experimental-strip-types --test evals/secret-discovery/test_*.mts
+      - name: Build host plugin for Standard eval integration
+        if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
+        run: |
+          node plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs --output plugins/codex-security/mcp --native host
+          ln -s ../../plugins/codex-security sdk/typescript/_bundled_plugin
+      - name: Test Standard time-budget eval contracts
+        if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12'
+        run: node --experimental-strip-types --test evals/standard-time-budget/test_*.mts
 
   required-test:
     name: ${{ matrix.os }} / node-22

evals/README.md

@@ -11,7 +11,10 @@ being embedded in its source tree or shipped npm runtime.
 - [Secret discovery](secret-discovery/README.md): checks whether the production
   core audit finds synthetic credentials in source and keeps them in its final
   findings, with deterministic grading and harness checks.
+- [Standard time budget](standard-time-budget/README.md): paired comparison of
+  production Standard scans with and without an advisory time allowance on
+  synthetic vulnerabilities and safe controls, with source-evidence grading.
 
-Model runs are opt-in. CI runs the deterministic triage and secret-discovery
-helper checks and the real-IPC reducer regression through the normal MCP test
-suite.
+Model runs are opt-in. CI runs the deterministic triage, secret-discovery, and
+time-budget helper checks and the real-IPC reducer regression through the normal
+MCP test suite.

evals/secret-discovery/fixtures.mts

@@ -165,7 +165,7 @@ CONTENT_SHA256 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcd
 /** Stage source only. Gold labels and secret-value lookup stay in the harness. */
 export async function writeFixture(
   repo: string,
-  fixture: ReturnType<typeof createFixture>,
+  fixture: Pick<ReturnType<typeof createFixture>, "files">,
 ) {
   for (const [relative, contents] of Object.entries(fixture.files)) {
     const destination = join(repo, relative);

evals/secret-discovery/runtime.mts

@@ -24,7 +24,7 @@ export { bundledCodexSdkEnvironment } from "../../sdk/typescript/src/codex-sdk-e
 /** Keep temporary source and the login link alive until cancelled work stops. */
 export async function withEvalState(
   createHome: (parent?: string) => Promise<string>,
-  ambientHome: string,
+  ambientHome: string | undefined,
   run: (state: {
     home: string;
     root: string;
@@ -47,11 +47,13 @@ export async function withEvalState(
     root = await realpath(await mkdtemp(join(tmpdir(), "source-audit-")));
     // Share saved login updates without importing the caller's configuration.
     let auth;
-    try {
-      const path = await realpath(join(ambientHome, "auth.json"));
-      if ((await stat(path)).isFile()) auth = path;
-    } catch (error) {
-      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+    if (ambientHome !== undefined) {
+      try {
+        const path = await realpath(join(ambientHome, "auth.json"));
+        if ((await stat(path)).isFile()) auth = path;
+      } catch (error) {
+        if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+      }
     }
     // A hard link must stay on the saved login's filesystem, including symlinks.
     const allocatedHome = await createHome(auth ? dirname(auth) : undefined);

evals/standard-time-budget/.gitignore

@@ -0,0 +1 @@
+/reports/

evals/standard-time-budget/README.md

@@ -0,0 +1,115 @@
+# Standard Scan time allowance comparison
+
+Compare production Standard scans with and without `maxTimeMinutes`. Both
+arms call `CodexSecurity.run`, including setup, delegation, canonical artifact
+validation, report generation, and completion. The treatment uses the shared
+[investigation-time-budget.md](../../plugins/codex-security/references/investigation-time-budget.md)
+policy, not an eval-only prompt.
+
+The SDK stamps one shared UTC deadline after setup, immediately before the
+first scan turn. The allowance is advisory: there is no abort timer, automatic
+continuation, or minimum runtime. Finalization is outside the allowance.
+
+## Run
+
+Use the repository's dependency and
+[native-artifact setup](../../plugins/codex-security/native/README.md), then run
+from the repository root:
+
+```sh
+pnpm --dir sdk/typescript run build:plugin
+pnpm --dir sdk/typescript run build:evals
+node --experimental-strip-types evals/standard-time-budget/run.mts
+```
+
+Rebuild the bundle after plugin changes. This developer eval requires an existing
+`OPENAI_API_KEY` or `CODEX_API_KEY` environment variable; saved Codex or
+`codex-security login` credentials are intentionally unsupported. It uses the
+bundled Codex SDK without reading saved logins, provisioning credentials, or
+changing persistent configuration. Production SDK and CLI login support is
+unchanged. **Model runs consume usage and have no enforced time or spend cap.**
+
+[experiment.json](experiment.json) selects an explicit model, two pairs
+(four scans including delegated work), and five advisory minutes. Both arms
+use `xhigh` reasoning and production worker settings. Pairs alternate
+baseline-first and treatment-first order. Override settings with a JSON file:
+
+```sh
+node --experimental-strip-types evals/standard-time-budget/run.mts /path/to/experiment.json
+```
+
+The fields are `model`, `pairs`, and `budgetMinutes`; positive fractional
+minutes are accepted. Production deadline validation runs before either arm
+starts. Ctrl+C saves the interrupted attempt, stops future arms, and waits for
+child processes before cleaning temporary state.
+
+## Fixture and isolation
+
+The synthetic application has tenant authorization, SQL injection, file
+traversal, and reflected HTML bugs, each with a safe control. Source and
+deployment context go to the model; gold labels and expected locations stay
+in the evaluator.
+
+Each arm runs in an opaque temporary directory outside the artifact tree; pair
+and variant labels appear only in evaluator reports and archives. After the
+client closes, its scan output is archived and its state is removed before the
+next arm. Gold labels and prior-arm results are not next to the model's working
+directory or included in prompts. This is layout separation, not a filesystem
+security boundary: production API-key authentication and root-read permissions
+remain unchanged. This eval does not test resumption or external publication.
+
+## Results
+
+Artifacts are saved under ignored `reports/`, updated after every attempt:
+
+- `experiment.json`: settings, evaluator/shared-helper, source, full-plugin,
+  and built-SDK file hashes, plus SDK/Codex package versions. Generated reports
+  are excluded from evaluator hashes.
+- `fixture.json`: synthetic source and held-out gold labels for review.
+- `pair-<number>-<variant>/prompt.txt`: the actual first dispatched scan prompt.
+- Per-arm `scan/`, `result.json`, and `report.json`: canonical scan artifacts,
+  semantic output, grading, usage, elapsed time, and exact failure text.
+- `runs.json` and `summary.json`: all attempts and the paired comparison.
+
+Elapsed time is monotonic and includes SDK setup, investigation, finalization,
+scan archival, cleanup, and grading, but not subsequent evaluator JSON writing.
+A canonical scan that misses a bug still completed successfully. The runner
+exits nonzero for execution failures, not low recall.
+
+The grader counts unique gold cases with matching CWE, a `root_control` or
+`sink` source location, and authentic source evidence. Other or missing roles
+do not earn automatic recall; unmatched findings require adjudication. It
+consumes the production contract's repository-relative POSIX paths. Duplicates
+do not increase recall.
+Safe-control overlaps, unmatched findings, and citation errors are separate
+signals; overlaps and unmatched findings require adjudication, not automatic
+false-positive labels. Free-text location roles can flag benign context
+citations. Preserve raw scores and record adjudication separately: source
+matching is a heuristic, not independent exploit validation.
+
+The summary separates execution success from completed-run quality. Paired
+deltas are treatment minus baseline and include only complete pairs; excluded
+pairs remain explicit. Each attempt's top-level `usage` includes delegated
+workers, even when completion or grading fails. Missing usage stays unknown,
+cached input is a subset of input tokens, and no dollar cost is inferred.
+
+## Interpretation and checks
+
+Two pairs are a smoke comparison, not statistical evidence. Review findings
+without variant labels and compare grounded recall, adjudicated false
+positives, completion, tokens, and elapsed time together. More findings or a
+longer run alone is not an improvement, and these arms do not have matched
+spend. Expand the corpus if both arms saturate the fixture; use more pairs and
+pinned public repository revisions before drawing quality/cost conclusions.
+
+After the builds above, run deterministic checks without model calls:
+
+```sh
+node --experimental-strip-types --test evals/standard-time-budget/test_*.mts evals/secret-discovery/test_*.mts
+```
+
+The integration test uses the real SDK and canonical artifact helpers with
+mocked model/workbench boundaries, verifying isolated state, completion, prompt
+capture, and parent-plus-worker usage. CI reuses its host native build, builds
+the plugin with `build_mcp_app.mjs --native host`, and links it at
+`sdk/typescript/_bundled_plugin`; universal release artifacts are not required.

evals/standard-time-budget/experiment.json

@@ -0,0 +1,5 @@
+{
+  "model": "gpt-5.6-sol",
+  "pairs": 2,
+  "budgetMinutes": 5
+}

evals/standard-time-budget/experiment.mts

@@ -0,0 +1,166 @@
+import { createHash } from "node:crypto";
+import { readdir, readFile } from "node:fs/promises";
+import { join } from "node:path";
+import type { GradeReport, Result } from "./grade.mts";
+import type { PreparedStandardEval } from "./fixtures.mts";
+import type { StandardRunner, StandardRunOptions } from "./production.mts";
+
+export interface ExperimentConfig {
+  model: string;
+  pairs: number;
+  budgetMinutes: number;
+}
+export type Variant = "baseline" | "time-aware";
+export type RunReport = GradeReport & {
+  scanId?: string;
+  scanDir?: string;
+  reportPath?: string;
+};
+interface RunMetadata {
+  pair: number;
+  variant: Variant;
+  startedAt: string;
+  advisoryBudgetMinutes: number | null;
+  promptSha256: string | null;
+  elapsedMs: number;
+  usage: unknown;
+}
+type RunOutcome =
+  | { status: "completed"; report: RunReport; error?: never }
+  | { status: "failed"; error: string; report?: never };
+export type RunRecord = RunMetadata & RunOutcome;
+export type CompletedRunRecord = Extract<RunRecord, { status: "completed" }>;
+interface ComparisonOptions {
+  prepared: PreparedStandardEval;
+  config: ExperimentConfig;
+  signal?: AbortSignal;
+  run: StandardRunner;
+  elapsedNow?: () => number;
+  onStart?: (record: Pick<RunMetadata, "pair" | "variant">) => Promise<unknown>;
+  onResult?: (
+    record: RunRecord,
+    result: Result | undefined,
+    records: RunRecord[],
+    prompt: string | undefined,
+  ) => Promise<unknown>;
+}
+
+export function validateExperiment(config: ExperimentConfig) {
+  if (typeof config?.model !== "string" || !config.model.trim()) {
+    throw new Error("The experiment requires an explicit model.");
+  }
+  if (!Number.isSafeInteger(config.pairs) || config.pairs < 1) {
+    throw new Error("Experiment pairs must be a positive integer.");
+  }
+  if (!Number.isFinite(config.budgetMinutes) || config.budgetMinutes <= 0) {
+    throw new Error("Experiment budgetMinutes must be a positive number.");
+  }
+  return {
+    model: config.model,
+    pairs: config.pairs,
+    budgetMinutes: config.budgetMinutes,
+  };
+}
+
+export function hashText(text: string | Uint8Array) {
+  return createHash("sha256").update(text).digest("hex");
+}
+
+export async function snapshotEvaluator(directory = import.meta.dirname) {
+  const files: Record<string, string> = {};
+  for (const path of [
+    "run.mts",
+    "production.mts",
+    "experiment.mts",
+    "fixtures.mts",
+    "grade.mts",
+    "summary.mts",
+    "../secret-discovery/fixtures.mts",
+    "../secret-discovery/runtime.mts",
+  ]) {
+    files[path] = hashText(await readFile(join(directory, path)));
+  }
+  return files;
+}
+
+export async function snapshotFiles(directory: string, prefix = "") {
+  const files: Record<string, string> = {};
+  const entries = await readdir(directory, { withFileTypes: true });
+  for (const entry of entries.sort((first, second) =>
+    first.name.localeCompare(second.name),
+  )) {
+    const relative = prefix ? `${prefix}/${entry.name}` : entry.name;
+    const path = join(directory, entry.name);
+    if (entry.isDirectory()) {
+      Object.assign(files, await snapshotFiles(path, relative));
+    } else {
+      files[relative] = hashText(await readFile(path));
+    }
+  }
+  return files;
+}
+
+export async function runComparison({
+  prepared,
+  config,
+  signal,
+  run,
+  elapsedNow = () => performance.now(),
+  onStart = async () => {},
+  onResult = async () => {},
+}: ComparisonOptions) {
+  config = validateExperiment(config);
+  const records: RunRecord[] = [];
+  for (let pair = 1; pair <= config.pairs; pair += 1) {
+    const variants: Variant[] =
+      pair % 2 === 1 ? ["baseline", "time-aware"] : ["time-aware", "baseline"];
+    for (const variant of variants) {
+      signal?.throwIfAborted();
+      await onStart({ pair, variant });
+      signal?.throwIfAborted();
+      const startedAt = new Date().toISOString();
+      const startedElapsed = elapsedNow();
+      const observation: StandardRunOptions["observation"] = {};
+      let outcome: RunOutcome;
+      let semanticResult: Result | undefined;
+      try {
+        const result = await run(prepared, {
+          model: config.model,
+          pair,
+          variant,
+          signal,
+          ...(variant === "time-aware"
+            ? { maxTimeMinutes: config.budgetMinutes }
+            : {}),
+          observation,
+        });
+        outcome = { status: "completed", report: result.report };
+        semanticResult = result.semanticResult;
+      } catch (error) {
+        outcome = {
+          status: "failed",
+          error: error instanceof Error ? error.message : String(error),
+        };
+      }
+      const elapsedMs = elapsedNow() - startedElapsed;
+      const record: RunRecord = {
+        pair,
+        variant,
+        startedAt,
+        advisoryBudgetMinutes:
+          variant === "time-aware" ? config.budgetMinutes : null,
+        promptSha256:
+          observation.prompt === undefined
+            ? null
+            : hashText(observation.prompt),
+        ...outcome,
+        elapsedMs,
+        usage: observation.usage ?? null,
+      };
+      records.push(record);
+      await onResult(record, semanticResult, records, observation.prompt);
+      signal?.throwIfAborted();
+    }
+  }
+  return records;
+}