diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml index 6a7ff6c9b..af57557f3 100644 --- a/.github/workflows/node-ci.yml +++ b/.github/workflows/node-ci.yml @@ -466,6 +466,14 @@ jobs: - name: Test secret discovery eval contracts if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12' run: node --experimental-strip-types --test evals/secret-discovery/test_*.mts + - name: Build host plugin for Standard eval integration + if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12' + run: | + node plugins/codex-security/mcp-app/scripts/build_mcp_app.mjs --output plugins/codex-security/mcp --native host + ln -s ../../plugins/codex-security sdk/typescript/_bundled_plugin + - name: Test Standard time-budget eval contracts + if: matrix.os == 'ubuntu-latest' && matrix.python == '3.12' + run: node --experimental-strip-types --test evals/standard-time-budget/test_*.mts required-test: name: ${{ matrix.os }} / node-22 diff --git a/evals/README.md b/evals/README.md index ae456816e..b1677d8d2 100644 --- a/evals/README.md +++ b/evals/README.md @@ -11,7 +11,10 @@ being embedded in its source tree or shipped npm runtime. - [Secret discovery](secret-discovery/README.md): checks whether the production core audit finds synthetic credentials in source and keeps them in its final findings, with deterministic grading and harness checks. +- [Standard time budget](standard-time-budget/README.md): paired comparison of + production Standard scans with and without an advisory time allowance on + synthetic vulnerabilities and safe controls, with source-evidence grading. -Model runs are opt-in. CI runs the deterministic triage and secret-discovery -helper checks and the real-IPC reducer regression through the normal MCP test -suite. +Model runs are opt-in. CI runs the deterministic triage, secret-discovery, and +time-budget helper checks and the real-IPC reducer regression through the normal +MCP test suite. diff --git a/evals/secret-discovery/fixtures.mts b/evals/secret-discovery/fixtures.mts index 5d90cf2ce..4a55a1463 100644 --- a/evals/secret-discovery/fixtures.mts +++ b/evals/secret-discovery/fixtures.mts @@ -165,7 +165,7 @@ CONTENT_SHA256 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcd /** Stage source only. Gold labels and secret-value lookup stay in the harness. */ export async function writeFixture( repo: string, - fixture: ReturnType, + fixture: Pick, "files">, ) { for (const [relative, contents] of Object.entries(fixture.files)) { const destination = join(repo, relative); diff --git a/evals/secret-discovery/runtime.mts b/evals/secret-discovery/runtime.mts index f2956e1fa..8cdb8b3b8 100644 --- a/evals/secret-discovery/runtime.mts +++ b/evals/secret-discovery/runtime.mts @@ -24,7 +24,7 @@ export { bundledCodexSdkEnvironment } from "../../sdk/typescript/src/codex-sdk-e /** Keep temporary source and the login link alive until cancelled work stops. */ export async function withEvalState( createHome: (parent?: string) => Promise, - ambientHome: string, + ambientHome: string | undefined, run: (state: { home: string; root: string; @@ -47,11 +47,13 @@ export async function withEvalState( root = await realpath(await mkdtemp(join(tmpdir(), "source-audit-"))); // Share saved login updates without importing the caller's configuration. let auth; - try { - const path = await realpath(join(ambientHome, "auth.json")); - if ((await stat(path)).isFile()) auth = path; - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + if (ambientHome !== undefined) { + try { + const path = await realpath(join(ambientHome, "auth.json")); + if ((await stat(path)).isFile()) auth = path; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } } // A hard link must stay on the saved login's filesystem, including symlinks. const allocatedHome = await createHome(auth ? dirname(auth) : undefined); diff --git a/evals/standard-time-budget/.gitignore b/evals/standard-time-budget/.gitignore new file mode 100644 index 000000000..30846059d --- /dev/null +++ b/evals/standard-time-budget/.gitignore @@ -0,0 +1 @@ +/reports/ diff --git a/evals/standard-time-budget/README.md b/evals/standard-time-budget/README.md new file mode 100644 index 000000000..70f30bf07 --- /dev/null +++ b/evals/standard-time-budget/README.md @@ -0,0 +1,115 @@ +# Standard Scan time allowance comparison + +Compare production Standard scans with and without `maxTimeMinutes`. Both +arms call `CodexSecurity.run`, including setup, delegation, canonical artifact +validation, report generation, and completion. The treatment uses the shared +[investigation-time-budget.md](../../plugins/codex-security/references/investigation-time-budget.md) +policy, not an eval-only prompt. + +The SDK stamps one shared UTC deadline after setup, immediately before the +first scan turn. The allowance is advisory: there is no abort timer, automatic +continuation, or minimum runtime. Finalization is outside the allowance. + +## Run + +Use the repository's dependency and +[native-artifact setup](../../plugins/codex-security/native/README.md), then run +from the repository root: + +```sh +pnpm --dir sdk/typescript run build:plugin +pnpm --dir sdk/typescript run build:evals +node --experimental-strip-types evals/standard-time-budget/run.mts +``` + +Rebuild the bundle after plugin changes. This developer eval requires an existing +`OPENAI_API_KEY` or `CODEX_API_KEY` environment variable; saved Codex or +`codex-security login` credentials are intentionally unsupported. It uses the +bundled Codex SDK without reading saved logins, provisioning credentials, or +changing persistent configuration. Production SDK and CLI login support is +unchanged. **Model runs consume usage and have no enforced time or spend cap.** + +[experiment.json](experiment.json) selects an explicit model, two pairs +(four scans including delegated work), and five advisory minutes. Both arms +use `xhigh` reasoning and production worker settings. Pairs alternate +baseline-first and treatment-first order. Override settings with a JSON file: + +```sh +node --experimental-strip-types evals/standard-time-budget/run.mts /path/to/experiment.json +``` + +The fields are `model`, `pairs`, and `budgetMinutes`; positive fractional +minutes are accepted. Production deadline validation runs before either arm +starts. Ctrl+C saves the interrupted attempt, stops future arms, and waits for +child processes before cleaning temporary state. + +## Fixture and isolation + +The synthetic application has tenant authorization, SQL injection, file +traversal, and reflected HTML bugs, each with a safe control. Source and +deployment context go to the model; gold labels and expected locations stay +in the evaluator. + +Each arm runs in an opaque temporary directory outside the artifact tree; pair +and variant labels appear only in evaluator reports and archives. After the +client closes, its scan output is archived and its state is removed before the +next arm. Gold labels and prior-arm results are not next to the model's working +directory or included in prompts. This is layout separation, not a filesystem +security boundary: production API-key authentication and root-read permissions +remain unchanged. This eval does not test resumption or external publication. + +## Results + +Artifacts are saved under ignored `reports/`, updated after every attempt: + +- `experiment.json`: settings, evaluator/shared-helper, source, full-plugin, + and built-SDK file hashes, plus SDK/Codex package versions. Generated reports + are excluded from evaluator hashes. +- `fixture.json`: synthetic source and held-out gold labels for review. +- `pair--/prompt.txt`: the actual first dispatched scan prompt. +- Per-arm `scan/`, `result.json`, and `report.json`: canonical scan artifacts, + semantic output, grading, usage, elapsed time, and exact failure text. +- `runs.json` and `summary.json`: all attempts and the paired comparison. + +Elapsed time is monotonic and includes SDK setup, investigation, finalization, +scan archival, cleanup, and grading, but not subsequent evaluator JSON writing. +A canonical scan that misses a bug still completed successfully. The runner +exits nonzero for execution failures, not low recall. + +The grader counts unique gold cases with matching CWE, a `root_control` or +`sink` source location, and authentic source evidence. Other or missing roles +do not earn automatic recall; unmatched findings require adjudication. It +consumes the production contract's repository-relative POSIX paths. Duplicates +do not increase recall. +Safe-control overlaps, unmatched findings, and citation errors are separate +signals; overlaps and unmatched findings require adjudication, not automatic +false-positive labels. Free-text location roles can flag benign context +citations. Preserve raw scores and record adjudication separately: source +matching is a heuristic, not independent exploit validation. + +The summary separates execution success from completed-run quality. Paired +deltas are treatment minus baseline and include only complete pairs; excluded +pairs remain explicit. Each attempt's top-level `usage` includes delegated +workers, even when completion or grading fails. Missing usage stays unknown, +cached input is a subset of input tokens, and no dollar cost is inferred. + +## Interpretation and checks + +Two pairs are a smoke comparison, not statistical evidence. Review findings +without variant labels and compare grounded recall, adjudicated false +positives, completion, tokens, and elapsed time together. More findings or a +longer run alone is not an improvement, and these arms do not have matched +spend. Expand the corpus if both arms saturate the fixture; use more pairs and +pinned public repository revisions before drawing quality/cost conclusions. + +After the builds above, run deterministic checks without model calls: + +```sh +node --experimental-strip-types --test evals/standard-time-budget/test_*.mts evals/secret-discovery/test_*.mts +``` + +The integration test uses the real SDK and canonical artifact helpers with +mocked model/workbench boundaries, verifying isolated state, completion, prompt +capture, and parent-plus-worker usage. CI reuses its host native build, builds +the plugin with `build_mcp_app.mjs --native host`, and links it at +`sdk/typescript/_bundled_plugin`; universal release artifacts are not required. diff --git a/evals/standard-time-budget/experiment.json b/evals/standard-time-budget/experiment.json new file mode 100644 index 000000000..dd10dd2a3 --- /dev/null +++ b/evals/standard-time-budget/experiment.json @@ -0,0 +1,5 @@ +{ + "model": "gpt-5.6-sol", + "pairs": 2, + "budgetMinutes": 5 +} diff --git a/evals/standard-time-budget/experiment.mts b/evals/standard-time-budget/experiment.mts new file mode 100644 index 000000000..cc56c0502 --- /dev/null +++ b/evals/standard-time-budget/experiment.mts @@ -0,0 +1,166 @@ +import { createHash } from "node:crypto"; +import { readdir, readFile } from "node:fs/promises"; +import { join } from "node:path"; +import type { GradeReport, Result } from "./grade.mts"; +import type { PreparedStandardEval } from "./fixtures.mts"; +import type { StandardRunner, StandardRunOptions } from "./production.mts"; + +export interface ExperimentConfig { + model: string; + pairs: number; + budgetMinutes: number; +} +export type Variant = "baseline" | "time-aware"; +export type RunReport = GradeReport & { + scanId?: string; + scanDir?: string; + reportPath?: string; +}; +interface RunMetadata { + pair: number; + variant: Variant; + startedAt: string; + advisoryBudgetMinutes: number | null; + promptSha256: string | null; + elapsedMs: number; + usage: unknown; +} +type RunOutcome = + | { status: "completed"; report: RunReport; error?: never } + | { status: "failed"; error: string; report?: never }; +export type RunRecord = RunMetadata & RunOutcome; +export type CompletedRunRecord = Extract; +interface ComparisonOptions { + prepared: PreparedStandardEval; + config: ExperimentConfig; + signal?: AbortSignal; + run: StandardRunner; + elapsedNow?: () => number; + onStart?: (record: Pick) => Promise; + onResult?: ( + record: RunRecord, + result: Result | undefined, + records: RunRecord[], + prompt: string | undefined, + ) => Promise; +} + +export function validateExperiment(config: ExperimentConfig) { + if (typeof config?.model !== "string" || !config.model.trim()) { + throw new Error("The experiment requires an explicit model."); + } + if (!Number.isSafeInteger(config.pairs) || config.pairs < 1) { + throw new Error("Experiment pairs must be a positive integer."); + } + if (!Number.isFinite(config.budgetMinutes) || config.budgetMinutes <= 0) { + throw new Error("Experiment budgetMinutes must be a positive number."); + } + return { + model: config.model, + pairs: config.pairs, + budgetMinutes: config.budgetMinutes, + }; +} + +export function hashText(text: string | Uint8Array) { + return createHash("sha256").update(text).digest("hex"); +} + +export async function snapshotEvaluator(directory = import.meta.dirname) { + const files: Record = {}; + for (const path of [ + "run.mts", + "production.mts", + "experiment.mts", + "fixtures.mts", + "grade.mts", + "summary.mts", + "../secret-discovery/fixtures.mts", + "../secret-discovery/runtime.mts", + ]) { + files[path] = hashText(await readFile(join(directory, path))); + } + return files; +} + +export async function snapshotFiles(directory: string, prefix = "") { + const files: Record = {}; + const entries = await readdir(directory, { withFileTypes: true }); + for (const entry of entries.sort((first, second) => + first.name.localeCompare(second.name), + )) { + const relative = prefix ? `${prefix}/${entry.name}` : entry.name; + const path = join(directory, entry.name); + if (entry.isDirectory()) { + Object.assign(files, await snapshotFiles(path, relative)); + } else { + files[relative] = hashText(await readFile(path)); + } + } + return files; +} + +export async function runComparison({ + prepared, + config, + signal, + run, + elapsedNow = () => performance.now(), + onStart = async () => {}, + onResult = async () => {}, +}: ComparisonOptions) { + config = validateExperiment(config); + const records: RunRecord[] = []; + for (let pair = 1; pair <= config.pairs; pair += 1) { + const variants: Variant[] = + pair % 2 === 1 ? ["baseline", "time-aware"] : ["time-aware", "baseline"]; + for (const variant of variants) { + signal?.throwIfAborted(); + await onStart({ pair, variant }); + signal?.throwIfAborted(); + const startedAt = new Date().toISOString(); + const startedElapsed = elapsedNow(); + const observation: StandardRunOptions["observation"] = {}; + let outcome: RunOutcome; + let semanticResult: Result | undefined; + try { + const result = await run(prepared, { + model: config.model, + pair, + variant, + signal, + ...(variant === "time-aware" + ? { maxTimeMinutes: config.budgetMinutes } + : {}), + observation, + }); + outcome = { status: "completed", report: result.report }; + semanticResult = result.semanticResult; + } catch (error) { + outcome = { + status: "failed", + error: error instanceof Error ? error.message : String(error), + }; + } + const elapsedMs = elapsedNow() - startedElapsed; + const record: RunRecord = { + pair, + variant, + startedAt, + advisoryBudgetMinutes: + variant === "time-aware" ? config.budgetMinutes : null, + promptSha256: + observation.prompt === undefined + ? null + : hashText(observation.prompt), + ...outcome, + elapsedMs, + usage: observation.usage ?? null, + }; + records.push(record); + await onResult(record, semanticResult, records, observation.prompt); + signal?.throwIfAborted(); + } + } + return records; +} diff --git a/evals/standard-time-budget/fixtures.mts b/evals/standard-time-budget/fixtures.mts new file mode 100644 index 000000000..590fa8bb0 --- /dev/null +++ b/evals/standard-time-budget/fixtures.mts @@ -0,0 +1,254 @@ +export interface SourceCase { + id: string; + path: string; + line: number; + cwes: string[]; +} + +export interface Fixture { + files: Record; + positives: SourceCase[]; + controls: SourceCase[]; +} + +export function createFixture(): Fixture { + const files: Record = { + "README.md": `# Account portal + +This Python Flask service serves authenticated tenant accounts. Both the account +and billing routes are currently supported. The sign-in service creates signed +Flask sessions containing a server-selected integer tenant_id; clients cannot +choose that value. The session signing key comes from deployment configuration. + +Invoices and catalog records belong to exactly one tenant. Account users may +read only their own tenant's records. Invoice identifiers are sequential. + +Export downloads are available to all authenticated users, but only files under +/srv/portal/exports may be downloaded. That directory and its symlinks are +deployment-managed; account users cannot create or modify files there. Other +server files, including configuration, must remain private. + +Preview endpoints render user-provided titles in HTML for the signed-in browser. +The deployment does not add an HTML sanitizer or Content-Security-Policy header. +The database and export files are deployment resources, not checked-in fixtures. +`, + "src/app.py": `import os +import sqlite3 + +from flask import Flask, abort, g, session + +from .downloads import downloads +from .invoices import invoices +from .preview import preview +from .search import search + +app = Flask(__name__) +app.secret_key = os.environ["SESSION_SIGNING_KEY"] +app.config.update(SESSION_COOKIE_SECURE=True, SESSION_COOKIE_HTTPONLY=True, + SESSION_COOKIE_SAMESITE="Lax") + + +@app.before_request +def authenticate(): + tenant_id = session.get("tenant_id") + if type(tenant_id) is not int: + abort(401) + g.tenant_id = tenant_id + g.database = sqlite3.connect(os.environ["PORTAL_DATABASE"]) + g.database.row_factory = sqlite3.Row + + +@app.teardown_request +def close_database(error): + database = g.pop("database", None) + if database is not None: + database.close() + + +app.register_blueprint(invoices) +app.register_blueprint(search) +app.register_blueprint(downloads) +app.register_blueprint(preview) +`, + "src/__init__.py": "", + "src/invoices.py": `from flask import Blueprint, abort, g, jsonify + +invoices = Blueprint("invoices", __name__) + + +@invoices.get("/account/invoices/") +def account_invoice(invoice_id): + row = g.database.execute( + "SELECT id, tenant_id, recipient, total FROM invoices WHERE id = ?", + (invoice_id,), + ).fetchone() + if row is None: + abort(404) + return jsonify(dict(row)) + + +@invoices.get("/billing/invoices/") +def billing_invoice(invoice_id): + row = g.database.execute( + "SELECT id, tenant_id, recipient, total FROM invoices WHERE id = ? AND tenant_id = ?", + (invoice_id, g.tenant_id), + ).fetchone() + if row is None: + abort(404) + return jsonify(dict(row)) +`, + "src/search.py": `from flask import Blueprint, g, jsonify, request + +search = Blueprint("search", __name__) + + +@search.get("/account/search") +def account_search(): + term = request.args.get("q", "") + query = f"SELECT id, name FROM catalog WHERE tenant_id = {g.tenant_id} AND name LIKE '%{term}%'" + rows = g.database.execute(query).fetchall() + return jsonify([dict(row) for row in rows]) + + +@search.get("/billing/search") +def billing_search(): + term = request.args.get("q", "") + rows = g.database.execute( + "SELECT id, name FROM catalog WHERE tenant_id = ? AND name LIKE ?", + (g.tenant_id, f"%{term}%"), + ).fetchall() + return jsonify([dict(row) for row in rows]) +`, + "src/downloads.py": `from pathlib import Path + +from flask import Blueprint, abort, request, send_file + +downloads = Blueprint("downloads", __name__) +EXPORT_ROOT = Path("/srv/portal/exports") + + +@downloads.get("/account/download") +def account_download(): + name = request.args["name"] + return send_file(EXPORT_ROOT / name, as_attachment=True) + + +@downloads.get("/billing/download") +def billing_download(): + name = request.args["name"] + root = EXPORT_ROOT.resolve() + target = (root / name).resolve() + if not target.is_relative_to(root): + abort(403) + return send_file(target, as_attachment=True) +`, + "src/preview.py": `from html import escape + +from flask import Blueprint, Response, request + +preview = Blueprint("preview", __name__) + + +@preview.get("/account/preview") +def account_preview(): + title = request.args.get("title", "") + return Response(f"

{title}

", content_type="text/html") + + +@preview.get("/billing/preview") +def billing_preview(): + title = escape(request.args.get("title", "")) + return Response(f"

{title}

", content_type="text/html") +`, + "schema.sql": `CREATE TABLE invoices ( + id INTEGER PRIMARY KEY, + tenant_id INTEGER NOT NULL, + recipient TEXT NOT NULL, + total INTEGER NOT NULL +); + +CREATE TABLE catalog ( + id INTEGER PRIMARY KEY, + tenant_id INTEGER NOT NULL, + name TEXT NOT NULL +); +`, + }; + const sourceCase = ( + id: string, + path: string, + marker: string, + cwes: string[], + occurrence = 0, + ): SourceCase => { + const matchingLines = files[path] + .split("\n") + .flatMap((line, index) => (line.includes(marker) ? [index + 1] : [])); + const line = matchingLines[occurrence]; + if (!line) throw new Error(`Fixture marker missing: ${id}`); + return { id, path, line, cwes }; + }; + return { + files, + positives: [ + sourceCase( + "invoice-tenant-isolation", + "src/invoices.py", + 'FROM invoices WHERE id = ?"', + ["CWE-639", "CWE-862", "CWE-863", "CWE-284"], + ), + sourceCase("catalog-sql-injection", "src/search.py", "query = f", [ + "CWE-89", + ]), + sourceCase( + "export-path-traversal", + "src/downloads.py", + "send_file(EXPORT_ROOT", + ["CWE-22", "CWE-23", "CWE-36", "CWE-73"], + ), + sourceCase( + "preview-reflected-html", + "src/preview.py", + "return Response", + ["CWE-79"], + ), + ], + controls: [ + sourceCase( + "invoice-scoped-query", + "src/invoices.py", + "AND tenant_id = ?", + ["CWE-639", "CWE-862", "CWE-863", "CWE-284"], + ), + sourceCase("catalog-bound-parameters", "src/search.py", "name LIKE ?", [ + "CWE-89", + ]), + sourceCase( + "export-contained-path", + "src/downloads.py", + "send_file(target", + ["CWE-22", "CWE-23", "CWE-36", "CWE-73"], + ), + sourceCase( + "preview-escaped-html", + "src/preview.py", + "return Response", + ["CWE-79"], + 1, + ), + ], + }; +} +import { join } from "node:path"; +import { writeFixture } from "../secret-discovery/fixtures.mts"; + +export async function prepareStandardEval(root: string, runtime: string) { + const fixture = createFixture(); + const repo = join(root, "repository"); + await writeFixture(repo, fixture); + return { repo, runtime, fixture }; +} + +export type PreparedStandardEval = Awaited< + ReturnType +>; diff --git a/evals/standard-time-budget/grade.mts b/evals/standard-time-budget/grade.mts new file mode 100644 index 000000000..985782d97 --- /dev/null +++ b/evals/standard-time-budget/grade.mts @@ -0,0 +1,205 @@ +import { posix } from "node:path"; +import type { + Finding as ScanFinding, + FindingCodeEvidence, + FindingLocation, +} from "../../sdk/typescript/src/models.js"; +import type { Fixture, SourceCase } from "./fixtures.mts"; + +export type SourceLocation = FindingLocation; +type Finding = Pick & { + taxonomy: Pick; + codeEvidence?: Pick[]; +}; + +export interface Result { + findings: Finding[]; + [key: string]: unknown; +} + +interface EvidenceError { + findingIndex?: number; + evidenceIndex?: number; + locationIndex?: number; + caseId?: string; + reason: string; +} + +export type GradeReport = ReturnType; + +function validRange(location: SourceLocation, fixture: Fixture) { + const source = fixture.files[posix.normalize(location.path)]; + const endLine = location.endLine ?? location.startLine; + return ( + typeof source === "string" && + Number.isInteger(location.startLine) && + Number.isInteger(endLine) && + location.startLine > 0 && + endLine >= location.startLine && + endLine <= source.trimEnd().split("\n").length + ); +} + +function overlaps(location: SourceLocation, expected: SourceCase) { + return ( + posix.normalize(location.path) === expected.path && + location.startLine <= expected.line && + (location.endLine ?? location.startLine) >= expected.line + ); +} + +function matches( + finding: Finding, + expected: SourceCase, + fixture: Fixture, + requirePrimaryRole: boolean, +) { + return ( + finding.taxonomy.cwe.some((cwe) => + expected.cwes.includes(String(cwe).trim().toUpperCase()), + ) && + finding.locations.some( + (location) => + (requirePrimaryRole + ? /^(root_control|sink)$/u.test(location.role ?? "") + : !/^(supporting|support|context|consumer|expected_control)$/u.test( + location.role ?? "", + )) && + validRange(location, fixture) && + overlaps(location, expected), + ) + ); +} + +function inspectEvidence( + finding: Finding, + fixture: Fixture, + findingIndex: number, +) { + const entries = finding.codeEvidence ?? []; + const ranges: SourceLocation[] = []; + const errors: EvidenceError[] = []; + if (!entries.length) { + errors.push({ findingIndex, reason: "missing code evidence" }); + } + for (const [evidenceIndex, entry] of entries.entries()) { + const code = entry.code.replaceAll("\r\n", "\n").replace(/\n$/u, ""); + const range = { + ...entry, + endLine: entry.startLine + code.split("\n").length - 1, + }; + const source = fixture.files[posix.normalize(entry.path)]; + if ( + !code.trim() || + !validRange(range, fixture) || + code !== + source! + .split("\n") + .slice(entry.startLine - 1, range.endLine) + .join("\n") + ) { + errors.push({ + findingIndex, + evidenceIndex, + reason: "code does not match source at the cited lines", + }); + } else { + ranges.push(range); + } + } + return { ranges, errors }; +} + +export function gradeResult(result: Result, fixture: Fixture) { + const { findings } = result; + const evidenceErrors: EvidenceError[] = []; + const unmatchedFindings = []; + const safeControlDetails = []; + const matchCounts = new Map( + fixture.positives.map((expected) => [expected.id, 0]), + ); + for (const [findingIndex, finding] of findings.entries()) { + const evidence = inspectEvidence(finding, fixture, findingIndex); + for (const [locationIndex, location] of finding.locations.entries()) { + if (!validRange(location, fixture)) { + evidence.errors.push({ + findingIndex, + locationIndex, + reason: "location path or line range does not exist in the source", + }); + } + } + evidenceErrors.push(...evidence.errors); + const candidates = fixture.positives.filter((expected) => + matches(finding, expected, fixture, true), + ); + const matched = candidates.filter((expected) => { + const grounded = evidence.ranges.some((range) => + overlaps(range, expected), + ); + if (!grounded) { + evidenceErrors.push({ + findingIndex, + caseId: expected.id, + reason: "code evidence does not cover the expected source line", + }); + } + return grounded && evidence.errors.length === 0; + }); + for (const expected of matched) { + matchCounts.set(expected.id, matchCounts.get(expected.id)! + 1); + } + const controls = fixture.controls.filter((expected) => + matches(finding, expected, fixture, false), + ); + if (controls.length) { + safeControlDetails.push({ + findingIndex, + title: finding.title, + controlIds: controls.map((entry) => entry.id), + }); + } + if (!matched.length && !controls.length) { + unmatchedFindings.push({ + findingIndex, + title: finding.title, + reason: candidates.length + ? "expected case lacks authentic evidence" + : "no matching source location and CWE in the gold cases", + }); + } + } + const cases = fixture.positives.map((expected) => ({ + id: expected.id, + found: matchCounts.get(expected.id)! > 0, + })); + const matchedIds = cases + .filter((entry) => entry.found) + .map((entry) => entry.id); + const foundCount = matchedIds.length; + const expectedCount = cases.length; + return { + passed: + foundCount === expectedCount && + safeControlDetails.length === 0 && + unmatchedFindings.length === 0 && + evidenceErrors.length === 0, + recall: foundCount / expectedCount, + foundCount, + expectedCount, + matchedIds, + cases, + duplicateFindings: [...matchCounts.values()].reduce( + (total, count) => total + Math.max(0, count - 1), + 0, + ), + safeControlFindings: safeControlDetails.length, + safeControlDetails, + unmatchedFindings, + evidenceErrors, + needsAdjudication: + unmatchedFindings.length > 0 || safeControlDetails.length > 0, + gradingMethod: + "Heuristic source-line, CWE, and verbatim evidence matching; not independent exploit validation. Unmatched findings require manual adjudication and are not automatically false positives.", + }; +} diff --git a/evals/standard-time-budget/production.mts b/evals/standard-time-budget/production.mts new file mode 100644 index 000000000..9c313bffa --- /dev/null +++ b/evals/standard-time-budget/production.mts @@ -0,0 +1,155 @@ +import { randomUUID } from "node:crypto"; +import { cp, mkdir, rm } from "node:fs/promises"; +import { join, relative } from "node:path"; +import { Codex } from "../../sdk/typescript/node_modules/@openai/codex-sdk/dist/index.js"; +import * as sdk from "../../sdk/typescript/dist/api.js"; +import { deepScanOptions } from "../../sdk/typescript/dist/deep-config.js"; +import { createProfileCodex } from "../../sdk/typescript/dist/provider-profile.js"; +import type { CodexSecurity as Client } from "../../sdk/typescript/src/api.js"; +import type { PreparedStandardEval } from "./fixtures.mts"; +import { gradeResult, type Result } from "./grade.mts"; +import type { RunReport, Variant } from "./experiment.mts"; + +const { CodexSecurity, scanAuthentication } = + sdk as unknown as typeof import("../../sdk/typescript/src/api.js"); +type Dependencies = NonNullable[1]>; + +export function preflightStandardEval( + environment: NodeJS.ProcessEnv, + budgetMinutes: number, +) { + if (scanAuthentication(environment).method !== "api_key") { + throw new Error( + "The Standard time-budget eval requires OPENAI_API_KEY or CODEX_API_KEY. Saved logins are not supported by this developer eval.", + ); + } + deepScanOptions({ mode: "standard", maxTimeMinutes: budgetMinutes }); +} + +export interface StandardRunOptions { + model: string; + pair: number; + variant: Variant; + maxTimeMinutes?: number; + signal?: AbortSignal; + observation: { prompt?: string; usage?: unknown }; +} + +export type StandardRunner = ( + prepared: PreparedStandardEval, + options: StandardRunOptions, +) => Promise<{ report: RunReport; semanticResult: Result }>; + +export function createStandardRunner({ + stateRoot, + outputRoot, + environment = process.env, + createClient = (config, dependencies) => + new CodexSecurity(config, dependencies, { surface: "sdk" }), + createCodex = ({ nativeProfile, ...settings }) => + nativeProfile === undefined + ? new Codex(settings) + : createProfileCodex(settings, nativeProfile), +}: { + stateRoot: string; + outputRoot: string; + environment?: NodeJS.ProcessEnv; + createClient?: ( + config: ConstructorParameters[0], + dependencies: Dependencies, + ) => Pick; + createCodex?: Dependencies["createCodex"]; +}): StandardRunner { + return async (prepared, options) => { + const stateDirectory = join(stateRoot, randomUUID()); + const scanDirectory = join(stateDirectory, "scan"); + const archiveDirectory = join( + outputRoot, + `pair-${options.pair}-${options.variant}`, + "scan", + ); + await mkdir(scanDirectory, { recursive: true, mode: 0o700 }); + const client = createClient( + { + pluginPath: prepared.runtime, + codexOverrides: { + model: options.model, + model_reasoning_effort: "xhigh", + }, + }, + { + environment: { + ...environment, + CODEX_SECURITY_STATE_DIR: stateDirectory, + }, + createCodex: async (settings) => { + const codex = await createCodex(settings); + return { + startThread: (threadOptions) => { + const thread = codex.startThread(threadOptions); + return { + get id() { + return thread.id; + }, + runStreamed: (prompt, turnOptions) => { + options.observation.prompt ??= prompt; + return thread.runStreamed(prompt, turnOptions); + }, + }; + }, + }; + }, + }, + ); + let failure: unknown; + try { + const result = await client.run(prepared.repo, { + mode: "standard", + maxTimeMinutes: options.maxTimeMinutes, + signal: options.signal, + onUsage: (usage) => { + options.observation.usage = usage; + }, + outputDir: scanDirectory, + }); + options.observation.usage = + result.turnResult.usage ?? options.observation.usage; + const semanticResult = { + findings: result.findings.findings, + coverage: result.coverage, + threatModel: result.threatModel, + }; + return { + semanticResult, + report: { + ...gradeResult(semanticResult, prepared.fixture), + scanId: result.manifest.scan.id, + scanDir: archiveDirectory, + reportPath: join( + archiveDirectory, + relative(result.scanDir, result.reportPath), + ), + }, + }; + } catch (error) { + failure = error; + throw error; + } finally { + try { + await client.close(); + } catch (error) { + if (failure === undefined) throw error; + throw new AggregateError( + [failure, error], + `${String(failure)}\n${String(error)}`, + ); + } finally { + try { + await cp(scanDirectory, archiveDirectory, { recursive: true }); + } finally { + await rm(stateDirectory, { recursive: true, force: true }); + } + } + } + }; +} diff --git a/evals/standard-time-budget/run.mts b/evals/standard-time-budget/run.mts new file mode 100644 index 000000000..9c3881464 --- /dev/null +++ b/evals/standard-time-budget/run.mts @@ -0,0 +1,106 @@ +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + bundledPluginRoot, + createIsolatedHome, +} from "../../sdk/typescript/dist/runtime.js"; +import { + CODEX_EXECUTABLE_VERSION, + CODEX_SDK_VERSION, + VERSION, +} from "../../sdk/typescript/dist/version.js"; +import { withEvalState } from "../secret-discovery/runtime.mts"; +import { createStandardRunner, preflightStandardEval } from "./production.mts"; +import { prepareStandardEval } from "./fixtures.mts"; +import { + runComparison, + snapshotEvaluator, + snapshotFiles, + validateExperiment, +} from "./experiment.mts"; +import { summarizeRuns } from "./summary.mts"; + +const config = validateExperiment( + JSON.parse( + await readFile( + process.argv[2] ?? new URL("./experiment.json", import.meta.url), + "utf8", + ), + ), +); +preflightStandardEval(process.env, config.budgetMinutes); +const pluginRoot = await bundledPluginRoot(); +const reports = fileURLToPath(new URL("./reports/", import.meta.url)); +await mkdir(reports, { recursive: true }); +const reportDirectory = await mkdtemp(join(reports, "run-")); +const saveJson = (path: string, value: unknown) => + writeFile(path, JSON.stringify(value, null, 2) + "\n"); +console.log(`Eval artifacts: ${reportDirectory}`); +await withEvalState( + createIsolatedHome, + undefined, + async ({ root, home, signal }) => { + const prepared = await prepareStandardEval(root, pluginRoot); + await saveJson(join(reportDirectory, "experiment.json"), { + ...config, + reasoningEffort: "xhigh", + execution: "CodexSecurity.run Standard production pipeline", + order: "alternating baseline-first and time-aware-first pairs", + evaluator: await snapshotEvaluator(), + source: await snapshotFiles(prepared.repo), + plugin: await snapshotFiles(pluginRoot), + sdk: await snapshotFiles( + fileURLToPath(new URL("../../sdk/typescript/dist/", import.meta.url)), + ), + packageVersions: { + "@openai/codex-security": VERSION, + "@openai/codex-sdk": CODEX_SDK_VERSION, + "@openai/codex": CODEX_EXECUTABLE_VERSION, + }, + }); + await saveJson(join(reportDirectory, "fixture.json"), prepared.fixture); + const records = await runComparison({ + prepared, + config, + signal, + run: createStandardRunner({ + stateRoot: join(root, "states"), + outputRoot: reportDirectory, + environment: { ...process.env, CODEX_HOME: home }, + }), + onStart: async (record) => { + const directory = join( + reportDirectory, + `pair-${record.pair}-${record.variant}`, + ); + await mkdir(directory); + console.log(`Pair ${record.pair}/${config.pairs}: ${record.variant}`); + }, + onResult: async (record, semanticResult, completedRecords, prompt) => { + const directory = join( + reportDirectory, + `pair-${record.pair}-${record.variant}`, + ); + if (prompt !== undefined) + await writeFile(join(directory, "prompt.txt"), prompt); + if (semanticResult !== undefined) { + await saveJson(join(directory, "result.json"), semanticResult); + } + await saveJson(join(directory, "report.json"), record); + await saveJson(join(reportDirectory, "runs.json"), completedRecords); + await saveJson( + join(reportDirectory, "summary.json"), + summarizeRuns(completedRecords), + ); + console.log( + `${record.variant}: ${record.status}, ${(record.elapsedMs / 1000).toFixed(1)}s`, + ); + }, + }); + console.log(JSON.stringify(summarizeRuns(records), null, 2)); + process.exitCode = records.some((record) => record.status === "failed") + ? 1 + : 0; + }, +); diff --git a/evals/standard-time-budget/summary.mts b/evals/standard-time-budget/summary.mts new file mode 100644 index 000000000..45a8c18ab --- /dev/null +++ b/evals/standard-time-budget/summary.mts @@ -0,0 +1,143 @@ +import type { + CompletedRunRecord, + RunRecord, + RunReport, + Variant, +} from "./experiment.mts"; + +const variants = ["baseline", "time-aware"] as const; +const usageFields = [ + "input_tokens", + "cached_input_tokens", + "output_tokens", +] as const; +type UsageField = (typeof usageFields)[number]; + +function mean(values: number[]) { + return values.length + ? values.reduce((total, value) => total + value, 0) / values.length + : null; +} + +function usageValue(record: RunRecord, field: UsageField) { + const usage = record.usage as Partial> | undefined; + const value = usage?.[field]; + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function totalTokens(record: RunRecord) { + const input = usageValue(record, "input_tokens"); + const output = usageValue(record, "output_tokens"); + return input === null || output === null ? null : input + output; +} + +function sumKnown(values: (number | null)[]) { + return values.length && + values.every((value): value is number => value !== null) + ? values.reduce((total, value) => total + value, 0) + : null; +} + +function summarizeArm(records: RunRecord[]) { + const completed = records.filter( + (record): record is CompletedRunRecord => record.status === "completed", + ); + const qualityCount = (getValue: (report: RunReport) => number) => + completed.length + ? completed.reduce((total, record) => total + getValue(record.report), 0) + : null; + return { + attempted: records.length, + completed: completed.length, + failed: records.length - completed.length, + successRate: records.length ? completed.length / records.length : null, + meanElapsedMs: mean(records.map((record) => record.elapsedMs)), + meanRecall: mean(completed.map((record) => record.report.recall)), + foundCount: qualityCount((report) => report.foundCount), + expectedCount: qualityCount((report) => report.expectedCount), + safeControlFindings: qualityCount((report) => report.safeControlFindings), + unmatchedFindings: qualityCount( + (report) => report.unmatchedFindings.length, + ), + evidenceErrors: qualityCount((report) => report.evidenceErrors.length), + usage: { + ...(Object.fromEntries( + usageFields.map((field) => [ + field, + sumKnown(records.map((record) => usageValue(record, field))), + ]), + ) as Record), + total_tokens: sumKnown(records.map(totalTokens)), + missingUsageRuns: records.filter((record) => + usageFields.some((field) => usageValue(record, field) === null), + ).length, + }, + }; +} + +export function summarizeRuns(records: RunRecord[]) { + const pairs = new Map>>(); + for (const record of records) { + const variant = record.variant; + const pair = pairs.get(record.pair) ?? {}; + if (pair[variant]) { + throw new Error( + `Duplicate run for pair ${record.pair}/${record.variant}`, + ); + } + pair[variant] = record; + pairs.set(record.pair, pair); + } + const completedPairs = [...pairs.values()].filter( + (pair): pair is Record => + variants.every((variant) => pair[variant]?.status === "completed"), + ); + const meanDelta = ( + getValue: (record: CompletedRunRecord) => number | null, + ) => { + const deltas = completedPairs.map((pair) => { + const baseline = getValue(pair.baseline); + const timeAware = getValue(pair["time-aware"]); + return baseline === null || timeAware === null + ? null + : timeAware - baseline; + }); + return deltas.every((delta): delta is number => delta !== null) + ? mean(deltas) + : null; + }; + return { + arms: Object.fromEntries( + variants.map((variant) => [ + variant, + summarizeArm(records.filter((record) => record.variant === variant)), + ]), + ), + paired: { + totalPairs: pairs.size, + completedPairs: completedPairs.length, + excludedPairs: pairs.size - completedPairs.length, + meanDelta: { + recall: meanDelta((record) => record.report.recall), + elapsedMs: meanDelta((record) => record.elapsedMs), + foundCount: meanDelta((record) => record.report.foundCount), + safeControlFindings: meanDelta( + (record) => record.report.safeControlFindings, + ), + unmatchedFindings: meanDelta( + (record) => record.report.unmatchedFindings.length, + ), + evidenceErrors: meanDelta( + (record) => record.report.evidenceErrors.length, + ), + ...(Object.fromEntries( + usageFields.map((field) => [ + field, + meanDelta((record) => usageValue(record, field)), + ]), + ) as Record), + total_tokens: meanDelta(totalTokens), + }, + }, + }; +} diff --git a/evals/standard-time-budget/test_experiment.mts b/evals/standard-time-budget/test_experiment.mts new file mode 100644 index 000000000..97583bc6d --- /dev/null +++ b/evals/standard-time-budget/test_experiment.mts @@ -0,0 +1,204 @@ +import assert from "node:assert/strict"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { writeFixture } from "../secret-discovery/fixtures.mts"; +import { + createFixture, + prepareStandardEval, + type PreparedStandardEval, +} from "./fixtures.mts"; +import { gradeResult, type Result } from "./grade.mts"; +import { + hashText, + runComparison, + snapshotEvaluator, + snapshotFiles, + validateExperiment, + type RunRecord, +} from "./experiment.mts"; + +const config = { model: "eval-model", pairs: 2, budgetMinutes: 5 }; +const preparedStub: PreparedStandardEval = { + repo: "/synthetic/repo", + runtime: "/synthetic/runtime", + fixture: createFixture(), +}; + +function mockResult() { + return { + report: { + ...gradeResult({ findings: [] }, preparedStub.fixture), + }, + semanticResult: { findings: [] }, + }; +} + +test("evaluator provenance tracks implementation and shared helpers, not reports", async (context) => { + const root = await mkdtemp(join(tmpdir(), "time-budget-provenance-")); + context.after(() => rm(root, { recursive: true, force: true })); + const directory = join(root, "standard-time-budget"); + const files = Object.fromEntries( + [ + "run.mts", + "production.mts", + "experiment.mts", + "fixtures.mts", + "grade.mts", + "summary.mts", + "../secret-discovery/fixtures.mts", + "../secret-discovery/runtime.mts", + ].map((path) => [path, path]), + ); + await writeFixture(directory, { files }); + const before = await snapshotEvaluator(directory); + assert.deepEqual( + before, + Object.fromEntries( + Object.entries(files).map(([path, text]) => [path, hashText(text)]), + ), + ); + await writeFixture(directory, { + files: { + "grade.mts": "changed grader", + "../secret-discovery/runtime.mts": "changed state helper", + "reports/run-123/experiment.json": "generated report", + }, + }); + assert.deepEqual(await snapshotEvaluator(directory), { + ...before, + "grade.mts": hashText("changed grader"), + "../secret-discovery/runtime.mts": hashText("changed state helper"), + }); +}); + +test("pairs stage source without labels, alternate order, and opt in only for treatment", async (context) => { + const root = await mkdtemp(join(tmpdir(), "time-budget-eval-")); + context.after(() => rm(root, { recursive: true, force: true })); + const prepared = await prepareStandardEval(root, "/synthetic/plugin"); + const sourceBefore = await snapshotFiles(prepared.repo); + assert.deepEqual( + Object.keys(sourceBefore).sort(), + Object.keys(prepared.fixture.files).sort(), + ); + const requests: (number | undefined)[] = []; + const records = await runComparison({ + prepared, + config, + run: async (actual, options) => { + assert.equal(actual, prepared); + requests.push(options.maxTimeMinutes); + options.observation.prompt = "Captured production prompt"; + return mockResult(); + }, + }); + assert.deepEqual(requests, [undefined, 5, 5, undefined]); + assert.deepEqual( + records.map(({ pair, variant }) => [pair, variant]), + [ + [1, "baseline"], + [1, "time-aware"], + [2, "time-aware"], + [2, "baseline"], + ], + ); + assert.equal(records[0].promptSha256, hashText("Captured production prompt")); +}); + +test("monotonic elapsed time includes runner work but not artifact callbacks", async () => { + let elapsed = 0; + let calls = 0; + const records = await runComparison({ + prepared: preparedStub, + config: { ...config, pairs: 1 }, + elapsedNow: () => elapsed, + onStart: async () => { + elapsed += 60_000; + }, + run: async () => { + calls += 1; + elapsed += calls === 1 ? 1_000 : 400_000; + return mockResult(); + }, + onResult: async () => { + elapsed += 60_000; + }, + }); + assert.equal(calls, 2); + assert.deepEqual( + records.map((record) => record.elapsedMs), + [1_000, 400_000], + ); + assert.equal(records[1].status, "completed"); +}); + +test("retains exact failures and still runs the paired arm", async () => { + const saved: { + record: RunRecord; + result: Result | undefined; + count: number; + }[] = []; + let calls = 0; + const records = await runComparison({ + prepared: preparedStub, + config: { ...config, pairs: 1 }, + run: async () => { + calls += 1; + if (calls === 1) + throw new Error("model stream failed: synthetic diagnostic"); + return mockResult(); + }, + onResult: async (record, result, records) => + saved.push({ record, result, count: records.length }), + }); + assert.equal(records[0].status, "failed"); + assert.equal(records[0].error, "model stream failed: synthetic diagnostic"); + assert.equal(records[0].promptSha256, null); + assert.equal(records[1].status, "completed"); + assert.deepEqual( + saved.map(({ count }) => count), + [1, 2], + ); + assert.equal(saved[0].result, undefined); +}); + +test("explicit cancellation saves the interrupted attempt and stops the comparison", async () => { + const controller = new AbortController(); + const saved: RunRecord[] = []; + let calls = 0; + await assert.rejects( + runComparison({ + prepared: preparedStub, + config, + signal: controller.signal, + run: async (_prepared, { signal }) => { + calls += 1; + controller.abort(new Error("user canceled")); + signal!.throwIfAborted(); + throw new Error("Cancellation should abort the runner"); + }, + onResult: async (record) => saved.push(record), + }), + /user canceled/, + ); + assert.equal(calls, 1); + assert.equal(saved.length, 1); + assert.equal(saved[0].status, "failed"); +}); + +test("requires an explicit model and valid experiment sizes without imposing a cap", () => { + for (const change of [ + { model: "" }, + { pairs: 0 }, + { pairs: 1.5 }, + { budgetMinutes: 0 }, + { budgetMinutes: Infinity }, + ]) { + assert.throws(() => validateExperiment({ ...config, ...change })); + } + assert.deepEqual( + validateExperiment({ ...config, pairs: 100, budgetMinutes: 0.25 }), + { ...config, pairs: 100, budgetMinutes: 0.25 }, + ); +}); diff --git a/evals/standard-time-budget/test_grading.mts b/evals/standard-time-budget/test_grading.mts new file mode 100644 index 000000000..a98ad1e41 --- /dev/null +++ b/evals/standard-time-budget/test_grading.mts @@ -0,0 +1,230 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { createFixture, type Fixture, type SourceCase } from "./fixtures.mts"; +import { gradeResult, type SourceLocation } from "./grade.mts"; + +function findingFor(fixture: Fixture, expected: SourceCase) { + return { + title: expected.id, + taxonomy: { cwe: [expected.cwes[0]] }, + locations: [ + { path: expected.path, startLine: expected.line, role: "root_control" }, + ] as SourceLocation[], + codeEvidence: [ + { + path: expected.path, + startLine: expected.line, + code: fixture.files[expected.path].split("\n")[expected.line - 1], + }, + ], + }; +} + +type FixtureFinding = ReturnType; + +function completeResult(fixture: Fixture) { + return { + findings: fixture.positives.map((expected) => + findingFor(fixture, expected), + ), + }; +} + +test("authentic source evidence matches all four vulnerability families", () => { + const fixture = createFixture(); + const result = completeResult(fixture); + result.findings[1].locations[0].role = "sink"; + const report = gradeResult(result, fixture); + assert.equal(report.passed, true); + assert.equal(report.recall, 1); + assert.equal(report.expectedCount, 4); + assert.equal(report.foundCount, 4); + assert.deepEqual(report.evidenceErrors, []); +}); + +test("a missed bug reduces recall", () => { + const fixture = createFixture(); + const result = completeResult(fixture); + result.findings.pop(); + const report = gradeResult(result, fixture); + assert.equal(report.recall, 0.75); + assert.equal(report.passed, false); + assert.deepEqual(report.unmatchedFindings, []); +}); + +test("additional fabricated locations invalidate an otherwise grounded finding", () => { + const fixture = createFixture(); + for (const role of ["root_control", "supporting"]) { + for (const location of [ + { path: "src/invented.py", startLine: 500 }, + { path: "src/invoices.py", startLine: 500 }, + { path: "src/invoices.py", startLine: 0 }, + { path: "src/invoices.py", startLine: 8, endLine: 7 }, + ]) { + const result = completeResult(fixture); + result.findings[0].locations.push({ ...location, role }); + const report = gradeResult(result, fixture); + assert.equal(report.passed, false); + assert.equal(report.recall, 0.75); + assert.equal(report.needsAdjudication, true); + assert.deepEqual(report.evidenceErrors, [ + { + findingIndex: 0, + locationIndex: 1, + reason: "location path or line range does not exist in the source", + }, + ]); + } + } +}); + +test("duplicate reports cannot increase recall", () => { + const fixture = createFixture(); + const result = completeResult(fixture); + result.findings.push(structuredClone(result.findings[0])); + const report = gradeResult(result, fixture); + assert.equal(report.recall, 1); + assert.equal(report.foundCount, 4); + assert.equal(report.duplicateFindings, 1); +}); + +test("a different CWE, source line, or non-primary role does not earn a gold match", () => { + const fixture = createFixture(); + const alterations: Array<(finding: FixtureFinding) => void> = [ + (finding) => { + finding.taxonomy.cwe = ["CWE-79"]; + }, + (finding) => { + finding.locations[0].path = "README.md"; + }, + (finding) => { + finding.locations[0].startLine = 1; + }, + (finding) => { + finding.locations[0].role = "supporting"; + }, + (finding) => { + finding.locations[0].role = "supporting_context"; + finding.locations.unshift({ + path: "README.md", + startLine: 1, + role: "root_control", + }); + }, + ]; + for (const alter of alterations) { + const result = completeResult(fixture); + alter(result.findings[0]); + const report = gradeResult(result, fixture); + assert.equal(report.recall, 0.75); + assert.equal(report.unmatchedFindings.length, 1); + assert.equal(report.needsAdjudication, true); + assert.equal(report.safeControlFindings, 0); + assert.deepEqual(report.evidenceErrors, []); + } +}); + +test("missing, fabricated, misplaced, or unrelated evidence cannot earn recall", () => { + const fixture = createFixture(); + const alterations: Array<(finding: FixtureFinding) => void> = [ + (finding) => { + finding.codeEvidence = []; + }, + (finding) => { + finding.codeEvidence[0].code = "invented source text"; + }, + (finding) => { + finding.codeEvidence[0].startLine++; + }, + (finding) => { + finding.codeEvidence[0].path = "src/search.py"; + }, + (finding) => { + finding.codeEvidence = [ + { + path: "src/invoices.py", + startLine: 1, + code: fixture.files["src/invoices.py"].split("\n")[0], + }, + ]; + }, + ]; + for (const alter of alterations) { + const result = completeResult(fixture); + alter(result.findings[0]); + const report = gradeResult(result, fixture); + assert.equal(report.recall, 0.75); + assert.equal(report.passed, false); + assert.ok(report.evidenceErrors.length > 0); + } +}); + +test("control accusations and unmatched findings have separate adjudication queues", () => { + const fixture = createFixture(); + const result = completeResult(fixture); + result.findings.push(findingFor(fixture, fixture.controls[0])); + const unmatched = findingFor(fixture, fixture.positives[1]); + unmatched.taxonomy.cwe = ["CWE-400"]; + result.findings.push(unmatched); + const report = gradeResult(result, fixture); + assert.equal(report.recall, 1); + assert.equal(report.safeControlFindings, 1); + assert.deepEqual(report.safeControlDetails[0].controlIds, [ + fixture.controls[0].id, + ]); + assert.equal(report.unmatchedFindings.length, 1); + assert.equal(report.passed, false); + assert.equal(report.needsAdjudication, true); +}); + +test("safe-control context does not become a control accusation", () => { + const fixture = createFixture(); + const result = completeResult(fixture); + const context = findingFor(fixture, fixture.controls[0]); + context.locations[0].role = "expected_control"; + result.findings[0].locations.push(...context.locations); + result.findings[0].codeEvidence.push(...context.codeEvidence); + assert.equal(gradeResult(result, fixture).passed, true); +}); + +test("descriptive safe-control roles require adjudication rather than a false-positive verdict", () => { + const fixture = createFixture(); + const result = completeResult(fixture); + const context = findingFor(fixture, fixture.controls[0]); + context.locations[0].role = + "safe sibling showing the expected tenant control"; + result.findings[0].locations.push(...context.locations); + result.findings[0].codeEvidence.push(...context.codeEvidence); + const report = gradeResult(result, fixture); + assert.equal(report.recall, 1); + assert.equal(report.safeControlFindings, 1); + assert.equal(report.needsAdjudication, true); + assert.equal(report.passed, false); + assert.deepEqual(report.safeControlDetails[0].controlIds, [ + fixture.controls[0].id, + ]); +}); + +test("source snippets may span the sink", () => { + const fixture = createFixture(); + const result = completeResult(fixture); + for (const finding of result.findings) { + const evidence = finding.codeEvidence[0]; + const source = fixture.files[evidence.path]; + evidence.startLine--; + evidence.code = source + .split("\n") + .slice(evidence.startLine - 1, evidence.startLine + 1) + .join("\n"); + } + assert.equal(gradeResult(result, fixture).passed, true); +}); + +test("fixture source is deterministic and contains no gold identifiers", () => { + const fixture = createFixture(); + assert.deepEqual(createFixture(), fixture); + const source = Object.values(fixture.files).join("\n"); + for (const expected of [...fixture.positives, ...fixture.controls]) { + assert.equal(source.includes(expected.id), false); + } +}); diff --git a/evals/standard-time-budget/test_production.mts b/evals/standard-time-budget/test_production.mts new file mode 100644 index 000000000..6aba3f47b --- /dev/null +++ b/evals/standard-time-budget/test_production.mts @@ -0,0 +1,279 @@ +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { cp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { join, relative } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { temporaryDirectory } from "../../plugins/codex-security/mcp-app/tests/support/temporary-directories.ts"; +import type { ThreadEvent } from "../../sdk/typescript/node_modules/@openai/codex-sdk/dist/index.js"; +import type { JsonObject } from "../../sdk/typescript/src/config.js"; +import * as sdk from "../../sdk/typescript/dist/api.js"; +import { createIsolatedHome } from "../../sdk/typescript/dist/runtime.js"; +import { withEvalState } from "../secret-discovery/runtime.mts"; +import { hashText, runComparison } from "./experiment.mts"; +import { createStandardRunner, preflightStandardEval } from "./production.mts"; +import { prepareStandardEval } from "./fixtures.mts"; +import { summarizeRuns } from "./summary.mts"; + +const { CodexSecurity } = + sdk as unknown as typeof import("../../sdk/typescript/src/api.js"); + +test("the developer eval preflights API-key authentication and the production budget before staging", async () => { + assert.throws( + () => preflightStandardEval({}, 5), + /requires OPENAI_API_KEY or CODEX_API_KEY/, + ); + for (const name of ["OPENAI_API_KEY", "CODEX_API_KEY"]) { + preflightStandardEval({ [name]: "synthetic-environment" }, 0.25); + } + assert.throws( + () => + preflightStandardEval({ OPENAI_API_KEY: "synthetic-environment" }, 1e12), + /deadline within the supported date range/, + ); + await withEvalState(createIsolatedHome, undefined, async ({ home }) => { + assert.equal(existsSync(join(home, "auth.json")), false); + }); +}); + +async function writeUsageSession( + home: string, + threadId: string, + usage: Record, + parent?: string, +) { + const directory = join(home, "sessions", "2026", "01", "01"); + await mkdir(directory, { recursive: true }); + await writeFile( + join(directory, `rollout-${threadId}.jsonl`), + [ + JSON.stringify({ + type: "session_meta", + payload: { + id: threadId, + ...(parent === undefined ? {} : { parent_thread_id: parent }), + }, + }), + JSON.stringify({ + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: usage }, + }, + }), + "", + ].join("\n"), + ); +} + +test("the paired evaluator preserves aggregate usage across production completion and failure", async (context) => { + const root = await temporaryDirectory("time-budget-production-"); + context.after(() => rm(root, { recursive: true, force: true })); + const pluginRoot = fileURLToPath( + new URL("../../plugins/codex-security/", import.meta.url), + ); + const prepared = await prepareStandardEval(root, pluginRoot); + const outputRoot = join(root, "results"); + await mkdir(outputRoot); + await writeFile( + join(outputRoot, "fixture.json"), + JSON.stringify(prepared.fixture), + ); + const states: string[] = []; + const commands: string[] = []; + const prompts: string[] = []; + const records = await runComparison({ + prepared, + config: { model: "gpt-5.6-sol", pairs: 2, budgetMinutes: 5 }, + run: createStandardRunner({ + stateRoot: join(root, "states"), + outputRoot, + environment: { OPENAI_API_KEY: "synthetic-environment" }, + createClient: (config, dependencies) => { + const home = dependencies.environment["CODEX_SECURITY_STATE_DIR"]!; + states.push(home); + const client = new CodexSecurity( + config, + { + ...dependencies, + now: () => Date.UTC(2026, 0, 1), + probeCodexSandbox: async () => {}, + prepareScanArtifactRestorer: async () => ({ + restore: async () => {}, + }), + prepareRuntime: async () => ({ + codexHome: home, + plugin: { + pluginRoot, + marketplaceRoot: pluginRoot, + installedRoot: pluginRoot, + marketplaceName: "codex-security-sdk", + name: "codex-security", + version: "0.1.0", + }, + environment: {}, + credentialsAvailable: true, + }), + resolvePluginPython: async () => "/managed/python", + repositoryRevision: async () => "deadbeef", + runWorkbench: async (_options, args): Promise => { + commands.push(args[0]!); + if ( + args[0] === "prepare-scan-completion" && + states.length === 1 + ) { + throw new Error( + "Synthetic completion failure after model usage", + ); + } + if (args[0] === "register-cli-scan") { + return { + scanId: "scan_example_001", + targetId: "target_sha256_example", + targetRevision: "deadbeef", + scanDir: args[args.indexOf("--scan-dir") + 1]!, + contract: { target: { allowedKinds: ["git_revision"] } }, + }; + } + if (args[0] === "get-scan-feedback") { + return { + scanId: "scan_example_001", + targetId: "target_sha256_example", + falsePositives: [], + }; + } + return {}; + }, + }, + { surface: "sdk" }, + ); + return { + run: client.run.bind(client), + close: async () => { + await client.close(); + if (states.length <= 2) + throw new Error("Synthetic cleanup failure"); + }, + }; + }, + createCodex: async (options) => ({ + startThread: (threadOptions) => ({ + id: "thread-1", + runStreamed: async (prompt) => { + prompts.push(prompt); + const scanDir = options.env!["CODEX_SECURITY_SCAN_DIR"]!; + const home = options.env!["CODEX_HOME"]!; + const nearby = join(scanDir, "..", ".."); + assert.equal(threadOptions?.workingDirectory, scanDir); + for (const path of [ + scanDir, + home, + options.env!["CODEX_SECURITY_STATE_DIR"]!, + ]) { + assert.doesNotMatch(relative(root, path), /baseline|time-aware/u); + } + assert.equal(existsSync(join(nearby, "fixture.json")), false); + assert.equal( + existsSync(join(nearby, "pair-1-time-aware", "result.json")), + false, + ); + for (const previous of states.slice(0, -1)) { + assert.equal(existsSync(previous), false); + } + await writeUsageSession(home, "thread-1", { + input_tokens: 100, + output_tokens: 20, + }); + await writeUsageSession( + home, + "worker-thread", + { input_tokens: 50, output_tokens: 7 }, + "thread-1", + ); + await cp(join(pluginRoot, "examples", "completed-scan"), scanDir, { + recursive: true, + }); + await writeFile( + join(scanDir, "report.md"), + "# Finalized synthetic report\n", + ); + async function* events(): AsyncGenerator { + yield { type: "thread.started", thread_id: "thread-1" }; + yield { type: "turn.started" }; + yield { + type: "turn.completed", + usage: { + input_tokens: 10, + cached_input_tokens: 2, + cache_write_input_tokens: 0, + output_tokens: 3, + reasoning_output_tokens: 1, + }, + }; + } + return { events: events() }; + }, + }), + }), + }), + onResult: async (record, result) => { + assert.equal(existsSync(states.at(-1)!), false); + if (result !== undefined) { + await writeFile( + join( + outputRoot, + `pair-${record.pair}-${record.variant}`, + "result.json", + ), + JSON.stringify(result), + ); + } + }, + }); + assert.equal(new Set(states).size, 4); + assert.equal(prompts.length, 4, JSON.stringify(records)); + assert.ok(!prompts[0].includes("# Investigation time allowance")); + assert.ok(prompts[1].includes("2026-01-01T00:05:00.000Z")); + assert.deepEqual( + records.map((record) => record.status), + ["failed", "failed", "completed", "completed"], + JSON.stringify(records), + ); + assert.match( + records[0].error!, + /Synthetic completion failure after model usage/, + ); + assert.match(records[0].error!, /Synthetic cleanup failure/); + assert.equal(records[1].error, "Synthetic cleanup failure"); + assert.equal( + commands.filter((command) => command === "prepare-scan-completion").length, + 4, + ); + assert.equal( + commands.filter((command) => command === "complete-scan").length, + 3, + ); + for (const [index, record] of records.entries()) { + assert.partialDeepStrictEqual(record.usage, { + input_tokens: 150, + output_tokens: 27, + }); + const archivedReport = join( + outputRoot, + `pair-${record.pair}-${record.variant}`, + "scan", + "report.md", + ); + assert.match( + await readFile(archivedReport, "utf8"), + /Finalized synthetic report/, + ); + if (record.status === "completed") + assert.equal(record.report.reportPath, archivedReport); + assert.equal(record.promptSha256, hashText(prompts[index])); + } + const summary = summarizeRuns(records); + assert.equal(summary.arms.baseline.usage.total_tokens, 354); + assert.equal(summary.paired.completedPairs, 1); + assert.equal(summary.paired.excludedPairs, 1); +}); diff --git a/evals/standard-time-budget/test_summary.mts b/evals/standard-time-budget/test_summary.mts new file mode 100644 index 000000000..8ea3678d5 --- /dev/null +++ b/evals/standard-time-budget/test_summary.mts @@ -0,0 +1,189 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { summarizeRuns } from "./summary.mts"; +import { createFixture } from "./fixtures.mts"; +import { gradeResult } from "./grade.mts"; +import type { CompletedRunRecord, RunReport, Variant } from "./experiment.mts"; + +function attempt(pair: number, variant: Variant) { + return { + pair, + variant, + elapsedMs: 100, + startedAt: "2026-01-01T00:00:00.000Z", + advisoryBudgetMinutes: variant === "time-aware" ? 5 : null, + promptSha256: null, + usage: null, + }; +} + +function completed( + pair: number, + variant: Variant, + overrides: Partial = {}, +): CompletedRunRecord { + return { + ...attempt(pair, variant), + status: "completed", + usage: { input_tokens: 100, cached_input_tokens: 40, output_tokens: 20 }, + report: { + ...gradeResult({ findings: [] }, createFixture()), + recall: 0.5, + foundCount: 1, + expectedCount: 2, + safeControlFindings: 0, + unmatchedFindings: [], + evidenceErrors: [], + ...overrides, + }, + }; +} + +test("pairs runs by identifier and reports time-aware minus baseline", () => { + const result = summarizeRuns([ + completed(2, "time-aware", { recall: 1, foundCount: 2 }), + completed(1, "baseline"), + completed(2, "baseline"), + { + ...completed(1, "time-aware", { + recall: 0, + foundCount: 0, + safeControlFindings: 1, + unmatchedFindings: [ + { findingIndex: 0, title: "unmatched", reason: "unknown source" }, + ], + evidenceErrors: [ + { reason: "bad source" }, + { reason: "missing evidence" }, + ], + }), + elapsedMs: 300, + }, + ]); + assert.equal(result.arms.baseline.attempted, 2); + assert.equal(result.arms.baseline.successRate, 1); + assert.equal(result.arms["time-aware"].meanElapsedMs, 200); + assert.equal(result.arms["time-aware"].meanRecall, 0.5); + assert.equal(result.arms["time-aware"].foundCount, 2); + assert.equal(result.arms["time-aware"].expectedCount, 4); + assert.equal(result.arms["time-aware"].safeControlFindings, 1); + assert.equal(result.arms["time-aware"].unmatchedFindings, 1); + assert.equal(result.arms["time-aware"].evidenceErrors, 2); + assert.equal(result.paired.completedPairs, 2); + assert.equal(result.paired.excludedPairs, 0); + assert.equal(result.paired.meanDelta.recall, 0); + assert.equal(result.paired.meanDelta.elapsedMs, 100); + assert.equal(result.paired.meanDelta.safeControlFindings, 0.5); + assert.equal(result.paired.meanDelta.evidenceErrors, 1); +}); + +test("shows failures and missing arms alongside completed-pair improvements", () => { + const result = summarizeRuns([ + completed(1, "baseline"), + completed(1, "time-aware", { recall: 1, foundCount: 2 }), + completed(2, "baseline"), + { + ...attempt(2, "time-aware"), + status: "failed", + error: "scan failed", + elapsedMs: 900, + }, + completed(3, "baseline"), + ]); + assert.equal(result.arms["time-aware"].attempted, 2); + assert.equal(result.arms["time-aware"].completed, 1); + assert.equal(result.arms["time-aware"].failed, 1); + assert.equal(result.arms["time-aware"].successRate, 0.5); + assert.equal(result.arms["time-aware"].meanElapsedMs, 500); + assert.equal(result.arms["time-aware"].meanRecall, 1); + assert.equal(result.paired.totalPairs, 3); + assert.equal(result.paired.completedPairs, 1); + assert.equal(result.paired.excludedPairs, 2); + assert.equal(result.paired.meanDelta.recall, 0.5); + assert.equal(result.arms["time-aware"].usage.input_tokens, null); + assert.equal(result.arms["time-aware"].usage.total_tokens, null); + assert.equal(result.arms["time-aware"].usage.missingUsageRuns, 1); +}); + +test("cached input tokens are a subset and never added to total tokens", () => { + const result = summarizeRuns([ + completed(1, "baseline"), + { + ...completed(1, "time-aware"), + usage: { input_tokens: 150, cached_input_tokens: 80, output_tokens: 40 }, + }, + ]); + assert.deepEqual(result.arms.baseline.usage, { + input_tokens: 100, + cached_input_tokens: 40, + output_tokens: 20, + total_tokens: 120, + missingUsageRuns: 0, + }); + assert.equal(result.paired.meanDelta.input_tokens, 50); + assert.equal(result.paired.meanDelta.cached_input_tokens, 40); + assert.equal(result.paired.meanDelta.output_tokens, 20); + assert.equal(result.paired.meanDelta.total_tokens, 70); +}); + +test("missing usage remains unknown without discarding completed pairs", () => { + const result = summarizeRuns([ + completed(1, "baseline"), + { ...completed(1, "time-aware"), usage: null }, + completed(2, "baseline"), + completed(2, "time-aware"), + ]); + assert.equal(result.arms["time-aware"].usage.input_tokens, null); + assert.equal(result.arms["time-aware"].usage.cached_input_tokens, null); + assert.equal(result.arms["time-aware"].usage.output_tokens, null); + assert.equal(result.arms["time-aware"].usage.total_tokens, null); + assert.equal(result.arms["time-aware"].usage.missingUsageRuns, 1); + assert.equal(result.paired.completedPairs, 2); + assert.equal(result.paired.meanDelta.total_tokens, null); + assert.equal(result.paired.meanDelta.recall, 0); +}); + +test("a missing cache count does not hide known input and output totals", () => { + const result = summarizeRuns([ + { + ...completed(1, "baseline"), + usage: { input_tokens: 100, output_tokens: 20 }, + }, + ]); + assert.equal(result.arms.baseline.usage.input_tokens, 100); + assert.equal(result.arms.baseline.usage.cached_input_tokens, null); + assert.equal(result.arms.baseline.usage.total_tokens, 120); + assert.equal(result.arms.baseline.usage.missingUsageRuns, 1); +}); + +test("empty and entirely failed arms have no measured quality or deltas", () => { + const empty = summarizeRuns([]); + assert.equal(empty.arms.baseline.attempted, 0); + assert.equal(empty.arms.baseline.successRate, null); + assert.equal(empty.arms.baseline.meanElapsedMs, null); + assert.equal(empty.arms.baseline.usage.total_tokens, null); + assert.equal(empty.paired.totalPairs, 0); + assert.equal(empty.paired.meanDelta.recall, null); + const failed = summarizeRuns([ + { ...attempt(1, "baseline"), status: "failed", error: "scan failed" }, + { + ...attempt(1, "time-aware"), + status: "failed", + error: "scan failed", + elapsedMs: 200, + }, + ]); + assert.equal(failed.arms.baseline.successRate, 0); + assert.equal(failed.arms.baseline.meanRecall, null); + assert.equal(failed.arms.baseline.safeControlFindings, null); + assert.equal(failed.paired.completedPairs, 0); + assert.equal(failed.paired.excludedPairs, 1); + assert.equal(failed.paired.meanDelta.elapsedMs, null); +}); + +test("duplicate arms cannot silently bias results", () => { + assert.throws( + () => summarizeRuns([completed(1, "baseline"), completed(1, "baseline")]), + /Duplicate run for pair 1\/baseline/, + ); +}); diff --git a/evals/tsconfig.json b/evals/tsconfig.json index 0aed99eab..55dc7f6ca 100644 --- a/evals/tsconfig.json +++ b/evals/tsconfig.json @@ -1,11 +1,13 @@ { "extends": "../plugins/codex-security/mcp-app/tsconfig.json", "compilerOptions": { + "lib": ["esnext", "dom", "dom.iterable"], "typeRoots": ["../sdk/typescript/node_modules/@types"] }, "include": [ "deep-reducer/*.mts", "secret-discovery/*.mts", + "standard-time-budget/*.mts", "triage-finding/assertions/*.mts", "triage-finding/scripts/*.mts", "triage-finding/sastbench/assertions/*.mts", diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index e70c21788..d25fad7b2 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -64,7 +64,7 @@ "build:plugin": "node scripts/build-plugin.mjs", "check:plugin-source": "node scripts/check-plugin-source.mjs", "check:package": "node scripts/check-package.mjs", - "format": "prettier --check --ignore-path .gitignore --ignore-path .prettierignore \"**/*.{cjs,mjs,js,ts,mts,tsx,json}\" \"../../.github/scripts/*.mts\" \"../../examples/custom-validation/*.mts\" \"../../plugins/codex-security/native/*.mts\" \"../../evals/{deep-reducer,secret-discovery}/*.mts\" \"../../evals/triage-finding/{assertions,scripts,sastbench/assertions,sastbench/scripts}/*.mts\" \"../../evals/triage-finding/types.ts\" \"../../evals/tsconfig.json\" && node scripts/check-markdown.mjs", + "format": "prettier --check --ignore-path .gitignore --ignore-path .prettierignore \"**/*.{cjs,mjs,js,ts,mts,tsx,json}\" \"../../.github/scripts/*.mts\" \"../../examples/custom-validation/*.mts\" \"../../plugins/codex-security/native/*.mts\" \"../../evals/{deep-reducer,secret-discovery,standard-time-budget}/*.mts\" \"../../evals/triage-finding/{assertions,scripts,sastbench/assertions,sastbench/scripts}/*.mts\" \"../../evals/triage-finding/types.ts\" \"../../evals/tsconfig.json\" && node scripts/check-markdown.mjs", "generate:models": "node scripts/generate-models.cjs", "generate:models:check": "node scripts/generate-models.cjs --check", "lint": "tsc --noEmit",