From 2814739a0fa59b849a84151e5480918cd839d8f6 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 8 Oct 2026 08:11:50 +0000 Subject: [PATCH 01/43] feat(cloud): publish selected Wiz findings with resumable receipts --- plugins/codex-security/plugin-files.json | 2 + .../skills/publish-findings/SKILL.md | 22 + .../publish-findings/agents/openai.yaml | 6 + sdk/typescript/docs/cli.md | 50 + sdk/typescript/package.json | 4 +- .../schemas/external-findings.schema.json | 929 ++++++++++++++++++ sdk/typescript/scripts/check-package.mjs | 5 + .../generate-external-import-models.cjs | 35 + sdk/typescript/src/cli.ts | 100 ++ sdk/typescript/src/cloud-publish.ts | 3 +- .../src/external-findings-publish.ts | 459 +++++++++ .../src/external-import-contract.ts | 59 ++ sdk/typescript/src/external-import-models.ts | 277 ++++++ sdk/typescript/src/wiz-findings.ts | 196 ++++ .../external-findings-publish.test.ts | 546 ++++++++++ 15 files changed, 2691 insertions(+), 2 deletions(-) create mode 100644 plugins/codex-security/skills/publish-findings/SKILL.md create mode 100644 plugins/codex-security/skills/publish-findings/agents/openai.yaml create mode 100644 sdk/typescript/schemas/external-findings.schema.json create mode 100644 sdk/typescript/scripts/generate-external-import-models.cjs create mode 100644 sdk/typescript/src/external-findings-publish.ts create mode 100644 sdk/typescript/src/external-import-contract.ts create mode 100644 sdk/typescript/src/external-import-models.ts create mode 100644 sdk/typescript/src/wiz-findings.ts create mode 100644 sdk/typescript/tests-ts/external-findings-publish.test.ts diff --git a/plugins/codex-security/plugin-files.json b/plugins/codex-security/plugin-files.json index f5a412f060..29b44ff023 100644 --- a/plugins/codex-security/plugin-files.json +++ b/plugins/codex-security/plugin-files.json @@ -130,6 +130,8 @@ "skills/security-scan/references/scan-artifacts-and-ledger.md", "skills/threat-model/SKILL.md", "skills/threat-model/agents/openai.yaml", + "skills/publish-findings/SKILL.md", + "skills/publish-findings/agents/openai.yaml", "skills/track-findings/SKILL.md", "skills/track-findings/agents/openai.yaml", "skills/track-findings/references/github-security-advisories.md", diff --git a/plugins/codex-security/skills/publish-findings/SKILL.md b/plugins/codex-security/skills/publish-findings/SKILL.md new file mode 100644 index 0000000000..837f7e2bd8 --- /dev/null +++ b/plugins/codex-security/skills/publish-findings/SKILL.md @@ -0,0 +1,22 @@ +--- +name: publish-findings +description: Publish selected external scanner findings to Codex Security Cloud after previewing the destination and evidence. Supports saved Wiz vulnerability findings and normalized JSONL; does not run a scan or assess findings. +--- + +# Publish external findings + +Use the installed `codex-security publish findings` command for both preview and upload. It owns source mapping, Cloud authentication, source versions, and saved request retries. Do not implement another HTTP uploader or put credentials in a command argument. + +Obtain the Cloud repository ID from the user's destination. Read selected Wiz vulnerability findings through the user's configured connection or supplied export. Save the actual selected JSON records locally, retaining their IDs and evidence. Finish the requested pagination; if the user selected a subset, save that explicit subset as an array. The publisher also accepts JSONL records with `source_finding_id` and `evidence` as described in the CLI documentation. + +Use a stable Wiz tenant/finding namespace as `--source-key`. A project filter is selection context, not a new identity. Confirm the selected assets or builds belong to the destination repository; do not infer a scanner branch or revision from the current checkout. Do not publish vendor credentials, unrelated records, or local assessments as original scanner evidence. + +```sh +codex-security publish findings /path/to/selected-wiz-findings.json \ + --to cloud --repository REPOSITORY_ID --provider wiz \ + --source-key TENANT_ID/vulnerability-finding --dry-run --format json +``` + +Show the destination, selected findings, exclusions, and evidence to the user. Obtain explicit confirmation for this upload, then run the same command without `--dry-run`, adding `--yes`. Existing authorization to inspect or assess findings alone does not authorize sending them to Cloud. A user who already approved the concrete preview need not approve it again. + +After a lost response or retryable failure, repeat the same command with unchanged input and destination. The publisher resends its saved request IDs and bodies. After a repository reset, stop and show the error; require fresh discovery and confirmation before a new publication. Report partial failures and retain their explanations. A successful import stores vendor evidence; it does not establish that Codex assessed the finding or that the vulnerability is exploitable. diff --git a/plugins/codex-security/skills/publish-findings/agents/openai.yaml b/plugins/codex-security/skills/publish-findings/agents/openai.yaml new file mode 100644 index 0000000000..c6d611bc38 --- /dev/null +++ b/plugins/codex-security/skills/publish-findings/agents/openai.yaml @@ -0,0 +1,6 @@ +interface: + display_name: "Publish Findings" + short_description: "Preview and publish vendor findings to Cloud" + default_prompt: "Use $publish-findings to preview selected Wiz findings for my Cloud repository." +policy: + allow_implicit_invocation: true diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md index 846f301941..94a705069f 100644 --- a/sdk/typescript/docs/cli.md +++ b/sdk/typescript/docs/cli.md @@ -773,6 +773,56 @@ again when rerunning. ## Publish findings to Cloud +Publish selected external Wiz vulnerability findings without creating a scan: + +```bash +codex-security publish findings selected-wiz.json \ + --to cloud --repository REPOSITORY_ID --provider wiz \ + --source-key TENANT_ID/vulnerability-finding --dry-run --format json +``` + +This command reads the authorized destination and current source versions during +preview. It uses the same saved ChatGPT file login as scan publication. The Cloud +repository needs an existing environment; no native scan is required. Review the +destination, findings, and exclusions, then rerun without `--dry-run`. The default +terminal prompt is No; `--yes` confirms a previously reviewed input for scripts or +the plugin. Without a terminal or `--yes`, no upload occurs. + +Input can be a Wiz vulnerability finding, an array, a complete +`data.vulnerabilityFindings.nodes` response, or JSONL with one vendor record per +line. A response that advertises another page is rejected: save the explicitly +selected records as an array. Normalized JSONL is also accepted: + +```json +{ + "source_finding_id": "vendor-finding-42", + "evidence": { + "title": "Vulnerable example package", + "severity": "high", + "source_data": { "id": "vendor-finding-42" } + } +} +``` + +Use Wiz's finding `id`, not a CVE, as identity. Keep the source key stable across +exports; a selected project filter does not change the vendor namespace. Confirm +that the selected assets or builds map to this repository. The raw Wiz adapter +retains each record in `source_data`, uses reported severity, and leaves unknown +branch/revision and repository locations empty. Container paths are retained as +vendor evidence rather than interpreted as source-code locations. It does not +fetch from Wiz, assess findings, or change vendor or Cloud triage decisions. + +Requests contain at most 100 findings and respect the Cloud payload limits. +Before uploading, the publisher saves request IDs and bodies privately under +the configured Codex Security state directory. After an uncertain response, +repeat the same command with unchanged input to resume those requests. A reset +retires the old request without republishing; review and explicitly approve a +fresh invocation. Final item errors are reported individually with exit code 1; +correct the input or source conflict before a fresh submission. Transport or +readback failures return exit code 2 and retain the resumable request. Successful +receipts are saved locally; the CLI verifies Cloud source-report reads and +returns a Findings link. Search indexing can lag an accepted import. + Preview selected completed scans before uploading: ```bash diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index 5f06b9fc5b..a80d2f4d8b 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -63,6 +63,8 @@ "format": "prettier --check --ignore-path .gitignore --ignore-path .prettierignore \"**/*.{cjs,mjs,js,ts,mts,tsx,json}\" \"../../.github/scripts/*.mts\" \"../../examples/custom-validation/*.mts\" \"../../plugins/codex-security/native/*.mts\" \"../../evals/{deep-reducer,secret-discovery}/*.mts\" \"../../evals/triage-finding/{assertions,scripts,sastbench/assertions,sastbench/scripts}/*.mts\" \"../../evals/triage-finding/types.ts\" \"../../evals/tsconfig.json\" && node scripts/check-markdown.mjs", "generate:models": "node scripts/generate-models.cjs", "generate:models:check": "node scripts/generate-models.cjs --check", + "generate:external-import-models": "node scripts/generate-external-import-models.cjs", + "generate:external-import-models:check": "node scripts/generate-external-import-models.cjs --check", "lint": "tsc --noEmit", "prepack": "node --run build:plugin && node --run build", "start:server": "node dist/server/index.js", @@ -71,7 +73,7 @@ "test:mcp": "node --run build:plugin && pnpm --dir ../../plugins/codex-security/mcp-app run test:mcp", "test:mutation": "stryker run", "test:package": "node scripts/smoke-package.mjs", - "types": "node scripts/generate-deep-defaults.mjs --check && node scripts/generate-project-config-schema.mjs --check && pnpm run generate:models:check && pnpm --dir ../../plugins/codex-security/mcp-app run typecheck && tsc --noEmit" + "types": "node scripts/generate-deep-defaults.mjs --check && node scripts/generate-project-config-schema.mjs --check && pnpm run generate:models:check && pnpm run generate:external-import-models:check && pnpm --dir ../../plugins/codex-security/mcp-app run typecheck && tsc --noEmit" }, "dependencies": { "@inquirer/prompts": "8.7.2", diff --git a/sdk/typescript/schemas/external-findings.schema.json b/sdk/typescript/schemas/external-findings.schema.json new file mode 100644 index 0000000000..497b6a6bd3 --- /dev/null +++ b/sdk/typescript/schemas/external-findings.schema.json @@ -0,0 +1,929 @@ +{ + "$defs": { + "FindingImportItem": { + "additionalProperties": false, + "properties": { + "client_id": { + "maxLength": 128, + "minLength": 1, + "title": "Client Id", + "type": "string" + }, + "source_finding_id": { + "maxLength": 512, + "minLength": 1, + "title": "Source Finding Id", + "type": "string" + }, + "expected_version": { + "minimum": 0, + "title": "Expected Version", + "type": "integer" + }, + "evidence": { + "$ref": "#/$defs/ImportedFindingEvidence" + } + }, + "required": [ + "client_id", + "source_finding_id", + "expected_version", + "evidence" + ], + "title": "FindingImportItem", + "type": "object" + }, + "FindingImportReceipt": { + "additionalProperties": false, + "properties": { + "id": { + "format": "uuid", + "title": "Id", + "type": "string" + }, + "object": { + "const": "security.finding_import", + "default": "security.finding_import", + "title": "Object", + "type": "string" + }, + "repository": { + "$ref": "#/$defs/ImportDestination" + }, + "source": { + "$ref": "#/$defs/ImportSource" + }, + "actor": { + "title": "Actor", + "type": "string" + }, + "created_at": { + "title": "Created At", + "type": "integer" + }, + "item_count": { + "maximum": 100, + "minimum": 1, + "title": "Item Count", + "type": "integer" + }, + "counts": { + "$ref": "#/$defs/ImportCounts" + }, + "results": { + "items": { + "$ref": "#/$defs/FindingImportResult" + }, + "title": "Results", + "type": "array" + } + }, + "required": [ + "id", + "repository", + "source", + "actor", + "created_at", + "item_count", + "counts", + "results" + ], + "title": "FindingImportReceipt", + "type": "object" + }, + "FindingImportRequest": { + "additionalProperties": false, + "properties": { + "request_id": { + "format": "uuid", + "title": "Request Id", + "type": "string" + }, + "repository": { + "$ref": "#/$defs/ImportDestination" + }, + "source": { + "$ref": "#/$defs/ImportSource" + }, + "items": { + "items": { + "$ref": "#/$defs/FindingImportItem" + }, + "maxItems": 100, + "minItems": 1, + "title": "Items", + "type": "array" + } + }, + "required": ["request_id", "repository", "source", "items"], + "title": "FindingImportRequest", + "type": "object" + }, + "FindingImportResult": { + "additionalProperties": false, + "properties": { + "client_id": { + "title": "Client Id", + "type": "string" + }, + "outcome": { + "enum": ["created", "updated", "unchanged", "error"], + "title": "Outcome", + "type": "string" + }, + "source_report_id": { + "anyOf": [ + { + "pattern": "^aif_[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Report Id" + }, + "observation_id": { + "anyOf": [ + { + "pattern": "^aio_[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Observation Id" + }, + "canonical_finding_id": { + "anyOf": [ + { + "pattern": "^acf_[0-9a-f]{32}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Canonical Finding Id" + }, + "version": { + "anyOf": [ + { + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Version" + }, + "error": { + "anyOf": [ + { + "$ref": "#/$defs/ImportItemError" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": ["client_id", "outcome"], + "title": "FindingImportResult", + "type": "object" + }, + "ImportCounts": { + "additionalProperties": false, + "properties": { + "created": { + "default": 0, + "minimum": 0, + "title": "Created", + "type": "integer" + }, + "updated": { + "default": 0, + "minimum": 0, + "title": "Updated", + "type": "integer" + }, + "unchanged": { + "default": 0, + "minimum": 0, + "title": "Unchanged", + "type": "integer" + }, + "error": { + "default": 0, + "minimum": 0, + "title": "Error", + "type": "integer" + } + }, + "title": "ImportCounts", + "type": "object" + }, + "ImportDestination": { + "additionalProperties": false, + "properties": { + "id": { + "maxLength": 512, + "minLength": 1, + "title": "Id", + "type": "string" + }, + "repo_connector_id": { + "maxLength": 512, + "minLength": 1, + "title": "Repo Connector Id", + "type": "string" + }, + "environment_id": { + "maxLength": 512, + "minLength": 1, + "title": "Environment Id", + "type": "string" + }, + "reset_marker": { + "maxLength": 512, + "minLength": 1, + "title": "Reset Marker", + "type": "string" + } + }, + "required": ["id", "repo_connector_id", "environment_id", "reset_marker"], + "title": "ImportDestination", + "type": "object" + }, + "ImportItemError": { + "additionalProperties": false, + "properties": { + "code": { + "enum": [ + "version_conflict", + "scope_conflict", + "stale_source_snapshot" + ], + "title": "Code", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + } + }, + "required": ["code", "message"], + "title": "ImportItemError", + "type": "object" + }, + "ImportRepository": { + "additionalProperties": false, + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "object": { + "const": "security.repository", + "default": "security.repository", + "title": "Object", + "type": "string" + }, + "repo_connector_id": { + "title": "Repo Connector Id", + "type": "string" + }, + "url": { + "title": "Url", + "type": "string" + }, + "default_branch": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Default Branch" + }, + "reset_marker": { + "title": "Reset Marker", + "type": "string" + }, + "import_environment_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Import Environment Id" + } + }, + "required": ["id", "repo_connector_id", "url", "reset_marker"], + "title": "ImportRepository", + "type": "object" + }, + "ImportRepositoryPage": { + "additionalProperties": false, + "properties": { + "object": { + "const": "page", + "default": "page", + "title": "Object", + "type": "string" + }, + "data": { + "items": { + "$ref": "#/$defs/ImportRepository" + }, + "title": "Data", + "type": "array" + }, + "has_more": { + "title": "Has More", + "type": "boolean" + }, + "next": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Next" + } + }, + "required": ["data", "has_more", "next"], + "title": "ImportRepositoryPage", + "type": "object" + }, + "ImportSource": { + "additionalProperties": false, + "properties": { + "provider": { + "enum": ["wiz", "endor", "snyk"], + "title": "Provider", + "type": "string" + }, + "source_key": { + "maxLength": 512, + "minLength": 1, + "title": "Source Key", + "type": "string" + } + }, + "required": ["provider", "source_key"], + "title": "ImportSource", + "type": "object" + }, + "ImportedAssessment": { + "additionalProperties": false, + "properties": { + "state": { + "const": "not_assessed", + "default": "not_assessed", + "title": "State", + "type": "string" + } + }, + "title": "ImportedAssessment", + "type": "object" + }, + "ImportedFindingEvidence": { + "additionalProperties": false, + "properties": { + "title": { + "maxLength": 512, + "minLength": 1, + "title": "Title", + "type": "string" + }, + "severity": { + "enum": ["critical", "high", "medium", "low", "informational"], + "title": "Severity", + "type": "string" + }, + "description": { + "anyOf": [ + { + "maxLength": 65536, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Description" + }, + "url": { + "anyOf": [ + { + "maxLength": 2048, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Url" + }, + "advisory_ids": { + "items": { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + "maxItems": 100, + "title": "Advisory Ids", + "type": "array" + }, + "packages": { + "items": { + "$ref": "#/$defs/ImportedPackage" + }, + "maxItems": 100, + "title": "Packages", + "type": "array" + }, + "locations": { + "items": { + "$ref": "#/$defs/ImportedLocation" + }, + "maxItems": 100, + "title": "Locations", + "type": "array" + }, + "branch": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Branch" + }, + "code_revision": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Code Revision" + }, + "source_scan_id": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Scan Id" + }, + "image_digests": { + "items": { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + "maxItems": 100, + "title": "Image Digests", + "type": "array" + }, + "source_updated_at": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Updated At" + }, + "source_data": { + "additionalProperties": true, + "title": "Source Data", + "type": "object" + } + }, + "required": ["title", "severity"], + "title": "ImportedFindingEvidence", + "type": "object" + }, + "ImportedLocation": { + "additionalProperties": false, + "properties": { + "path": { + "maxLength": 2048, + "minLength": 1, + "title": "Path", + "type": "string" + }, + "line": { + "anyOf": [ + { + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Line" + } + }, + "required": ["path"], + "title": "ImportedLocation", + "type": "object" + }, + "ImportedPackage": { + "additionalProperties": false, + "properties": { + "name": { + "maxLength": 512, + "minLength": 1, + "title": "Name", + "type": "string" + }, + "ecosystem": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Ecosystem" + }, + "installed_version": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Installed Version" + }, + "manifest_path": { + "anyOf": [ + { + "maxLength": 2048, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Manifest Path" + }, + "fixed_versions": { + "items": { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + "maxItems": 100, + "title": "Fixed Versions", + "type": "array" + } + }, + "required": ["name"], + "title": "ImportedPackage", + "type": "object" + }, + "LastImport": { + "additionalProperties": false, + "properties": { + "id": { + "format": "uuid", + "title": "Id", + "type": "string" + }, + "actor": { + "title": "Actor", + "type": "string" + }, + "created_at": { + "title": "Created At", + "type": "integer" + } + }, + "required": ["id", "actor", "created_at"], + "title": "LastImport", + "type": "object" + }, + "SourceReport": { + "additionalProperties": false, + "properties": { + "id": { + "pattern": "^aif_[0-9a-f]{64}$", + "title": "Id", + "type": "string" + }, + "object": { + "const": "security.source_report", + "default": "security.source_report", + "title": "Object", + "type": "string" + }, + "origin": { + "const": "imported", + "default": "imported", + "title": "Origin", + "type": "string" + }, + "repo_id": { + "title": "Repo Id", + "type": "string" + }, + "repo_connector_id": { + "title": "Repo Connector Id", + "type": "string" + }, + "environment_id": { + "title": "Environment Id", + "type": "string" + }, + "canonical_finding_id": { + "pattern": "^acf_[0-9a-f]{32}$", + "title": "Canonical Finding Id", + "type": "string" + }, + "source": { + "$ref": "#/$defs/ImportSource" + }, + "source_finding_id": { + "title": "Source Finding Id", + "type": "string" + }, + "observation_id": { + "pattern": "^aio_[0-9a-f]{64}$", + "title": "Observation Id", + "type": "string" + }, + "version": { + "minimum": 1, + "title": "Version", + "type": "integer" + }, + "evidence": { + "$ref": "#/$defs/ImportedFindingEvidence" + }, + "assessment": { + "$ref": "#/$defs/ImportedAssessment" + }, + "last_import": { + "anyOf": [ + { + "$ref": "#/$defs/LastImport" + }, + { + "type": "null" + } + ], + "default": null + }, + "created_at": { + "title": "Created At", + "type": "integer" + }, + "updated_at": { + "title": "Updated At", + "type": "integer" + } + }, + "required": [ + "id", + "repo_id", + "repo_connector_id", + "environment_id", + "canonical_finding_id", + "source", + "source_finding_id", + "observation_id", + "version", + "evidence", + "created_at", + "updated_at" + ], + "title": "SourceReport", + "type": "object" + }, + "SourceReportPage": { + "additionalProperties": false, + "properties": { + "object": { + "const": "page", + "default": "page", + "title": "Object", + "type": "string" + }, + "data": { + "items": { + "$ref": "#/$defs/SourceReportSummary" + }, + "title": "Data", + "type": "array" + }, + "has_more": { + "title": "Has More", + "type": "boolean" + }, + "next": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Next" + } + }, + "required": ["data", "has_more", "next"], + "title": "SourceReportPage", + "type": "object" + }, + "SourceReportSummary": { + "additionalProperties": false, + "properties": { + "id": { + "pattern": "^aif_[0-9a-f]{64}$", + "title": "Id", + "type": "string" + }, + "object": { + "const": "security.source_report_summary", + "default": "security.source_report_summary", + "title": "Object", + "type": "string" + }, + "origin": { + "const": "imported", + "default": "imported", + "title": "Origin", + "type": "string" + }, + "repo_id": { + "title": "Repo Id", + "type": "string" + }, + "repo_connector_id": { + "title": "Repo Connector Id", + "type": "string" + }, + "environment_id": { + "title": "Environment Id", + "type": "string" + }, + "canonical_finding_id": { + "pattern": "^acf_[0-9a-f]{32}$", + "title": "Canonical Finding Id", + "type": "string" + }, + "source": { + "$ref": "#/$defs/ImportSource" + }, + "source_finding_id": { + "title": "Source Finding Id", + "type": "string" + }, + "observation_id": { + "pattern": "^aio_[0-9a-f]{64}$", + "title": "Observation Id", + "type": "string" + }, + "version": { + "minimum": 1, + "title": "Version", + "type": "integer" + }, + "title": { + "title": "Title", + "type": "string" + }, + "severity": { + "enum": ["critical", "high", "medium", "low", "informational"], + "title": "Severity", + "type": "string" + }, + "source_updated_at": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Source Updated At" + }, + "assessment": { + "$ref": "#/$defs/ImportedAssessment" + }, + "created_at": { + "title": "Created At", + "type": "integer" + }, + "updated_at": { + "title": "Updated At", + "type": "integer" + } + }, + "required": [ + "id", + "repo_id", + "repo_connector_id", + "environment_id", + "canonical_finding_id", + "source", + "source_finding_id", + "observation_id", + "version", + "title", + "severity", + "source_updated_at", + "created_at", + "updated_at" + ], + "title": "SourceReportSummary", + "type": "object" + } + }, + "additionalProperties": false, + "description": "Schema export root; transport endpoints return the individual models.", + "properties": { + "request": { + "$ref": "#/$defs/FindingImportRequest" + }, + "receipt": { + "$ref": "#/$defs/FindingImportReceipt" + }, + "repositories": { + "$ref": "#/$defs/ImportRepositoryPage" + }, + "source_report": { + "$ref": "#/$defs/SourceReport" + }, + "source_reports": { + "$ref": "#/$defs/SourceReportPage" + } + }, + "required": [ + "request", + "receipt", + "repositories", + "source_report", + "source_reports" + ], + "title": "ExternalFindingContracts", + "type": "object" +} diff --git a/sdk/typescript/scripts/check-package.mjs b/sdk/typescript/scripts/check-package.mjs index 6cd8748e5d..ce390c817c 100644 --- a/sdk/typescript/scripts/check-package.mjs +++ b/sdk/typescript/scripts/check-package.mjs @@ -124,6 +124,7 @@ const required = [ "package/dist/index.d.ts", "package/dist/cli.js", "package/schemas/project-config.schema.json", + "package/schemas/external-findings.schema.json", "package/_bundled_plugin/.codex-plugin/plugin.json", ]; @@ -157,6 +158,10 @@ const allowedFiles = new Set([ "classify-scan-severity", "severity-store", "cloud-publish", + "external-findings-publish", + "external-import-contract", + "external-import-models", + "wiz-findings", "codex-prompt", "codex-sdk-environment", "component-plan", diff --git a/sdk/typescript/scripts/generate-external-import-models.cjs b/sdk/typescript/scripts/generate-external-import-models.cjs new file mode 100644 index 0000000000..cec33f0434 --- /dev/null +++ b/sdk/typescript/scripts/generate-external-import-models.cjs @@ -0,0 +1,35 @@ +const { readFileSync, writeFileSync } = require("node:fs"); +const { resolve } = require("node:path"); +const { compile } = require("json-schema-to-typescript"); +const { format } = require("prettier"); + +async function generate() { + const schema = JSON.parse( + readFileSync( + resolve(__dirname, "../schemas/external-findings.schema.json"), + "utf8", + ), + ); + const types = await compile(schema, "ExternalFindingContracts", { + bannerComment: + "/* Generated from schemas/external-findings.schema.json. Run pnpm generate:external-import-models. */", + format: false, + ignoreMinAndMaxItems: true, + unknownAny: true, + }); + const output = await format( + `${types}\nexport type ExternalFindingEvidence = ImportedFindingEvidence;\n`, + { parser: "typescript" }, + ); + const target = resolve(__dirname, "../src/external-import-models.ts"); + if (process.argv.includes("--check")) { + if (readFileSync(target, "utf8").replaceAll("\r\n", "\n") !== output) + throw new Error( + "External import models are stale. Run pnpm generate:external-import-models.", + ); + } else writeFileSync(target, output); +} +generate().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 1774c4820d..f18bd6b7b9 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -92,6 +92,7 @@ import { isRecord as isJsonObject } from "./record.js"; import { suggestOwnersInternal } from "./suggest-owners.js"; import { parseImportedFindings } from "./findings-import.js"; import { publishScanToCustom } from "./custom-publish.js"; +import { prepareExternalPublication } from "./external-findings-publish.js"; import { DEFAULT_DEDUPE_CONCURRENCY } from "./deduplication/deduplication.js"; import { savedScanWorkbench } from "./saved-scan-bootstrap.js"; import { deduplicateScanInternal } from "./deduplication/scan.js"; @@ -358,6 +359,8 @@ const PROJECT_CONFIG_OPTION = optionValue("--config") ); const EXPORT_DEFAULT_OUTPUTS = ARTIFACT_EXPORT_FILENAMES; const VALUE_OPTIONS = new Set([ + "--repository", + "--source-key", "--config", "-c", "--port", @@ -1168,6 +1171,7 @@ interface CliDependencies { scanInput?: ConstructorParameters[1]["input"]; publishPrompt?: Pick & Partial>; + externalPublicationPrompt?: Pick; checkScanPublication?: typeof checkScanPublication; publishScan?: typeof publishScan; deduplicateScan?: typeof deduplicateScanInternal; @@ -3170,6 +3174,102 @@ export async function main( } }, }); + publication.command("findings", { + description: "Preview and publish selected Wiz findings to Cloud.", + destructive: true, + mcp: false, + args: z.object({ + file: z + .string() + .describe("Saved Wiz vulnerability findings JSON or normalized JSONL."), + }), + options: z.object({ + to: z.literal("cloud").describe("Publication destination."), + repository: optionValue("--repository").describe( + "Authorized Cloud repository ID.", + ), + provider: z + .literal("wiz") + .describe("Vendor that produced these findings."), + sourceKey: optionValue("--source-key").describe( + "Stable vendor tenant and finding namespace; not a project selection filter.", + ), + dryRun: z + .boolean() + .default(false) + .describe( + "Read and preview the destination and findings without uploading.", + ), + yes: z + .boolean() + .default(false) + .describe( + "Confirm publication of the previewed input without a terminal prompt.", + ), + }), + output: z.record(z.string(), z.unknown()).optional(), + async run({ args, options, format, formatExplicit }) { + const controller = new AbortController(); + const removeSignals = listenForAbort(dependencies, controller); + const structured = formatExplicit && format !== "toon"; + try { + const prepared = await prepareExternalPublication( + resolveCliPath(dependencies.currentDirectory(), args.file), + options, + { + environment: dependencies.environment, + fetch: dependencies.cloudFetch, + signal: controller.signal, + }, + ); + const { preview } = prepared; + const showPreview = () => { + errorOutput.write( + `Source: ${preview.source.provider} / ${diagnosticValue(preview.source.source_key)}\nDestination: ${diagnosticValue(preview.destination.url)} (${diagnosticValue(preview.destination.id)})\nRead: ${preview.read} Ready: ${preview.findings.length} Excluded: ${preview.excluded.length}${preview.resumed ? "\nResuming the saved submission." : ""}\n`, + ); + for (const excluded of preview.excluded) + errorOutput.write( + `Excluded item ${excluded.position}: ${diagnosticValue(excluded.reason)}\n`, + ); + }; + if (!structured) showPreview(); + if (options.dryRun) return { ...preview, dryRun: true }; + if (!options.yes) { + const prompt = + dependencies.externalPublicationPrompt ?? + createTerminalPrompt(errorOutput); + if (!prompt.isInteractive()) + throw new CodexSecurityError( + "Publication needs confirmation. Run --dry-run, review the results, then use --yes to approve this input.", + ); + if (structured) showPreview(); + if ( + !(await prompt.confirm( + `Publish these ${preview.findings.length} findings?`, + false, + controller.signal, + )) + ) + return { published: false, preview }; + } + const result = await prepared.publish(); + if (result.counts.error) exitCode = 1; + if (!structured) + errorOutput.write( + `Created: ${result.counts.created} Updated: ${result.counts.updated} Unchanged: ${result.counts.unchanged} Failed: ${result.counts.error}\nOpen in Cloud: ${result.cloudUrl}\n`, + ); + return { ...result, excluded: preview.excluded }; + } catch (error) { + reportPublicationError( + error, + controller.signal.aborted ? controller.signal.reason : undefined, + ); + return undefined; + } finally { + removeSignals(); + } + }, + }); publication.command("check", { description: "Check saved scan history and Linear access without creating issues.", diff --git a/sdk/typescript/src/cloud-publish.ts b/sdk/typescript/src/cloud-publish.ts index 5d3d9fe26f..1f8b9b60c0 100644 --- a/sdk/typescript/src/cloud-publish.ts +++ b/sdk/typescript/src/cloud-publish.ts @@ -251,7 +251,8 @@ async function publishCloudPayload( }; } -async function readCloudCredentials(environment: NodeJS.ProcessEnv) { +/** @internal Shared sign-in for local Cloud publishers. */ +export async function readCloudCredentials(environment: NodeJS.ProcessEnv) { const configuredHome = environment["CODEX_HOME"]; let home = expandHome( configuredHome?.trim() ? configuredHome : "~/.codex", diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts new file mode 100644 index 0000000000..9f8c53f76c --- /dev/null +++ b/sdk/typescript/src/external-findings-publish.ts @@ -0,0 +1,459 @@ +import { hash, randomUUID } from "node:crypto"; +import { link, mkdir, open, readFile, rename, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { readCloudCredentials } from "./cloud-publish.js"; +import { CodexSecurityError } from "./errors.js"; +import { + codexSecurityStateDirectory, + validatePreparedOutputDir, +} from "./runtime.js"; +import { readVendorFindings, type VendorFindings } from "./wiz-findings.js"; +import { + validateImportRequest, + validateImportReceipt, + validateRepositories, + validateSourceReports, + validateSourceReport, +} from "./external-import-contract.js"; +import type { + FindingImportRequest, + FindingImportReceipt, + ImportRepository, +} from "./external-import-models.js"; + +const BASE_URL = "https://chatgpt.com/backend-api/aardvark/external"; +const MAX_REQUEST_BYTES = 5 * 1024 * 1024; + +/** CLI and plugin callers prepare the same immutable request before approval. */ +export interface ExternalPublicationOptions { + repository: string; + provider: "wiz"; + sourceKey: string; +} + +/** @internal Injectable transport and account for offline integration tests. */ +export interface ExternalPublicationDependencies { + environment?: NodeJS.ProcessEnv; + fetch?: (url: string, options: RequestInit) => Promise; + credentials?: () => Promise<{ access_token: string; account_id: string }>; + signal?: AbortSignal; +} + +interface SavedSubmission { + accountId: string; + requests: FindingImportRequest[]; +} + +export interface ExternalPublicationPreview extends VendorFindings { + destination: ImportRepository; + source: FindingImportRequest["source"]; + resumed: boolean; + requests: FindingImportRequest[]; +} + +export interface PreparedExternalPublication { + preview: ExternalPublicationPreview; + publish(): Promise<{ + receipts: FindingImportReceipt[]; + counts: FindingImportReceipt["counts"]; + cloudUrl: string; + }>; +} + +function canonicalJson(value: unknown): string { + return JSON.stringify(value, (_key, child: unknown) => { + if (child !== null && typeof child === "object" && !Array.isArray(child)) { + return Object.fromEntries( + Object.entries(child).sort(([a], [b]) => a.localeCompare(b)), + ); + } + return child; + }); +} + +function sourcePath(repository: ImportRepository): string { + return `/repository_connectors/${encodeURIComponent(repository.repo_connector_id)}/repositories/${encodeURIComponent(repository.id)}/source_reports`; +} + +class CloudImportError extends CodexSecurityError { + constructor( + public readonly status: number, + message: string, + ) { + super(message); + } +} + +export async function prepareExternalPublication( + path: string, + options: ExternalPublicationOptions, + dependencies: ExternalPublicationDependencies = {}, +): Promise { + if (!options.sourceKey.trim() || Buffer.byteLength(options.sourceKey) > 512) { + throw new CodexSecurityError( + "The source key must contain 1–512 UTF-8 bytes and identify the stable vendor namespace.", + ); + } + const parsed = await readVendorFindings(path); + if (parsed.findings.length === 0) { + throw new CodexSecurityError( + `No supported findings are ready to publish. ${parsed.excluded.map((item) => `Item ${item.position}: ${item.reason}`).join(" ")}`, + ); + } + const environment = dependencies.environment ?? process.env; + const credentials = await ( + dependencies.credentials ?? (() => readCloudCredentials(environment)) + )(); + async function request( + endpoint: string, + body?: FindingImportRequest, + ): Promise { + dependencies.signal?.throwIfAborted(); + const response = await (dependencies.fetch ?? globalThis.fetch)( + `${BASE_URL}${endpoint}`, + { + method: body ? "POST" : "GET", + headers: { + Authorization: `Bearer ${credentials.access_token}`, + "ChatGPT-Account-ID": credentials.account_id, + Accept: "application/json", + ...(body ? { "Content-Type": "application/json" } : {}), + }, + ...(body ? { body: JSON.stringify(body) } : {}), + redirect: "error", + signal: dependencies.signal + ? AbortSignal.any([dependencies.signal, AbortSignal.timeout(30_000)]) + : AbortSignal.timeout(30_000), + }, + ); + if (!response.ok) { + const payload = (await response.json().catch(() => null)) as { + error?: { message?: string }; + } | null; + const retryAfter = response.headers.get("Retry-After"); + throw new CloudImportError( + response.status, + `Cloud import failed (HTTP ${response.status}). ${payload?.error?.message ?? response.statusText}${retryAfter ? ` Retry-After: ${retryAfter}.` : ""}`, + ); + } + return response.json(); + } + const destinations: ImportRepository[] = []; + let page: string | null = null; + const seenPages = new Set(); + do { + const result = validateRepositories( + await request( + `/repositories?limit=100${page ? `&page=${encodeURIComponent(page)}` : ""}`, + ), + ); + destinations.push( + ...result.data.filter((item) => item.id === options.repository), + ); + page = result.next; + if ( + result.has_more !== (page !== null) || + (page !== null && seenPages.has(page)) + ) + throw new CodexSecurityError( + "Cloud returned invalid repository pagination.", + ); + if (page !== null) seenPages.add(page); + } while (page !== null); + if (destinations.length !== 1) + throw new CodexSecurityError( + destinations.length + ? "The repository ID matches multiple connectors. Select an unambiguous Cloud repository." + : "The destination repository is not available to this Cloud account.", + ); + const destination = destinations[0]!; + if (destination.import_environment_id == null) + throw new CodexSecurityError( + "Configure an authorized Cloud environment for this repository before importing findings.", + ); + const source = { provider: options.provider, source_key: options.sourceKey }; + const state = join( + codexSecurityStateDirectory(environment), + "external-finding-publications", + ); + const key = hash( + "sha256", + canonicalJson([ + credentials.account_id, + options.repository, + destination.repo_connector_id, + source, + parsed.findings, + ]), + ); + const pendingPath = join(state, `${key}.pending.json`); + let saved: SavedSubmission | undefined; + try { + const content = JSON.parse( + await readFile(pendingPath, "utf8"), + ) as SavedSubmission; + if ( + content.accountId !== credentials.account_id || + !Array.isArray(content.requests) || + content.requests.length === 0 + ) + throw new Error("Saved import account or requests do not match."); + content.requests = content.requests.map(validateImportRequest); + if ( + canonicalJson( + content.requests.flatMap((batch) => + batch.items.map(({ source_finding_id, evidence }) => ({ + source_finding_id, + evidence, + })), + ), + ) !== canonicalJson(parsed.findings) + ) { + throw new Error( + "Saved import evidence does not match the selected input.", + ); + } + for (const submission of content.requests) { + if ( + submission.repository.id !== destination.id || + submission.repository.repo_connector_id !== + destination.repo_connector_id || + canonicalJson(submission.source) !== canonicalJson(source) + ) + throw new Error("Saved import destination does not match."); + if ( + submission.repository.environment_id !== + destination.import_environment_id + ) { + throw new Error( + "The Cloud environment changed since this submission. Restore the original destination before resuming the saved request.", + ); + } + if (submission.repository.reset_marker !== destination.reset_marker) { + // The old request is never sent after reset. A subsequent explicit + // invocation prepares a fresh submission and asks for approval again. + await rm(pendingPath); + throw new CloudImportError( + 409, + "The repository was reset after this submission. The saved request was retired without uploading. Review the destination and run the command again to approve a fresh publication.", + ); + } + } + saved = content; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + let requests = saved?.requests; + if (!requests) { + const repository = { + id: destination.id, + repo_connector_id: destination.repo_connector_id, + environment_id: destination.import_environment_id, + reset_marker: destination.reset_marker, + }; + requests = []; + let current: FindingImportRequest = { + request_id: randomUUID(), + repository, + source, + items: [], + }; + for (const [index, finding] of parsed.findings.entries()) { + const query = new URLSearchParams({ + provider: source.provider, + source_key: source.source_key, + source_finding_id: finding.source_finding_id, + limit: "2", + }); + const summaries = validateSourceReports( + await request(`${sourcePath(destination)}?${query}`), + ); + if (summaries.has_more || summaries.data.length > 1) + throw new CodexSecurityError( + "Cloud returned more than one report for a source identity.", + ); + const previous = summaries.data[0]; + if ( + previous && + (previous.source_finding_id !== finding.source_finding_id || + previous.source.provider !== source.provider || + previous.source.source_key !== source.source_key || + previous.repo_id !== destination.id || + previous.repo_connector_id !== destination.repo_connector_id) + ) + throw new CodexSecurityError( + "Cloud returned a different source identity.", + ); + const item = { + client_id: `item-${index + 1}`, + source_finding_id: finding.source_finding_id, + expected_version: previous?.version ?? 0, + evidence: finding.evidence, + }; + if ( + current.items.length > 0 && + (current.items.length === 100 || + Buffer.byteLength( + JSON.stringify({ ...current, items: [...current.items, item] }), + ) > MAX_REQUEST_BYTES) + ) { + requests.push(validateImportRequest(current)); + current = { request_id: randomUUID(), repository, source, items: [] }; + } + current.items.push(item); + } + if (current.items.length) requests.push(validateImportRequest(current)); + } + const submission: SavedSubmission = { + accountId: credentials.account_id, + requests, + }; + const preview: ExternalPublicationPreview = { + ...parsed, + destination, + source, + resumed: saved !== undefined, + requests, + }; + return { + preview, + async publish() { + await mkdir(state, { recursive: true, mode: 0o700 }); + await validatePreparedOutputDir(state, undefined, true); + // Install a fully written file atomically. A crash during serialization + // must not leave a partial request that cannot be resumed. + const pendingTemporary = `${pendingPath}.${randomUUID()}.tmp`; + const pendingFile = await open(pendingTemporary, "wx", 0o600); + try { + await pendingFile.writeFile(JSON.stringify(submission)); + await pendingFile.sync(); + } finally { + await pendingFile.close(); + } + try { + await link(pendingTemporary, pendingPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + const other = JSON.parse( + await readFile(pendingPath, "utf8"), + ) as SavedSubmission; + if (canonicalJson(other) !== canonicalJson(submission)) { + throw new CodexSecurityError( + "Another publication prepared this input. Run the command again to review and resume that saved request.", + ); + } + } finally { + await rm(pendingTemporary, { force: true }); + } + const receipts: FindingImportReceipt[] = []; + try { + for (const batch of requests) { + const receipt = validateImportReceipt( + await request("/finding_imports", batch), + ); + if ( + receipt.id !== batch.request_id || + canonicalJson(receipt.repository) !== + canonicalJson(batch.repository) || + canonicalJson(receipt.source) !== canonicalJson(batch.source) || + receipt.item_count !== batch.items.length || + receipt.results.length !== batch.items.length || + receipt.results.some( + (item, index) => item.client_id !== batch.items[index]!.client_id, + ) + ) + throw new CodexSecurityError( + "Cloud returned a receipt for a different publication.", + ); + const actual = { created: 0, updated: 0, unchanged: 0, error: 0 }; + for (const result of receipt.results) { + actual[result.outcome]++; + if (result.outcome === "error") { + if ( + !result.error || + result.source_report_id !== null || + result.observation_id !== null || + result.canonical_finding_id !== null || + result.version !== null + ) + throw new CodexSecurityError( + "Cloud returned inconsistent item error fields.", + ); + } else if ( + result.error !== null || + !result.source_report_id || + !result.observation_id || + !result.canonical_finding_id || + result.version === null + ) + throw new CodexSecurityError( + "Cloud returned an incomplete saved finding.", + ); + } + if (canonicalJson(receipt.counts) !== canonicalJson(actual)) + throw new CodexSecurityError( + "Cloud returned inconsistent publication counts.", + ); + receipts.push(receipt); + } + // Verify readable source records without mistaking a newer concurrent + // observation for failure of the original, immutable import receipt. + for (const [batchIndex, receipt] of receipts.entries()) { + for (const [index, item] of receipt.results.entries()) { + if (item.outcome === "error") continue; + const report = validateSourceReport( + await request( + `${sourcePath(destination)}/${encodeURIComponent(item.source_report_id!)}`, + ), + ); + if ( + report.canonical_finding_id !== item.canonical_finding_id || + report.version < item.version! || + (report.version === item.version && + (report.observation_id !== item.observation_id || + canonicalJson(report.evidence) !== + canonicalJson( + requests[batchIndex]!.items[index]!.evidence, + ))) || + report.source_finding_id !== + requests[batchIndex]!.items[index]!.source_finding_id || + report.repo_id !== destination.id || + report.repo_connector_id !== destination.repo_connector_id || + canonicalJson(report.source) !== canonicalJson(source) + ) + throw new CodexSecurityError( + "Cloud readback did not match the saved finding identity.", + ); + } + } + } catch (error) { + if (error instanceof CloudImportError && error.status === 409) + await rm(pendingPath, { force: true }); + throw new CodexSecurityError( + `${error instanceof Error ? error.message : String(error)} ${error instanceof CloudImportError && error.status === 409 ? "The old submission was retired. Rediscover and approve a fresh publication." : `Repeat the same command to resume the saved request. Saved submission: ${pendingPath}`}`, + { cause: error }, + ); + } + const counts = { created: 0, updated: 0, unchanged: 0, error: 0 }; + for (const receipt of receipts) + for (const outcome of Object.keys(counts) as (keyof typeof counts)[]) + counts[outcome] += receipt.counts[outcome] ?? 0; + const result = { + receipts, + counts, + cloudUrl: `https://chatgpt.com/codex/cloud/security/findings?repo=${encodeURIComponent(destination.url)}&source=imported&provider=${source.provider}`, + }; + const temporary = `${pendingPath}.${randomUUID()}.tmp`; + const file = await open(temporary, "wx", 0o600); + try { + await file.writeFile(JSON.stringify(result)); + await file.sync(); + } finally { + await file.close(); + } + await rename(temporary, join(state, `${key}.result.json`)); + await rm(pendingPath, { force: true }); + return result; + }, + }; +} diff --git a/sdk/typescript/src/external-import-contract.ts b/sdk/typescript/src/external-import-contract.ts new file mode 100644 index 0000000000..58bdb01167 --- /dev/null +++ b/sdk/typescript/src/external-import-contract.ts @@ -0,0 +1,59 @@ +import Ajv2020 from "ajv/dist/2020.js"; +import schema from "../schemas/external-findings.schema.json" with { type: "json" }; +import type { + ExternalFindingEvidence, + FindingImportRequest, + FindingImportReceipt, + ImportRepositoryPage, + SourceReportPage, + SourceReport, +} from "./external-import-models.js"; + +const ajv = new Ajv2020({ + strict: false, + validateFormats: false, + useDefaults: true, +}); + +function validator(name: keyof typeof schema.$defs) { + const check = ajv.compile({ + $defs: schema.$defs, + $ref: `#/$defs/${name}`, + }); + return (input: unknown): T => { + if (!check(input)) + throw new Error(`${name}: ${ajv.errorsText(check.errors)}`); + return input; + }; +} + +const evidence = validator("ImportedFindingEvidence"); +const request = validator("FindingImportRequest"); +export const validateImportReceipt = validator( + "FindingImportReceipt", +); +export const validateRepositories = validator( + "ImportRepositoryPage", +); +export const validateSourceReports = + validator("SourceReportPage"); +export const validateSourceReport = validator("SourceReport"); + +export function validateExternalEvidence( + input: unknown, +): ExternalFindingEvidence { + const result = evidence(input); + if (!result.title.trim()) + throw new Error("Evidence title must not be empty."); + if (Buffer.byteLength(JSON.stringify(result)) > 256 * 1024) + throw new Error("Evidence exceeds the Cloud limit of 256 KiB."); + return result; +} + +export function validateImportRequest(input: unknown): FindingImportRequest { + const result = request(input); + for (const item of result.items) validateExternalEvidence(item.evidence); + if (Buffer.byteLength(JSON.stringify(result)) > 5 * 1024 * 1024) + throw new Error("Import exceeds the Cloud limit of 5 MiB."); + return result; +} diff --git a/sdk/typescript/src/external-import-models.ts b/sdk/typescript/src/external-import-models.ts new file mode 100644 index 0000000000..b2eb4d7897 --- /dev/null +++ b/sdk/typescript/src/external-import-models.ts @@ -0,0 +1,277 @@ +/* Generated from schemas/external-findings.schema.json. Run pnpm generate:external-import-models. */ + +export type RequestId = string; +export type Id = string; +export type RepoConnectorId = string; +export type EnvironmentId = string; +export type ResetMarker = string; +export type Provider = "wiz" | "endor" | "snyk"; +export type SourceKey = string; +export type ClientId = string; +export type SourceFindingId = string; +export type ExpectedVersion = number; +export type Title = string; +export type Severity = "critical" | "high" | "medium" | "low" | "informational"; +export type Description = string | null; +export type Url = string | null; +/** + * @maxItems 100 + */ +export type AdvisoryIds = string[]; +export type Name = string; +export type Ecosystem = string | null; +export type InstalledVersion = string | null; +export type ManifestPath = string | null; +/** + * @maxItems 100 + */ +export type FixedVersions = string[]; +/** + * @maxItems 100 + */ +export type Packages = ImportedPackage[]; +export type Path = string; +export type Line = number | null; +/** + * @maxItems 100 + */ +export type Locations = ImportedLocation[]; +export type Branch = string | null; +export type CodeRevision = string | null; +export type SourceScanId = string | null; +/** + * @maxItems 100 + */ +export type ImageDigests = string[]; +export type SourceUpdatedAt = number | null; +/** + * @minItems 1 + * @maxItems 100 + */ +export type Items = FindingImportItem[]; +export type Id1 = string; +export type Object = "security.finding_import"; +export type Actor = string; +export type CreatedAt = number; +export type ItemCount = number; +export type Created = number; +export type Updated = number; +export type Unchanged = number; +export type Error = number; +export type ClientId1 = string; +export type Outcome = "created" | "updated" | "unchanged" | "error"; +export type SourceReportId = string | null; +export type ObservationId = string | null; +export type CanonicalFindingId = string | null; +export type Version = number | null; +export type Code = + "version_conflict" | "scope_conflict" | "stale_source_snapshot"; +export type Message = string; +export type Results = FindingImportResult[]; +export type Object1 = "page"; +export type Id2 = string; +export type Object2 = "security.repository"; +export type RepoConnectorId1 = string; +export type Url1 = string; +export type DefaultBranch = string | null; +export type ResetMarker1 = string; +export type ImportEnvironmentId = string | null; +export type Data = ImportRepository[]; +export type HasMore = boolean; +export type Next = string | null; +export type Id3 = string; +export type Object3 = "security.source_report"; +export type Origin = "imported"; +export type RepoId = string; +export type RepoConnectorId2 = string; +export type EnvironmentId1 = string; +export type CanonicalFindingId1 = string; +export type SourceFindingId1 = string; +export type ObservationId1 = string; +export type Version1 = number; +export type State = "not_assessed"; +export type Id4 = string; +export type Actor1 = string; +export type CreatedAt1 = number; +export type CreatedAt2 = number; +export type UpdatedAt = number; +export type Object4 = "page"; +export type Id5 = string; +export type Object5 = "security.source_report_summary"; +export type Origin1 = "imported"; +export type RepoId1 = string; +export type RepoConnectorId3 = string; +export type EnvironmentId2 = string; +export type CanonicalFindingId2 = string; +export type SourceFindingId2 = string; +export type ObservationId2 = string; +export type Version2 = number; +export type Title1 = string; +export type Severity1 = + "critical" | "high" | "medium" | "low" | "informational"; +export type SourceUpdatedAt1 = number | null; +export type CreatedAt3 = number; +export type UpdatedAt1 = number; +export type Data1 = SourceReportSummary[]; +export type HasMore1 = boolean; +export type Next1 = string | null; + +/** + * Schema export root; transport endpoints return the individual models. + */ +export interface ExternalFindingContracts { + request: FindingImportRequest; + receipt: FindingImportReceipt; + repositories: ImportRepositoryPage; + source_report: SourceReport; + source_reports: SourceReportPage; +} +export interface FindingImportRequest { + request_id: RequestId; + repository: ImportDestination; + source: ImportSource; + items: Items; +} +export interface ImportDestination { + id: Id; + repo_connector_id: RepoConnectorId; + environment_id: EnvironmentId; + reset_marker: ResetMarker; +} +export interface ImportSource { + provider: Provider; + source_key: SourceKey; +} +export interface FindingImportItem { + client_id: ClientId; + source_finding_id: SourceFindingId; + expected_version: ExpectedVersion; + evidence: ImportedFindingEvidence; +} +export interface ImportedFindingEvidence { + title: Title; + severity: Severity; + description?: Description; + url?: Url; + advisory_ids?: AdvisoryIds; + packages?: Packages; + locations?: Locations; + branch?: Branch; + code_revision?: CodeRevision; + source_scan_id?: SourceScanId; + image_digests?: ImageDigests; + source_updated_at?: SourceUpdatedAt; + source_data?: SourceData; +} +export interface ImportedPackage { + name: Name; + ecosystem?: Ecosystem; + installed_version?: InstalledVersion; + manifest_path?: ManifestPath; + fixed_versions?: FixedVersions; +} +export interface ImportedLocation { + path: Path; + line?: Line; +} +export interface SourceData { + [k: string]: unknown; +} +export interface FindingImportReceipt { + id: Id1; + object?: Object; + repository: ImportDestination; + source: ImportSource; + actor: Actor; + created_at: CreatedAt; + item_count: ItemCount; + counts: ImportCounts; + results: Results; +} +export interface ImportCounts { + created?: Created; + updated?: Updated; + unchanged?: Unchanged; + error?: Error; +} +export interface FindingImportResult { + client_id: ClientId1; + outcome: Outcome; + source_report_id?: SourceReportId; + observation_id?: ObservationId; + canonical_finding_id?: CanonicalFindingId; + version?: Version; + error?: ImportItemError | null; +} +export interface ImportItemError { + code: Code; + message: Message; +} +export interface ImportRepositoryPage { + object?: Object1; + data: Data; + has_more: HasMore; + next: Next; +} +export interface ImportRepository { + id: Id2; + object?: Object2; + repo_connector_id: RepoConnectorId1; + url: Url1; + default_branch?: DefaultBranch; + reset_marker: ResetMarker1; + import_environment_id?: ImportEnvironmentId; +} +export interface SourceReport { + id: Id3; + object?: Object3; + origin?: Origin; + repo_id: RepoId; + repo_connector_id: RepoConnectorId2; + environment_id: EnvironmentId1; + canonical_finding_id: CanonicalFindingId1; + source: ImportSource; + source_finding_id: SourceFindingId1; + observation_id: ObservationId1; + version: Version1; + evidence: ImportedFindingEvidence; + assessment?: ImportedAssessment; + last_import?: LastImport | null; + created_at: CreatedAt2; + updated_at: UpdatedAt; +} +export interface ImportedAssessment { + state?: State; +} +export interface LastImport { + id: Id4; + actor: Actor1; + created_at: CreatedAt1; +} +export interface SourceReportPage { + object?: Object4; + data: Data1; + has_more: HasMore1; + next: Next1; +} +export interface SourceReportSummary { + id: Id5; + object?: Object5; + origin?: Origin1; + repo_id: RepoId1; + repo_connector_id: RepoConnectorId3; + environment_id: EnvironmentId2; + canonical_finding_id: CanonicalFindingId2; + source: ImportSource; + source_finding_id: SourceFindingId2; + observation_id: ObservationId2; + version: Version2; + title: Title1; + severity: Severity1; + source_updated_at: SourceUpdatedAt1; + assessment?: ImportedAssessment; + created_at: CreatedAt3; + updated_at: UpdatedAt1; +} + +export type ExternalFindingEvidence = ImportedFindingEvidence; diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts new file mode 100644 index 0000000000..5c79088e79 --- /dev/null +++ b/sdk/typescript/src/wiz-findings.ts @@ -0,0 +1,196 @@ +import { readFile } from "node:fs/promises"; +import { CodexSecurityError } from "./errors.js"; +import { validateExternalEvidence } from "./external-import-contract.js"; +import type { ExternalFindingEvidence } from "./external-import-models.js"; + +export interface VendorFinding { + source_finding_id: string; + evidence: ExternalFindingEvidence; +} + +export interface VendorFindings { + read: number; + findings: VendorFinding[]; + excluded: { + position: number; + source_finding_id: string | null; + reason: string; + }[]; +} + +function object(value: unknown): Record | undefined { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +function text(value: unknown): string | null { + return typeof value === "string" && value.trim() ? value : null; +} + +/** Keep a Wiz vulnerability occurrence intact; advisory names are not identities. */ +function wizFinding(record: Record): VendorFinding { + const id = text(record["id"]); + if (!id) throw new Error("Wiz vulnerability finding is missing its id."); + const name = text(record["name"]); + const packageName = text(record["detailedName"]); + const version = text(record["version"]); + const severity = text(record["vendorSeverity"]) ?? text(record["severity"]); + const asset = object(record["vulnerableAsset"]); + if (!name || !packageName || !text(asset?.["id"])) { + throw new Error( + "Expected a Wiz package vulnerability finding with name, detailedName, and vulnerableAsset.id. Other finding classes need a separate mapping.", + ); + } + const digest = text(asset?.["imageDigest"]) ?? text(record["imageDigest"]); + const updated = text(record["updatedAt"]); + const updatedSeconds = updated ? Date.parse(updated) / 1000 : null; + if ( + updatedSeconds !== null && + (!Number.isFinite(updatedSeconds) || updatedSeconds < 0) + ) { + throw new Error("Wiz updatedAt is not a valid timestamp."); + } + // Container paths and lastDetectedAt are retained in source_data; neither is a + // repository location or proof that the entire vendor record was updated. + const evidence = { + title: + text(record["title"]) ?? + [packageName, version, name].filter(Boolean).join(" "), + description: text(record["description"]), + severity: severity?.toLowerCase(), + url: text(record["portalUrl"]), + advisory_ids: name ? [name] : [], + packages: packageName + ? [ + { + name: packageName, + ecosystem: text(record["packageManager"]), + installed_version: version, + manifest_path: null, + fixed_versions: text(record["fixedVersion"]) + ? [record["fixedVersion"]] + : [], + }, + ] + : [], + locations: [], + branch: null, + code_revision: null, + source_scan_id: null, + image_digests: digest ? [digest] : [], + source_updated_at: + updatedSeconds === null ? null : Math.floor(updatedSeconds), + source_data: record, + }; + return { + source_finding_id: id, + evidence: validateExternalEvidence(evidence), + }; +} + +function records(payload: unknown): unknown[] { + if (Array.isArray(payload)) return payload; + const envelope = object(payload); + if (!envelope) + throw new Error( + "Expected finding objects or a Wiz vulnerabilityFindings response.", + ); + if (Array.isArray(envelope["errors"]) && envelope["errors"].length > 0) { + throw new Error( + "Wiz returned GraphQL errors. Resolve them before publishing this response.", + ); + } + if ( + envelope["source_finding_id"] !== undefined || + envelope["id"] !== undefined + ) + return [envelope]; + const data = object(envelope["data"]) ?? envelope; + const findings = object(data["vulnerabilityFindings"]); + if (findings && Array.isArray(findings["nodes"])) { + const page = object(findings["pageInfo"]); + if (page?.["hasNextPage"] === true) { + throw new Error( + "Wiz response has another page. Save the explicitly selected records as an array before publishing.", + ); + } + return findings["nodes"]; + } + throw new Error( + "Unsupported Wiz export. Supply vulnerability finding objects or normalized JSONL.", + ); +} + +export async function readVendorFindings( + path: string, +): Promise { + const contents = (await readFile(path, "utf8")).replace(/^\uFEFF/u, ""); + let input: unknown[]; + try { + let payload: unknown; + try { + payload = JSON.parse(contents); + } catch { + payload = contents + .split(/\r?\n/u) + .filter((line) => line.trim()) + .map((line) => JSON.parse(line)); + } + input = records(payload); + } catch (cause) { + throw new CodexSecurityError( + `Could not read vendor findings: ${cause instanceof Error ? cause.message : String(cause)}`, + { cause }, + ); + } + const result: VendorFindings = { + read: input.length, + findings: [], + excluded: [], + }; + const identities = new Set(); + for (const [index, item] of input.entries()) { + const record = object(item); + try { + if (!record) throw new Error("Finding must be an object."); + let finding: VendorFinding; + if ("source_finding_id" in record) { + if ( + Object.keys(record).some( + (key) => !["source_finding_id", "evidence"].includes(key), + ) + ) { + throw new Error( + "Normalized input permits only source_finding_id and evidence.", + ); + } + const id = text(record["source_finding_id"]); + if (!id || Buffer.byteLength(id) > 512) + throw new Error("Source finding id must contain 1–512 UTF-8 bytes."); + finding = { + source_finding_id: id, + evidence: validateExternalEvidence(record["evidence"]), + }; + } else { + finding = wizFinding(record); + } + if (identities.has(finding.source_finding_id)) { + throw new CodexSecurityError( + `Duplicate source finding id ${JSON.stringify(finding.source_finding_id)}. Select each vendor occurrence once.`, + ); + } + identities.add(finding.source_finding_id); + result.findings.push(finding); + } catch (error) { + if (error instanceof CodexSecurityError) throw error; + result.excluded.push({ + position: index + 1, + source_finding_id: + text(record?.["source_finding_id"]) ?? text(record?.["id"]), + reason: error instanceof Error ? error.message : String(error), + }); + } + } + return result; +} diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts new file mode 100644 index 0000000000..9a61459c13 --- /dev/null +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -0,0 +1,546 @@ +import { createServer, type ServerResponse } from "node:http"; +import { hash } from "node:crypto"; +import { + mkdtemp, + mkdir, + readFile, + readdir, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { afterEach, expect, test } from "bun:test"; +import { prepareExternalPublication } from "../src/external-findings-publish.js"; +import { readVendorFindings } from "../src/wiz-findings.js"; +import { validateImportRequest } from "../src/external-import-contract.js"; +import type { + FindingImportReceipt, + FindingImportRequest, + SourceReport, +} from "../src/external-import-models.js"; +import { main } from "../src/cli.js"; +import { dependencies } from "./cli-fixtures.js"; +import { createCliTest } from "./support/cli-run.js"; + +const cleanups: (() => Promise)[] = []; +afterEach(async () => { + for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); +}); +const options = { + repository: "repo-example", + provider: "wiz" as const, + sourceKey: "tenant-example/vulnerability-finding", +}; +const normalized = (id = "vendor-1", severity = "high") => ({ + source_finding_id: id, + evidence: { + title: "Example vulnerable package", + severity, + source_data: { id }, + }, +}); + +async function fixture(records: unknown = [normalized()]) { + const root = await mkdtemp(join(tmpdir(), "external-findings-")); + cleanups.push(() => rm(root, { recursive: true, force: true })); + const file = join(root, "selected findings.json"); + await writeFile(file, JSON.stringify(records)); + const environment = { + ...process.env, + CODEX_HOME: join(root, "credentials"), + CODEX_SECURITY_STATE_DIR: join(root, "state"), + }; + await mkdir(environment.CODEX_HOME, { mode: 0o700 }); + await writeFile( + join(environment.CODEX_HOME, "config.toml"), + 'cli_auth_credentials_store = "file"\n', + ); + await writeFile( + join(environment.CODEX_HOME, "auth.json"), + JSON.stringify({ + tokens: { + access_token: "synthetic-token", + account_id: "synthetic-account", + }, + }), + ); + const reports = new Map(); + const receipts = new Map(); + const posts: string[] = []; + const state = { + marker: "generation-1", + loseResponse: false, + brokenReadback: false, + throttle: false, + finalError: false, + }; + const destination = () => ({ + id: options.repository, + object: "security.repository", + repo_connector_id: "github", + url: "https://github.com/example/project", + default_branch: "main", + reset_marker: state.marker, + import_environment_id: "environment-example", + }); + function json(response: ServerResponse, value: unknown, status = 200) { + response.writeHead(status, { "Content-Type": "application/json" }); + response.end(JSON.stringify(value)); + } + const server = createServer(async (incoming, response) => { + try { + expect(incoming.headers.authorization).toBe("Bearer synthetic-token"); + expect(incoming.headers["chatgpt-account-id"]).toBe("synthetic-account"); + const url = new URL(incoming.url!, "http://localhost"); + if (url.pathname.endsWith("/repositories")) + return json(response, { + object: "page", + data: [destination()], + has_more: false, + next: null, + }); + if (url.pathname.endsWith("/source_reports")) { + const report = reports.get(url.searchParams.get("source_finding_id")!); + let summary; + if (report) { + const { evidence, last_import: _lastImport, ...rest } = report; + summary = { + ...rest, + object: "security.source_report_summary", + title: evidence.title, + severity: evidence.severity, + source_updated_at: evidence.source_updated_at ?? null, + }; + } + return json(response, { + object: "page", + data: summary ? [summary] : [], + has_more: false, + next: null, + }); + } + if (url.pathname.includes("/source_reports/")) { + if (state.brokenReadback) + return json( + response, + { error: { message: "Readback unavailable" } }, + 503, + ); + const report = [...reports.values()].find( + (item) => + item.id === decodeURIComponent(url.pathname.split("/").at(-1)!), + ); + return json(response, report, report ? 200 : 404); + } + if (url.pathname.endsWith("/finding_imports")) { + let body = ""; + for await (const chunk of incoming) body += chunk; + posts.push(body); + const request = validateImportRequest(JSON.parse(body)); + if (state.throttle) { + response.setHeader("Retry-After", "1"); + return json(response, { error: { message: "Slow down" } }, 429); + } + if (request.repository.reset_marker !== state.marker) + return json( + response, + { error: { message: "stale_reset_marker" } }, + 409, + ); + let receipt = receipts.get(request.request_id); + if (!receipt) { + receipt = { + id: request.request_id, + object: "security.finding_import", + repository: request.repository, + source: request.source, + actor: "synthetic-user", + created_at: 1, + item_count: request.items.length, + counts: { created: 0, updated: 0, unchanged: 0, error: 0 }, + results: [], + }; + for (const item of request.items) { + const previous = reports.get(item.source_finding_id); + if (state.finalError) { + receipt.counts.error = (receipt.counts.error ?? 0) + 1; + receipt.results.push({ + client_id: item.client_id, + outcome: "error", + source_report_id: null, + observation_id: null, + canonical_finding_id: null, + version: null, + error: { + code: "version_conflict", + message: "Reload current evidence", + }, + }); + continue; + } + const outcome = previous + ? JSON.stringify(previous.evidence) === + JSON.stringify(item.evidence) + ? "unchanged" + : "updated" + : "created"; + if (outcome === "updated") + expect(item.expected_version).toBe(previous!.version); + const version = previous + ? previous.version + (outcome === "updated" ? 1 : 0) + : 1; + const identity = hash("sha256", item.source_finding_id); + const report: SourceReport = { + id: `aif_${identity}`, + object: "security.source_report", + origin: "imported", + repo_id: request.repository.id, + repo_connector_id: request.repository.repo_connector_id, + environment_id: request.repository.environment_id, + canonical_finding_id: `acf_${identity.slice(0, 32)}`, + source: request.source, + source_finding_id: item.source_finding_id, + observation_id: `aio_${hash("sha256", `${identity}:${version}`)}`, + version, + evidence: item.evidence, + assessment: { state: "not_assessed" }, + last_import: null, + created_at: 1, + updated_at: version, + }; + reports.set(item.source_finding_id, report); + receipt.counts[outcome] = (receipt.counts[outcome] ?? 0) + 1; + receipt.results.push({ + client_id: item.client_id, + outcome, + source_report_id: report.id, + observation_id: report.observation_id, + canonical_finding_id: report.canonical_finding_id, + version, + error: null, + }); + } + receipts.set(request.request_id, receipt); + } + if (state.loseResponse) { + state.loseResponse = false; + response.destroy(); + return; + } + return json(response, receipt); + } + json(response, { error: { message: "Unexpected fixture route" } }, 404); + } catch (error) { + json(response, { error: { message: String(error) } }, 500); + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + cleanups.push( + () => + new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + server.closeAllConnections(); + }), + ); + const port = (server.address() as { port: number }).port; + const transport = (url: string, init: RequestInit) => + fetch( + `http://127.0.0.1:${port}${new URL(url).pathname}${new URL(url).search}`, + init, + ); + const deps = { environment, fetch: transport }; + const cliDeps = { ...dependencies(), environment, cloudFetch: transport }; + const command = [ + "publish", + "findings", + file, + "--to", + "cloud", + "--repository", + options.repository, + "--provider", + options.provider, + "--source-key", + options.sourceKey, + "--format", + "json", + ]; + return { + root, + file, + reports, + receipts, + posts, + state, + destination, + environment, + port, + deps, + cliDeps, + command, + }; +} + +test("Wiz occurrence mapping preserves original evidence without inventing repository provenance", async () => { + const record = { + id: "wiz-occurrence", + name: "CVE-2099-0001", + detailedName: "example-lib", + version: "1.2.0", + fixedVersion: "1.2.1", + vendorSeverity: "HIGH", + portalUrl: "https://example.test/finding", + locationPath: "/usr/lib/example", + lastDetectedAt: "2026-10-01T00:00:00Z", + projects: [{ id: "project-a" }, { id: "project-b" }], + vulnerableAsset: { id: "image-resource", imageDigest: "sha256:example" }, + }; + const f = await fixture([record]); + const parsed = await readVendorFindings(f.file); + expect(parsed.findings[0]).toMatchObject({ + source_finding_id: record.id, + evidence: { + severity: "high", + locations: [], + code_revision: null, + branch: null, + source_updated_at: null, + source_data: record, + image_digests: ["sha256:example"], + }, + }); +}); + +test("unsupported records are visible exclusions; duplicate identities and unfinished pages stop publication", async () => { + const f = await fixture([normalized(), normalized("unknown", "unknown")]); + const parsed = await readVendorFindings(f.file); + expect(parsed.findings).toHaveLength(1); + expect(parsed.excluded).toHaveLength(1); + await writeFile(f.file, JSON.stringify([normalized(), normalized()])); + await expect(readVendorFindings(f.file)).rejects.toThrow("Duplicate source"); + await writeFile( + f.file, + JSON.stringify({ + data: { + vulnerabilityFindings: { + nodes: [normalized()], + pageInfo: { hasNextPage: true }, + }, + }, + }), + ); + await expect(readVendorFindings(f.file)).rejects.toThrow("another page"); +}); + +test("CLI preview and rejected confirmation never POST or persist an upload", async () => { + const f = await fixture(); + const cli = createCliTest(main); + const code = await cli.runCli([...f.command, "--dry-run"], f.cliDeps); + expect({ code, error: cli.stderr.text() }).toEqual({ code: 0, error: "" }); + expect(JSON.parse(cli.stdout.text())).toMatchObject({ + dryRun: true, + destination: { id: options.repository }, + read: 1, + }); + expect(f.posts).toHaveLength(0); + expect(await readdir(join(f.root, "state")).catch(() => [])).toEqual([]); + const rejected = createCliTest(main); + expect( + await rejected.runCli(f.command, { + ...f.cliDeps, + externalPublicationPrompt: { + isInteractive: () => true, + confirm: async () => false, + }, + }), + ).toBe(0); + expect(f.posts).toHaveLength(0); + const headless = createCliTest(main); + expect(await headless.runCli(f.command, f.cliDeps)).toBe(2); + expect(headless.stderr.text()).toContain("needs confirmation"); +}); + +test("CLI creates, reimports unchanged, and updates the same canonical finding", async () => { + const f = await fixture(); + const first = createCliTest(main); + expect(await first.runCli([...f.command, "--yes"], f.cliDeps)).toBe(0); + const canonical = f.reports.get("vendor-1")!.canonical_finding_id; + expect(JSON.parse(first.stdout.text()).counts.created).toBe(1); + const again = await prepareExternalPublication(f.file, options, f.deps); + expect((await again.publish()).counts.unchanged).toBe(1); + await writeFile(f.file, JSON.stringify([normalized("vendor-1", "critical")])); + const updated = await prepareExternalPublication(f.file, options, f.deps); + expect(updated.preview.requests[0]!.items[0]!.expected_version).toBe(1); + expect((await updated.publish()).counts.updated).toBe(1); + expect(f.reports.get("vendor-1")).toMatchObject({ + canonical_finding_id: canonical, + version: 2, + }); +}); + +test("interactive JSON publication shows the destination and exclusions before confirmation", async () => { + const f = await fixture([normalized(), normalized("unsupported", "unknown")]); + const cli = createCliTest(main); + let prompted = false; + expect( + await cli.runCli(f.command, { + ...f.cliDeps, + externalPublicationPrompt: { + isInteractive: () => true, + confirm: async (_question, defaultValue) => { + prompted = true; + expect(defaultValue).toBe(false); + expect(cli.stderr.text()).toContain( + "https://github.com/example/project", + ); + expect(cli.stderr.text()).toContain(options.sourceKey); + expect(cli.stderr.text()).toContain("Excluded: 1"); + expect(f.posts).toHaveLength(0); + return true; + }, + }, + }), + ).toBe(0); + expect(prompted).toBe(true); + expect(JSON.parse(cli.stdout.text()).counts.created).toBe(1); +}); + +test("saved request survives an actual publisher process restart after a committed response is lost", async () => { + const f = await fixture(); + const runner = join(f.root, "run-cli.ts"); + const cliUrl = pathToFileURL(join(import.meta.dir, "../src/cli.ts")).href; + const fixturesUrl = pathToFileURL( + join(import.meta.dir, "cli-fixtures.ts"), + ).href; + await writeFile( + runner, + `import { main } from ${JSON.stringify(cliUrl)};\nimport { dependencies } from ${JSON.stringify(fixturesUrl)};\nprocess.exitCode = await main(process.argv.slice(2), process.stdout, process.stderr, { ...dependencies({ environment: process.env }), cloudFetch: (url, init) => fetch('http://127.0.0.1:${f.port}' + new URL(url).pathname + new URL(url).search, init) });\n`, + ); + async function run() { + const process = Bun.spawn( + [Bun.which("bun")!, runner, ...f.command, "--yes"], + { env: f.environment, stdout: "pipe", stderr: "pipe" }, + ); + const [code, stdout, stderr] = await Promise.all([ + process.exited, + new Response(process.stdout).text(), + new Response(process.stderr).text(), + ]); + return { code, stdout, stderr }; + } + f.state.loseResponse = true; + const lost = await run(); + expect(lost.code).toBe(2); + expect({ + size: f.reports.size, + error: lost.stderr.includes("resume the saved request") + ? "resumable" + : lost.stderr, + }).toEqual({ size: 1, error: "resumable" }); + const firstBody = f.posts[0]; + const second = await run(); + expect(second.code).toBe(0); + expect(f.posts[1]).toBe(firstBody); + expect(f.receipts.size).toBe(1); + expect(JSON.parse(second.stdout).counts.created).toBe(1); +}); + +test("readback failure retains the accepted request; a retry replays its outcome", async () => { + const f = await fixture(); + f.state.brokenReadback = true; + const prepared = await prepareExternalPublication(f.file, options, f.deps); + await expect(prepared.publish()).rejects.toThrow("resume the saved request"); + f.state.brokenReadback = false; + const retry = await prepareExternalPublication(f.file, options, f.deps); + expect(retry.preview.resumed).toBe(true); + expect((await retry.publish()).counts.created).toBe(1); + expect(f.posts[0]).toBe(f.posts[1]); +}); + +test("an older pending receipt can resume after newer evidence without rolling it back", async () => { + const f = await fixture(); + f.state.loseResponse = true; + await expect( + (await prepareExternalPublication(f.file, options, f.deps)).publish(), + ).rejects.toThrow("resume the saved request"); + const original = f.posts[0]; + await writeFile(f.file, JSON.stringify([normalized("vendor-1", "critical")])); + expect( + ( + await ( + await prepareExternalPublication(f.file, options, f.deps) + ).publish() + ).counts.updated, + ).toBe(1); + await writeFile(f.file, JSON.stringify([normalized()])); + const replay = await ( + await prepareExternalPublication(f.file, options, f.deps) + ).publish(); + expect(f.posts.at(-1)).toBe(original); + expect(replay.receipts[0]!.results[0]!.version).toBe(1); + expect(f.reports.get("vendor-1")).toMatchObject({ + version: 2, + evidence: { severity: "critical" }, + }); +}); + +test("repository reset never republishes a pending old submission", async () => { + const f = await fixture(); + f.state.throttle = true; + await expect( + (await prepareExternalPublication(f.file, options, f.deps)).publish(), + ).rejects.toThrow("Retry-After: 1"); + f.state.marker = "generation-2"; + await expect( + prepareExternalPublication(f.file, options, f.deps), + ).rejects.toThrow("retired without uploading"); + expect(f.posts).toHaveLength(1); + f.state.throttle = false; + const fresh = await prepareExternalPublication(f.file, options, f.deps); + expect(fresh.preview.resumed).toBe(false); + expect(fresh.preview.requests[0]!.repository.reset_marker).toBe( + "generation-2", + ); +}); + +test("batching preserves item order and keeps final item errors visible", async () => { + const f = await fixture( + Array.from({ length: 101 }, (_, index) => normalized(`vendor-${index}`)), + ); + const prepared = await prepareExternalPublication(f.file, options, f.deps); + expect( + prepared.preview.requests.map((request) => request.items.length), + ).toEqual([100, 1]); + expect((await prepared.publish()).counts.created).toBe(101); + const errorFixture = await fixture(); + errorFixture.state.finalError = true; + const cli = createCliTest(main); + expect( + await cli.runCli([...errorFixture.command, "--yes"], errorFixture.cliDeps), + ).toBe(1); + expect(JSON.parse(cli.stdout.text()).receipts[0].results[0]).toMatchObject({ + outcome: "error", + error: { code: "version_conflict" }, + }); +}); + +test("local receipts contain no credential and pending bodies match transmitted bytes", async () => { + const f = await fixture(); + f.state.throttle = true; + await expect( + (await prepareExternalPublication(f.file, options, f.deps)).publish(), + ).rejects.toThrow("Slow down"); + const directory = join(f.root, "state", "external-finding-publications"); + const name = (await readdir(directory)).find((name) => + name.endsWith(".pending.json"), + )!; + const saved = await readFile(join(directory, name), "utf8"); + expect(saved).not.toContain("synthetic-token"); + expect( + JSON.stringify( + (JSON.parse(saved) as { requests: FindingImportRequest[] }).requests[0], + ), + ).toBe(f.posts[0]!); +}); From b6bb46ed4564f539cd1668bb3fe8810b493074bb Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 8 Oct 2026 08:42:29 +0000 Subject: [PATCH 02/43] fix(cloud): normalize import readback and recover after resets --- sdk/typescript/docs/cli.md | 7 +- .../schemas/external-findings.schema.json | 6 + sdk/typescript/src/cli.ts | 2 +- .../src/external-findings-publish.ts | 26 +++-- .../external-findings-publish.test.ts | 107 ++++++++++++++---- 5 files changed, 112 insertions(+), 36 deletions(-) diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md index 94a705069f..2d14807ff6 100644 --- a/sdk/typescript/docs/cli.md +++ b/sdk/typescript/docs/cli.md @@ -784,9 +784,10 @@ codex-security publish findings selected-wiz.json \ This command reads the authorized destination and current source versions during preview. It uses the same saved ChatGPT file login as scan publication. The Cloud repository needs an existing environment; no native scan is required. Review the -destination, findings, and exclusions, then rerun without `--dry-run`. The default -terminal prompt is No; `--yes` confirms a previously reviewed input for scripts or -the plugin. Without a terminal or `--yes`, no upload occurs. +account, repository, environment, findings, and exclusions, then rerun without +`--dry-run`. The default terminal prompt is No; `--yes` confirms a previously +reviewed input for scripts or the plugin. Without a terminal or `--yes`, no upload +occurs. Input can be a Wiz vulnerability finding, an array, a complete `data.vulnerabilityFindings.nodes` response, or JSONL with one vendor record per diff --git a/sdk/typescript/schemas/external-findings.schema.json b/sdk/typescript/schemas/external-findings.schema.json index 497b6a6bd3..d08dd510ec 100644 --- a/sdk/typescript/schemas/external-findings.schema.json +++ b/sdk/typescript/schemas/external-findings.schema.json @@ -446,6 +446,7 @@ "title": "Url" }, "advisory_ids": { + "default": [], "items": { "maxLength": 512, "minLength": 1, @@ -456,6 +457,7 @@ "type": "array" }, "packages": { + "default": [], "items": { "$ref": "#/$defs/ImportedPackage" }, @@ -464,6 +466,7 @@ "type": "array" }, "locations": { + "default": [], "items": { "$ref": "#/$defs/ImportedLocation" }, @@ -514,6 +517,7 @@ "title": "Source Scan Id" }, "image_digests": { + "default": [], "items": { "maxLength": 512, "minLength": 1, @@ -538,6 +542,7 @@ }, "source_data": { "additionalProperties": true, + "default": {}, "title": "Source Data", "type": "object" } @@ -624,6 +629,7 @@ "title": "Manifest Path" }, "fixed_versions": { + "default": [], "items": { "maxLength": 512, "minLength": 1, diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index f18bd6b7b9..74c37ad3b4 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -3225,7 +3225,7 @@ export async function main( const { preview } = prepared; const showPreview = () => { errorOutput.write( - `Source: ${preview.source.provider} / ${diagnosticValue(preview.source.source_key)}\nDestination: ${diagnosticValue(preview.destination.url)} (${diagnosticValue(preview.destination.id)})\nRead: ${preview.read} Ready: ${preview.findings.length} Excluded: ${preview.excluded.length}${preview.resumed ? "\nResuming the saved submission." : ""}\n`, + `Account: ${diagnosticValue(preview.accountId)}\nDestination: ${diagnosticValue(preview.destination.url)} (${diagnosticValue(preview.destination.id)})\nEnvironment: ${diagnosticValue(preview.destination.import_environment_id)}\nSource: ${preview.source.provider} / ${diagnosticValue(preview.source.source_key)}\nRead: ${preview.read} Ready: ${preview.findings.length} Excluded: ${preview.excluded.length}${preview.resumed ? "\nResuming the saved submission." : ""}\n`, ); for (const excluded of preview.excluded) errorOutput.write( diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 9f8c53f76c..b038ccdd4a 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -45,6 +45,7 @@ interface SavedSubmission { } export interface ExternalPublicationPreview extends VendorFindings { + accountId: string; destination: ImportRepository; source: FindingImportRequest["source"]; resumed: boolean; @@ -167,10 +168,6 @@ export async function prepareExternalPublication( : "The destination repository is not available to this Cloud account.", ); const destination = destinations[0]!; - if (destination.import_environment_id == null) - throw new CodexSecurityError( - "Configure an authorized Cloud environment for this repository before importing findings.", - ); const source = { provider: options.provider, source_key: options.sourceKey }; const state = join( codexSecurityStateDirectory(environment), @@ -221,14 +218,6 @@ export async function prepareExternalPublication( canonicalJson(submission.source) !== canonicalJson(source) ) throw new Error("Saved import destination does not match."); - if ( - submission.repository.environment_id !== - destination.import_environment_id - ) { - throw new Error( - "The Cloud environment changed since this submission. Restore the original destination before resuming the saved request.", - ); - } if (submission.repository.reset_marker !== destination.reset_marker) { // The old request is never sent after reset. A subsequent explicit // invocation prepares a fresh submission and asks for approval again. @@ -238,11 +227,23 @@ export async function prepareExternalPublication( "The repository was reset after this submission. The saved request was retired without uploading. Review the destination and run the command again to approve a fresh publication.", ); } + if ( + submission.repository.environment_id !== + destination.import_environment_id + ) { + throw new Error( + "The Cloud environment changed since this submission. Restore the original destination before resuming the saved request.", + ); + } } saved = content; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } + if (destination.import_environment_id == null) + throw new CodexSecurityError( + "Configure an authorized Cloud environment for this repository before importing findings.", + ); let requests = saved?.requests; if (!requests) { const repository = { @@ -310,6 +311,7 @@ export async function prepareExternalPublication( }; const preview: ExternalPublicationPreview = { ...parsed, + accountId: credentials.account_id, destination, source, resumed: saved !== undefined, diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 9a61459c13..4bfa5d86e9 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -71,6 +71,8 @@ async function fixture(records: unknown = [normalized()]) { const posts: string[] = []; const state = { marker: "generation-1", + environmentId: "environment-example" as string | null, + materializeDefaults: false, loseResponse: false, brokenReadback: false, throttle: false, @@ -83,7 +85,7 @@ async function fixture(records: unknown = [normalized()]) { url: "https://github.com/example/project", default_branch: "main", reset_marker: state.marker, - import_environment_id: "environment-example", + import_environment_id: state.environmentId, }); function json(response: ServerResponse, value: unknown, status = 200) { response.writeHead(status, { "Content-Type": "application/json" }); @@ -132,6 +134,21 @@ async function fixture(records: unknown = [normalized()]) { (item) => item.id === decodeURIComponent(url.pathname.split("/").at(-1)!), ); + if (report && state.materializeDefaults) + return json(response, { + ...report, + evidence: { + advisory_ids: [], + locations: [], + image_digests: [], + source_data: {}, + ...report.evidence, + packages: (report.evidence.packages ?? []).map((entry) => ({ + fixed_versions: [], + ...entry, + })), + }, + }); return json(response, report, report ? 200 : 404); } if (url.pathname.endsWith("/finding_imports")) { @@ -341,7 +358,11 @@ test("CLI preview and rejected confirmation never POST or persist an upload", as expect({ code, error: cli.stderr.text() }).toEqual({ code: 0, error: "" }); expect(JSON.parse(cli.stdout.text())).toMatchObject({ dryRun: true, - destination: { id: options.repository }, + accountId: "synthetic-account", + destination: { + id: options.repository, + import_environment_id: "environment-example", + }, read: 1, }); expect(f.posts).toHaveLength(0); @@ -380,6 +401,33 @@ test("CLI creates, reimports unchanged, and updates the same canonical finding", }); }); +test("normalized input accepts server-materialized optional evidence defaults", async () => { + const f = await fixture([ + { + source_finding_id: "vendor-1", + evidence: { + title: "Example vulnerable package", + severity: "high", + packages: [{ name: "example-package" }], + }, + }, + ]); + f.state.materializeDefaults = true; + const prepared = await prepareExternalPublication(f.file, options, f.deps); + expect((await prepared.publish()).counts.created).toBe(1); + expect(f.reports.get("vendor-1")?.evidence).toMatchObject({ + advisory_ids: [], + packages: [{ name: "example-package", fixed_versions: [] }], + locations: [], + image_digests: [], + source_data: {}, + }); + const stateFiles = await readdir( + join(f.root, "state", "external-finding-publications"), + ); + expect(stateFiles.some((name) => name.endsWith(".pending.json"))).toBe(false); +}); + test("interactive JSON publication shows the destination and exclusions before confirmation", async () => { const f = await fixture([normalized(), normalized("unsupported", "unknown")]); const cli = createCliTest(main); @@ -396,7 +444,10 @@ test("interactive JSON publication shows the destination and exclusions before c "https://github.com/example/project", ); expect(cli.stderr.text()).toContain(options.sourceKey); + expect(cli.stderr.text()).toContain("synthetic-account"); + expect(cli.stderr.text()).toContain("environment-example"); expect(cli.stderr.text()).toContain("Excluded: 1"); + expect(cli.stderr.text()).not.toContain("synthetic-token"); expect(f.posts).toHaveLength(0); return true; }, @@ -486,24 +537,40 @@ test("an older pending receipt can resume after newer evidence without rolling i }); }); -test("repository reset never republishes a pending old submission", async () => { - const f = await fixture(); - f.state.throttle = true; - await expect( - (await prepareExternalPublication(f.file, options, f.deps)).publish(), - ).rejects.toThrow("Retry-After: 1"); - f.state.marker = "generation-2"; - await expect( - prepareExternalPublication(f.file, options, f.deps), - ).rejects.toThrow("retired without uploading"); - expect(f.posts).toHaveLength(1); - f.state.throttle = false; - const fresh = await prepareExternalPublication(f.file, options, f.deps); - expect(fresh.preview.resumed).toBe(false); - expect(fresh.preview.requests[0]!.repository.reset_marker).toBe( - "generation-2", - ); -}); +test.each(["environment-example", "replacement-environment", null])( + "repository reset retires a pending request with environment %p", + async (environmentId) => { + const f = await fixture(); + f.state.throttle = true; + await expect( + (await prepareExternalPublication(f.file, options, f.deps)).publish(), + ).rejects.toThrow("Retry-After: 1"); + f.state.marker = "generation-2"; + f.state.environmentId = environmentId; + await expect( + prepareExternalPublication(f.file, options, f.deps), + ).rejects.toThrow("retired without uploading"); + expect(f.posts).toHaveLength(1); + if (environmentId === null) { + await expect( + prepareExternalPublication(f.file, options, f.deps), + ).rejects.toThrow("Configure an authorized Cloud environment"); + f.state.environmentId = "replacement-environment"; + } + f.state.throttle = false; + const fresh = await prepareExternalPublication(f.file, options, f.deps); + expect(fresh.preview.resumed).toBe(false); + expect(fresh.preview.requests[0]!.repository).toMatchObject({ + reset_marker: "generation-2", + environment_id: f.state.environmentId, + }); + expect((await fresh.publish()).counts.created).toBe(1); + expect(f.posts).toHaveLength(2); + expect(JSON.parse(f.posts[1]!).request_id).not.toBe( + JSON.parse(f.posts[0]!).request_id, + ); + }, +); test("batching preserves item order and keeps final item errors visible", async () => { const f = await fixture( From 01d29ecdc76802cf8c27c806e143aa7b338e5115 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 8 Oct 2026 09:04:02 +0000 Subject: [PATCH 03/43] fix(cloud): serialize resumable finding publications --- .../src/external-findings-publish.ts | 322 +++++++++++------- .../external-findings-publish.test.ts | 242 +++++++++++-- 2 files changed, 400 insertions(+), 164 deletions(-) diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index b038ccdd4a..3bc6c9b974 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -1,6 +1,17 @@ import { hash, randomUUID } from "node:crypto"; -import { link, mkdir, open, readFile, rename, rm } from "node:fs/promises"; +import { chmodSync } from "node:fs"; +import { + link, + lstat, + mkdir, + open, + readFile, + rename, + rm, +} from "node:fs/promises"; +import { createRequire } from "node:module"; import { join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; import { readCloudCredentials } from "./cloud-publish.js"; import { CodexSecurityError } from "./errors.js"; import { @@ -85,6 +96,51 @@ class CloudImportError extends CodexSecurityError { } } +async function withImportLock( + state: string, + key: string, + signal: AbortSignal | undefined, + operation: () => Promise, +): Promise { + await mkdir(state, { recursive: true, mode: 0o700 }); + await validatePreparedOutputDir(state, undefined, true); + const path = join(state, `${key}.lock.sqlite`); + const metadata = await lstat(path).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return undefined; + throw error; + }); + if (metadata && !metadata.isFile()) + throw new CodexSecurityError("The publication lock is not a regular file."); + const require = createRequire(import.meta.url); + const Database = ( + process.versions["bun"] + ? require("bun:sqlite").Database + : require("node:sqlite").DatabaseSync + ) as new (path: string) => { exec(sql: string): void; close(): void }; + const guard = new Database(path); + try { + if (process.platform !== "win32") chmodSync(path, 0o600); + guard.exec("PRAGMA busy_timeout = 0"); + for (;;) { + signal?.throwIfAborted(); + try { + // Keep this inode across invocations. SQLite releases the transaction + // on process exit, so a paused publisher cannot lose its lock to a lease. + guard.exec("BEGIN EXCLUSIVE"); + break; + } catch (error) { + const sqliteError = error as { code?: string; errcode?: number }; + if (sqliteError.errcode !== 5 && sqliteError.code !== "SQLITE_BUSY") + throw error; + await delay(50, undefined, { signal }); + } + } + return await operation(); + } finally { + guard.close(); + } +} + export async function prepareExternalPublication( path: string, options: ExternalPublicationOptions, @@ -186,9 +242,8 @@ export async function prepareExternalPublication( const pendingPath = join(state, `${key}.pending.json`); let saved: SavedSubmission | undefined; try { - const content = JSON.parse( - await readFile(pendingPath, "utf8"), - ) as SavedSubmission; + const serialized = await readFile(pendingPath, "utf8"); + const content = JSON.parse(serialized) as SavedSubmission; if ( content.accountId !== credentials.account_id || !Array.isArray(content.requests) || @@ -221,7 +276,15 @@ export async function prepareExternalPublication( if (submission.repository.reset_marker !== destination.reset_marker) { // The old request is never sent after reset. A subsequent explicit // invocation prepares a fresh submission and asks for approval again. - await rm(pendingPath); + await withImportLock(state, key, dependencies.signal, async () => { + const current = await readFile(pendingPath, "utf8").catch( + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return undefined; + throw error; + }, + ); + if (current === serialized) await rm(pendingPath); + }); throw new CloudImportError( 409, "The repository was reset after this submission. The saved request was retired without uploading. Review the destination and run the command again to approve a fresh publication.", @@ -320,142 +383,143 @@ export async function prepareExternalPublication( return { preview, async publish() { - await mkdir(state, { recursive: true, mode: 0o700 }); - await validatePreparedOutputDir(state, undefined, true); - // Install a fully written file atomically. A crash during serialization - // must not leave a partial request that cannot be resumed. - const pendingTemporary = `${pendingPath}.${randomUUID()}.tmp`; - const pendingFile = await open(pendingTemporary, "wx", 0o600); - try { - await pendingFile.writeFile(JSON.stringify(submission)); - await pendingFile.sync(); - } finally { - await pendingFile.close(); - } - try { - await link(pendingTemporary, pendingPath); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; - const other = JSON.parse( - await readFile(pendingPath, "utf8"), - ) as SavedSubmission; - if (canonicalJson(other) !== canonicalJson(submission)) { - throw new CodexSecurityError( - "Another publication prepared this input. Run the command again to review and resume that saved request.", - ); + return await withImportLock(state, key, dependencies.signal, async () => { + // Install a fully written file atomically. A crash during serialization + // must not leave a partial request that cannot be resumed. + const pendingTemporary = `${pendingPath}.${randomUUID()}.tmp`; + const pendingFile = await open(pendingTemporary, "wx", 0o600); + try { + await pendingFile.writeFile(JSON.stringify(submission)); + await pendingFile.sync(); + } finally { + await pendingFile.close(); } - } finally { - await rm(pendingTemporary, { force: true }); - } - const receipts: FindingImportReceipt[] = []; - try { - for (const batch of requests) { - const receipt = validateImportReceipt( - await request("/finding_imports", batch), - ); - if ( - receipt.id !== batch.request_id || - canonicalJson(receipt.repository) !== - canonicalJson(batch.repository) || - canonicalJson(receipt.source) !== canonicalJson(batch.source) || - receipt.item_count !== batch.items.length || - receipt.results.length !== batch.items.length || - receipt.results.some( - (item, index) => item.client_id !== batch.items[index]!.client_id, - ) - ) + try { + await link(pendingTemporary, pendingPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + const other = JSON.parse( + await readFile(pendingPath, "utf8"), + ) as SavedSubmission; + if (canonicalJson(other) !== canonicalJson(submission)) { throw new CodexSecurityError( - "Cloud returned a receipt for a different publication.", + "Another publication prepared this input. Run the command again to review and resume that saved request.", ); - const actual = { created: 0, updated: 0, unchanged: 0, error: 0 }; - for (const result of receipt.results) { - actual[result.outcome]++; - if (result.outcome === "error") { - if ( - !result.error || - result.source_report_id !== null || - result.observation_id !== null || - result.canonical_finding_id !== null || - result.version !== null - ) - throw new CodexSecurityError( - "Cloud returned inconsistent item error fields.", - ); - } else if ( - result.error !== null || - !result.source_report_id || - !result.observation_id || - !result.canonical_finding_id || - result.version === null - ) - throw new CodexSecurityError( - "Cloud returned an incomplete saved finding.", - ); } - if (canonicalJson(receipt.counts) !== canonicalJson(actual)) - throw new CodexSecurityError( - "Cloud returned inconsistent publication counts.", - ); - receipts.push(receipt); + } finally { + await rm(pendingTemporary, { force: true }); } - // Verify readable source records without mistaking a newer concurrent - // observation for failure of the original, immutable import receipt. - for (const [batchIndex, receipt] of receipts.entries()) { - for (const [index, item] of receipt.results.entries()) { - if (item.outcome === "error") continue; - const report = validateSourceReport( - await request( - `${sourcePath(destination)}/${encodeURIComponent(item.source_report_id!)}`, - ), + const receipts: FindingImportReceipt[] = []; + try { + for (const batch of requests) { + const receipt = validateImportReceipt( + await request("/finding_imports", batch), ); if ( - report.canonical_finding_id !== item.canonical_finding_id || - report.version < item.version! || - (report.version === item.version && - (report.observation_id !== item.observation_id || - canonicalJson(report.evidence) !== - canonicalJson( - requests[batchIndex]!.items[index]!.evidence, - ))) || - report.source_finding_id !== - requests[batchIndex]!.items[index]!.source_finding_id || - report.repo_id !== destination.id || - report.repo_connector_id !== destination.repo_connector_id || - canonicalJson(report.source) !== canonicalJson(source) + receipt.id !== batch.request_id || + canonicalJson(receipt.repository) !== + canonicalJson(batch.repository) || + canonicalJson(receipt.source) !== canonicalJson(batch.source) || + receipt.item_count !== batch.items.length || + receipt.results.length !== batch.items.length || + receipt.results.some( + (item, index) => + item.client_id !== batch.items[index]!.client_id, + ) ) throw new CodexSecurityError( - "Cloud readback did not match the saved finding identity.", + "Cloud returned a receipt for a different publication.", ); + const actual = { created: 0, updated: 0, unchanged: 0, error: 0 }; + for (const result of receipt.results) { + actual[result.outcome]++; + if (result.outcome === "error") { + if ( + !result.error || + result.source_report_id !== null || + result.observation_id !== null || + result.canonical_finding_id !== null || + result.version !== null + ) + throw new CodexSecurityError( + "Cloud returned inconsistent item error fields.", + ); + } else if ( + result.error !== null || + !result.source_report_id || + !result.observation_id || + !result.canonical_finding_id || + result.version === null + ) + throw new CodexSecurityError( + "Cloud returned an incomplete saved finding.", + ); + } + if (canonicalJson(receipt.counts) !== canonicalJson(actual)) + throw new CodexSecurityError( + "Cloud returned inconsistent publication counts.", + ); + receipts.push(receipt); + } + // Verify readable source records without mistaking a newer concurrent + // observation for failure of the original, immutable import receipt. + for (const [batchIndex, receipt] of receipts.entries()) { + for (const [index, item] of receipt.results.entries()) { + if (item.outcome === "error") continue; + const report = validateSourceReport( + await request( + `${sourcePath(destination)}/${encodeURIComponent(item.source_report_id!)}`, + ), + ); + if ( + report.canonical_finding_id !== item.canonical_finding_id || + report.version < item.version! || + (report.version === item.version && + (report.observation_id !== item.observation_id || + canonicalJson(report.evidence) !== + canonicalJson( + requests[batchIndex]!.items[index]!.evidence, + ))) || + report.source_finding_id !== + requests[batchIndex]!.items[index]!.source_finding_id || + report.repo_id !== destination.id || + report.repo_connector_id !== destination.repo_connector_id || + canonicalJson(report.source) !== canonicalJson(source) + ) + throw new CodexSecurityError( + "Cloud readback did not match the saved finding identity.", + ); + } } + } catch (error) { + if (error instanceof CloudImportError && error.status === 409) + await rm(pendingPath, { force: true }); + throw new CodexSecurityError( + `${error instanceof Error ? error.message : String(error)} ${error instanceof CloudImportError && error.status === 409 ? "The old submission was retired. Rediscover and approve a fresh publication." : `Repeat the same command to resume the saved request. Saved submission: ${pendingPath}`}`, + { cause: error }, + ); } - } catch (error) { - if (error instanceof CloudImportError && error.status === 409) - await rm(pendingPath, { force: true }); - throw new CodexSecurityError( - `${error instanceof Error ? error.message : String(error)} ${error instanceof CloudImportError && error.status === 409 ? "The old submission was retired. Rediscover and approve a fresh publication." : `Repeat the same command to resume the saved request. Saved submission: ${pendingPath}`}`, - { cause: error }, - ); - } - const counts = { created: 0, updated: 0, unchanged: 0, error: 0 }; - for (const receipt of receipts) - for (const outcome of Object.keys(counts) as (keyof typeof counts)[]) - counts[outcome] += receipt.counts[outcome] ?? 0; - const result = { - receipts, - counts, - cloudUrl: `https://chatgpt.com/codex/cloud/security/findings?repo=${encodeURIComponent(destination.url)}&source=imported&provider=${source.provider}`, - }; - const temporary = `${pendingPath}.${randomUUID()}.tmp`; - const file = await open(temporary, "wx", 0o600); - try { - await file.writeFile(JSON.stringify(result)); - await file.sync(); - } finally { - await file.close(); - } - await rename(temporary, join(state, `${key}.result.json`)); - await rm(pendingPath, { force: true }); - return result; + const counts = { created: 0, updated: 0, unchanged: 0, error: 0 }; + for (const receipt of receipts) + for (const outcome of Object.keys(counts) as (keyof typeof counts)[]) + counts[outcome] += receipt.counts[outcome] ?? 0; + const result = { + receipts, + counts, + cloudUrl: `https://chatgpt.com/codex/cloud/security/findings?repo=${encodeURIComponent(destination.url)}&source=imported&provider=${source.provider}`, + }; + const temporary = `${pendingPath}.${randomUUID()}.tmp`; + const file = await open(temporary, "wx", 0o600); + try { + await file.writeFile(JSON.stringify(result)); + await file.sync(); + } finally { + await file.close(); + } + await rename(temporary, join(state, `${key}.result.json`)); + await rm(pendingPath, { force: true }); + return result; + }); }, }; } diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 4bfa5d86e9..2ee1f755fc 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -11,7 +11,8 @@ import { import { tmpdir } from "node:os"; import { join } from "node:path"; import { pathToFileURL } from "node:url"; -import { afterEach, expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import { afterEach, expect, spyOn, test } from "bun:test"; import { prepareExternalPublication } from "../src/external-findings-publish.js"; import { readVendorFindings } from "../src/wiz-findings.js"; import { validateImportRequest } from "../src/external-import-contract.js"; @@ -73,6 +74,7 @@ async function fixture(records: unknown = [normalized()]) { marker: "generation-1", environmentId: "environment-example" as string | null, materializeDefaults: false, + holdReadback: null as (() => void) | null, loseResponse: false, brokenReadback: false, throttle: false, @@ -124,6 +126,7 @@ async function fixture(records: unknown = [normalized()]) { }); } if (url.pathname.includes("/source_reports/")) { + if (state.holdReadback) return state.holdReadback(); if (state.brokenReadback) return json( response, @@ -458,44 +461,213 @@ test("interactive JSON publication shows the destination and exclusions before c expect(JSON.parse(cli.stdout.text()).counts.created).toBe(1); }); -test("saved request survives an actual publisher process restart after a committed response is lost", async () => { - const f = await fixture(); - const runner = join(f.root, "run-cli.ts"); - const cliUrl = pathToFileURL(join(import.meta.dir, "../src/cli.ts")).href; - const fixturesUrl = pathToFileURL( - join(import.meta.dir, "cli-fixtures.ts"), - ).href; - await writeFile( - runner, - `import { main } from ${JSON.stringify(cliUrl)};\nimport { dependencies } from ${JSON.stringify(fixturesUrl)};\nprocess.exitCode = await main(process.argv.slice(2), process.stdout, process.stderr, { ...dependencies({ environment: process.env }), cloudFetch: (url, init) => fetch('http://127.0.0.1:${f.port}' + new URL(url).pathname + new URL(url).search, init) });\n`, - ); - async function run() { - const process = Bun.spawn( - [Bun.which("bun")!, runner, ...f.command, "--yes"], - { env: f.environment, stdout: "pipe", stderr: "pipe" }, +test.each([false, true])( + "saved request survives a publisher process restart (abrupt: %p)", + async (abrupt) => { + const f = await fixture(); + const runner = join(f.root, "run-cli.ts"); + const cliUrl = pathToFileURL(join(import.meta.dir, "../src/cli.ts")).href; + const fixturesUrl = pathToFileURL( + join(import.meta.dir, "cli-fixtures.ts"), + ).href; + await writeFile( + runner, + `import { main } from ${JSON.stringify(cliUrl)};\nimport { dependencies } from ${JSON.stringify(fixturesUrl)};\nprocess.exitCode = await main(process.argv.slice(2), process.stdout, process.stderr, { ...dependencies({ environment: process.env }), cloudFetch: (url, init) => fetch('http://127.0.0.1:${f.port}' + new URL(url).pathname + new URL(url).search, init) });\n`, ); - const [code, stdout, stderr] = await Promise.all([ - process.exited, - new Response(process.stdout).text(), - new Response(process.stderr).text(), + function run() { + const process = Bun.spawn( + [Bun.which("bun")!, runner, ...f.command, "--yes"], + { env: f.environment, stdout: "pipe", stderr: "pipe" }, + ); + const result = Promise.all([ + process.exited, + new Response(process.stdout).text(), + new Response(process.stderr).text(), + ]).then(([code, stdout, stderr]) => ({ code, stdout, stderr })); + return { process, result }; + } + const readback = Promise.withResolvers(); + if (abrupt) f.state.holdReadback = () => readback.resolve(); + else f.state.loseResponse = true; + const first = run(); + if (abrupt) { + await readback.promise; + first.process.kill("SIGKILL"); + } + const lost = await first.result; + expect(lost.code).not.toBe(0); + if (!abrupt) { + expect(lost.code).toBe(2); + expect(lost.stderr).toContain("resume the saved request"); + } + expect(f.reports.size).toBe(1); + const firstBody = f.posts[0]; + f.state.holdReadback = null; + const second = await run().result; + expect(second.code).toBe(0); + expect(f.posts[1]).toBe(firstBody); + expect(f.receipts.size).toBe(1); + expect(JSON.parse(second.stdout).counts.created).toBe(1); + }, +); + +function observeLockContention() { + const contended = Promise.withResolvers(); + const original = Database.prototype.exec; + const spy = spyOn(Database.prototype, "exec").mockImplementation(function ( + this: Database, + ...args: Parameters + ) { + try { + return original.apply(this, args); + } catch (error) { + if ( + args[0] === "BEGIN EXCLUSIVE" && + (error as { code?: string }).code === "SQLITE_BUSY" + ) + contended.resolve(); + throw error; + } + }); + return { contended: contended.promise, restore: () => spy.mockRestore() }; +} + +async function pauseAtReadback(f: Awaited>) { + const arrived = Promise.withResolvers(); + const release = Promise.withResolvers(); + const completion = ( + await prepareExternalPublication(f.file, options, { + ...f.deps, + fetch: async (url, init) => { + if (new URL(url).pathname.includes("/source_reports/")) { + arrived.resolve(); + await release.promise; + } + return f.deps.fetch(url, init); + }, + }) + ).publish(); + await Promise.race([ + arrived.promise, + completion.then(() => { + throw new Error("Publication completed without a source readback."); + }), + ]); + return { completion, release }; +} + +test("overlapping retries cannot delete a newer pending submission or roll back newer evidence", async () => { + const f = await fixture(); + f.state.brokenReadback = true; + await expect( + (await prepareExternalPublication(f.file, options, f.deps)).publish(), + ).rejects.toThrow("resume the saved request"); + f.state.brokenReadback = false; + const { completion: first, release } = await pauseAtReadback(f); + const observer = observeLockContention(); + const second = ( + await prepareExternalPublication(f.file, options, f.deps) + ).publish(); + try { + await Promise.race([ + observer.contended, + second.then(() => { + throw new Error("Concurrent publication skipped the active lock."); + }), ]); - return { code, stdout, stderr }; + expect(f.posts).toHaveLength(2); + release.resolve(); + await Promise.all([first, second]); + } finally { + release.resolve(); + observer.restore(); + await Promise.allSettled([first, second]); } + const newer = await prepareExternalPublication(f.file, options, f.deps); f.state.loseResponse = true; - const lost = await run(); - expect(lost.code).toBe(2); - expect({ - size: f.reports.size, - error: lost.stderr.includes("resume the saved request") - ? "resumable" - : lost.stderr, - }).toEqual({ size: 1, error: "resumable" }); - const firstBody = f.posts[0]; - const second = await run(); - expect(second.code).toBe(0); - expect(f.posts[1]).toBe(firstBody); - expect(f.receipts.size).toBe(1); - expect(JSON.parse(second.stdout).counts.created).toBe(1); + await expect(newer.publish()).rejects.toThrow("resume the saved request"); + const acceptedBody = f.posts.at(-1); + await writeFile(f.file, JSON.stringify([normalized("vendor-1", "critical")])); + await (await prepareExternalPublication(f.file, options, f.deps)).publish(); + await writeFile(f.file, JSON.stringify([normalized()])); + const retry = await prepareExternalPublication(f.file, options, f.deps); + expect(retry.preview.resumed).toBe(true); + expect(retry.preview.requests[0]!.request_id).toBe( + newer.preview.requests[0]!.request_id, + ); + await retry.publish(); + expect(f.posts.at(-1)).toBe(acceptedBody); + expect(f.reports.get("vendor-1")?.evidence.severity).toBe("critical"); +}); + +test("reset retirement waits for the active publisher before removing its saved request", async () => { + const f = await fixture(); + const { completion: active, release } = await pauseAtReadback(f); + f.state.marker = "generation-2"; + f.state.environmentId = "replacement-environment"; + f.state.brokenReadback = true; + const observer = observeLockContention(); + const reset = prepareExternalPublication(f.file, options, f.deps); + const resetResult = reset.then( + () => "unexpected-success", + (error: Error) => error.message, + ); + try { + await Promise.race([ + observer.contended, + resetResult.then(() => { + throw new Error("Reset skipped the active publication lock."); + }), + ]); + expect(f.posts).toHaveLength(1); + release.resolve(); + await expect(active).rejects.toThrow("resume the saved request"); + expect(await resetResult).toContain("retired without uploading"); + } finally { + release.resolve(); + observer.restore(); + await Promise.allSettled([active, resetResult]); + } + f.state.brokenReadback = false; + f.reports.clear(); + const fresh = await prepareExternalPublication(f.file, options, f.deps); + expect(fresh.preview.resumed).toBe(false); + expect((await fresh.publish()).counts.created).toBe(1); +}); + +test("canceling a waiting publisher preserves the active request", async () => { + const f = await fixture(); + const { completion: active, release } = await pauseAtReadback(f); + const controller = new AbortController(); + const observer = observeLockContention(); + const waiting = ( + await prepareExternalPublication(f.file, options, { + ...f.deps, + signal: controller.signal, + }) + ).publish(); + try { + await Promise.race([ + observer.contended, + waiting.then(() => { + throw new Error("Concurrent publication skipped the active lock."); + }), + ]); + controller.abort(); + await expect(waiting).rejects.toMatchObject({ name: "AbortError" }); + expect(f.posts).toHaveLength(1); + const retry = await prepareExternalPublication(f.file, options, f.deps); + expect(retry.preview.resumed).toBe(true); + expect(retry.preview.requests[0]!.request_id).toBe( + JSON.parse(f.posts[0]!).request_id, + ); + release.resolve(); + expect((await active).counts.created).toBe(1); + } finally { + release.resolve(); + observer.restore(); + await Promise.allSettled([active, waiting]); + } }); test("readback failure retains the accepted request; a retry replays its outcome", async () => { From e6a2d4849ffbcb6e00f9e9087e0b66509706b635 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 8 Oct 2026 10:41:10 +0000 Subject: [PATCH 04/43] fix(cloud): align import response cursor limits --- sdk/typescript/schemas/external-findings.schema.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/typescript/schemas/external-findings.schema.json b/sdk/typescript/schemas/external-findings.schema.json index d08dd510ec..9ba52049ef 100644 --- a/sdk/typescript/schemas/external-findings.schema.json +++ b/sdk/typescript/schemas/external-findings.schema.json @@ -360,6 +360,7 @@ "next": { "anyOf": [ { + "maxLength": 4096, "type": "string" }, { @@ -785,6 +786,7 @@ "next": { "anyOf": [ { + "maxLength": 4096, "type": "string" }, { From b5ef54c861d1c1c79b5287b2e1d9469bc74b85a2 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 8 Oct 2026 17:18:14 +0000 Subject: [PATCH 05/43] fix(cloud): checkpoint completed finding import batches --- .../src/external-findings-publish.ts | 60 ++++++- .../external-findings-publish.test.ts | 152 ++++++++++++++++-- 2 files changed, 193 insertions(+), 19 deletions(-) diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 3bc6c9b974..08b759f2cc 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -53,6 +53,7 @@ export interface ExternalPublicationDependencies { interface SavedSubmission { accountId: string; requests: FindingImportRequest[]; + receipts?: FindingImportReceipt[]; } export interface ExternalPublicationPreview extends VendorFindings { @@ -83,6 +84,35 @@ function canonicalJson(value: unknown): string { }); } +function sameSubmission( + left: SavedSubmission, + right: SavedSubmission, +): boolean { + return ( + left.accountId === right.accountId && + canonicalJson(left.requests) === canonicalJson(right.requests) + ); +} + +async function checkpointSubmission( + path: string, + submission: SavedSubmission, +): Promise { + const temporary = `${path}.${randomUUID()}.tmp`; + try { + const file = await open(temporary, "wx", 0o600); + try { + await file.writeFile(JSON.stringify(submission)); + await file.sync(); + } finally { + await file.close(); + } + await rename(temporary, path); + } finally { + await rm(temporary, { force: true }); + } +} + function sourcePath(repository: ImportRepository): string { return `/repository_connectors/${encodeURIComponent(repository.repo_connector_id)}/repositories/${encodeURIComponent(repository.id)}/source_reports`; } @@ -283,7 +313,11 @@ export async function prepareExternalPublication( throw error; }, ); - if (current === serialized) await rm(pendingPath); + if ( + current !== undefined && + sameSubmission(JSON.parse(current) as SavedSubmission, content) + ) + await rm(pendingPath); }); throw new CloudImportError( 409, @@ -394,6 +428,7 @@ export async function prepareExternalPublication( } finally { await pendingFile.close(); } + let receipts: FindingImportReceipt[] = []; try { await link(pendingTemporary, pendingPath); } catch (error) { @@ -401,19 +436,24 @@ export async function prepareExternalPublication( const other = JSON.parse( await readFile(pendingPath, "utf8"), ) as SavedSubmission; - if (canonicalJson(other) !== canonicalJson(submission)) { + if (!sameSubmission(other, submission)) { throw new CodexSecurityError( "Another publication prepared this input. Run the command again to review and resume that saved request.", ); } + receipts = (other.receipts ?? []).map(validateImportReceipt); + if (receipts.length > requests.length) + throw new CodexSecurityError( + "Saved publication has more receipts than requests.", + ); } finally { await rm(pendingTemporary, { force: true }); } - const receipts: FindingImportReceipt[] = []; try { - for (const batch of requests) { + for (const [batchIndex, batch] of requests.entries()) { const receipt = validateImportReceipt( - await request("/finding_imports", batch), + receipts[batchIndex] ?? + (await request("/finding_imports", batch)), ); if ( receipt.id !== batch.request_id || @@ -459,7 +499,15 @@ export async function prepareExternalPublication( throw new CodexSecurityError( "Cloud returned inconsistent publication counts.", ); - receipts.push(receipt); + if (batchIndex === receipts.length) { + receipts.push(receipt); + // Acknowledged batches must not consume another POST quota on retry. + // Readback can still resume after this checkpoint without replaying writes. + await checkpointSubmission(pendingPath, { + ...submission, + receipts, + }); + } } // Verify readable source records without mistaking a newer concurrent // observation for failure of the original, immutable import receipt. diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 2ee1f755fc..f2e1c14a21 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -1,5 +1,6 @@ import { createServer, type ServerResponse } from "node:http"; import { hash } from "node:crypto"; +import * as fs from "node:fs/promises"; import { mkdtemp, mkdir, @@ -78,6 +79,7 @@ async function fixture(records: unknown = [normalized()]) { loseResponse: false, brokenReadback: false, throttle: false, + postBudget: Infinity, finalError: false, }; const destination = () => ({ @@ -159,10 +161,11 @@ async function fixture(records: unknown = [normalized()]) { for await (const chunk of incoming) body += chunk; posts.push(body); const request = validateImportRequest(JSON.parse(body)); - if (state.throttle) { + if (state.throttle || state.postBudget === 0) { response.setHeader("Retry-After", "1"); return json(response, { error: { message: "Slow down" } }, 429); } + state.postBudget--; if (request.repository.reset_marker !== state.marker) return json( response, @@ -505,7 +508,8 @@ test.each([false, true])( f.state.holdReadback = null; const second = await run().result; expect(second.code).toBe(0); - expect(f.posts[1]).toBe(firstBody); + if (abrupt) expect(f.posts).toHaveLength(1); + else expect(f.posts[1]).toBe(firstBody); expect(f.receipts.size).toBe(1); expect(JSON.parse(second.stdout).counts.created).toBe(1); }, @@ -532,25 +536,34 @@ function observeLockContention() { return { contended: contended.promise, restore: () => spy.mockRestore() }; } -async function pauseAtReadback(f: Awaited>) { +async function pauseAtPublication( + f: Awaited>, + stage: "receipt" | "readback" = "readback", +) { const arrived = Promise.withResolvers(); const release = Promise.withResolvers(); const completion = ( await prepareExternalPublication(f.file, options, { ...f.deps, fetch: async (url, init) => { - if (new URL(url).pathname.includes("/source_reports/")) { + const response = + stage === "receipt" ? await f.deps.fetch(url, init) : undefined; + if ( + (stage === "receipt" && init.method === "POST") || + (stage === "readback" && + new URL(url).pathname.includes("/source_reports/")) + ) { arrived.resolve(); await release.promise; } - return f.deps.fetch(url, init); + return response ?? f.deps.fetch(url, init); }, }) ).publish(); await Promise.race([ arrived.promise, completion.then(() => { - throw new Error("Publication completed without a source readback."); + throw new Error("Publication completed without reaching the pause."); }), ]); return { completion, release }; @@ -563,7 +576,7 @@ test("overlapping retries cannot delete a newer pending submission or roll back (await prepareExternalPublication(f.file, options, f.deps)).publish(), ).rejects.toThrow("resume the saved request"); f.state.brokenReadback = false; - const { completion: first, release } = await pauseAtReadback(f); + const { completion: first, release } = await pauseAtPublication(f); const observer = observeLockContention(); const second = ( await prepareExternalPublication(f.file, options, f.deps) @@ -575,7 +588,7 @@ test("overlapping retries cannot delete a newer pending submission or roll back throw new Error("Concurrent publication skipped the active lock."); }), ]); - expect(f.posts).toHaveLength(2); + expect(f.posts).toHaveLength(1); release.resolve(); await Promise.all([first, second]); } finally { @@ -600,9 +613,12 @@ test("overlapping retries cannot delete a newer pending submission or roll back expect(f.reports.get("vendor-1")?.evidence.severity).toBe("critical"); }); -test("reset retirement waits for the active publisher before removing its saved request", async () => { +test("reset retirement tolerates a checkpoint written while waiting for the active publisher", async () => { const f = await fixture(); - const { completion: active, release } = await pauseAtReadback(f); + const { completion: active, release } = await pauseAtPublication( + f, + "receipt", + ); f.state.marker = "generation-2"; f.state.environmentId = "replacement-environment"; f.state.brokenReadback = true; @@ -637,7 +653,7 @@ test("reset retirement waits for the active publisher before removing its saved test("canceling a waiting publisher preserves the active request", async () => { const f = await fixture(); - const { completion: active, release } = await pauseAtReadback(f); + const { completion: active, release } = await pauseAtPublication(f); const controller = new AbortController(); const observer = observeLockContention(); const waiting = ( @@ -670,7 +686,7 @@ test("canceling a waiting publisher preserves the active request", async () => { } }); -test("readback failure retains the accepted request; a retry replays its outcome", async () => { +test("readback failure retries acknowledged receipts without another POST", async () => { const f = await fixture(); f.state.brokenReadback = true; const prepared = await prepareExternalPublication(f.file, options, f.deps); @@ -679,7 +695,7 @@ test("readback failure retains the accepted request; a retry replays its outcome const retry = await prepareExternalPublication(f.file, options, f.deps); expect(retry.preview.resumed).toBe(true); expect((await retry.publish()).counts.created).toBe(1); - expect(f.posts[0]).toBe(f.posts[1]); + expect(f.posts).toHaveLength(1); }); test("an older pending receipt can resume after newer evidence without rolling it back", async () => { @@ -783,3 +799,113 @@ test("local receipts contain no credential and pending bodies match transmitted ), ).toBe(f.posts[0]!); }); + +test("a restarted large publication resumes after the ten-request rate window", async () => { + const f = await fixture( + Array.from({ length: 1001 }, (_, i) => normalized(`bulk-${i}`)), + ); + const runner = join(f.root, "run-batched-cli.ts"); + const cliUrl = pathToFileURL(join(import.meta.dir, "../src/cli.ts")).href; + const fixturesUrl = pathToFileURL( + join(import.meta.dir, "cli-fixtures.ts"), + ).href; + await writeFile( + runner, + `import { main } from ${JSON.stringify(cliUrl)};\nimport { dependencies } from ${JSON.stringify(fixturesUrl)};\nprocess.exitCode = await main(process.argv.slice(2), process.stdout, process.stderr, { ...dependencies({ environment: process.env }), cloudFetch: (url, init) => fetch('http://127.0.0.1:${f.port}' + new URL(url).pathname + new URL(url).search, init) });\n`, + ); + async function run() { + const child = Bun.spawn( + [Bun.which("bun")!, runner, ...f.command, "--yes"], + { + env: f.environment, + stdout: "pipe", + stderr: "pipe", + }, + ); + const [code, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]); + return { code, stdout, stderr }; + } + f.state.postBudget = 10; + const first = await run(); + expect(first.code).toBe(2); + expect(first.stderr).toContain("HTTP 429"); + expect(f.reports.size).toBe(1000); + expect(f.posts).toHaveLength(11); + const rejectedBody = f.posts[10]; + f.state.postBudget = 10; + const second = await run(); + expect(second.code).toBe(0); + expect(JSON.parse(second.stdout).counts.created).toBe(1001); + expect(f.posts).toHaveLength(12); + expect(f.posts[11]).toBe(rejectedBody); + expect(f.reports.size).toBe(1001); + expect(f.receipts.size).toBe(11); +}); + +test("a failed checkpoint leaves the original pending request recoverable", async () => { + const f = await fixture(); + const rename = fs.rename; + const checkpoint = spyOn(fs, "rename").mockImplementation( + async (source, destination) => { + if (String(destination).endsWith(".pending.json")) + throw new Error("Checkpoint interrupted"); + return rename(source, destination); + }, + ); + try { + await expect( + (await prepareExternalPublication(f.file, options, f.deps)).publish(), + ).rejects.toThrow("Checkpoint interrupted"); + } finally { + checkpoint.mockRestore(); + } + const directory = join(f.root, "state", "external-finding-publications"); + const name = (await readdir(directory)).find((name) => + name.endsWith(".pending.json"), + )!; + const saved = JSON.parse(await readFile(join(directory, name), "utf8")); + expect(saved.receipts).toBeUndefined(); + expect(JSON.stringify(saved.requests[0])).toBe(f.posts[0]!); + await writeFile(join(directory, `${name}.orphan.tmp`), "{"); + expect( + ( + await ( + await prepareExternalPublication(f.file, options, f.deps) + ).publish() + ).counts.created, + ).toBe(1); + expect(f.posts).toHaveLength(2); + expect(f.posts[1]).toBe(f.posts[0]); + expect(f.receipts.size).toBe(1); +}); + +test.each(["request", "counts", "length"])( + "cached receipts retain publication validation (%s)", + async (invalid) => { + const f = await fixture(); + f.state.brokenReadback = true; + await expect( + (await prepareExternalPublication(f.file, options, f.deps)).publish(), + ).rejects.toThrow("resume the saved request"); + const directory = join(f.root, "state", "external-finding-publications"); + const name = (await readdir(directory)).find((name) => + name.endsWith(".pending.json"), + )!; + const pending = join(directory, name); + const saved = JSON.parse(await readFile(pending, "utf8")); + if (invalid === "request") + saved.receipts[0].id = "00000000-0000-4000-8000-000000000000"; + if (invalid === "counts") saved.receipts[0].counts.error = 1; + if (invalid === "length") saved.receipts.push(saved.receipts[0]); + await writeFile(pending, JSON.stringify(saved)); + f.state.brokenReadback = false; + await expect( + (await prepareExternalPublication(f.file, options, f.deps)).publish(), + ).rejects.toThrow(); + expect(f.posts).toHaveLength(1); + }, +); From d27aea029cb3ddb4ab6f32fa6917aec100b8086f Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 8 Oct 2026 17:41:22 +0000 Subject: [PATCH 06/43] fix(cloud): retain checkpoints across overlapping retries --- .../src/external-findings-publish.ts | 10 +++-- .../external-findings-publish.test.ts | 39 ++++++++++++++++++- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 08b759f2cc..b887563ee9 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -405,6 +405,7 @@ export async function prepareExternalPublication( const submission: SavedSubmission = { accountId: credentials.account_id, requests, + receipts: saved?.receipts, }; const preview: ExternalPublicationPreview = { ...parsed, @@ -430,6 +431,7 @@ export async function prepareExternalPublication( } let receipts: FindingImportReceipt[] = []; try { + receipts = (submission.receipts ?? []).map(validateImportReceipt); await link(pendingTemporary, pendingPath); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; @@ -442,13 +444,13 @@ export async function prepareExternalPublication( ); } receipts = (other.receipts ?? []).map(validateImportReceipt); - if (receipts.length > requests.length) - throw new CodexSecurityError( - "Saved publication has more receipts than requests.", - ); } finally { await rm(pendingTemporary, { force: true }); } + if (receipts.length > requests.length) + throw new CodexSecurityError( + "Saved publication has more receipts than requests.", + ); try { for (const [batchIndex, batch] of requests.entries()) { const receipt = validateImportReceipt( diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index f2e1c14a21..855507df81 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -591,6 +591,7 @@ test("overlapping retries cannot delete a newer pending submission or roll back expect(f.posts).toHaveLength(1); release.resolve(); await Promise.all([first, second]); + expect(f.posts).toHaveLength(1); } finally { release.resolve(); observer.restore(); @@ -883,7 +884,7 @@ test("a failed checkpoint leaves the original pending request recoverable", asyn expect(f.receipts.size).toBe(1); }); -test.each(["request", "counts", "length"])( +test.each(["request", "counts", "length", "schema"])( "cached receipts retain publication validation (%s)", async (invalid) => { const f = await fixture(); @@ -901,11 +902,47 @@ test.each(["request", "counts", "length"])( saved.receipts[0].id = "00000000-0000-4000-8000-000000000000"; if (invalid === "counts") saved.receipts[0].counts.error = 1; if (invalid === "length") saved.receipts.push(saved.receipts[0]); + if (invalid === "schema") saved.receipts[0].source.provider = "unsupported"; await writeFile(pending, JSON.stringify(saved)); f.state.brokenReadback = false; await expect( (await prepareExternalPublication(f.file, options, f.deps)).publish(), ).rejects.toThrow(); expect(f.posts).toHaveLength(1); + expect( + (await readdir(directory)).filter((name) => name.endsWith(".tmp")), + ).toEqual([]); + }, +); + +test.each([1, 101])( + "a prepared retry keeps its receipts after another resumer completes (%i items)", + async (count) => { + const f = await fixture( + Array.from({ length: count }, (_, index) => normalized(`item-${index}`)), + ); + f.state.postBudget = 1; + f.state.brokenReadback = true; + await expect( + (await prepareExternalPublication(f.file, options, f.deps)).publish(), + ).rejects.toThrow("resume the saved request"); + f.state.brokenReadback = false; + const first = await prepareExternalPublication(f.file, options, f.deps); + const second = await prepareExternalPublication(f.file, options, f.deps); + f.state.postBudget = Infinity; + expect((await first.publish()).counts.created).toBe(count); + const completedPosts = f.posts.length; + f.state.postBudget = count === 1 ? 0 : 1; + expect((await second.publish()).counts.created).toBe(count); + expect(f.posts.slice(completedPosts)).toEqual( + count === 1 ? [] : [JSON.stringify(second.preview.requests[1])], + ); + const fresh = await prepareExternalPublication(f.file, options, f.deps); + expect(fresh.preview.resumed).toBe(false); + expect(fresh.preview.requests[0]!.request_id).not.toBe( + first.preview.requests[0]!.request_id, + ); + f.state.postBudget = Infinity; + expect((await fresh.publish()).counts.unchanged).toBe(count); }, ); From 30b2e95cd1ff663866a04783865add91e24a62c0 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 8 Oct 2026 18:05:47 +0000 Subject: [PATCH 07/43] fix(cloud): validate imports before approval and simplify persistence --- .../src/external-findings-publish.ts | 75 +++----- .../src/external-import-contract.ts | 87 ++++++++- sdk/typescript/src/wiz-findings.ts | 17 +- .../external-findings-publish.test.ts | 181 +++++++++++++++++- 4 files changed, 298 insertions(+), 62 deletions(-) diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index b887563ee9..0eeb14b19c 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -1,14 +1,6 @@ import { hash, randomUUID } from "node:crypto"; import { chmodSync } from "node:fs"; -import { - link, - lstat, - mkdir, - open, - readFile, - rename, - rm, -} from "node:fs/promises"; +import { lstat, mkdir, open, readFile, rename, rm } from "node:fs/promises"; import { createRequire } from "node:module"; import { join } from "node:path"; import { setTimeout as delay } from "node:timers/promises"; @@ -94,15 +86,12 @@ function sameSubmission( ); } -async function checkpointSubmission( - path: string, - submission: SavedSubmission, -): Promise { +async function writeAtomicJson(path: string, value: unknown): Promise { const temporary = `${path}.${randomUUID()}.tmp`; try { const file = await open(temporary, "wx", 0o600); try { - await file.writeFile(JSON.stringify(submission)); + await file.writeFile(JSON.stringify(value)); await file.sync(); } finally { await file.close(); @@ -419,38 +408,28 @@ export async function prepareExternalPublication( preview, async publish() { return await withImportLock(state, key, dependencies.signal, async () => { - // Install a fully written file atomically. A crash during serialization - // must not leave a partial request that cannot be resumed. - const pendingTemporary = `${pendingPath}.${randomUUID()}.tmp`; - const pendingFile = await open(pendingTemporary, "wx", 0o600); - try { - await pendingFile.writeFile(JSON.stringify(submission)); - await pendingFile.sync(); - } finally { - await pendingFile.close(); - } - let receipts: FindingImportReceipt[] = []; - try { - receipts = (submission.receipts ?? []).map(validateImportReceipt); - await link(pendingTemporary, pendingPath); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; - const other = JSON.parse( - await readFile(pendingPath, "utf8"), - ) as SavedSubmission; - if (!sameSubmission(other, submission)) { - throw new CodexSecurityError( - "Another publication prepared this input. Run the command again to review and resume that saved request.", - ); - } - receipts = (other.receipts ?? []).map(validateImportReceipt); - } finally { - await rm(pendingTemporary, { force: true }); - } + // Re-read under the lock to keep checkpoints from another publisher. + const pending = await readFile(pendingPath, "utf8").catch( + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return undefined; + throw error; + }, + ); + const current = + pending === undefined + ? submission + : (JSON.parse(pending) as SavedSubmission); + if (!sameSubmission(current, submission)) + throw new CodexSecurityError( + "Another publication prepared this input. Run the command again to review and resume that saved request.", + ); + const receipts = (current.receipts ?? []).map(validateImportReceipt); if (receipts.length > requests.length) throw new CodexSecurityError( "Saved publication has more receipts than requests.", ); + if (pending === undefined) + await writeAtomicJson(pendingPath, submission); try { for (const [batchIndex, batch] of requests.entries()) { const receipt = validateImportReceipt( @@ -505,7 +484,7 @@ export async function prepareExternalPublication( receipts.push(receipt); // Acknowledged batches must not consume another POST quota on retry. // Readback can still resume after this checkpoint without replaying writes. - await checkpointSubmission(pendingPath, { + await writeAtomicJson(pendingPath, { ...submission, receipts, }); @@ -558,15 +537,7 @@ export async function prepareExternalPublication( counts, cloudUrl: `https://chatgpt.com/codex/cloud/security/findings?repo=${encodeURIComponent(destination.url)}&source=imported&provider=${source.provider}`, }; - const temporary = `${pendingPath}.${randomUUID()}.tmp`; - const file = await open(temporary, "wx", 0o600); - try { - await file.writeFile(JSON.stringify(result)); - await file.sync(); - } finally { - await file.close(); - } - await rename(temporary, join(state, `${key}.result.json`)); + await writeAtomicJson(join(state, `${key}.result.json`), result); await rm(pendingPath, { force: true }); return result; }); diff --git a/sdk/typescript/src/external-import-contract.ts b/sdk/typescript/src/external-import-contract.ts index 58bdb01167..3cb03a3e12 100644 --- a/sdk/typescript/src/external-import-contract.ts +++ b/sdk/typescript/src/external-import-contract.ts @@ -14,6 +14,45 @@ const ajv = new Ajv2020({ validateFormats: false, useDefaults: true, }); +const STRING_BYTE_LIMITS: Readonly> = { + description: 65536, + path: 2048, + url: 2048, + manifest_path: 2048, + client_id: 128, + next: 4096, +}; + +// These model constraints are not represented by JSON Schema character limits. +function validateModelStrings(input: unknown): void { + if (input === null || typeof input !== "object") return; + if (Array.isArray(input)) { + for (const item of input) validateModelStrings(item); + return; + } + for (const [field, value] of Object.entries(input)) { + if (field === "source_data") continue; + if (typeof value === "string") { + if (!value.trim() || value.includes("\0")) + throw new Error( + `${field} must be nonempty and contain no NUL characters.`, + ); + const limit = STRING_BYTE_LIMITS[field] ?? 512; + if (Buffer.byteLength(value) > limit) + throw new Error( + `${field} exceeds its Cloud limit of ${limit} UTF-8 bytes.`, + ); + } else if ( + Array.isArray(value) && + value.every((item) => typeof item === "string") + ) { + if (value.some((item) => Buffer.byteLength(item) > 512)) + throw new Error( + `${field} entries exceed the Cloud limit of 512 UTF-8 bytes.`, + ); + } else validateModelStrings(value); + } +} function validator(name: keyof typeof schema.$defs) { const check = ajv.compile({ @@ -23,6 +62,7 @@ function validator(name: keyof typeof schema.$defs) { return (input: unknown): T => { if (!check(input)) throw new Error(`${name}: ${ajv.errorsText(check.errors)}`); + validateModelStrings(input); return input; }; } @@ -43,8 +83,41 @@ export function validateExternalEvidence( input: unknown, ): ExternalFindingEvidence { const result = evidence(input); - if (!result.title.trim()) - throw new Error("Evidence title must not be empty."); + for (const location of result.locations ?? []) { + const path = location.path.replaceAll("\\", "/"); + if ( + path.startsWith("/") || + path.split("/")[0]!.includes(":") || + path.split("/").includes("..") + ) + throw new Error("Locations must use repository-relative paths."); + } + if (result.url !== null && result.url !== undefined) { + const url = result.url + .replace(/[\t\r\n]/gu, "") + .replace(/^[\u0000-\u0020]+/u, ""); + const authority = /^https?:\/\/([^/?#]+)/iu.exec(url)?.[1]; + if ( + !authority || + authority.includes("@") || + authority.includes("[") !== authority.includes("]") + ) + throw new Error( + "Evidence URLs must be HTTP(S) links without credentials.", + ); + } + const pending: [unknown, number][] = [[result, 0]]; + while (pending.length > 0) { + const [value, depth] = pending.pop()!; + if (depth > 20) + throw new Error("Evidence exceeds the Cloud JSON depth limit of 20."); + if (value !== null && typeof value === "object") { + for (const child of Object.values(value)) + pending.push([child, depth + 1]); + } else if (typeof value === "number" && !Number.isFinite(value)) { + throw new Error("Evidence must contain finite JSON numbers."); + } + } if (Buffer.byteLength(JSON.stringify(result)) > 256 * 1024) throw new Error("Evidence exceeds the Cloud limit of 256 KiB."); return result; @@ -52,6 +125,16 @@ export function validateExternalEvidence( export function validateImportRequest(input: unknown): FindingImportRequest { const result = request(input); + if ( + new Set(result.items.map((item) => item.client_id)).size !== + result.items.length + ) + throw new Error("Import contains duplicate client IDs."); + if ( + new Set(result.items.map((item) => item.source_finding_id)).size !== + result.items.length + ) + throw new Error("Import contains duplicate source finding identities."); for (const item of result.items) validateExternalEvidence(item.evidence); if (Buffer.byteLength(JSON.stringify(result)) > 5 * 1024 * 1024) throw new Error("Import exceeds the Cloud limit of 5 MiB."); diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts index 5c79088e79..4640bfa265 100644 --- a/sdk/typescript/src/wiz-findings.ts +++ b/sdk/typescript/src/wiz-findings.ts @@ -28,10 +28,18 @@ function text(value: unknown): string | null { return typeof value === "string" && value.trim() ? value : null; } +function sourceFindingId(value: unknown): string { + const id = text(value); + if (!id || id.includes("\0") || Buffer.byteLength(id) > 512) + throw new Error( + "Source finding id must contain 1–512 UTF-8 bytes and no NUL characters.", + ); + return id; +} + /** Keep a Wiz vulnerability occurrence intact; advisory names are not identities. */ function wizFinding(record: Record): VendorFinding { - const id = text(record["id"]); - if (!id) throw new Error("Wiz vulnerability finding is missing its id."); + const id = sourceFindingId(record["id"]); const name = text(record["name"]); const packageName = text(record["detailedName"]); const version = text(record["version"]); @@ -165,11 +173,8 @@ export async function readVendorFindings( "Normalized input permits only source_finding_id and evidence.", ); } - const id = text(record["source_finding_id"]); - if (!id || Buffer.byteLength(id) > 512) - throw new Error("Source finding id must contain 1–512 UTF-8 bytes."); finding = { - source_finding_id: id, + source_finding_id: sourceFindingId(record["source_finding_id"]), evidence: validateExternalEvidence(record["evidence"]), }; } else { diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 855507df81..2c83cea070 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -16,7 +16,11 @@ import { Database } from "bun:sqlite"; import { afterEach, expect, spyOn, test } from "bun:test"; import { prepareExternalPublication } from "../src/external-findings-publish.js"; import { readVendorFindings } from "../src/wiz-findings.js"; -import { validateImportRequest } from "../src/external-import-contract.js"; +import { + validateExternalEvidence, + validateImportRequest, + validateRepositories, +} from "../src/external-import-contract.js"; import type { FindingImportReceipt, FindingImportRequest, @@ -357,6 +361,142 @@ test("unsupported records are visible exclusions; duplicate identities and unfin await expect(readVendorFindings(f.file)).rejects.toThrow("another page"); }); +test("Cloud evidence constraints exclude invalid normalized records before approval", async () => { + let nested: unknown = "leaf"; + for (let index = 0; index < 20; index++) nested = { child: nested }; + const invalid = [ + { title: "é".repeat(300) }, + { description: "" }, + { packages: [{ name: "example", installed_version: "1.0\0" }] }, + { advisory_ids: ["é".repeat(300)] }, + { locations: [{ path: "../outside.ts" }] }, + { url: "https://user@example.test/finding" }, + { source_data: { content: "x".repeat(6 * 1024 * 1024) } }, + { source_data: { nested } }, + { url: "https://[example.test" }, + ].map((evidence, index) => ({ + source_finding_id: `invalid-${index}`, + evidence: { ...normalized().evidence, ...evidence }, + })); + const f = await fixture([normalized(), ...invalid]); + const cli = createCliTest(main); + expect(await cli.runCli([...f.command, "--dry-run"], f.cliDeps)).toBe(0); + const preview = JSON.parse(cli.stdout.text()); + expect(preview.findings).toHaveLength(1); + expect(preview.excluded).toHaveLength(invalid.length); + expect(preview.excluded[6].reason).toContain("256 KiB"); + expect(f.posts).toHaveLength(0); + await writeFile(f.file, JSON.stringify(invalid)); + const rejected = createCliTest(main); + expect(await rejected.runCli([...f.command, "--yes"], f.cliDeps)).toBe(2); + expect(rejected.stderr.text()).toContain("No supported findings"); + expect(f.posts).toHaveLength(0); + expect(await readdir(join(f.root, "state")).catch(() => [])).toEqual([]); +}); + +test("raw Wiz records apply Cloud string and URL constraints before approval", async () => { + const wiz = { + id: "wiz-valid", + name: "CVE-2099-0001", + detailedName: "example-package", + vendorSeverity: "HIGH", + vulnerableAsset: { id: "example-image" }, + }; + const f = await fixture([ + wiz, + { ...wiz, id: "wiz-title", title: "é".repeat(300) }, + { ...wiz, id: "é".repeat(300) }, + { ...wiz, id: "wiz\0id" }, + { ...wiz, id: "wiz-url", portalUrl: "https://@example.test" }, + ]); + const cli = createCliTest(main); + expect(await cli.runCli([...f.command, "--dry-run"], f.cliDeps)).toBe(0); + const preview = JSON.parse(cli.stdout.text()); + expect(preview.findings).toHaveLength(1); + expect(preview.excluded).toHaveLength(4); + expect(f.posts).toHaveLength(0); +}); + +test("Cloud validation preserves allowed evidence boundaries and raw vendor strings", () => { + let nested: unknown = "leaf"; + for (let index = 0; index < 18; index++) nested = { child: nested }; + const evidence = { + title: "é".repeat(256), + severity: "high" as const, + description: "é".repeat(32768), + url: "https://example.test:non-numeric/finding", + advisory_ids: [" ", "\0", "é".repeat(256)], + packages: [{ name: "example", manifest_path: "../vendor-manifest" }], + locations: [{ path: "src\\example.ts" }], + source_data: { nested, title: "", path: "\0", content: "x".repeat(4096) }, + }; + expect(validateExternalEvidence(evidence)).toEqual(evidence); + for (const url of [ + "ftp://example.test", + "https:///finding", + "https:\\example.test", + "http://@example.test", + ]) { + expect(() => validateExternalEvidence({ ...evidence, url })).toThrow( + "HTTP(S)", + ); + } + for (const path of ["/absolute.ts", "C:\\source.ts", "src\\..\\outside.ts"]) { + expect(() => + validateExternalEvidence({ ...evidence, locations: [{ path }] }), + ).toThrow("repository-relative"); + } + expect(() => + validateExternalEvidence({ ...evidence, source_data: { value: Infinity } }), + ).toThrow("finite JSON"); +}); + +test("request and pagination metadata retain Cloud byte limits and distinct identities", async () => { + const f = await fixture(); + const prepared = await prepareExternalPublication(f.file, options, f.deps); + const request = prepared.preview.requests[0]!; + expect(() => + validateImportRequest({ + ...request, + source: { ...request.source, source_key: "tenant\0" }, + }), + ).toThrow("NUL"); + expect(() => + validateImportRequest({ + ...request, + repository: { ...request.repository, id: "é".repeat(300) }, + }), + ).toThrow("UTF-8"); + const item = request.items[0]!; + expect(() => + validateImportRequest({ + ...request, + items: [{ ...item, client_id: "é".repeat(100) }], + }), + ).toThrow("128 UTF-8"); + expect(() => + validateImportRequest({ + ...request, + items: [item, { ...item, source_finding_id: "another" }], + }), + ).toThrow("duplicate client"); + expect(() => + validateImportRequest({ + ...request, + items: [item, { ...item, client_id: "another" }], + }), + ).toThrow("duplicate source"); + const page = { + data: [f.destination()], + has_more: true, + next: "é".repeat(2048), + }; + expect(validateRepositories(page).next).toBe(page.next); + expect(() => + validateRepositories({ ...page, next: `${page.next}é` }), + ).toThrow("4096 UTF-8"); +}); + test("CLI preview and rejected confirmation never POST or persist an upload", async () => { const f = await fixture(); const cli = createCliTest(main); @@ -852,7 +992,7 @@ test("a failed checkpoint leaves the original pending request recoverable", asyn const rename = fs.rename; const checkpoint = spyOn(fs, "rename").mockImplementation( async (source, destination) => { - if (String(destination).endsWith(".pending.json")) + if (String(destination).endsWith(".pending.json") && f.posts.length > 0) throw new Error("Checkpoint interrupted"); return rename(source, destination); }, @@ -884,6 +1024,43 @@ test("a failed checkpoint leaves the original pending request recoverable", asyn expect(f.receipts.size).toBe(1); }); +test.each(["pending", "result"])( + "a failed %s file installation cleans up temporary files and can be retried", + async (stage) => { + const f = await fixture(); + const rename = fs.rename; + const installation = spyOn(fs, "rename").mockImplementation( + async (source, destination) => { + if (String(destination).endsWith(`.${stage}.json`)) + throw new Error("File installation interrupted"); + return rename(source, destination); + }, + ); + try { + await expect( + (await prepareExternalPublication(f.file, options, f.deps)).publish(), + ).rejects.toThrow("File installation interrupted"); + } finally { + installation.mockRestore(); + } + const directory = join(f.root, "state", "external-finding-publications"); + const files = await readdir(directory); + expect(files.filter((name) => name.endsWith(".tmp"))).toEqual([]); + expect(files.some((name) => name.endsWith(".pending.json"))).toBe( + stage === "result", + ); + expect(f.posts).toHaveLength(stage === "result" ? 1 : 0); + expect( + ( + await ( + await prepareExternalPublication(f.file, options, f.deps) + ).publish() + ).counts.created, + ).toBe(1); + expect(f.posts).toHaveLength(1); + }, +); + test.each(["request", "counts", "length", "schema"])( "cached receipts retain publication validation (%s)", async (invalid) => { From 7b132cf1da405762ba321ff268975eda8d878fee Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 8 Oct 2026 18:11:26 +0000 Subject: [PATCH 08/43] fix(cloud): show import evidence and clean failed writes --- sdk/typescript/src/cli.ts | 3 + .../src/external-findings-publish.ts | 54 +++--- .../external-findings-publish.test.ts | 155 ++++++++++++++---- 3 files changed, 160 insertions(+), 52 deletions(-) diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 74c37ad3b4..7953f05186 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -3243,6 +3243,9 @@ export async function main( "Publication needs confirmation. Run --dry-run, review the results, then use --yes to approve this input.", ); if (structured) showPreview(); + errorOutput.write( + `Selected findings and evidence:\n${JSON.stringify(preview.findings, null, 2).split("\n").map(diagnosticValue).join("\n")}\n`, + ); if ( !(await prompt.confirm( `Publish these ${preview.findings.length} findings?`, diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index b887563ee9..802992f5b3 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -422,28 +422,30 @@ export async function prepareExternalPublication( // Install a fully written file atomically. A crash during serialization // must not leave a partial request that cannot be resumed. const pendingTemporary = `${pendingPath}.${randomUUID()}.tmp`; - const pendingFile = await open(pendingTemporary, "wx", 0o600); - try { - await pendingFile.writeFile(JSON.stringify(submission)); - await pendingFile.sync(); - } finally { - await pendingFile.close(); - } let receipts: FindingImportReceipt[] = []; try { - receipts = (submission.receipts ?? []).map(validateImportReceipt); - await link(pendingTemporary, pendingPath); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; - const other = JSON.parse( - await readFile(pendingPath, "utf8"), - ) as SavedSubmission; - if (!sameSubmission(other, submission)) { - throw new CodexSecurityError( - "Another publication prepared this input. Run the command again to review and resume that saved request.", - ); + const pendingFile = await open(pendingTemporary, "wx", 0o600); + try { + await pendingFile.writeFile(JSON.stringify(submission)); + await pendingFile.sync(); + } finally { + await pendingFile.close(); + } + try { + receipts = (submission.receipts ?? []).map(validateImportReceipt); + await link(pendingTemporary, pendingPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + const other = JSON.parse( + await readFile(pendingPath, "utf8"), + ) as SavedSubmission; + if (!sameSubmission(other, submission)) { + throw new CodexSecurityError( + "Another publication prepared this input. Run the command again to review and resume that saved request.", + ); + } + receipts = (other.receipts ?? []).map(validateImportReceipt); } - receipts = (other.receipts ?? []).map(validateImportReceipt); } finally { await rm(pendingTemporary, { force: true }); } @@ -559,14 +561,18 @@ export async function prepareExternalPublication( cloudUrl: `https://chatgpt.com/codex/cloud/security/findings?repo=${encodeURIComponent(destination.url)}&source=imported&provider=${source.provider}`, }; const temporary = `${pendingPath}.${randomUUID()}.tmp`; - const file = await open(temporary, "wx", 0o600); try { - await file.writeFile(JSON.stringify(result)); - await file.sync(); + const file = await open(temporary, "wx", 0o600); + try { + await file.writeFile(JSON.stringify(result)); + await file.sync(); + } finally { + await file.close(); + } + await rename(temporary, join(state, `${key}.result.json`)); } finally { - await file.close(); + await rm(temporary, { force: true }); } - await rename(temporary, join(state, `${key}.result.json`)); await rm(pendingPath, { force: true }); return result; }); diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 855507df81..98025d9a2c 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -395,6 +395,7 @@ test("CLI creates, reimports unchanged, and updates the same canonical finding", expect(await first.runCli([...f.command, "--yes"], f.cliDeps)).toBe(0); const canonical = f.reports.get("vendor-1")!.canonical_finding_id; expect(JSON.parse(first.stdout.text()).counts.created).toBe(1); + expect(first.stderr.text()).not.toContain("Selected findings and evidence:"); const again = await prepareExternalPublication(f.file, options, f.deps); expect((await again.publish()).counts.unchanged).toBe(1); await writeFile(f.file, JSON.stringify([normalized("vendor-1", "critical")])); @@ -434,35 +435,58 @@ test("normalized input accepts server-materialized optional evidence defaults", expect(stateFiles.some((name) => name.endsWith(".pending.json"))).toBe(false); }); -test("interactive JSON publication shows the destination and exclusions before confirmation", async () => { - const f = await fixture([normalized(), normalized("unsupported", "unknown")]); - const cli = createCliTest(main); - let prompted = false; - expect( - await cli.runCli(f.command, { - ...f.cliDeps, - externalPublicationPrompt: { - isInteractive: () => true, - confirm: async (_question, defaultValue) => { - prompted = true; - expect(defaultValue).toBe(false); - expect(cli.stderr.text()).toContain( - "https://github.com/example/project", - ); - expect(cli.stderr.text()).toContain(options.sourceKey); - expect(cli.stderr.text()).toContain("synthetic-account"); - expect(cli.stderr.text()).toContain("environment-example"); - expect(cli.stderr.text()).toContain("Excluded: 1"); - expect(cli.stderr.text()).not.toContain("synthetic-token"); - expect(f.posts).toHaveLength(0); - return true; - }, +test.each(["human", "json"])( + "interactive %s publication shows selected evidence before confirmation", + async (format) => { + const selected = { + ...normalized("vendor-selected"), + evidence: { + ...normalized("vendor-selected").evidence, + title: "Selected \u001b[31m evidence", + description: "Description to approve", + source_data: { id: "vendor-selected", note: "Original vendor detail" }, }, - }), - ).toBe(0); - expect(prompted).toBe(true); - expect(JSON.parse(cli.stdout.text()).counts.created).toBe(1); -}); + }; + const f = await fixture([selected, normalized("unsupported", "unknown")]); + const expected = (await readVendorFindings(f.file)).findings; + const cli = createCliTest(main); + let prompted = false; + expect( + await cli.runCli(format === "json" ? f.command : f.command.slice(0, -2), { + ...f.cliDeps, + externalPublicationPrompt: { + isInteractive: () => true, + confirm: async (_question, defaultValue) => { + prompted = true; + expect(defaultValue).toBe(false); + expect(cli.stderr.text()).toContain( + "https://github.com/example/project", + ); + expect(cli.stderr.text()).toContain(options.sourceKey); + expect(cli.stderr.text()).toContain("synthetic-account"); + expect(cli.stderr.text()).toContain("environment-example"); + expect(cli.stderr.text()).toContain("Excluded: 1"); + expect(cli.stderr.text()).toContain( + "Selected findings and evidence:", + ); + expect(cli.stderr.text()).toContain( + JSON.stringify(expected, null, 2), + ); + expect(cli.stderr.text()).not.toContain("\u001b"); + expect(cli.stderr.text()).not.toContain("synthetic-token"); + expect(cli.stdout.text()).toBe(""); + expect(f.posts).toHaveLength(0); + return true; + }, + }, + }), + ).toBe(0); + expect(prompted).toBe(true); + expect(f.reports.has("vendor-selected")).toBe(true); + if (format === "json") + expect(JSON.parse(cli.stdout.text()).counts.created).toBe(1); + }, +); test.each([false, true])( "saved request survives a publisher process restart (abrupt: %p)", @@ -946,3 +970,78 @@ test.each([1, 101])( expect((await fresh.publish()).counts.unchanged).toBe(count); }, ); + +test.each([ + ["initial", "write"], + ["initial", "sync"], + ["result", "write"], + ["result", "sync"], + ["result", "rename"], +])( + "failed %s staging cleans temporary data and remains retryable (%s)", + async (stage, fault) => { + const f = await fixture(); + const prepared = await prepareExternalPublication(f.file, options, f.deps); + const originalOpen = fs.open; + const originalRename = fs.rename; + let temporaryOpens = 0; + let injected = false; + const openSpy = spyOn(fs, "open").mockImplementation(async (...args) => { + const handle = await originalOpen(...args); + // A fresh publication stages the pending body, its checkpoint, then its result. + if ( + String(args[0]).endsWith(".tmp") && + ++temporaryOpens === (stage === "initial" ? 1 : 3) + ) { + if (fault === "write") { + const originalWrite = handle.writeFile.bind(handle); + handle.writeFile = async (data) => { + const serialized = String(data); + await originalWrite( + serialized.slice(0, Math.ceil(serialized.length / 2)), + ); + injected = true; + throw new Error("Synthetic disk write failed"); + }; + } else if (fault === "sync") { + handle.sync = async () => { + injected = true; + throw new Error("Synthetic file sync failed"); + }; + } + } + return handle; + }); + const renameSpy = spyOn(fs, "rename").mockImplementation( + async (from, to) => { + if (fault === "rename" && String(to).endsWith(".result.json")) { + injected = true; + throw new Error("Synthetic result rename failed"); + } + return originalRename(from, to); + }, + ); + try { + await expect(prepared.publish()).rejects.toThrow("Synthetic"); + } finally { + openSpy.mockRestore(); + renameSpy.mockRestore(); + } + expect(injected).toBe(true); + const directory = join(f.root, "state", "external-finding-publications"); + const files = await readdir(directory); + expect(files.filter((name) => name.endsWith(".tmp"))).toEqual([]); + expect(files.some((name) => name.endsWith(".pending.json"))).toBe( + stage === "result", + ); + expect(f.posts).toHaveLength(stage === "initial" ? 0 : 1); + const retry = await prepareExternalPublication(f.file, options, f.deps); + expect(retry.preview.resumed).toBe(stage === "result"); + expect((await retry.publish()).counts.created).toBe(1); + expect(f.posts).toHaveLength(1); + expect(f.receipts.size).toBe(1); + expect( + (await readdir(directory)).some((name) => name.endsWith(".tmp")), + ).toBe(false); + }, +); From f7987468b18df2ffe3ea5848c1814f3b28a67a34 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 8 Oct 2026 18:28:48 +0000 Subject: [PATCH 09/43] fix(cloud): format malformed import diagnostics --- sdk/typescript/src/cli.ts | 2 +- .../external-findings-publish.test.ts | 56 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 7953f05186..4369a1e239 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -3264,7 +3264,7 @@ export async function main( return { ...result, excluded: preview.excluded }; } catch (error) { reportPublicationError( - error, + diagnosticValue(error), controller.signal.aborted ? controller.signal.reason : undefined, ); return undefined; diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 98025d9a2c..1a79b32bb9 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -1045,3 +1045,59 @@ test.each([ ).toBe(false); }, ); + +test.each(["JSON", "JSONL"])( + "malformed %s input keeps Node parser diagnostics terminal-safe", + async (format) => { + const f = await fixture(); + await writeFile( + f.file, + `${format === "JSONL" ? "{}\n" : ""}\u001b[2Jmalformed`, + ); + // Bundle beneath the SDK so the real Node process resolves installed packages. + const bundle = await mkdtemp( + join(import.meta.dir, "..", ".import-parser-"), + ); + cleanups.push(() => rm(bundle, { recursive: true, force: true })); + const runner = join(bundle, "parse-input.mts"); + await writeFile( + runner, + ` + import { main } from "../src/cli.js"; + import { dependencies } from "../tests-ts/cli-fixtures.js"; + process.exitCode = await main(process.argv.slice(2), process.stdout, process.stderr, { + ...dependencies({ environment: process.env }), + cloudFetch: async () => { throw new Error("Unexpected Cloud request"); }, + }); + `, + ); + const built = await Bun.build({ + entrypoints: [runner], + outdir: bundle, + target: "node", + format: "esm", + packages: "external", + }); + expect(built.success).toBe(true); + const child = Bun.spawn( + [Bun.which("node")!, built.outputs[0]!.path, ...f.command, "--dry-run"], + { + env: f.environment, + stdout: "pipe", + stderr: "pipe", + }, + ); + const [code, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]); + expect(code).toBe(2); + expect(stdout).toBe(""); + expect(stderr).toContain("Could not read vendor findings:"); + expect(stderr).toContain("malformed"); + expect(stderr).not.toContain("\u001b"); + expect(f.posts).toHaveLength(0); + expect(await readdir(join(f.root, "state")).catch(() => [])).toEqual([]); + }, +); From 54913d3410ab92814323e1cc1720250686acbc1d Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 8 Oct 2026 19:35:18 +0000 Subject: [PATCH 10/43] fix(cloud): map Wiz container and package metadata --- sdk/typescript/src/wiz-findings.ts | 35 +++++----- .../external-findings-publish.test.ts | 66 +++++++++++++++++++ 2 files changed, 86 insertions(+), 15 deletions(-) diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts index 4640bfa265..63e122027e 100644 --- a/sdk/typescript/src/wiz-findings.ts +++ b/sdk/typescript/src/wiz-findings.ts @@ -45,12 +45,16 @@ function wizFinding(record: Record): VendorFinding { const version = text(record["version"]); const severity = text(record["vendorSeverity"]) ?? text(record["severity"]); const asset = object(record["vulnerableAsset"]); + const artifactType = object(record["artifactType"]); if (!name || !packageName || !text(asset?.["id"])) { throw new Error( "Expected a Wiz package vulnerability finding with name, detailedName, and vulnerableAsset.id. Other finding classes need a separate mapping.", ); } - const digest = text(asset?.["imageDigest"]) ?? text(record["imageDigest"]); + const digest = + text(asset?.["imageDigest"]) ?? + text(record["imageDigest"]) ?? + (asset?.["type"] === "CONTAINER_IMAGE" ? text(asset["imageId"]) : null); const updated = text(record["updatedAt"]); const updatedSeconds = updated ? Date.parse(updated) / 1000 : null; if ( @@ -68,20 +72,21 @@ function wizFinding(record: Record): VendorFinding { description: text(record["description"]), severity: severity?.toLowerCase(), url: text(record["portalUrl"]), - advisory_ids: name ? [name] : [], - packages: packageName - ? [ - { - name: packageName, - ecosystem: text(record["packageManager"]), - installed_version: version, - manifest_path: null, - fixed_versions: text(record["fixedVersion"]) - ? [record["fixedVersion"]] - : [], - }, - ] - : [], + advisory_ids: [name], + packages: [ + { + name: packageName, + ecosystem: + text(record["packageManager"]) ?? + text(artifactType?.["osPackageManager"]) ?? + text(artifactType?.["codeLibraryLanguage"]), + installed_version: version, + manifest_path: null, + fixed_versions: text(record["fixedVersion"]) + ? [record["fixedVersion"]] + : [], + }, + ], locations: [], branch: null, code_revision: null, diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 91d2100c98..922a2633ab 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -340,6 +340,72 @@ test("Wiz occurrence mapping preserves original evidence without inventing repos }); }); +test("Wiz API nodes preserve artifact ecosystems and distinguish container digests from VM image IDs", async () => { + const finding = { + name: "CVE-2099-0001", + detailedName: "example-package", + version: "1.2.0", + vendorSeverity: "HIGH", + }; + const records = [ + { + ...finding, + id: "container-package", + artifactType: { osPackageManager: "DPKG", codeLibraryLanguage: null }, + vulnerableAsset: { + id: "container-image", + type: "CONTAINER_IMAGE", + imageId: `sha256:${"a".repeat(64)}`, + }, + }, + { + ...finding, + id: "vm-library", + artifactType: { osPackageManager: null, codeLibraryLanguage: "PYTHON" }, + vulnerableAsset: { + id: "virtual-machine", + type: "VIRTUAL_MACHINE", + imageId: "ami-synthetic", + }, + }, + { + ...finding, + id: "explicit-metadata", + packageManager: "reported-manager", + imageDigest: `sha256:${"b".repeat(64)}`, + artifactType: { osPackageManager: "DPKG" }, + vulnerableAsset: { + id: "explicit-container", + type: "CONTAINER_IMAGE", + imageId: `sha256:${"c".repeat(64)}`, + }, + }, + ]; + const f = await fixture({ + data: { + vulnerabilityFindings: { + nodes: records, + pageInfo: { hasNextPage: false }, + }, + }, + }); + const parsed = await readVendorFindings(f.file); + expect(parsed.excluded).toEqual([]); + expect( + parsed.findings.map(({ evidence }) => ({ + ecosystem: evidence.packages?.[0]?.ecosystem, + digests: evidence.image_digests, + })), + ).toEqual([ + { ecosystem: "DPKG", digests: [`sha256:${"a".repeat(64)}`] }, + { ecosystem: "PYTHON", digests: [] }, + { ecosystem: "reported-manager", digests: [`sha256:${"b".repeat(64)}`] }, + ]); + expect(parsed.findings.map(({ evidence }) => evidence.source_data)).toEqual( + records, + ); +}); + test("unsupported records are visible exclusions; duplicate identities and unfinished pages stop publication", async () => { const f = await fixture([normalized(), normalized("unknown", "unknown")]); const parsed = await readVendorFindings(f.file); From 89648f805e9adde91c768f051e50c19249b1f637 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 8 Oct 2026 19:36:39 +0000 Subject: [PATCH 11/43] fix(cloud): preserve Wiz metadata and match import result IDs --- .../src/external-findings-publish.ts | 23 +-- sdk/typescript/src/wiz-findings.ts | 12 +- .../external-findings-publish.test.ts | 139 ++++++++++++++++++ 3 files changed, 162 insertions(+), 12 deletions(-) diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 0eeb14b19c..c29d4b3699 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -436,6 +436,9 @@ export async function prepareExternalPublication( receipts[batchIndex] ?? (await request("/finding_imports", batch)), ); + const batchItems = new Map( + batch.items.map((item) => [item.client_id, item]), + ); if ( receipt.id !== batch.request_id || canonicalJson(receipt.repository) !== @@ -443,10 +446,9 @@ export async function prepareExternalPublication( canonicalJson(receipt.source) !== canonicalJson(batch.source) || receipt.item_count !== batch.items.length || receipt.results.length !== batch.items.length || - receipt.results.some( - (item, index) => - item.client_id !== batch.items[index]!.client_id, - ) + new Set(receipt.results.map((item) => item.client_id)).size !== + batch.items.length || + receipt.results.some((item) => !batchItems.has(item.client_id)) ) throw new CodexSecurityError( "Cloud returned a receipt for a different publication.", @@ -493,8 +495,12 @@ export async function prepareExternalPublication( // Verify readable source records without mistaking a newer concurrent // observation for failure of the original, immutable import receipt. for (const [batchIndex, receipt] of receipts.entries()) { - for (const [index, item] of receipt.results.entries()) { + const batchItems = new Map( + requests[batchIndex]!.items.map((item) => [item.client_id, item]), + ); + for (const item of receipt.results) { if (item.outcome === "error") continue; + const expected = batchItems.get(item.client_id)!; const report = validateSourceReport( await request( `${sourcePath(destination)}/${encodeURIComponent(item.source_report_id!)}`, @@ -506,11 +512,8 @@ export async function prepareExternalPublication( (report.version === item.version && (report.observation_id !== item.observation_id || canonicalJson(report.evidence) !== - canonicalJson( - requests[batchIndex]!.items[index]!.evidence, - ))) || - report.source_finding_id !== - requests[batchIndex]!.items[index]!.source_finding_id || + canonicalJson(expected.evidence))) || + report.source_finding_id !== expected.source_finding_id || report.repo_id !== destination.id || report.repo_connector_id !== destination.repo_connector_id || canonicalJson(report.source) !== canonicalJson(source) diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts index 4640bfa265..8cb6f27ee8 100644 --- a/sdk/typescript/src/wiz-findings.ts +++ b/sdk/typescript/src/wiz-findings.ts @@ -45,12 +45,17 @@ function wizFinding(record: Record): VendorFinding { const version = text(record["version"]); const severity = text(record["vendorSeverity"]) ?? text(record["severity"]); const asset = object(record["vulnerableAsset"]); + const artifact = object(record["artifactType"]); if (!name || !packageName || !text(asset?.["id"])) { throw new Error( "Expected a Wiz package vulnerability finding with name, detailedName, and vulnerableAsset.id. Other finding classes need a separate mapping.", ); } - const digest = text(asset?.["imageDigest"]) ?? text(record["imageDigest"]); + const digest = + text(asset?.["imageId"]) ?? + text(asset?.["ImageExternalId"]) ?? + text(asset?.["imageDigest"]) ?? + text(record["imageDigest"]); const updated = text(record["updatedAt"]); const updatedSeconds = updated ? Date.parse(updated) / 1000 : null; if ( @@ -73,7 +78,10 @@ function wizFinding(record: Record): VendorFinding { ? [ { name: packageName, - ecosystem: text(record["packageManager"]), + ecosystem: + text(artifact?.["osPackageManager"]) ?? + text(artifact?.["codeLibraryLanguage"]) ?? + text(record["packageManager"]), installed_version: version, manifest_path: null, fixed_versions: text(record["fixedVersion"]) diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 91d2100c98..79b9b9398f 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -322,6 +322,7 @@ test("Wiz occurrence mapping preserves original evidence without inventing repos locationPath: "/usr/lib/example", lastDetectedAt: "2026-10-01T00:00:00Z", projects: [{ id: "project-a" }, { id: "project-b" }], + packageManager: "legacy-manager", vulnerableAsset: { id: "image-resource", imageDigest: "sha256:example" }, }; const f = await fixture([record]); @@ -336,6 +337,7 @@ test("Wiz occurrence mapping preserves original evidence without inventing repos source_updated_at: null, source_data: record, image_digests: ["sha256:example"], + packages: [{ name: "example-lib", ecosystem: "legacy-manager" }], }, }); }); @@ -1278,3 +1280,140 @@ test.each(["JSON", "JSONL"])( expect(await readdir(join(f.root, "state")).catch(() => [])).toEqual([]); }, ); + +test.each([ + { + kind: "OS package container image", + artifactType: { group: "OS_PACKAGE", osPackageManager: "RPM" }, + vulnerableAsset: { id: "image-os", imageId: "sha256:os-image" }, + ecosystem: "RPM", + digest: "sha256:os-image", + }, + { + kind: "code library container image", + artifactType: { group: "CODE_LIBRARY", codeLibraryLanguage: "PYTHON" }, + vulnerableAsset: { id: "image-library", imageId: "sha256:library-image" }, + ecosystem: "PYTHON", + digest: "sha256:library-image", + }, + { + kind: "running container", + artifactType: { group: "CODE_LIBRARY", codeLibraryLanguage: "JAVA" }, + vulnerableAsset: { + id: "container-library", + ImageExternalId: "sha256:running-image", + }, + ecosystem: "JAVA", + digest: "sha256:running-image", + }, +])( + "Wiz GraphQL nodes retain $kind metadata", + async ({ artifactType, vulnerableAsset, ecosystem, digest }) => { + const record = { + id: "synthetic-occurrence", + name: "CVE-2099-0002", + detailedName: "example-lib", + version: "1.0.0", + vendorSeverity: "HIGH", + artifactType, + vulnerableAsset, + }; + const f = await fixture({ + data: { + vulnerabilityFindings: { + nodes: [record], + pageInfo: { hasNextPage: false }, + }, + }, + }); + const parsed = await readVendorFindings(f.file); + expect(parsed.excluded).toEqual([]); + expect(parsed.findings[0]).toMatchObject({ + source_finding_id: record.id, + evidence: { + packages: [ + { name: "example-lib", ecosystem, installed_version: "1.0.0" }, + ], + image_digests: [digest], + source_data: record, + }, + }); + }, +); + +test.each([false, true])( + "receipt client IDs correlate reordered results (cached: %p)", + async (cached) => { + const f = await fixture([ + normalized("vendor-first", "high"), + normalized("vendor-second", "low"), + ]); + const deps = { + ...f.deps, + fetch: async (url: string, init: RequestInit) => { + const response = await f.deps.fetch(url, init); + if (init.method !== "POST" || !response.ok) return response; + const receipt = (await response.json()) as FindingImportReceipt; + receipt.results.reverse(); + return new Response(JSON.stringify(receipt), { + headers: { "Content-Type": "application/json" }, + }); + }, + }; + let prepared = await prepareExternalPublication(f.file, options, deps); + if (cached) { + f.state.brokenReadback = true; + await expect(prepared.publish()).rejects.toThrow("Readback unavailable"); + f.state.brokenReadback = false; + f.state.postBudget = 0; + prepared = await prepareExternalPublication(f.file, options, deps); + expect(prepared.preview.resumed).toBe(true); + } + const result = await prepared.publish(); + expect(result.counts.created).toBe(2); + expect(result.receipts[0]!.results.map((item) => item.client_id)).toEqual([ + "item-2", + "item-1", + ]); + expect(f.posts).toHaveLength(1); + expect(f.receipts.size).toBe(1); + }, +); + +test.each(["duplicate", "missing", "unexpected"])( + "receipt client IDs reject %s results before checkpointing", + async (invalid) => { + const f = await fixture([ + normalized("vendor-first"), + normalized("vendor-second"), + ]); + const deps = { + ...f.deps, + fetch: async (url: string, init: RequestInit) => { + const response = await f.deps.fetch(url, init); + if (init.method !== "POST" || !response.ok) return response; + const receipt = (await response.json()) as FindingImportReceipt; + if (invalid === "duplicate") + receipt.results[1]!.client_id = receipt.results[0]!.client_id; + if (invalid === "missing") receipt.results.pop(); + if (invalid === "unexpected") + receipt.results[0]!.client_id = "unexpected-client"; + return new Response(JSON.stringify(receipt), { + headers: { "Content-Type": "application/json" }, + }); + }, + }; + await expect( + (await prepareExternalPublication(f.file, options, deps)).publish(), + ).rejects.toThrow("Cloud returned a receipt for a different publication"); + const directory = join(f.root, "state", "external-finding-publications"); + const pending = (await readdir(directory)).find((name) => + name.endsWith(".pending.json"), + )!; + expect( + JSON.parse(await readFile(join(directory, pending), "utf8")).receipts ?? + [], + ).toEqual([]); + expect(f.posts).toHaveLength(1); + }, +); From d775fe15d549256dc20ccdeddf592fd442327e54 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 8 Oct 2026 20:18:59 +0000 Subject: [PATCH 12/43] fix(cloud): preserve imported finding metadata and environment bindings --- sdk/typescript/docs/cli.md | 17 +- sdk/typescript/src/cli.ts | 11 +- .../src/external-findings-publish.ts | 43 +++-- .../src/external-import-contract.ts | 22 +++ sdk/typescript/src/wiz-findings.ts | 10 +- .../external-findings-publish.test.ts | 151 +++++++++++++++++- 6 files changed, 214 insertions(+), 40 deletions(-) diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md index 2d14807ff6..f353adab85 100644 --- a/sdk/typescript/docs/cli.md +++ b/sdk/typescript/docs/cli.md @@ -783,11 +783,12 @@ codex-security publish findings selected-wiz.json \ This command reads the authorized destination and current source versions during preview. It uses the same saved ChatGPT file login as scan publication. The Cloud -repository needs an existing environment; no native scan is required. Review the -account, repository, environment, findings, and exclusions, then rerun without -`--dry-run`. The default terminal prompt is No; `--yes` confirms a previously -reviewed input for scripts or the plugin. Without a terminal or `--yes`, no upload -occurs. +repository needs an existing environment; no native scan is required. Existing +source reports retain their environment; new reports use the discovered default. +Review the account, repository, prepared environments, findings, and exclusions, +then rerun without `--dry-run`. The default terminal prompt is No; `--yes` +confirms a previously reviewed input for scripts or the plugin. Without a +terminal or `--yes`, no upload occurs. Input can be a Wiz vulnerability finding, an array, a complete `data.vulnerabilityFindings.nodes` response, or JSONL with one vendor record per @@ -812,6 +813,8 @@ retains each record in `source_data`, uses reported severity, and leaves unknown branch/revision and repository locations empty. Container paths are retained as vendor evidence rather than interpreted as source-code locations. It does not fetch from Wiz, assess findings, or change vendor or Cloud triage decisions. +Findings from external network scans are excluded from this package vulnerability +mapping. Requests contain at most 100 findings and respect the Cloud payload limits. Before uploading, the publisher saves request IDs and bodies privately under @@ -821,8 +824,8 @@ retires the old request without republishing; review and explicitly approve a fresh invocation. Final item errors are reported individually with exit code 1; correct the input or source conflict before a fresh submission. Transport or readback failures return exit code 2 and retain the resumable request. Successful -receipts are saved locally; the CLI verifies Cloud source-report reads and -returns a Findings link. Search indexing can lag an accepted import. +receipts are saved locally; the CLI verifies Cloud source-report reads before +reporting success. The current Cloud findings page does not display these imports. Preview selected completed scans before uploading: diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 4369a1e239..72a977cb4a 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -3223,9 +3223,16 @@ export async function main( }, ); const { preview } = prepared; + const environmentIds = [ + ...new Set( + preview.requests.map( + (request) => request.repository.environment_id, + ), + ), + ]; const showPreview = () => { errorOutput.write( - `Account: ${diagnosticValue(preview.accountId)}\nDestination: ${diagnosticValue(preview.destination.url)} (${diagnosticValue(preview.destination.id)})\nEnvironment: ${diagnosticValue(preview.destination.import_environment_id)}\nSource: ${preview.source.provider} / ${diagnosticValue(preview.source.source_key)}\nRead: ${preview.read} Ready: ${preview.findings.length} Excluded: ${preview.excluded.length}${preview.resumed ? "\nResuming the saved submission." : ""}\n`, + `Account: ${diagnosticValue(preview.accountId)}\nDestination: ${diagnosticValue(preview.destination.url)} (${diagnosticValue(preview.destination.id)})\nEnvironment: ${environmentIds.map(diagnosticValue).join(", ")}\nSource: ${preview.source.provider} / ${diagnosticValue(preview.source.source_key)}\nRead: ${preview.read} Ready: ${preview.findings.length} Excluded: ${preview.excluded.length}${preview.resumed ? "\nResuming the saved submission." : ""}\n`, ); for (const excluded of preview.excluded) errorOutput.write( @@ -3259,7 +3266,7 @@ export async function main( if (result.counts.error) exitCode = 1; if (!structured) errorOutput.write( - `Created: ${result.counts.created} Updated: ${result.counts.updated} Unchanged: ${result.counts.unchanged} Failed: ${result.counts.error}\nOpen in Cloud: ${result.cloudUrl}\n`, + `Created: ${result.counts.created} Updated: ${result.counts.updated} Unchanged: ${result.counts.unchanged} Failed: ${result.counts.error}\n`, ); return { ...result, excluded: preview.excluded }; } catch (error) { diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 0eeb14b19c..3fe5517884 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -61,7 +61,6 @@ export interface PreparedExternalPublication { publish(): Promise<{ receipts: FindingImportReceipt[]; counts: FindingImportReceipt["counts"]; - cloudUrl: string; }>; } @@ -313,38 +312,20 @@ export async function prepareExternalPublication( "The repository was reset after this submission. The saved request was retired without uploading. Review the destination and run the command again to approve a fresh publication.", ); } - if ( - submission.repository.environment_id !== - destination.import_environment_id - ) { - throw new Error( - "The Cloud environment changed since this submission. Restore the original destination before resuming the saved request.", - ); - } } saved = content; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } - if (destination.import_environment_id == null) - throw new CodexSecurityError( - "Configure an authorized Cloud environment for this repository before importing findings.", - ); let requests = saved?.requests; if (!requests) { const repository = { id: destination.id, repo_connector_id: destination.repo_connector_id, - environment_id: destination.import_environment_id, reset_marker: destination.reset_marker, }; requests = []; - let current: FindingImportRequest = { - request_id: randomUUID(), - repository, - source, - items: [], - }; + let current: FindingImportRequest | undefined; for (const [index, finding] of parsed.findings.entries()) { const query = new URLSearchParams({ provider: source.provider, @@ -371,6 +352,12 @@ export async function prepareExternalPublication( throw new CodexSecurityError( "Cloud returned a different source identity.", ); + const environmentId = + previous?.environment_id ?? destination.import_environment_id; + if (environmentId == null) + throw new CodexSecurityError( + "Configure an authorized Cloud environment for this repository before importing findings.", + ); const item = { client_id: `item-${index + 1}`, source_finding_id: finding.source_finding_id, @@ -378,18 +365,25 @@ export async function prepareExternalPublication( evidence: finding.evidence, }; if ( - current.items.length > 0 && - (current.items.length === 100 || + current && + (current.repository.environment_id !== environmentId || + current.items.length === 100 || Buffer.byteLength( JSON.stringify({ ...current, items: [...current.items, item] }), ) > MAX_REQUEST_BYTES) ) { requests.push(validateImportRequest(current)); - current = { request_id: randomUUID(), repository, source, items: [] }; + current = undefined; } + current ??= { + request_id: randomUUID(), + repository: { ...repository, environment_id: environmentId }, + source, + items: [], + }; current.items.push(item); } - if (current.items.length) requests.push(validateImportRequest(current)); + if (current) requests.push(validateImportRequest(current)); } const submission: SavedSubmission = { accountId: credentials.account_id, @@ -535,7 +529,6 @@ export async function prepareExternalPublication( const result = { receipts, counts, - cloudUrl: `https://chatgpt.com/codex/cloud/security/findings?repo=${encodeURIComponent(destination.url)}&source=imported&provider=${source.provider}`, }; await writeAtomicJson(join(state, `${key}.result.json`), result); await rm(pendingPath, { force: true }); diff --git a/sdk/typescript/src/external-import-contract.ts b/sdk/typescript/src/external-import-contract.ts index 3cb03a3e12..0ca3a41318 100644 --- a/sdk/typescript/src/external-import-contract.ts +++ b/sdk/typescript/src/external-import-contract.ts @@ -1,3 +1,4 @@ +import { isIP } from "node:net"; import Ajv2020 from "ajv/dist/2020.js"; import schema from "../schemas/external-findings.schema.json" with { type: "json" }; import type { @@ -105,6 +106,27 @@ export function validateExternalEvidence( throw new Error( "Evidence URLs must be HTTP(S) links without credentials.", ); + if (authority.includes("[")) { + const closing = authority.indexOf("]"); + const host = authority.slice(1, closing); + const port = authority.slice(closing + 1); + const [address, scope, ...extraScopes] = host.split("%"); + const validHost = host.startsWith("v") + ? /^v[\da-fA-F]+\.[\s\S]+$/u.test(host) + : isIP(address!) === 6 && scope !== "" && extraScopes.length === 0; + if ( + !authority.startsWith("[") || + (port !== "" && !port.startsWith(":")) || + !validHost + ) + throw new Error("Evidence URL has an invalid bracketed host."); + } + const hostname = authority.replace(/[@:#?]/gu, ""); + const normalized = hostname.normalize("NFKC"); + if (normalized !== hostname && /[/?:#@]/u.test(normalized)) + throw new Error( + "Evidence URL authority contains an invalid normalized separator.", + ); } const pending: [unknown, number][] = [[result, 0]]; while (pending.length > 0) { diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts index 63e122027e..50f9608bf0 100644 --- a/sdk/typescript/src/wiz-findings.ts +++ b/sdk/typescript/src/wiz-findings.ts @@ -39,6 +39,10 @@ function sourceFindingId(value: unknown): string { /** Keep a Wiz vulnerability occurrence intact; advisory names are not identities. */ function wizFinding(record: Record): VendorFinding { + if (record["detectionMethod"] === "EXTERNAL_NETWORK_SCAN") + throw new Error( + "Wiz external network findings are not supported by the package vulnerability mapping.", + ); const id = sourceFindingId(record["id"]); const name = text(record["name"]); const packageName = text(record["detailedName"]); @@ -54,7 +58,9 @@ function wizFinding(record: Record): VendorFinding { const digest = text(asset?.["imageDigest"]) ?? text(record["imageDigest"]) ?? - (asset?.["type"] === "CONTAINER_IMAGE" ? text(asset["imageId"]) : null); + (asset?.["type"] === "CONTAINER_IMAGE" + ? (text(asset["imageId"]) ?? text(asset["containerImageId"])) + : null); const updated = text(record["updatedAt"]); const updatedSeconds = updated ? Date.parse(updated) / 1000 : null; if ( @@ -72,7 +78,7 @@ function wizFinding(record: Record): VendorFinding { description: text(record["description"]), severity: severity?.toLowerCase(), url: text(record["portalUrl"]), - advisory_ids: [name], + advisory_ids: [text(record["vulnerabilityExternalId"]) ?? name], packages: [ { name: packageName, diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 922a2633ab..e78a2ec8ba 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -191,7 +191,10 @@ async function fixture(records: unknown = [normalized()]) { }; for (const item of request.items) { const previous = reports.get(item.source_finding_id); - if (state.finalError) { + const scopeConflict = + previous !== undefined && + previous.environment_id !== request.repository.environment_id; + if (state.finalError || scopeConflict) { receipt.counts.error = (receipt.counts.error ?? 0) + 1; receipt.results.push({ client_id: item.client_id, @@ -201,8 +204,10 @@ async function fixture(records: unknown = [normalized()]) { canonical_finding_id: null, version: null, error: { - code: "version_conflict", - message: "Reload current evidence", + code: scopeConflict ? "scope_conflict" : "version_conflict", + message: scopeConflict + ? "Source report belongs to a different Cloud environment" + : "Reload current evidence", }, }); continue; @@ -336,13 +341,15 @@ test("Wiz occurrence mapping preserves original evidence without inventing repos source_updated_at: null, source_data: record, image_digests: ["sha256:example"], + advisory_ids: [record.name], }, }); }); test("Wiz API nodes preserve artifact ecosystems and distinguish container digests from VM image IDs", async () => { const finding = { - name: "CVE-2099-0001", + name: "Vendor display advisory", + vulnerabilityExternalId: "CVE-2099-0001", detailedName: "example-package", version: "1.2.0", vendorSeverity: "HIGH", @@ -366,6 +373,16 @@ test("Wiz API nodes preserve artifact ecosystems and distinguish container diges id: "virtual-machine", type: "VIRTUAL_MACHINE", imageId: "ami-synthetic", + containerImageId: "ami-synthetic-alias", + }, + }, + { + ...finding, + id: "container-alias", + vulnerableAsset: { + id: "aliased-image", + type: "CONTAINER_IMAGE", + containerImageId: `sha256:${"d".repeat(64)}`, }, }, { @@ -391,6 +408,10 @@ test("Wiz API nodes preserve artifact ecosystems and distinguish container diges }); const parsed = await readVendorFindings(f.file); expect(parsed.excluded).toEqual([]); + expect(parsed.findings[0]!.evidence.title).toContain(finding.name); + expect(parsed.findings.map(({ evidence }) => evidence.advisory_ids)).toEqual( + records.map(() => [finding.vulnerabilityExternalId]), + ); expect( parsed.findings.map(({ evidence }) => ({ ecosystem: evidence.packages?.[0]?.ecosystem, @@ -399,6 +420,7 @@ test("Wiz API nodes preserve artifact ecosystems and distinguish container diges ).toEqual([ { ecosystem: "DPKG", digests: [`sha256:${"a".repeat(64)}`] }, { ecosystem: "PYTHON", digests: [] }, + { ecosystem: null, digests: [`sha256:${"d".repeat(64)}`] }, { ecosystem: "reported-manager", digests: [`sha256:${"b".repeat(64)}`] }, ]); expect(parsed.findings.map(({ evidence }) => evidence.source_data)).toEqual( @@ -406,6 +428,36 @@ test("Wiz API nodes preserve artifact ecosystems and distinguish container diges ); }); +test("Wiz network-scan findings are excluded without requiring a detection method on package exports", async () => { + const supported = { + id: "package-occurrence", + name: "CVE-2099-0001", + detailedName: "example-package", + vendorSeverity: "HIGH", + vulnerableAsset: { id: "synthetic-asset" }, + }; + const network = { + ...supported, + id: "network-occurrence", + name: "CWE-89", + detailedName: "GET /synthetic-endpoint", + detectionMethod: "EXTERNAL_NETWORK_SCAN", + }; + const f = await fixture([supported, network]); + const prepared = await prepareExternalPublication(f.file, options, f.deps); + expect( + prepared.preview.findings.map((finding) => finding.source_finding_id), + ).toEqual([supported.id]); + expect(prepared.preview.excluded).toEqual([ + expect.objectContaining({ + source_finding_id: network.id, + reason: expect.stringContaining("external network"), + }), + ]); + expect((await prepared.publish()).counts.created).toBe(1); + expect(f.reports.has(network.id)).toBe(false); +}); + test("unsupported records are visible exclusions; duplicate identities and unfinished pages stop publication", async () => { const f = await fixture([normalized(), normalized("unknown", "unknown")]); const parsed = await readVendorFindings(f.file); @@ -517,6 +569,35 @@ test("Cloud validation preserves allowed evidence boundaries and raw vendor stri ).toThrow("finite JSON"); }); +test("evidence URL authorities match Cloud bracket and Unicode separator checks", () => { + for (const url of [ + "https://[::1]", + "https://[fe80::1%eth0]", + "https://[fe80::1%25eth0]", + "https://[v1.example]", + "https://[vF.example]:service", + "https://éxample.test", + "https://example.test:non-numeric/finding", + ]) { + expect( + validateExternalEvidence({ ...normalized().evidence, url }).url, + ).toBe(url); + } + for (const url of [ + "https://[example.test]", + "https://[127.0.0.1]", + "https://prefix[::1]", + "https://[::1]suffix", + "https://[::1%]", + "https://[::1%a%b]", + "https://example.com／path", + ]) { + expect(() => + validateExternalEvidence({ ...normalized().evidence, url }), + ).toThrow(); + } +}); + test("request and pagination metadata retain Cloud byte limits and distinct identities", async () => { const f = await fixture(); const prepared = await prepareExternalPublication(f.file, options, f.deps); @@ -601,6 +682,8 @@ test("CLI creates, reimports unchanged, and updates the same canonical finding", expect(await first.runCli([...f.command, "--yes"], f.cliDeps)).toBe(0); const canonical = f.reports.get("vendor-1")!.canonical_finding_id; expect(JSON.parse(first.stdout.text()).counts.created).toBe(1); + expect(JSON.parse(first.stdout.text())).not.toHaveProperty("cloudUrl"); + expect(first.stderr.text()).not.toContain("/codex/cloud/security/findings"); expect(first.stderr.text()).not.toContain("Selected findings and evidence:"); const again = await prepareExternalPublication(f.file, options, f.deps); expect((await again.publish()).counts.unchanged).toBe(1); @@ -614,6 +697,66 @@ test("CLI creates, reimports unchanged, and updates the same canonical finding", }); }); +test("publication preserves existing environments and resumes ordered mixed batches after the default changes", async () => { + const f = await fixture([ + normalized("existing-first"), + normalized("existing-last"), + ]); + const originalEnvironment = f.state.environmentId!; + await (await prepareExternalPublication(f.file, options, f.deps)).publish(); + f.state.environmentId = "new-default"; + const selected = [ + normalized("existing-first", "critical"), + normalized("new"), + normalized("existing-last", "critical"), + ]; + await writeFile(f.file, JSON.stringify(selected)); + const prepared = await prepareExternalPublication(f.file, options, f.deps); + expect( + prepared.preview.requests.map( + (request) => request.repository.environment_id, + ), + ).toEqual([originalEnvironment, "new-default", originalEnvironment]); + expect( + prepared.preview.requests.flatMap((request) => + request.items.map((item) => item.source_finding_id), + ), + ).toEqual(selected.map((finding) => finding.source_finding_id)); + f.state.throttle = true; + await expect(prepared.publish()).rejects.toThrow("Retry-After"); + f.state.throttle = false; + f.state.environmentId = "changed-default"; + const cli = createCliTest(main); + expect( + await cli.runCli(f.command, { + ...f.cliDeps, + externalPublicationPrompt: { + isInteractive: () => true, + confirm: async () => { + expect(cli.stderr.text()).toContain( + `Environment: ${originalEnvironment}, new-default`, + ); + expect(cli.stderr.text()).not.toContain("changed-default"); + return true; + }, + }, + }), + ).toBe(0); + expect(JSON.parse(cli.stdout.text()).counts).toEqual({ + created: 1, + updated: 2, + unchanged: 0, + error: 0, + }); + expect(f.reports.get("existing-first")!.environment_id).toBe( + originalEnvironment, + ); + expect(f.reports.get("existing-last")!.environment_id).toBe( + originalEnvironment, + ); + expect(f.reports.get("new")!.environment_id).toBe("new-default"); +}); + test("normalized input accepts server-materialized optional evidence defaults", async () => { const f = await fixture([ { From 7759a2ca465bf61c227f102ceb4b07c158526d32 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 8 Oct 2026 20:23:53 +0000 Subject: [PATCH 13/43] fix(cloud): preserve import scope across environment changes --- .../skills/publish-findings/SKILL.md | 2 +- sdk/typescript/docs/cli.md | 10 +- sdk/typescript/src/cli.ts | 4 +- .../src/external-findings-publish.ts | 59 ++--- .../src/external-import-contract.ts | 17 +- .../external-findings-publish.test.ts | 243 +++++++++++++++++- 6 files changed, 293 insertions(+), 42 deletions(-) diff --git a/plugins/codex-security/skills/publish-findings/SKILL.md b/plugins/codex-security/skills/publish-findings/SKILL.md index 837f7e2bd8..c51b488475 100644 --- a/plugins/codex-security/skills/publish-findings/SKILL.md +++ b/plugins/codex-security/skills/publish-findings/SKILL.md @@ -19,4 +19,4 @@ codex-security publish findings /path/to/selected-wiz-findings.json \ Show the destination, selected findings, exclusions, and evidence to the user. Obtain explicit confirmation for this upload, then run the same command without `--dry-run`, adding `--yes`. Existing authorization to inspect or assess findings alone does not authorize sending them to Cloud. A user who already approved the concrete preview need not approve it again. -After a lost response or retryable failure, repeat the same command with unchanged input and destination. The publisher resends its saved request IDs and bodies. After a repository reset, stop and show the error; require fresh discovery and confirmation before a new publication. Report partial failures and retain their explanations. A successful import stores vendor evidence; it does not establish that Codex assessed the finding or that the vulnerability is exploitable. +After a lost response or retryable failure, repeat the same command with unchanged input and destination. The publisher resends only unacknowledged requests, using their saved IDs and bodies. Existing findings keep their original Cloud environment even if the repository’s default changes. After a repository reset, stop and show the error; require fresh discovery and confirmation before a new publication. Report partial failures and retain their explanations. A successful import stores vendor evidence; it does not establish that Codex assessed the finding or that the vulnerability is exploitable. Direct the user to the main Findings view in the Codex Security Cloud app to find the repository’s imported Wiz findings; do not invent a direct Findings URL. diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md index 2d14807ff6..8be06e6932 100644 --- a/sdk/typescript/docs/cli.md +++ b/sdk/typescript/docs/cli.md @@ -784,10 +784,12 @@ codex-security publish findings selected-wiz.json \ This command reads the authorized destination and current source versions during preview. It uses the same saved ChatGPT file login as scan publication. The Cloud repository needs an existing environment; no native scan is required. Review the -account, repository, environment, findings, and exclusions, then rerun without +account, repository, environments, findings, and exclusions, then rerun without `--dry-run`. The default terminal prompt is No; `--yes` confirms a previously reviewed input for scripts or the plugin. Without a terminal or `--yes`, no upload -occurs. +occurs. Existing source findings keep their Cloud environment; new findings use +the repository’s current authorized default. Preview shows the environments that +will receive the selected findings. Input can be a Wiz vulnerability finding, an array, a complete `data.vulnerabilityFindings.nodes` response, or JSONL with one vendor record per @@ -822,7 +824,9 @@ fresh invocation. Final item errors are reported individually with exit code 1; correct the input or source conflict before a fresh submission. Transport or readback failures return exit code 2 and retain the resumable request. Successful receipts are saved locally; the CLI verifies Cloud source-report reads and -returns a Findings link. Search indexing can lag an accepted import. +returns publication receipts and counts. Open the Codex Security Cloud app, go +to its main Findings view, and find the repository’s imported Wiz findings. +Search indexing can lag an accepted import. Preview selected completed scans before uploading: diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 4369a1e239..7d05ed8b96 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -3225,7 +3225,7 @@ export async function main( const { preview } = prepared; const showPreview = () => { errorOutput.write( - `Account: ${diagnosticValue(preview.accountId)}\nDestination: ${diagnosticValue(preview.destination.url)} (${diagnosticValue(preview.destination.id)})\nEnvironment: ${diagnosticValue(preview.destination.import_environment_id)}\nSource: ${preview.source.provider} / ${diagnosticValue(preview.source.source_key)}\nRead: ${preview.read} Ready: ${preview.findings.length} Excluded: ${preview.excluded.length}${preview.resumed ? "\nResuming the saved submission." : ""}\n`, + `Account: ${diagnosticValue(preview.accountId)}\nDestination: ${diagnosticValue(preview.destination.url)} (${diagnosticValue(preview.destination.id)})\nEnvironment: ${[...new Set(preview.requests.map((request) => request.repository.environment_id))].map(diagnosticValue).join(", ")}\nSource: ${preview.source.provider} / ${diagnosticValue(preview.source.source_key)}\nRead: ${preview.read} Ready: ${preview.findings.length} Excluded: ${preview.excluded.length}${preview.resumed ? "\nResuming the saved submission." : ""}\n`, ); for (const excluded of preview.excluded) errorOutput.write( @@ -3259,7 +3259,7 @@ export async function main( if (result.counts.error) exitCode = 1; if (!structured) errorOutput.write( - `Created: ${result.counts.created} Updated: ${result.counts.updated} Unchanged: ${result.counts.unchanged} Failed: ${result.counts.error}\nOpen in Cloud: ${result.cloudUrl}\n`, + `Created: ${result.counts.created} Updated: ${result.counts.updated} Unchanged: ${result.counts.unchanged} Failed: ${result.counts.error}\nOpen the Codex Security Cloud app, go to Findings, and find this repository’s imported Wiz findings.\n`, ); return { ...result, excluded: preview.excluded }; } catch (error) { diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index c29d4b3699..3f11a043a8 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -61,7 +61,6 @@ export interface PreparedExternalPublication { publish(): Promise<{ receipts: FindingImportReceipt[]; counts: FindingImportReceipt["counts"]; - cloudUrl: string; }>; } @@ -185,6 +184,8 @@ export async function prepareExternalPublication( body?: FindingImportRequest, ): Promise { dependencies.signal?.throwIfAborted(); + // Allow upload and response transport around the server's publication budget. + const timeout = AbortSignal.timeout(body ? 60_000 : 30_000); const response = await (dependencies.fetch ?? globalThis.fetch)( `${BASE_URL}${endpoint}`, { @@ -198,8 +199,8 @@ export async function prepareExternalPublication( ...(body ? { body: JSON.stringify(body) } : {}), redirect: "error", signal: dependencies.signal - ? AbortSignal.any([dependencies.signal, AbortSignal.timeout(30_000)]) - : AbortSignal.timeout(30_000), + ? AbortSignal.any([dependencies.signal, timeout]) + : timeout, }, ); if (!response.ok) { @@ -313,38 +314,15 @@ export async function prepareExternalPublication( "The repository was reset after this submission. The saved request was retired without uploading. Review the destination and run the command again to approve a fresh publication.", ); } - if ( - submission.repository.environment_id !== - destination.import_environment_id - ) { - throw new Error( - "The Cloud environment changed since this submission. Restore the original destination before resuming the saved request.", - ); - } } saved = content; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } - if (destination.import_environment_id == null) - throw new CodexSecurityError( - "Configure an authorized Cloud environment for this repository before importing findings.", - ); let requests = saved?.requests; if (!requests) { - const repository = { - id: destination.id, - repo_connector_id: destination.repo_connector_id, - environment_id: destination.import_environment_id, - reset_marker: destination.reset_marker, - }; requests = []; - let current: FindingImportRequest = { - request_id: randomUUID(), - repository, - source, - items: [], - }; + let current: FindingImportRequest | undefined; for (const [index, finding] of parsed.findings.entries()) { const query = new URLSearchParams({ provider: source.provider, @@ -371,6 +349,18 @@ export async function prepareExternalPublication( throw new CodexSecurityError( "Cloud returned a different source identity.", ); + const environmentId = + previous?.environment_id ?? destination.import_environment_id; + if (environmentId == null) + throw new CodexSecurityError( + "Configure an authorized Cloud environment for this repository before importing findings.", + ); + const repository = { + id: destination.id, + repo_connector_id: destination.repo_connector_id, + environment_id: environmentId, + reset_marker: destination.reset_marker, + }; const item = { client_id: `item-${index + 1}`, source_finding_id: finding.source_finding_id, @@ -378,18 +368,20 @@ export async function prepareExternalPublication( evidence: finding.evidence, }; if ( - current.items.length > 0 && - (current.items.length === 100 || + current && + (current.repository.environment_id !== environmentId || + current.items.length === 100 || Buffer.byteLength( JSON.stringify({ ...current, items: [...current.items, item] }), ) > MAX_REQUEST_BYTES) ) { requests.push(validateImportRequest(current)); - current = { request_id: randomUUID(), repository, source, items: [] }; + current = undefined; } + current ??= { request_id: randomUUID(), repository, source, items: [] }; current.items.push(item); } - if (current.items.length) requests.push(validateImportRequest(current)); + if (current) requests.push(validateImportRequest(current)); } const submission: SavedSubmission = { accountId: credentials.account_id, @@ -495,8 +487,9 @@ export async function prepareExternalPublication( // Verify readable source records without mistaking a newer concurrent // observation for failure of the original, immutable import receipt. for (const [batchIndex, receipt] of receipts.entries()) { + const batch = requests[batchIndex]!; const batchItems = new Map( - requests[batchIndex]!.items.map((item) => [item.client_id, item]), + batch.items.map((item) => [item.client_id, item]), ); for (const item of receipt.results) { if (item.outcome === "error") continue; @@ -516,6 +509,7 @@ export async function prepareExternalPublication( report.source_finding_id !== expected.source_finding_id || report.repo_id !== destination.id || report.repo_connector_id !== destination.repo_connector_id || + report.environment_id !== batch.repository.environment_id || canonicalJson(report.source) !== canonicalJson(source) ) throw new CodexSecurityError( @@ -538,7 +532,6 @@ export async function prepareExternalPublication( const result = { receipts, counts, - cloudUrl: `https://chatgpt.com/codex/cloud/security/findings?repo=${encodeURIComponent(destination.url)}&source=imported&provider=${source.provider}`, }; await writeAtomicJson(join(state, `${key}.result.json`), result); await rm(pendingPath, { force: true }); diff --git a/sdk/typescript/src/external-import-contract.ts b/sdk/typescript/src/external-import-contract.ts index 3cb03a3e12..b126d1d115 100644 --- a/sdk/typescript/src/external-import-contract.ts +++ b/sdk/typescript/src/external-import-contract.ts @@ -1,3 +1,4 @@ +import { isIP } from "node:net"; import Ajv2020 from "ajv/dist/2020.js"; import schema from "../schemas/external-findings.schema.json" with { type: "json" }; import type { @@ -97,10 +98,24 @@ export function validateExternalEvidence( .replace(/[\t\r\n]/gu, "") .replace(/^[\u0000-\u0020]+/u, ""); const authority = /^https?:\/\/([^/?#]+)/iu.exec(url)?.[1]; + const normalizedAuthority = (authority ?? "") + .replace(/[@:#?]/gu, "") + .normalize("NFKC"); + let validHost = true; + if (authority?.includes("[") || authority?.includes("]")) { + const hostname = /^\[([^\]]+)\](?::[\s\S]*)?$/u.exec(authority)?.[1]; + const [address, scope, extraScope] = (hostname ?? "").split("%"); + // Cloud accepts scoped IPv6 and IPvFuture hosts in brackets. + validHost = + hostname !== undefined && + (/^v[a-fA-F0-9]+\.[\s\S]+$/u.test(hostname) || + (isIP(address!) === 6 && scope !== "" && extraScope === undefined)); + } if ( !authority || authority.includes("@") || - authority.includes("[") !== authority.includes("]") + /[/?#@:]/u.test(normalizedAuthority) || + !validHost ) throw new Error( "Evidence URLs must be HTTP(S) links without credentials.", diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 79b9b9398f..d77ba0f1dd 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -438,11 +438,27 @@ test("Cloud validation preserves allowed evidence boundaries and raw vendor stri "https:///finding", "https:\\example.test", "http://@example.test", + "https://[127.0.0.1]/finding", + "https://[example.test]/finding", + "https://prefix[::1]/finding", + "https://[::1]suffix/finding", + "https://example.test\uFF1A443/finding", + "https://exam\u2100ple.test/finding", ]) { expect(() => validateExternalEvidence({ ...evidence, url })).toThrow( "HTTP(S)", ); } + for (const url of [ + "https://[::1]/finding", + "https://[fe80::1%25eth0]/finding", + "https://[fe80::1%zone!]/finding", + "https://[v1.example]/finding", + "https://[::1]:non-numeric/finding", + "https://bücher.example/finding", + ]) { + expect(validateExternalEvidence({ ...evidence, url }).url).toBe(url); + } for (const path of ["/absolute.ts", "C:\\source.ts", "src\\..\\outside.ts"]) { expect(() => validateExternalEvidence({ ...evidence, locations: [{ path }] }), @@ -625,8 +641,14 @@ test.each(["human", "json"])( ).toBe(0); expect(prompted).toBe(true); expect(f.reports.has("vendor-selected")).toBe(true); - if (format === "json") - expect(JSON.parse(cli.stdout.text()).counts.created).toBe(1); + if (format === "json") { + const result = JSON.parse(cli.stdout.text()); + expect(result.counts.created).toBe(1); + expect(result).not.toHaveProperty("cloudUrl"); + } else { + expect(cli.stderr.text()).toContain("Findings"); + expect(cli.stderr.text()).not.toContain("/codex/cloud/security/findings"); + } }, ); @@ -1417,3 +1439,220 @@ test.each(["duplicate", "missing", "unexpected"])( expect(f.posts).toHaveLength(1); }, ); + +test.each(["replacement-environment", null])( + "saved publication preserves its environment when discovery changes to %p", + async (environmentId) => { + const f = await fixture(); + f.state.loseResponse = true; + await expect( + (await prepareExternalPublication(f.file, options, f.deps)).publish(), + ).rejects.toThrow("resume the saved request"); + const original = f.posts[0]!; + f.state.environmentId = environmentId; + const retry = await prepareExternalPublication(f.file, options, f.deps); + expect(retry.preview.resumed).toBe(true); + expect(retry.preview.requests[0]!.repository.environment_id).toBe( + "environment-example", + ); + expect((await retry.publish()).counts.created).toBe(1); + expect(f.posts.at(-1)).toBe(original); + }, +); + +test("mixed environments preserve input order and are shown before confirmation", async () => { + const f = await fixture([normalized("existing-a"), normalized("existing-c")]); + await (await prepareExternalPublication(f.file, options, f.deps)).publish(); + const originalIds = ["existing-a", "existing-c"].map( + (id) => f.reports.get(id)!.canonical_finding_id, + ); + f.state.environmentId = "replacement-environment"; + await writeFile( + f.file, + JSON.stringify([ + normalized("existing-a", "critical"), + normalized("new-b"), + normalized("existing-c", "critical"), + ]), + ); + const prepared = await prepareExternalPublication(f.file, options, f.deps); + expect( + prepared.preview.requests.map( + (request) => request.repository.environment_id, + ), + ).toEqual([ + "environment-example", + "replacement-environment", + "environment-example", + ]); + expect( + prepared.preview.requests.flatMap((request) => + request.items.map((item) => [ + item.source_finding_id, + item.expected_version, + ]), + ), + ).toEqual([ + ["existing-a", 1], + ["new-b", 0], + ["existing-c", 1], + ]); + const cli = createCliTest(main); + expect( + await cli.runCli(f.command, { + ...f.cliDeps, + externalPublicationPrompt: { + isInteractive: () => true, + confirm: async () => { + expect(cli.stderr.text()).toContain( + "Environment: environment-example, replacement-environment", + ); + expect(f.posts).toHaveLength(1); + return true; + }, + }, + }), + ).toBe(0); + expect(JSON.parse(cli.stdout.text()).counts).toEqual({ + created: 1, + updated: 2, + unchanged: 0, + error: 0, + }); + expect( + ["existing-a", "new-b", "existing-c"].map( + (id) => f.reports.get(id)!.environment_id, + ), + ).toEqual([ + "environment-example", + "replacement-environment", + "environment-example", + ]); + expect( + ["existing-a", "existing-c"].map( + (id) => f.reports.get(id)!.canonical_finding_id, + ), + ).toEqual(originalIds); +}); + +test("existing source updates do not need a newly discovered default environment", async () => { + const f = await fixture(); + await (await prepareExternalPublication(f.file, options, f.deps)).publish(); + f.state.environmentId = null; + await writeFile(f.file, JSON.stringify([normalized("vendor-1", "critical")])); + const prepared = await prepareExternalPublication(f.file, options, f.deps); + expect(prepared.preview.requests[0]!.repository.environment_id).toBe( + "environment-example", + ); + expect((await prepared.publish()).counts.updated).toBe(1); + const posts = f.posts.length; + await writeFile(f.file, JSON.stringify([normalized("new-vendor")])); + await expect( + prepareExternalPublication(f.file, options, f.deps), + ).rejects.toThrow("Configure an authorized Cloud environment"); + expect(f.posts).toHaveLength(posts); +}); + +test("source readback must preserve the acknowledged batch environment", async () => { + const f = await fixture(); + const prepared = await prepareExternalPublication(f.file, options, { + ...f.deps, + fetch: async (url, init) => { + const response = await f.deps.fetch(url, init); + if (new URL(url).pathname.includes("/source_reports/")) { + return Response.json({ + ...(await response.json()), + environment_id: "unexpected-environment", + }); + } + return response; + }, + }); + await expect(prepared.publish()).rejects.toThrow( + "Cloud readback did not match the saved finding identity", + ); + expect(f.posts).toHaveLength(1); + const retry = await prepareExternalPublication(f.file, options, f.deps); + expect((await retry.publish()).counts.created).toBe(1); + expect(f.posts).toHaveLength(1); +}); + +test("malformed bracketed URLs exclude only their own finding before upload", async () => { + const invalid = normalized("invalid-url"); + const f = await fixture([ + normalized("valid-url"), + { + ...invalid, + evidence: { ...invalid.evidence, url: "https://[127.0.0.1]/finding" }, + }, + ]); + const prepared = await prepareExternalPublication(f.file, options, f.deps); + expect( + prepared.preview.findings.map((finding) => finding.source_finding_id), + ).toEqual(["valid-url"]); + expect(prepared.preview.excluded).toHaveLength(1); + expect(prepared.preview.excluded[0]!.reason).toContain("HTTP(S)"); + expect((await prepared.publish()).counts.created).toBe(1); + expect(f.reports.has("invalid-url")).toBe(false); +}); + +test.each([false, true])( + "publication waits through the server budget while retaining cancellation: %p", + async (cancel) => { + const f = await fixture(); + const arrived = Promise.withResolvers(); + const release = Promise.withResolvers(); + const controller = new AbortController(); + const prepared = await prepareExternalPublication(f.file, options, { + ...f.deps, + signal: controller.signal, + fetch: async (url, init) => { + if (init.method === "POST") { + arrived.resolve(); + await release.promise; + } + return f.deps.fetch(url, init); + }, + }); + const deadlines: { milliseconds: number; controller: AbortController }[] = + []; + const timeout = spyOn(AbortSignal, "timeout").mockImplementation( + (milliseconds) => { + const controller = new AbortController(); + deadlines.push({ milliseconds, controller }); + return controller.signal; + }, + ); + const completion = prepared.publish(); + try { + await Promise.race([ + arrived.promise, + completion.then(() => { + throw new Error( + "Publication completed before the POST was released.", + ); + }), + ]); + // Advance a controlled clock through the server's publication budget. + for (const deadline of deadlines) { + if (deadline.milliseconds <= 45_000) + deadline.controller.abort( + new DOMException("Publication deadline elapsed", "TimeoutError"), + ); + } + if (cancel) controller.abort(new Error("Synthetic cancellation")); + release.resolve(); + if (cancel) { + await expect(completion).rejects.toThrow("Synthetic cancellation"); + expect(f.posts).toHaveLength(0); + } else { + expect((await completion).counts.created).toBe(1); + expect(f.posts).toHaveLength(1); + } + } finally { + release.resolve(); + await completion.catch(() => undefined); + timeout.mockRestore(); + } + }, +); From f293e77aca5ba620dbcc083659b040de141cc0dd Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 8 Oct 2026 20:34:56 +0000 Subject: [PATCH 14/43] fix(cloud): allow publication processing time and preserve legacy ecosystems --- .../src/external-findings-publish.ts | 5 +- sdk/typescript/src/wiz-findings.ts | 3 +- .../external-findings-publish.test.ts | 65 +++++++++++++++++++ 3 files changed, 70 insertions(+), 3 deletions(-) diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 3fe5517884..5b55b2b52e 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -184,6 +184,7 @@ export async function prepareExternalPublication( body?: FindingImportRequest, ): Promise { dependencies.signal?.throwIfAborted(); + const requestTimeout = AbortSignal.timeout(body ? 60_000 : 30_000); const response = await (dependencies.fetch ?? globalThis.fetch)( `${BASE_URL}${endpoint}`, { @@ -197,8 +198,8 @@ export async function prepareExternalPublication( ...(body ? { body: JSON.stringify(body) } : {}), redirect: "error", signal: dependencies.signal - ? AbortSignal.any([dependencies.signal, AbortSignal.timeout(30_000)]) - : AbortSignal.timeout(30_000), + ? AbortSignal.any([dependencies.signal, requestTimeout]) + : requestTimeout, }, ); if (!response.ok) { diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts index 50f9608bf0..4fb1244a77 100644 --- a/sdk/typescript/src/wiz-findings.ts +++ b/sdk/typescript/src/wiz-findings.ts @@ -85,7 +85,8 @@ function wizFinding(record: Record): VendorFinding { ecosystem: text(record["packageManager"]) ?? text(artifactType?.["osPackageManager"]) ?? - text(artifactType?.["codeLibraryLanguage"]), + text(artifactType?.["codeLibraryLanguage"]) ?? + text(record["codeLibraryLanguage"]), installed_version: version, manifest_path: null, fixed_versions: text(record["fixedVersion"]) diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index e78a2ec8ba..189689813d 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -359,6 +359,7 @@ test("Wiz API nodes preserve artifact ecosystems and distinguish container diges ...finding, id: "container-package", artifactType: { osPackageManager: "DPKG", codeLibraryLanguage: null }, + codeLibraryLanguage: "JAVA", vulnerableAsset: { id: "container-image", type: "CONTAINER_IMAGE", @@ -369,6 +370,7 @@ test("Wiz API nodes preserve artifact ecosystems and distinguish container diges ...finding, id: "vm-library", artifactType: { osPackageManager: null, codeLibraryLanguage: "PYTHON" }, + codeLibraryLanguage: "JAVA", vulnerableAsset: { id: "virtual-machine", type: "VIRTUAL_MACHINE", @@ -391,12 +393,19 @@ test("Wiz API nodes preserve artifact ecosystems and distinguish container diges packageManager: "reported-manager", imageDigest: `sha256:${"b".repeat(64)}`, artifactType: { osPackageManager: "DPKG" }, + codeLibraryLanguage: "JAVA", vulnerableAsset: { id: "explicit-container", type: "CONTAINER_IMAGE", imageId: `sha256:${"c".repeat(64)}`, }, }, + { + ...finding, + id: "legacy-library", + codeLibraryLanguage: "JAVASCRIPT", + vulnerableAsset: { id: "legacy-image" }, + }, ]; const f = await fixture({ data: { @@ -422,6 +431,7 @@ test("Wiz API nodes preserve artifact ecosystems and distinguish container diges { ecosystem: "PYTHON", digests: [] }, { ecosystem: null, digests: [`sha256:${"d".repeat(64)}`] }, { ecosystem: "reported-manager", digests: [`sha256:${"b".repeat(64)}`] }, + { ecosystem: "JAVASCRIPT", digests: [] }, ]); expect(parsed.findings.map(({ evidence }) => evidence.source_data)).toEqual( records, @@ -1060,6 +1070,61 @@ test("canceling a waiting publisher preserves the active request", async () => { } }); +test("longer publications remain cancelable and resume their saved request", async () => { + const f = await fixture(); + let elapsed = 0; + const deadlines: { at: number; controller: AbortController }[] = []; + const timeout = spyOn(AbortSignal, "timeout").mockImplementation((delay) => { + const controller = new AbortController(); + deadlines.push({ at: elapsed + delay, controller }); + return controller.signal; + }); + const controller = new AbortController(); + let cancel = false; + const deps = { + ...f.deps, + signal: controller.signal, + fetch: async (url: string, init: RequestInit) => { + if (init.method === "POST") { + // Advance request deadlines without waiting for the server's processing budget. + elapsed += 45_000; + for (const deadline of deadlines) + if (deadline.at <= elapsed) + deadline.controller.abort( + new DOMException("Timed out", "TimeoutError"), + ); + if (cancel) controller.abort(new Error("Publication canceled")); + } + return f.deps.fetch(url, init); + }, + }; + try { + expect( + ( + await ( + await prepareExternalPublication(f.file, options, deps) + ).publish() + ).counts.created, + ).toBe(1); + await writeFile( + f.file, + JSON.stringify([normalized("vendor-1", "critical")]), + ); + cancel = true; + const prepared = await prepareExternalPublication(f.file, options, deps); + await expect(prepared.publish()).rejects.toThrow("Publication canceled"); + expect(f.posts).toHaveLength(1); + const retry = await prepareExternalPublication(f.file, options, f.deps); + expect(retry.preview.resumed).toBe(true); + expect(retry.preview.requests[0]!.request_id).toBe( + prepared.preview.requests[0]!.request_id, + ); + expect((await retry.publish()).counts.updated).toBe(1); + } finally { + timeout.mockRestore(); + } +}); + test("readback failure retries acknowledged receipts without another POST", async () => { const f = await fixture(); f.state.brokenReadback = true; From b50aa7173c78885a586ccfb7d4523461ea1ffcfb Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 8 Oct 2026 20:50:08 +0000 Subject: [PATCH 15/43] Match numeric evidence sizes to Cloud validation --- .../src/external-import-contract.ts | 19 ++++++++-- .../external-findings-publish.test.ts | 35 +++++++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/sdk/typescript/src/external-import-contract.ts b/sdk/typescript/src/external-import-contract.ts index b126d1d115..13dbcbf7a3 100644 --- a/sdk/typescript/src/external-import-contract.ts +++ b/sdk/typescript/src/external-import-contract.ts @@ -121,6 +121,7 @@ export function validateExternalEvidence( "Evidence URLs must be HTTP(S) links without credentials.", ); } + let numericByteAdjustment = 0; const pending: [unknown, number][] = [[result, 0]]; while (pending.length > 0) { const [value, depth] = pending.pop()!; @@ -129,11 +130,23 @@ export function validateExternalEvidence( if (value !== null && typeof value === "object") { for (const child of Object.values(value)) pending.push([child, depth + 1]); - } else if (typeof value === "number" && !Number.isFinite(value)) { - throw new Error("Evidence must contain finite JSON numbers."); + } else if (typeof value === "number") { + if (!Number.isFinite(value)) + throw new Error("Evidence must contain finite JSON numbers."); + const wire = JSON.stringify(value); + // Cloud pads short exponents and formats small fractions scientifically. + const cloud = ( + wire.includes(".") && Math.abs(value) < 1e-4 + ? value.toExponential() + : wire + ).replace(/e-(\d)$/u, "e-0$1"); + numericByteAdjustment += cloud.length - wire.length; } } - if (Buffer.byteLength(JSON.stringify(result)) > 256 * 1024) + if ( + Buffer.byteLength(JSON.stringify(result)) + numericByteAdjustment > + 256 * 1024 + ) throw new Error("Evidence exceeds the Cloud limit of 256 KiB."); return result; } diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index d77ba0f1dd..eb7f017540 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -1656,3 +1656,38 @@ test.each([false, true])( } }, ); + +test("numeric evidence size excludes an oversized item and preserves a valid mixed-format sibling", async () => { + const oversized = { + ...normalized("oversized-numbers"), + evidence: { + ...normalized().evidence, + source_data: { metrics: Array(48_000).fill(1e-7) }, + }, + }; + const accepted = { + ...normalized("accepted-numbers"), + evidence: { + ...normalized().evidence, + source_data: { + metrics: [...Array(10_000).fill(1e-7), ...Array(30_000).fill(1e-6)], + }, + }, + }; + expect(Buffer.byteLength(JSON.stringify(accepted.evidence))).toBeGreaterThan( + 256 * 1024, + ); + const f = await fixture([oversized, accepted]); + const prepared = await prepareExternalPublication(f.file, options, f.deps); + expect( + prepared.preview.findings.map((finding) => finding.source_finding_id), + ).toEqual(["accepted-numbers"]); + expect(prepared.preview.excluded).toHaveLength(1); + expect(prepared.preview.excluded[0]!.reason).toContain("256 KiB"); + expect((await prepared.publish()).counts.created).toBe(1); + expect(f.posts).toHaveLength(1); + expect(JSON.parse(f.posts[0]!).items[0].evidence.source_data).toEqual( + accepted.evidence.source_data, + ); + expect(f.reports.has("oversized-numbers")).toBe(false); +}); From 6786d35366c5916be75208caf3a2067faa393064 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Thu, 8 Oct 2026 23:53:09 -0400 Subject: [PATCH 16/43] fix(cloud): preserve import evidence and report partial uploads --- .../skills/publish-findings/SKILL.md | 10 +- sdk/typescript/docs/cli.md | 76 ++++- sdk/typescript/src/cli.ts | 128 ++++++- .../src/external-findings-publish.ts | 317 +++++++++++++----- .../src/external-import-contract.ts | 4 + sdk/typescript/src/wiz-findings.ts | 7 +- .../external-findings-publish.test.ts | 300 +++++++++++++++-- 7 files changed, 712 insertions(+), 130 deletions(-) diff --git a/plugins/codex-security/skills/publish-findings/SKILL.md b/plugins/codex-security/skills/publish-findings/SKILL.md index c51b488475..f6f059f2de 100644 --- a/plugins/codex-security/skills/publish-findings/SKILL.md +++ b/plugins/codex-security/skills/publish-findings/SKILL.md @@ -1,22 +1,24 @@ --- name: publish-findings -description: Publish selected external scanner findings to Codex Security Cloud after previewing the destination and evidence. Supports saved Wiz vulnerability findings and normalized JSONL; does not run a scan or assess findings. +description: Import selected Wiz package vulnerabilities to Codex Security Cloud after previewing the destination and evidence. Supports saved vulnerability JSON and normalized JSONL; does not import raw SAST, secrets, or IaC findings, run a scan, or assess findings. --- # Publish external findings Use the installed `codex-security publish findings` command for both preview and upload. It owns source mapping, Cloud authentication, source versions, and saved request retries. Do not implement another HTTP uploader or put credentials in a command argument. -Obtain the Cloud repository ID from the user's destination. Read selected Wiz vulnerability findings through the user's configured connection or supplied export. Save the actual selected JSON records locally, retaining their IDs and evidence. Finish the requested pagination; if the user selected a subset, save that explicit subset as an array. The publisher also accepts JSONL records with `source_finding_id` and `evidence` as described in the CLI documentation. +Resolve the user's destination to its repository URL or Cloud repository ID; the command accepts either. Confirm that the repository is connected in the intended Cloud account and has an authorized environment. Use the existing file-backed ChatGPT login; explain a missing or incompatible login rather than changing credential storage automatically. Read selected Wiz package vulnerabilities through the user's configured connection or supplied export. Save the actual selected JSON records locally, retaining their IDs and evidence. Finish the requested pagination; if the user selected a subset, save that explicit subset as an array. The command does not fetch from Wiz or accept CSV, SAST, secrets, IaC, or external network exports. The publisher also accepts JSONL records with `source_finding_id` and `evidence` as described in the CLI documentation. Use a stable Wiz tenant/finding namespace as `--source-key`. A project filter is selection context, not a new identity. Confirm the selected assets or builds belong to the destination repository; do not infer a scanner branch or revision from the current checkout. Do not publish vendor credentials, unrelated records, or local assessments as original scanner evidence. ```sh codex-security publish findings /path/to/selected-wiz-findings.json \ - --to cloud --repository REPOSITORY_ID --provider wiz \ + --to cloud --repository https://github.com/example/project --provider wiz \ --source-key TENANT_ID/vulnerability-finding --dry-run --format json ``` -Show the destination, selected findings, exclusions, and evidence to the user. Obtain explicit confirmation for this upload, then run the same command without `--dry-run`, adding `--yes`. Existing authorization to inspect or assess findings alone does not authorize sending them to Cloud. A user who already approved the concrete preview need not approve it again. +Show a concise preview of the account, repository, environments, source namespace, severity counts, and exclusions. Identify the saved evidence file and let the user inspect the complete JSON preview; do not paste an entire large report into the conversation. Explain that imported vendor evidence is not a Codex assessment. Obtain explicit confirmation for this upload, including any skipped records, then run the same command without `--dry-run`, adding `--yes`. Existing authorization to inspect or assess findings alone does not authorize sending them to Cloud. A user who already approved the concrete preview need not approve it again. After a lost response or retryable failure, repeat the same command with unchanged input and destination. The publisher resends only unacknowledged requests, using their saved IDs and bodies. Existing findings keep their original Cloud environment even if the repository’s default changes. After a repository reset, stop and show the error; require fresh discovery and confirmation before a new publication. Report partial failures and retain their explanations. A successful import stores vendor evidence; it does not establish that Codex assessed the finding or that the vulnerability is exploitable. Direct the user to the main Findings view in the Codex Security Cloud app to find the repository’s imported Wiz findings; do not invent a direct Findings URL. + +Read the result even when the process exits nonzero. `complete` exits 0, `partial` exits 1 for local exclusions or final item failures, and `interrupted` exits 2 while preserving acknowledged counts and receipts. Report `failures` by vendor source ID, distinguish `verified` from `unacknowledged`, and retain `savedSubmission` and the recovery error. A lost response does not mean that no findings reached Cloud. Preparation failures use `failed`; cancellation keeps the normal signal exit code. Terminal progress goes to stderr and does not replace the JSON result. diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md index 332835f369..722d1d4360 100644 --- a/sdk/typescript/docs/cli.md +++ b/sdk/typescript/docs/cli.md @@ -791,15 +791,41 @@ again when rerunning. ## Publish findings to Cloud -Publish selected external Wiz vulnerability findings without creating a scan: +### Import Wiz package vulnerabilities + +Import selected Wiz package vulnerabilities without creating a scan. Imported +records retain vendor evidence and remain **not assessed by Codex**. Raw SAST, +secret, IaC, and external network findings need separate mappings; this is not +a synchronization of every finding in Wiz. + +Before the first import: + +1. Select an existing repository in Codex Security Cloud and copy its repository + URL (for example, a GitHub URL) or Cloud repository ID. The repository needs an + authorized environment, but does not need a completed native scan. +2. Use your authorized Wiz connection to read selected package vulnerability + records and save their JSON. Include `id`, `name`, `detailedName`, + `vendorSeverity` (or `severity`), and `vulnerableAsset.id`; retain the available + package, image, and source metadata. The command does not fetch Wiz data or + accept CSV exports. See the accepted input formats below. +3. Choose a stable source key such as `TENANT_ID/vulnerability-finding`. Reuse it + for that Wiz tenant and finding class across exports. Do not create a new key + for each project filter or import; that creates different source identities. +4. Check the ChatGPT account used by your existing file-backed Codex Security + login. API-key-only and keyring-only logins are not supported for Cloud + publication. See the credential-storage instructions below. + +Preview the selection and destination: ```bash codex-security publish findings selected-wiz.json \ - --to cloud --repository REPOSITORY_ID --provider wiz \ + --to cloud --repository https://github.com/example/project --provider wiz \ --source-key TENANT_ID/vulnerability-finding --dry-run --format json ``` -This command reads the authorized destination and current source versions during +`--repository` accepts the repository's URL or ID. Both resolve to the same Cloud +destination and resume the same saved submission. This command reads the +authorized destination and current source versions during preview. It uses the same saved ChatGPT file login as scan publication. The Cloud repository needs an existing environment; no native scan is required. Review the account, repository, environments, findings, and exclusions, then rerun without @@ -807,7 +833,11 @@ account, repository, environments, findings, and exclusions, then rerun without reviewed input for scripts or the plugin. Without a terminal or `--yes`, no upload occurs. Existing source findings keep their Cloud environment; new findings use the repository’s current authorized default. Preview shows the environments that -will receive the selected findings. +will receive the selected findings. The terminal preview summarizes severities, +shows up to five finding IDs and titles, and identifies the evidence file and +exclusions. `--dry-run --format json` retains all normalized findings and complete +evidence for inspection. A prompt explicitly includes any records that will be +skipped. Input can be a Wiz vulnerability finding, an array, a complete `data.vulnerabilityFindings.nodes` response, or JSONL with one vendor record per @@ -835,19 +865,47 @@ fetch from Wiz, assess findings, or change vendor or Cloud triage decisions. Findings from external network scans are excluded from this package vulnerability mapping. +Inputs must be valid UTF-8. Integer-valued evidence outside JavaScript's safe +integer range is rejected; export those values as strings to preserve them +exactly. Invalid records appear in `excluded` with their source ID and reason. + Requests contain at most 100 findings and respect the Cloud payload limits. Before uploading, the publisher saves request IDs and bodies privately under the configured Codex Security state directory. After an uncertain response, repeat the same command with unchanged input to resume those requests. A reset retires the old request without republishing; review and explicitly approve a -fresh invocation. Final item errors are reported individually with exit code 1; -correct the input or source conflict before a fresh submission. Transport or -readback failures return exit code 2 and retain the resumable request. Successful -receipts are saved locally; the CLI verifies Cloud source-report reads and -returns publication receipts and counts. Open the Codex Security Cloud app, go +fresh invocation. + +Results distinguish completion from partial or interrupted work: + +| Status | Exit code | Meaning | +| ------------- | --------- | -------------------------------------------------------------------------------------------------- | +| `complete` | 0 | All selected records were acknowledged and verified. | +| `partial` | 1 | The import finished, but some records were excluded locally or rejected by Cloud. | +| `interrupted` | 2 | Upload, verification, or receipt persistence stopped; acknowledged work is included in the result. | +| `failed` | 2 | Preparation failed before uploading; structured output includes the error. | + +Ctrl-C and SIGTERM retain exit codes 130 and 143. Interrupted JSON output preserves +`receipts`, `counts`, `failures` with vendor source IDs, `excluded`, `verified`, and +`unacknowledged`. A lost response can mean unacknowledged findings were accepted; +resume the saved submission rather than treating them as never uploaded. When +available, `savedSubmission` identifies the checkpoint to resume by repeating the +same command. Review `error` before retrying; a repository reset retires the old +submission and requires fresh approval. Correct local exclusions or final source +conflicts before preparing a new submission. + +Counts are cumulative for the saved submission: a resumed result includes +batches acknowledged by earlier invocations, not just the latest upload attempt. + +Terminal progress reports preparation, acknowledged uploads, and verification +on stderr; JSON results stay on stdout. Lookups and readback use at most four +concurrent requests, while uploads remain sequential. Successful receipts are +saved locally. Open the Codex Security Cloud app, go to its main Findings view, and find the repository’s imported Wiz findings. Search indexing can lag an accepted import. +### Publish completed scans + Preview selected completed scans before uploading: ```bash diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 39e3594a02..54b866e592 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -92,7 +92,12 @@ import { isRecord as isJsonObject } from "./record.js"; import { suggestOwnersInternal } from "./suggest-owners.js"; import { parseImportedFindings } from "./findings-import.js"; import { publishScanToCustom } from "./custom-publish.js"; -import { prepareExternalPublication } from "./external-findings-publish.js"; +import { + ExternalPublicationError, + prepareExternalPublication, + type ExternalPublicationProgress, + type ExternalPublicationResult, +} from "./external-findings-publish.js"; import { DEFAULT_DEDUPE_CONCURRENCY } from "./deduplication/deduplication.js"; import { savedScanWorkbench } from "./saved-scan-bootstrap.js"; import { deduplicateScanInternal } from "./deduplication/scan.js"; @@ -547,6 +552,34 @@ function publicationIssueUrl(value: string | undefined): string | undefined { return value; } +function renderExternalPublicationSummary( + result: ExternalPublicationResult, +): string { + return [ + `Import ${result.status}.`, + `Submission totals — Created: ${result.counts.created} Updated: ${result.counts.updated} Unchanged: ${result.counts.unchanged} Failed: ${result.counts.error}`, + `Excluded: ${result.excluded.length} Awaiting acknowledgement: ${result.unacknowledged} Verified: ${result.verified}`, + ...result.failures.map( + (item) => + `Failed ${diagnosticValue(item.source_finding_id)} (${item.code}): ${diagnosticValue(item.message)}`, + ), + ...result.excluded.map( + (item) => + `Excluded item ${item.position}${item.source_finding_id ? ` (${diagnosticValue(item.source_finding_id)})` : ""}: ${diagnosticValue(item.reason)}`, + ), + ...(result.savedSubmission + ? [`Saved submission: ${diagnosticValue(result.savedSubmission)}`] + : []), + ...(result.status === "interrupted" + ? [] + : [ + "Imported Wiz evidence remains not assessed by Codex.", + "Open the Codex Security Cloud app's Findings view and select this repository. Search indexing can lag an accepted import.", + ]), + "", + ].join("\n"); +} + function renderPublicationSummary( result: PublishScanResult, color: boolean, @@ -3114,7 +3147,9 @@ export async function main( }, }); publication.command("findings", { - description: "Preview and publish selected Wiz findings to Cloud.", + description: + "Preview and import selected Wiz package vulnerabilities to Cloud.", + hint: "Supply selected Wiz vulnerability JSON or normalized JSONL (not CSV, SAST, secrets, or IaC exports).\nExample: codex-security publish findings selected-wiz.json --to cloud --repository https://github.com/example/project --provider wiz --source-key TENANT_ID/vulnerability-finding --dry-run --format json", destructive: true, mcp: false, args: z.object({ @@ -3125,7 +3160,7 @@ export async function main( options: z.object({ to: z.literal("cloud").describe("Publication destination."), repository: optionValue("--repository").describe( - "Authorized Cloud repository ID.", + "Authorized Cloud repository ID or repository URL copied from Cloud.", ), provider: z .literal("wiz") @@ -3151,6 +3186,31 @@ export async function main( const controller = new AbortController(); const removeSignals = listenForAbort(dependencies, controller); const structured = formatExplicit && format !== "toon"; + const humanSummary = + format === "toon" && + !formatExplicit && + !argv.some((argument) => OUTPUT_OPTION.test(argument)); + let lastProgress = ""; + const onProgress = (event: ExternalPublicationProgress) => { + if (errorOutput.isTTY !== true) return; + const step = event.total + ? Math.floor((event.completed * 10) / event.total) + : 0; + const key = `${event.phase}:${step}`; + if (key === lastProgress) return; + lastProgress = key; + const labels = { + reading: "Reading selected findings", + discovering: "Checking Cloud account and repository", + preparing: "Preparing findings", + waiting: "Waiting for publication access", + uploading: "Acknowledged findings", + verifying: "Verifying imported findings", + }; + errorOutput.write( + `${labels[event.phase]}${event.total === undefined ? "..." : `: ${event.completed}/${event.total}`}\n`, + ); + }; try { const prepared = await prepareExternalPublication( resolveCliPath(dependencies.currentDirectory(), args.file), @@ -3159,6 +3219,7 @@ export async function main( environment: dependencies.environment, fetch: dependencies.cloudFetch, signal: controller.signal, + onProgress, }, ); const { preview } = prepared; @@ -3170,9 +3231,36 @@ export async function main( errorOutput.write( `Excluded item ${excluded.position}: ${diagnosticValue(excluded.reason)}\n`, ); + errorOutput.write( + `Evidence file: ${diagnosticValue(resolveCliPath(dependencies.currentDirectory(), args.file))}\nImported evidence remains not assessed by Codex.\n`, + ); + const severities = new Map(); + for (const finding of preview.findings) + severities.set( + finding.evidence.severity, + (severities.get(finding.evidence.severity) ?? 0) + 1, + ); + errorOutput.write( + `Severities: ${[...severities].map(([severity, count]) => `${severity}: ${count}`).join(" ")}\n`, + ); + for (const finding of preview.findings.slice(0, 5)) + errorOutput.write( + ` ${diagnosticValue(finding.source_finding_id)} [${finding.evidence.severity}] ${diagnosticValue(finding.evidence.title)}\n`, + ); + if (preview.findings.length > 5) + errorOutput.write( + `Showing 5 of ${preview.findings.length} findings.\n`, + ); + errorOutput.write( + "Use --dry-run --format json to inspect all normalized findings and the complete evidence before uploading.\n", + ); }; if (!structured) showPreview(); - if (options.dryRun) return { ...preview, dryRun: true }; + if (options.dryRun) { + if (humanSummary) + renderedPublication = "Dry run complete. No findings uploaded.\n"; + return { ...preview, dryRun: true }; + } if (!options.yes) { const prompt = dependencies.externalPublicationPrompt ?? @@ -3182,31 +3270,39 @@ export async function main( "Publication needs confirmation. Run --dry-run, review the results, then use --yes to approve this input.", ); if (structured) showPreview(); - errorOutput.write( - `Selected findings and evidence:\n${JSON.stringify(preview.findings, null, 2).split("\n").map(diagnosticValue).join("\n")}\n`, - ); if ( !(await prompt.confirm( - `Publish these ${preview.findings.length} findings?`, + `Publish these ${preview.findings.length} findings${preview.excluded.length ? ` and skip ${preview.excluded.length} excluded records` : ""}?`, false, controller.signal, )) - ) + ) { + if (humanSummary) renderedPublication = "No findings published.\n"; return { published: false, preview }; + } } const result = await prepared.publish(); - if (result.counts.error) exitCode = 1; - if (!structured) - errorOutput.write( - `Created: ${result.counts.created} Updated: ${result.counts.updated} Unchanged: ${result.counts.unchanged} Failed: ${result.counts.error}\nOpen the Codex Security Cloud app, go to Findings, and find this repository’s imported Wiz findings.\n`, - ); - return { ...result, excluded: preview.excluded }; + if (result.status === "partial") exitCode = 1; + if (humanSummary) + renderedPublication = renderExternalPublicationSummary(result); + else if (!structured) + errorOutput.write(renderExternalPublicationSummary(result)); + return { ...result }; } catch (error) { reportPublicationError( diagnosticValue(error), controller.signal.aborted ? controller.signal.reason : undefined, ); - return undefined; + if (error instanceof ExternalPublicationError) { + if (humanSummary) + renderedPublication = renderExternalPublicationSummary( + error.result, + ); + return { ...error.result }; + } + return structured + ? { status: "failed", error: diagnosticValue(error) } + : undefined; } finally { removeSignals(); } diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 3f11a043a8..a2b68dc9f2 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -22,6 +22,7 @@ import type { FindingImportRequest, FindingImportReceipt, ImportRepository, + SourceReportSummary, } from "./external-import-models.js"; const BASE_URL = "https://chatgpt.com/backend-api/aardvark/external"; @@ -40,6 +41,45 @@ export interface ExternalPublicationDependencies { fetch?: (url: string, options: RequestInit) => Promise; credentials?: () => Promise<{ access_token: string; account_id: string }>; signal?: AbortSignal; + onProgress?: (event: ExternalPublicationProgress) => void; +} + +export interface ExternalPublicationProgress { + phase: + | "reading" + | "discovering" + | "preparing" + | "waiting" + | "uploading" + | "verifying"; + completed: number; + total?: number; +} + +/** Counts describe acknowledged receipts, even when later transport or readback fails. */ +export interface ExternalPublicationResult { + status: "complete" | "partial" | "interrupted"; + read: number; + ready: number; + excluded: VendorFindings["excluded"]; + receipts: FindingImportReceipt[]; + counts: Required; + failures: { source_finding_id: string; code: string; message: string }[]; + unacknowledged: number; + verified: number; + error?: string; + savedSubmission?: string; +} + +/** Preserve accepted work in CLI JSON output when publication cannot finish. */ +export class ExternalPublicationError extends CodexSecurityError { + constructor( + message: string, + public readonly result: ExternalPublicationResult, + cause: unknown, + ) { + super(message, { cause }); + } } interface SavedSubmission { @@ -58,10 +98,30 @@ export interface ExternalPublicationPreview extends VendorFindings { export interface PreparedExternalPublication { preview: ExternalPublicationPreview; - publish(): Promise<{ - receipts: FindingImportReceipt[]; - counts: FindingImportReceipt["counts"]; - }>; + publish(): Promise; +} + +// Limit read concurrency to avoid one network round trip per finding in series. +// Wait for the entire batch on failure so callers never outlive their read tasks. +async function readInBatches( + items: T[], + read: (item: T) => Promise, +): Promise { + const output: U[] = []; + for (let start = 0; start < items.length; start += 4) { + const batch = await Promise.allSettled( + items.slice(start, start + 4).map(read), + ); + for (const result of batch) { + if (result.status === "rejected") throw result.reason; + output.push(result.value); + } + } + return output; +} + +function repositoryUrl(value: string): string { + return value.replace(/\/$/u, "").replace(/\.git$/u, ""); } function canonicalJson(value: unknown): string { @@ -164,17 +224,26 @@ export async function prepareExternalPublication( options: ExternalPublicationOptions, dependencies: ExternalPublicationDependencies = {}, ): Promise { + const progress = (event: ExternalPublicationProgress): void => { + try { + dependencies.onProgress?.(event); + } catch { + // Optional progress must not interrupt preparation or publication. + } + }; if (!options.sourceKey.trim() || Buffer.byteLength(options.sourceKey) > 512) { throw new CodexSecurityError( - "The source key must contain 1–512 UTF-8 bytes and identify the stable vendor namespace.", + "The source key must contain 1–512 UTF-8 bytes. Use a stable Wiz tenant/finding namespace, such as TENANT_ID/vulnerability-finding, across exports.", ); } + progress({ phase: "reading", completed: 0 }); const parsed = await readVendorFindings(path); if (parsed.findings.length === 0) { throw new CodexSecurityError( `No supported findings are ready to publish. ${parsed.excluded.map((item) => `Item ${item.position}: ${item.reason}`).join(" ")}`, ); } + progress({ phase: "discovering", completed: 0 }); const environment = dependencies.environment ?? process.env; const credentials = await ( dependencies.credentials ?? (() => readCloudCredentials(environment)) @@ -225,7 +294,11 @@ export async function prepareExternalPublication( ), ); destinations.push( - ...result.data.filter((item) => item.id === options.repository), + ...result.data.filter( + (item) => + item.id === options.repository || + repositoryUrl(item.url) === repositoryUrl(options.repository), + ), ); page = result.next; if ( @@ -240,8 +313,8 @@ export async function prepareExternalPublication( if (destinations.length !== 1) throw new CodexSecurityError( destinations.length - ? "The repository ID matches multiple connectors. Select an unambiguous Cloud repository." - : "The destination repository is not available to this Cloud account.", + ? "The repository matches multiple connectors. Select an unambiguous Cloud repository." + : "The destination repository is not available to this Cloud account. Copy its repository URL or ID from Codex Security Cloud and check that you are signed into the intended account.", ); const destination = destinations[0]!; const source = { provider: options.provider, source_key: options.sourceKey }; @@ -253,7 +326,7 @@ export async function prepareExternalPublication( "sha256", canonicalJson([ credentials.account_id, - options.repository, + destination.id, destination.repo_connector_id, source, parsed.findings, @@ -321,34 +394,52 @@ export async function prepareExternalPublication( } let requests = saved?.requests; if (!requests) { + let preparedCount = 0; + progress({ + phase: "preparing", + completed: 0, + total: parsed.findings.length, + }); + const previousReports = await readInBatches( + parsed.findings, + async (finding): Promise => { + const query = new URLSearchParams({ + provider: source.provider, + source_key: source.source_key, + source_finding_id: finding.source_finding_id, + limit: "2", + }); + const summaries = validateSourceReports( + await request(`${sourcePath(destination)}?${query}`), + ); + if (summaries.has_more || summaries.data.length > 1) + throw new CodexSecurityError( + "Cloud returned more than one report for a source identity.", + ); + const previous = summaries.data[0]; + if ( + previous && + (previous.source_finding_id !== finding.source_finding_id || + previous.source.provider !== source.provider || + previous.source.source_key !== source.source_key || + previous.repo_id !== destination.id || + previous.repo_connector_id !== destination.repo_connector_id) + ) + throw new CodexSecurityError( + "Cloud returned a different source identity.", + ); + progress({ + phase: "preparing", + completed: ++preparedCount, + total: parsed.findings.length, + }); + return previous; + }, + ); requests = []; let current: FindingImportRequest | undefined; for (const [index, finding] of parsed.findings.entries()) { - const query = new URLSearchParams({ - provider: source.provider, - source_key: source.source_key, - source_finding_id: finding.source_finding_id, - limit: "2", - }); - const summaries = validateSourceReports( - await request(`${sourcePath(destination)}?${query}`), - ); - if (summaries.has_more || summaries.data.length > 1) - throw new CodexSecurityError( - "Cloud returned more than one report for a source identity.", - ); - const previous = summaries.data[0]; - if ( - previous && - (previous.source_finding_id !== finding.source_finding_id || - previous.source.provider !== source.provider || - previous.source.source_key !== source.source_key || - previous.repo_id !== destination.id || - previous.repo_connector_id !== destination.repo_connector_id) - ) - throw new CodexSecurityError( - "Cloud returned a different source identity.", - ); + const previous = previousReports[index]; const environmentId = previous?.environment_id ?? destination.import_environment_id; if (environmentId == null) @@ -399,6 +490,7 @@ export async function prepareExternalPublication( return { preview, async publish() { + progress({ phase: "waiting", completed: 0 }); return await withImportLock(state, key, dependencies.signal, async () => { // Re-read under the lock to keep checkpoints from another publisher. const pending = await readFile(pendingPath, "utf8").catch( @@ -420,9 +512,56 @@ export async function prepareExternalPublication( throw new CodexSecurityError( "Saved publication has more receipts than requests.", ); - if (pending === undefined) - await writeAtomicJson(pendingPath, submission); + const acknowledged: FindingImportReceipt[] = []; + let verified = 0; + let persisted = pending !== undefined; + const result = (): ExternalPublicationResult => { + const counts = { created: 0, updated: 0, unchanged: 0, error: 0 }; + const failures: ExternalPublicationResult["failures"] = []; + for (const [index, receipt] of acknowledged.entries()) { + for (const outcome of Object.keys( + counts, + ) as (keyof typeof counts)[]) + counts[outcome] += receipt.counts[outcome] ?? 0; + const ids = new Map( + requests[index]!.items.map((item) => [ + item.client_id, + item.source_finding_id, + ]), + ); + for (const item of receipt.results) + if (item.error) + failures.push({ + source_finding_id: ids.get(item.client_id)!, + ...item.error, + }); + } + return { + status: + parsed.excluded.length || counts.error ? "partial" : "complete", + read: parsed.read, + ready: parsed.findings.length, + excluded: parsed.excluded, + receipts: acknowledged, + counts, + failures, + unacknowledged: + parsed.findings.length - + Object.values(counts).reduce((sum, count) => sum + count, 0), + verified, + }; + }; try { + if (pending === undefined) { + await writeAtomicJson(pendingPath, submission); + persisted = true; + } + progress({ + phase: "uploading", + completed: 0, + total: parsed.findings.length, + }); + let acknowledgedCount = 0; for (const [batchIndex, batch] of requests.entries()) { const receipt = validateImportReceipt( receipts[batchIndex] ?? @@ -474,6 +613,13 @@ export async function prepareExternalPublication( throw new CodexSecurityError( "Cloud returned inconsistent publication counts.", ); + acknowledged.push(receipt); + acknowledgedCount += receipt.item_count; + progress({ + phase: "uploading", + completed: acknowledgedCount, + total: parsed.findings.length, + }); if (batchIndex === receipts.length) { receipts.push(receipt); // Acknowledged batches must not consume another POST quota on retry. @@ -486,56 +632,73 @@ export async function prepareExternalPublication( } // Verify readable source records without mistaking a newer concurrent // observation for failure of the original, immutable import receipt. - for (const [batchIndex, receipt] of receipts.entries()) { + const readbacks = receipts.flatMap((receipt, batchIndex) => { const batch = requests[batchIndex]!; const batchItems = new Map( batch.items.map((item) => [item.client_id, item]), ); - for (const item of receipt.results) { - if (item.outcome === "error") continue; - const expected = batchItems.get(item.client_id)!; - const report = validateSourceReport( - await request( - `${sourcePath(destination)}/${encodeURIComponent(item.source_report_id!)}`, - ), + return receipt.results + .filter((item) => item.outcome !== "error") + .map((item) => ({ + item, + expected: batchItems.get(item.client_id)!, + batch, + })); + }); + progress({ + phase: "verifying", + completed: 0, + total: readbacks.length, + }); + await readInBatches(readbacks, async ({ item, expected, batch }) => { + const report = validateSourceReport( + await request( + `${sourcePath(destination)}/${encodeURIComponent(item.source_report_id!)}`, + ), + ); + if ( + report.canonical_finding_id !== item.canonical_finding_id || + report.version < item.version! || + (report.version === item.version && + (report.observation_id !== item.observation_id || + canonicalJson(report.evidence) !== + canonicalJson(expected.evidence))) || + report.source_finding_id !== expected.source_finding_id || + report.repo_id !== destination.id || + report.repo_connector_id !== destination.repo_connector_id || + report.environment_id !== batch.repository.environment_id || + canonicalJson(report.source) !== canonicalJson(source) + ) + throw new CodexSecurityError( + "Cloud readback did not match the saved finding identity.", ); - if ( - report.canonical_finding_id !== item.canonical_finding_id || - report.version < item.version! || - (report.version === item.version && - (report.observation_id !== item.observation_id || - canonicalJson(report.evidence) !== - canonicalJson(expected.evidence))) || - report.source_finding_id !== expected.source_finding_id || - report.repo_id !== destination.id || - report.repo_connector_id !== destination.repo_connector_id || - report.environment_id !== batch.repository.environment_id || - canonicalJson(report.source) !== canonicalJson(source) - ) - throw new CodexSecurityError( - "Cloud readback did not match the saved finding identity.", - ); - } - } + progress({ + phase: "verifying", + completed: ++verified, + total: readbacks.length, + }); + }); + const completed = result(); + await writeAtomicJson(join(state, `${key}.result.json`), completed); + await rm(pendingPath, { force: true }); + return completed; } catch (error) { - if (error instanceof CloudImportError && error.status === 409) + if (error instanceof CloudImportError && error.status === 409) { await rm(pendingPath, { force: true }); - throw new CodexSecurityError( - `${error instanceof Error ? error.message : String(error)} ${error instanceof CloudImportError && error.status === 409 ? "The old submission was retired. Rediscover and approve a fresh publication." : `Repeat the same command to resume the saved request. Saved submission: ${pendingPath}`}`, - { cause: error }, + persisted = false; + } + const message = `${error instanceof Error ? error.message : String(error)} ${error instanceof CloudImportError && error.status === 409 ? "The old submission was retired. Rediscover and approve a fresh publication." : persisted ? `Repeat the same command to resume the saved request. Saved submission: ${pendingPath}` : "No resumable request was saved. Correct the error and review the input before retrying."}`; + throw new ExternalPublicationError( + message, + { + ...result(), + status: "interrupted", + error: message, + ...(persisted ? { savedSubmission: pendingPath } : {}), + }, + error, ); } - const counts = { created: 0, updated: 0, unchanged: 0, error: 0 }; - for (const receipt of receipts) - for (const outcome of Object.keys(counts) as (keyof typeof counts)[]) - counts[outcome] += receipt.counts[outcome] ?? 0; - const result = { - receipts, - counts, - }; - await writeAtomicJson(join(state, `${key}.result.json`), result); - await rm(pendingPath, { force: true }); - return result; }); }, }; diff --git a/sdk/typescript/src/external-import-contract.ts b/sdk/typescript/src/external-import-contract.ts index 13dbcbf7a3..6e64723503 100644 --- a/sdk/typescript/src/external-import-contract.ts +++ b/sdk/typescript/src/external-import-contract.ts @@ -133,6 +133,10 @@ export function validateExternalEvidence( } else if (typeof value === "number") { if (!Number.isFinite(value)) throw new Error("Evidence must contain finite JSON numbers."); + if (Number.isInteger(value) && !Number.isSafeInteger(value)) + throw new Error( + "Evidence integers must be within JavaScript's safe integer range; export larger values as strings to preserve them exactly.", + ); const wire = JSON.stringify(value); // Cloud pads short exponents and formats small fractions scientifically. const cloud = ( diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts index 1ffe194859..77459527c5 100644 --- a/sdk/typescript/src/wiz-findings.ts +++ b/sdk/typescript/src/wiz-findings.ts @@ -152,9 +152,12 @@ function records(payload: unknown): unknown[] { export async function readVendorFindings( path: string, ): Promise { - const contents = (await readFile(path, "utf8")).replace(/^\uFEFF/u, ""); let input: unknown[]; try { + // A replacement character would change vendor identities and retained evidence. + const contents = new TextDecoder("utf-8", { fatal: true }).decode( + await readFile(path), + ); let payload: unknown; try { payload = JSON.parse(contents); @@ -167,7 +170,7 @@ export async function readVendorFindings( input = records(payload); } catch (cause) { throw new CodexSecurityError( - `Could not read vendor findings: ${cause instanceof Error ? cause.message : String(cause)}`, + `Could not read vendor findings as UTF-8 JSON or JSONL: ${cause instanceof Error ? cause.message : String(cause)}`, { cause }, ); } diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index c634c12bef..05078c823a 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -14,7 +14,11 @@ import { join } from "node:path"; import { pathToFileURL } from "node:url"; import { Database } from "bun:sqlite"; import { afterEach, expect, spyOn, test } from "bun:test"; -import { prepareExternalPublication } from "../src/external-findings-publish.js"; +import { + ExternalPublicationError, + prepareExternalPublication, + type ExternalPublicationProgress, +} from "../src/external-findings-publish.js"; import { readVendorFindings } from "../src/wiz-findings.js"; import { validateExternalEvidence, @@ -737,7 +741,7 @@ test("normalized input accepts server-materialized optional evidence defaults", }); test.each(["human", "json"])( - "interactive %s publication shows selected evidence before confirmation", + "interactive %s publication summarizes the input and confirms exclusions", async (format) => { const selected = { ...normalized("vendor-selected"), @@ -749,7 +753,6 @@ test.each(["human", "json"])( }, }; const f = await fixture([selected, normalized("unsupported", "unknown")]); - const expected = (await readVendorFindings(f.file)).findings; const cli = createCliTest(main); let prompted = false; expect( @@ -757,7 +760,7 @@ test.each(["human", "json"])( ...f.cliDeps, externalPublicationPrompt: { isInteractive: () => true, - confirm: async (_question, defaultValue) => { + confirm: async (question, defaultValue) => { prompted = true; expect(defaultValue).toBe(false); expect(cli.stderr.text()).toContain( @@ -767,12 +770,11 @@ test.each(["human", "json"])( expect(cli.stderr.text()).toContain("synthetic-account"); expect(cli.stderr.text()).toContain("environment-example"); expect(cli.stderr.text()).toContain("Excluded: 1"); - expect(cli.stderr.text()).toContain( - "Selected findings and evidence:", - ); - expect(cli.stderr.text()).toContain( - JSON.stringify(expected, null, 2), - ); + expect(question).toContain("skip 1 excluded"); + expect(cli.stderr.text()).toContain("vendor-selected [high]"); + expect(cli.stderr.text()).toContain(f.file); + expect(cli.stderr.text()).toContain("--dry-run --format json"); + expect(cli.stderr.text()).not.toContain("Original vendor detail"); expect(cli.stderr.text()).not.toContain("\u001b"); expect(cli.stderr.text()).not.toContain("synthetic-token"); expect(cli.stdout.text()).toBe(""); @@ -781,16 +783,17 @@ test.each(["human", "json"])( }, }, }), - ).toBe(0); + ).toBe(1); expect(prompted).toBe(true); expect(f.reports.has("vendor-selected")).toBe(true); if (format === "json") { const result = JSON.parse(cli.stdout.text()); + expect(result.status).toBe("partial"); expect(result.counts.created).toBe(1); expect(result).not.toHaveProperty("cloudUrl"); } else { - expect(cli.stderr.text()).toContain("Findings"); - expect(cli.stderr.text()).not.toContain("/codex/cloud/security/findings"); + expect(cli.stdout.text()).toContain("Findings"); + expect(cli.stdout.text()).not.toContain("/codex/cloud/security/findings"); } }, ); @@ -1407,12 +1410,8 @@ test.each(["JSON", "JSONL"])( await writeFile( runner, ` - import { main } from "../src/cli.js"; - import { dependencies } from "../tests-ts/cli-fixtures.js"; - process.exitCode = await main(process.argv.slice(2), process.stdout, process.stderr, { - ...dependencies({ environment: process.env }), - cloudFetch: async () => { throw new Error("Unexpected Cloud request"); }, - }); + export { main } from "../src/cli.js"; + export { dependencies } from "../tests-ts/cli-fixtures.js"; `, ); const built = await Bun.build({ @@ -1423,8 +1422,19 @@ test.each(["JSON", "JSONL"])( packages: "external", }); expect(built.success).toBe(true); + const invocation = join(bundle, "invoke.mjs"); + await writeFile( + invocation, + ` + import { main, dependencies } from ${JSON.stringify(pathToFileURL(built.outputs[0]!.path).href)}; + process.exitCode = await main(process.argv.slice(2), process.stdout, process.stderr, { + ...dependencies({ environment: process.env }), + cloudFetch: async () => { throw new Error("Unexpected Cloud request"); }, + }); + `, + ); const child = Bun.spawn( - [Bun.which("node")!, built.outputs[0]!.path, ...f.command, "--dry-run"], + [Bun.which("node")!, invocation, ...f.command, "--dry-run"], { env: f.environment, stdout: "pipe", @@ -1437,8 +1447,14 @@ test.each(["JSON", "JSONL"])( new Response(child.stderr).text(), ]); expect(code).toBe(2); - expect(stdout).toBe(""); - expect(stderr).toContain("Could not read vendor findings:"); + expect(JSON.parse(stdout)).toMatchObject({ + status: "failed", + error: expect.stringContaining("malformed"), + }); + expect(stdout).not.toContain("\u001b"); + expect(stderr).toContain( + "Could not read vendor findings as UTF-8 JSON or JSONL:", + ); expect(stderr).toContain("malformed"); expect(stderr).not.toContain("\u001b"); expect(f.posts).toHaveLength(0); @@ -1852,3 +1868,243 @@ test("numeric evidence size excludes an oversized item and preserves a valid mix ); expect(f.reports.has("oversized-numbers")).toBe(false); }); + +test.each([0xff, 0xfe])( + "invalid UTF-8 byte %p cannot change a source identity", + async (byte) => { + const f = await fixture(); + await writeFile( + f.file, + Buffer.concat([ + Buffer.from('{"source_finding_id":"vendor-'), + Buffer.from([byte]), + Buffer.from('","evidence":{"title":"Example","severity":"high"}}'), + ]), + ); + await expect( + prepareExternalPublication(f.file, options, f.deps), + ).rejects.toThrow("UTF-8"); + expect(f.posts).toHaveLength(0); + await writeFile( + f.file, + `\uFEFF${JSON.stringify(normalized("vendor-\uFFFD"))}`, + ); + expect( + (await readVendorFindings(f.file)).findings[0]!.source_finding_id, + ).toBe("vendor-\uFFFD"); + }, +); + +test("unsafe evidence integers are rejected while exact strings and safe integers survive", async () => { + const f = await fixture(); + await writeFile( + f.file, + `[ + {"source_finding_id":"unsafe-a","evidence":{"title":"Example","severity":"high","source_data":{"counter":9007199254740992}}}, + {"source_finding_id":"unsafe-b","evidence":{"title":"Example","severity":"high","source_data":{"counter":9007199254740993}}}, + {"source_finding_id":"unsafe-negative","evidence":{"title":"Example","severity":"high","source_data":{"counter":-9007199254740993}}}, + {"source_finding_id":"exact","evidence":{"title":"Example","severity":"high","source_data":{"counter":"9007199254740993","safe":9007199254740991,"fraction":0.125}}} + ]`, + ); + const parsed = await readVendorFindings(f.file); + expect(parsed.excluded.map((item) => item.source_finding_id)).toEqual([ + "unsafe-a", + "unsafe-b", + "unsafe-negative", + ]); + expect( + parsed.excluded.every((item) => item.reason.includes("safe integer")), + ).toBe(true); + expect(parsed.findings[0]!.evidence.source_data).toEqual({ + counter: "9007199254740993", + safe: Number.MAX_SAFE_INTEGER, + fraction: 0.125, + }); +}); + +test("mixed valid and rejected input reports partial completion with exit code 1", async () => { + const f = await fixture([ + normalized("invalid", "unknown"), + normalized("valid"), + ]); + const cli = createCliTest(main); + expect(await cli.runCli([...f.command, "--yes"], f.cliDeps)).toBe(1); + expect(JSON.parse(cli.stdout.text())).toMatchObject({ + status: "partial", + read: 2, + ready: 1, + counts: { created: 1, error: 0 }, + unacknowledged: 0, + verified: 1, + excluded: [{ source_finding_id: "invalid", position: 1 }], + }); +}); + +test("server item failures identify the vendor record after local exclusions", async () => { + const f = await fixture([ + normalized("invalid", "unknown"), + normalized("rejected-vendor"), + ]); + f.state.finalError = true; + const cli = createCliTest(main); + expect(await cli.runCli([...f.command, "--yes"], f.cliDeps)).toBe(1); + expect(JSON.parse(cli.stdout.text())).toMatchObject({ + status: "partial", + verified: 0, + unacknowledged: 0, + failures: [ + { + source_finding_id: "rejected-vendor", + code: "version_conflict", + message: "Reload current evidence", + }, + ], + }); +}); + +test("interrupted CLI output preserves acknowledged batches and resumes only unfinished uploads", async () => { + const f = await fixture( + Array.from({ length: 101 }, (_, i) => normalized(`vendor-${i}`)), + ); + f.state.postBudget = 1; + const first = createCliTest(main); + expect(await first.runCli([...f.command, "--yes"], f.cliDeps)).toBe(2); + const partial = JSON.parse(first.stdout.text()); + expect(partial).toMatchObject({ + status: "interrupted", + ready: 101, + counts: { created: 100 }, + unacknowledged: 1, + verified: 0, + }); + expect(partial.receipts).toHaveLength(1); + expect(partial.savedSubmission).toEndWith(".pending.json"); + expect(partial.error).toContain("Repeat the same command"); + expect(f.reports.size).toBe(100); + f.state.postBudget = Infinity; + const retry = createCliTest(main); + expect(await retry.runCli([...f.command, "--yes"], f.cliDeps)).toBe(0); + expect(JSON.parse(retry.stdout.text())).toMatchObject({ + status: "complete", + counts: { created: 101 }, + unacknowledged: 0, + verified: 101, + }); + expect(f.posts).toHaveLength(3); + expect(f.posts[2]).toBe(f.posts[1]); +}); + +test("readback interruption distinguishes acknowledged findings from verified findings", async () => { + const f = await fixture(); + f.state.brokenReadback = true; + const prepared = await prepareExternalPublication(f.file, options, f.deps); + try { + await prepared.publish(); + throw new Error("Expected readback interruption"); + } catch (error) { + expect(error).toBeInstanceOf(ExternalPublicationError); + expect((error as ExternalPublicationError).result).toMatchObject({ + status: "interrupted", + counts: { created: 1 }, + unacknowledged: 0, + verified: 0, + }); + } +}); + +test("repository URLs and IDs share the same resumable destination", async () => { + const f = await fixture(); + f.state.loseResponse = true; + const first = await prepareExternalPublication( + f.file, + { ...options, repository: `${f.destination().url}.git` }, + f.deps, + ); + expect(first.preview.destination.id).toBe(options.repository); + await expect(first.publish()).rejects.toThrow("resume the saved request"); + const retry = await prepareExternalPublication(f.file, options, f.deps); + expect(retry.preview.resumed).toBe(true); + expect((await retry.publish()).status).toBe("complete"); + expect(f.posts[1]).toBe(f.posts[0]); +}); + +test("read lookups and verification run concurrently with bounded requests and ordered input", async () => { + const f = await fixture( + Array.from({ length: 9 }, (_, i) => normalized(`vendor-${i}`)), + ); + const phases: ExternalPublicationProgress[] = []; + let active = 0; + let maximum = 0; + const prepared = await prepareExternalPublication(f.file, options, { + ...f.deps, + onProgress: (event) => phases.push(event), + fetch: async (url, init) => { + if (!url.includes("/source_reports")) return f.deps.fetch(url, init); + active++; + maximum = Math.max(maximum, active); + try { + await Promise.resolve(); + return await f.deps.fetch(url, init); + } finally { + active--; + } + }, + }); + expect( + prepared.preview.requests.flatMap((batch) => + batch.items.map((item) => item.source_finding_id), + ), + ).toEqual(Array.from({ length: 9 }, (_, i) => `vendor-${i}`)); + expect((await prepared.publish()).status).toBe("complete"); + expect(maximum).toBeGreaterThan(1); + expect(maximum).toBeLessThanOrEqual(4); + expect(active).toBe(0); + for (const phase of ["preparing", "uploading", "verifying"]) + expect( + phases.filter((event) => event.phase === phase).at(-1), + ).toMatchObject({ completed: 9, total: 9 }); +}); + +test("progress is optional and stderr progress never corrupts JSON results", async () => { + const f = await fixture(); + const prepared = await prepareExternalPublication(f.file, options, { + ...f.deps, + onProgress: () => { + throw new Error("Optional observer failed"); + }, + }); + expect((await prepared.publish()).status).toBe("complete"); + const cli = createCliTest(main, { stderr: true }); + expect(await cli.runCli([...f.command, "--yes"], f.cliDeps)).toBe(0); + expect(JSON.parse(cli.stdout.text())).toMatchObject({ + status: "complete", + counts: { unchanged: 1 }, + }); + expect(cli.stderr.text()).toContain("Preparing findings"); + expect(cli.stderr.text()).toContain("Verifying imported findings: 1/1"); +}); + +test("human previews stay compact while JSON previews retain complete evidence", async () => { + const records = Array.from({ length: 6 }, (_, i) => ({ + ...normalized(`vendor-${i}`), + evidence: { + ...normalized().evidence, + source_data: { detail: `complete-vendor-evidence-${i}` }, + }, + })); + const f = await fixture(records); + const human = createCliTest(main); + expect( + await human.runCli([...f.command.slice(0, -2), "--dry-run"], f.cliDeps), + ).toBe(0); + expect(human.stderr.text()).toContain("Showing 5 of 6 findings"); + expect(human.stderr.text()).not.toContain("complete-vendor-evidence-"); + expect(human.stdout.text()).not.toContain("request_id"); + const json = createCliTest(main); + expect(await json.runCli([...f.command, "--dry-run"], f.cliDeps)).toBe(0); + expect(JSON.parse(json.stdout.text()).findings).toHaveLength(6); + expect( + JSON.parse(json.stdout.text()).findings[5].evidence.source_data.detail, + ).toBe("complete-vendor-evidence-5"); + expect(f.posts).toHaveLength(0); +}); From 029e01434bc128a3934ce45bcc61bbcaa18238e8 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 9 Oct 2026 01:33:55 -0400 Subject: [PATCH 17/43] fix(cloud): isolate deployment routing and support Wiz report exports --- .../skills/publish-findings/SKILL.md | 8 +- sdk/typescript/docs/cli.md | 84 ++++++--- sdk/typescript/src/cli.ts | 4 +- sdk/typescript/src/cloud-endpoint.ts | 29 +++ sdk/typescript/src/cloud-publish.ts | 13 +- .../src/external-findings-publish.ts | 13 +- sdk/typescript/src/wiz-findings.ts | 37 +++- sdk/typescript/tests-ts/cloud-publish.test.ts | 27 +++ .../external-findings-publish.test.ts | 165 ++++++++++++++++++ 9 files changed, 330 insertions(+), 50 deletions(-) create mode 100644 sdk/typescript/src/cloud-endpoint.ts diff --git a/plugins/codex-security/skills/publish-findings/SKILL.md b/plugins/codex-security/skills/publish-findings/SKILL.md index f6f059f2de..9ae020d12d 100644 --- a/plugins/codex-security/skills/publish-findings/SKILL.md +++ b/plugins/codex-security/skills/publish-findings/SKILL.md @@ -1,6 +1,6 @@ --- name: publish-findings -description: Import selected Wiz package vulnerabilities to Codex Security Cloud after previewing the destination and evidence. Supports saved vulnerability JSON and normalized JSONL; does not import raw SAST, secrets, or IaC findings, run a scan, or assess findings. +description: Import selected Wiz package vulnerabilities to Codex Security Cloud after previewing the destination and evidence. Supports plain or gzip-compressed vulnerability JSON and normalized JSONL; does not import raw SAST, secrets, or IaC findings, run a scan, or assess findings. --- # Publish external findings @@ -11,13 +11,17 @@ Resolve the user's destination to its repository URL or Cloud repository ID; the Use a stable Wiz tenant/finding namespace as `--source-key`. A project filter is selection context, not a new identity. Confirm the selected assets or builds belong to the destination repository; do not infer a scanner branch or revision from the current checkout. Do not publish vendor credentials, unrelated records, or local assessments as original scanner evidence. +For Wiz UI exports, use **Vulnerability Findings → Save as → Report**, keeping the repository and finding filters. Select **Repository Branch** (remove the default Virtual Machine selection), **JSON**, and **Detailed** columns. Download the completed report; default gzip compression is supported, even with a `.json` filename. **Raw Event** on Code & Build Scans contains scan metadata and cannot supply findings. Do not create a scheduled report or enable notifications or external storage unless requested. + +For staging or another Cloud deployment, set `CODEX_SECURITY_CLOUD_BASE_URL` to its API root before both preview and upload. Use that deployment's login and an isolated state directory for tests. Verify `cloudApiUrl` in the preview; it is the API endpoint, not a Cloud UI link. Saved retries are separated by deployment. The native scan setting `CODEX_SECURITY_CLOUD_PUBLISH_URL` does not route vendor imports. Never fall back to production when staging is unavailable. + ```sh codex-security publish findings /path/to/selected-wiz-findings.json \ --to cloud --repository https://github.com/example/project --provider wiz \ --source-key TENANT_ID/vulnerability-finding --dry-run --format json ``` -Show a concise preview of the account, repository, environments, source namespace, severity counts, and exclusions. Identify the saved evidence file and let the user inspect the complete JSON preview; do not paste an entire large report into the conversation. Explain that imported vendor evidence is not a Codex assessment. Obtain explicit confirmation for this upload, including any skipped records, then run the same command without `--dry-run`, adding `--yes`. Existing authorization to inspect or assess findings alone does not authorize sending them to Cloud. A user who already approved the concrete preview need not approve it again. +Show a concise preview of the Cloud API endpoint, account, repository, environments, source namespace, severity counts, and exclusions. Identify the saved evidence file and let the user inspect the complete JSON preview; do not paste an entire large report into the conversation. Explain that imported vendor evidence is not a Codex assessment. Obtain explicit confirmation for this upload, including any skipped records, then run the same command without `--dry-run`, adding `--yes`. Existing authorization to inspect or assess findings alone does not authorize sending them to Cloud. A user who already approved the concrete preview need not approve it again. After a lost response or retryable failure, repeat the same command with unchanged input and destination. The publisher resends only unacknowledged requests, using their saved IDs and bodies. Existing findings keep their original Cloud environment even if the repository’s default changes. After a repository reset, stop and show the error; require fresh discovery and confirmation before a new publication. Report partial failures and retain their explanations. A successful import stores vendor evidence; it does not establish that Codex assessed the finding or that the vulnerability is exploitable. Direct the user to the main Findings view in the Codex Security Cloud app to find the repository’s imported Wiz findings; do not invent a direct Findings URL. diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md index 722d1d4360..2c643857f5 100644 --- a/sdk/typescript/docs/cli.md +++ b/sdk/typescript/docs/cli.md @@ -577,24 +577,25 @@ For filesystem and approval behavior, see the ### Environment variables -| Variable | Effect | -| --------------------------------------------------------------------------- | ------------------------------------------------------------------ | -| `OPENAI_API_KEY`, `CODEX_API_KEY` | Scan credentials; the first takes precedence. | -| `CODEX_SECURITY_STATE_DIR` | Private history, database, and default artifacts. | -| `CODEX_SECURITY_PROJECT_CONFIG` | Selected project file; `-c` overrides it. | -| `CODEX_HOME` | Ambient Codex home; default `~/.codex`. | -| `CODEX_CLI_PATH` | Codex executable for login, setup, scans, and workers. | -| `PYTHON` | Python interpreter unless an explicit option overrides it. | -| `CODEX_SECURITY_LOG_LEVEL` | CLI diagnostics; `debug` enables verbose output. | -| `LOG_LEVEL` | Fallback if `CODEX_SECURITY_LOG_LEVEL` is unset or blank. | -| `CODEX_SECURITY_LINEAR_TEAM`, `CODEX_SECURITY_LINEAR_PROJECT` | Default publication destination. | -| `CODEX_SECURITY_LINEAR_API_KEY` | Linear personal API key. | -| `CODEX_SECURITY_EMBEDDINGS_URL` | Local dedupe and findings service embeddings endpoint. | -| `GH_HOST` | GitHub Enterprise host for bulk discovery. | -| `CODEX_SECURITY_NO_UPDATE_NOTICE`, `NO_UPDATE_NOTIFIER` | Disable interactive update notices. | -| `CODEX_SECURITY_NPM_REGISTRY`, `npm_config_registry`, `NPM_CONFIG_REGISTRY` | Update registry, in precedence order. | -| `CI` | Disable interactive update notices. | -| `NO_COLOR`, `TERM` | Disable colored history when `NO_COLOR` is defined or `TERM=dumb`. | +| Variable | Effect | +| --------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | +| `OPENAI_API_KEY`, `CODEX_API_KEY` | Scan credentials; the first takes precedence. | +| `CODEX_SECURITY_STATE_DIR` | Private history, database, and default artifacts. | +| `CODEX_SECURITY_PROJECT_CONFIG` | Selected project file; `-c` overrides it. | +| `CODEX_HOME` | Ambient Codex home; default `~/.codex`. | +| `CODEX_CLI_PATH` | Codex executable for login, setup, scans, and workers. | +| `PYTHON` | Python interpreter unless an explicit option overrides it. | +| `CODEX_SECURITY_LOG_LEVEL` | CLI diagnostics; `debug` enables verbose output. | +| `LOG_LEVEL` | Fallback if `CODEX_SECURITY_LOG_LEVEL` is unset or blank. | +| `CODEX_SECURITY_LINEAR_TEAM`, `CODEX_SECURITY_LINEAR_PROJECT` | Default publication destination. | +| `CODEX_SECURITY_LINEAR_API_KEY` | Linear personal API key. | +| `CODEX_SECURITY_CLOUD_BASE_URL` | Cloud API root for discovery, imports, and scan publication. Defaults to `https://chatgpt.com/backend-api/aardvark`. | +| `CODEX_SECURITY_EMBEDDINGS_URL` | Local dedupe and findings service embeddings endpoint. | +| `GH_HOST` | GitHub Enterprise host for bulk discovery. | +| `CODEX_SECURITY_NO_UPDATE_NOTICE`, `NO_UPDATE_NOTIFIER` | Disable interactive update notices. | +| `CODEX_SECURITY_NPM_REGISTRY`, `npm_config_registry`, `NPM_CONFIG_REGISTRY` | Update registry, in precedence order. | +| `CI` | Disable interactive update notices. | +| `NO_COLOR`, `TERM` | Disable colored history when `NO_COLOR` is defined or `TERM=dumb`. | Custom Codex executables need thread source attribution for `exec` and `app-server` (Codex 0.149.1+). On Windows, use a native `.exe` or `.com`; @@ -803,11 +804,18 @@ Before the first import: 1. Select an existing repository in Codex Security Cloud and copy its repository URL (for example, a GitHub URL) or Cloud repository ID. The repository needs an authorized environment, but does not need a completed native scan. -2. Use your authorized Wiz connection to read selected package vulnerability - records and save their JSON. Include `id`, `name`, `detailedName`, - `vendorSeverity` (or `severity`), and `vulnerableAsset.id`; retain the available - package, image, and source metadata. The command does not fetch Wiz data or - accept CSV exports. See the accepted input formats below. +2. In Wiz **Vulnerability Findings**, filter to the intended repository and + findings. Choose **Save as → Report**, select **Repository Branch** as the + resource type (remove the default Virtual Machine selection), **JSON** format, + and **Detailed** columns. Retain the repository filter and download the completed + report. Gzip compression can stay enabled; the importer detects it even when + the download is named `.json`. **Raw Event** on Code & Build Scans exports scan + metadata, not finding records. CSV is not supported. + Alternatively, read selected package vulnerability records through your + authorized Wiz connection and save their JSON. Include `id`, `name`, + `detailedName`, `vendorSeverity` (or `severity`), and `vulnerableAsset.id`; + retain available package, image, and source metadata. The command does not + fetch from Wiz. 3. Choose a stable source key such as `TENANT_ID/vulnerability-finding`. Reuse it for that Wiz tenant and finding class across exports. Do not create a new key for each project filter or import; that creates different source identities. @@ -815,6 +823,16 @@ Before the first import: login. API-key-only and keyring-only logins are not supported for Cloud publication. See the credential-storage instructions below. +For an alternate Cloud deployment, set `CODEX_SECURITY_CLOUD_BASE_URL` to its +API root before previewing or uploading, for example +`https://cloud.example.test/backend-api/aardvark`. Use a login issued by that +deployment and an isolated `CODEX_SECURITY_STATE_DIR` for testing. The endpoint +does not change your login or sign you into another deployment. An invalid or +empty configured URL stops the import before authentication or network requests. +With the variable unset, the existing production endpoint remains the default. +The older `CODEX_SECURITY_CLOUD_PUBLISH_URL` overrides only the final native scan +publication endpoint; it does not route vendor imports. + Preview the selection and destination: ```bash @@ -828,7 +846,8 @@ destination and resume the same saved submission. This command reads the authorized destination and current source versions during preview. It uses the same saved ChatGPT file login as scan publication. The Cloud repository needs an existing environment; no native scan is required. Review the -account, repository, environments, findings, and exclusions, then rerun without +Cloud API endpoint, account, repository, environments, findings, and exclusions, +then rerun without `--dry-run`. The default terminal prompt is No; `--yes` confirms a previously reviewed input for scripts or the plugin. Without a terminal or `--yes`, no upload occurs. Existing source findings keep their Cloud environment; new findings use @@ -839,9 +858,11 @@ exclusions. `--dry-run --format json` retains all normalized findings and comple evidence for inspection. A prompt explicitly includes any records that will be skipped. -Input can be a Wiz vulnerability finding, an array, a complete +Input can be a downloaded Wiz JSON vulnerability report, a vulnerability finding, +an array, a complete `data.vulnerabilityFindings.nodes` response, or JSONL with one vendor record per -line. A response that advertises another page is rejected: save the explicitly +line. Plain and gzip-compressed files are accepted. A response that advertises +another page is rejected: save the explicitly selected records as an array. Normalized JSONL is also accepted: ```json @@ -859,8 +880,9 @@ Use Wiz's finding `id`, not a CVE, as identity. Keep the source key stable acros exports; a selected project filter does not change the vendor namespace. Confirm that the selected assets or builds map to this repository. The raw Wiz adapter retains each record in `source_data`, uses reported severity, and leaves unknown -branch/revision and repository locations empty. Container paths are retained as -vendor evidence rather than interpreted as source-code locations. It does not +branch/revision empty. Repository-branch findings retain their repository-relative +file paths; container and other workload paths remain vendor evidence rather than +source-code locations. It does not fetch from Wiz, assess findings, or change vendor or Cloud triage decisions. Findings from external network scans are excluded from this package vulnerability mapping. @@ -872,7 +894,11 @@ exactly. Invalid records appear in `excluded` with their source ID and reason. Requests contain at most 100 findings and respect the Cloud payload limits. Before uploading, the publisher saves request IDs and bodies privately under the configured Codex Security state directory. After an uncertain response, -repeat the same command with unchanged input to resume those requests. A reset +repeat the same command with unchanged input and Cloud API endpoint to resume those +requests. Different deployments have separate saved submissions, including when +their account and repository IDs match. The preview and result include +`cloudApiUrl` so scripts can verify which API received the findings; this is an +API endpoint, not a link to the Cloud UI. A reset retires the old request without republishing; review and explicitly approve a fresh invocation. diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 54b866e592..1a5e4f3f2c 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -3149,7 +3149,7 @@ export async function main( publication.command("findings", { description: "Preview and import selected Wiz package vulnerabilities to Cloud.", - hint: "Supply selected Wiz vulnerability JSON or normalized JSONL (not CSV, SAST, secrets, or IaC exports).\nExample: codex-security publish findings selected-wiz.json --to cloud --repository https://github.com/example/project --provider wiz --source-key TENANT_ID/vulnerability-finding --dry-run --format json", + hint: "Supply a selected Wiz vulnerability JSON report or normalized JSONL; gzip downloads are supported (not CSV, scan events, SAST, secrets, or IaC exports).\nExample: codex-security publish findings selected-wiz.json --to cloud --repository https://github.com/example/project --provider wiz --source-key TENANT_ID/vulnerability-finding --dry-run --format json", destructive: true, mcp: false, args: z.object({ @@ -3225,7 +3225,7 @@ export async function main( const { preview } = prepared; const showPreview = () => { errorOutput.write( - `Account: ${diagnosticValue(preview.accountId)}\nDestination: ${diagnosticValue(preview.destination.url)} (${diagnosticValue(preview.destination.id)})\nEnvironment: ${[...new Set(preview.requests.map((request) => request.repository.environment_id))].map(diagnosticValue).join(", ")}\nSource: ${preview.source.provider} / ${diagnosticValue(preview.source.source_key)}\nRead: ${preview.read} Ready: ${preview.findings.length} Excluded: ${preview.excluded.length}${preview.resumed ? "\nResuming the saved submission." : ""}\n`, + `Cloud API: ${diagnosticValue(preview.cloudApiUrl)}\nAccount: ${diagnosticValue(preview.accountId)}\nDestination: ${diagnosticValue(preview.destination.url)} (${diagnosticValue(preview.destination.id)})\nEnvironment: ${[...new Set(preview.requests.map((request) => request.repository.environment_id))].map(diagnosticValue).join(", ")}\nSource: ${preview.source.provider} / ${diagnosticValue(preview.source.source_key)}\nRead: ${preview.read} Ready: ${preview.findings.length} Excluded: ${preview.excluded.length}${preview.resumed ? "\nResuming the saved submission." : ""}\n`, ); for (const excluded of preview.excluded) errorOutput.write( diff --git a/sdk/typescript/src/cloud-endpoint.ts b/sdk/typescript/src/cloud-endpoint.ts new file mode 100644 index 0000000000..50dffdff01 --- /dev/null +++ b/sdk/typescript/src/cloud-endpoint.ts @@ -0,0 +1,29 @@ +import { CodexSecurityError } from "./errors.js"; + +export const DEFAULT_CLOUD_BASE_URL = + "https://chatgpt.com/backend-api/aardvark"; + +/** Resolve the deployment once so discovery, writes, and retries share a host. */ +export function cloudBaseUrl(environment: NodeJS.ProcessEnv): string { + const configured = environment["CODEX_SECURITY_CLOUD_BASE_URL"]; + if (configured === undefined) return DEFAULT_CLOUD_BASE_URL; + let url: URL; + try { + url = new URL(configured.trim()); + } catch { + throw new CodexSecurityError( + "CODEX_SECURITY_CLOUD_BASE_URL must be an absolute HTTP(S) Cloud API base URL.", + ); + } + if ( + !["https:", "http:"].includes(url.protocol) || + url.username || + url.password || + url.search || + url.hash + ) + throw new CodexSecurityError( + "CODEX_SECURITY_CLOUD_BASE_URL must be an HTTP(S) Cloud API base URL without credentials, a query, or a fragment.", + ); + return url.href.replace(/\/+$/u, ""); +} diff --git a/sdk/typescript/src/cloud-publish.ts b/sdk/typescript/src/cloud-publish.ts index 4150594a81..226016fd44 100644 --- a/sdk/typescript/src/cloud-publish.ts +++ b/sdk/typescript/src/cloud-publish.ts @@ -20,9 +20,7 @@ import { expandHome, } from "./runtime.js"; import { VERSION } from "./version.js"; - -const CLOUD_PUBLISH_URL = - "https://chatgpt.com/backend-api/aardvark/cli/findings"; +import { cloudBaseUrl } from "./cloud-endpoint.js"; const CHATGPT_LOGIN_REQUIRED = "Cloud publication requires a ChatGPT login already available to Codex Security. Run a scan or sign in with ChatGPT using Codex file credential storage, then retry."; @@ -181,12 +179,11 @@ async function publishCloudPayload( findings, }; } - const credentials = await readCloudCredentials( - dependencies.environment ?? process.env, - ); + const environment = dependencies.environment ?? process.env; + const credentials = await readCloudCredentials(environment); const publishUrl = - dependencies.environment?.["CODEX_SECURITY_CLOUD_PUBLISH_URL"]?.trim() || - CLOUD_PUBLISH_URL; + environment["CODEX_SECURITY_CLOUD_PUBLISH_URL"]?.trim() || + `${cloudBaseUrl(environment)}/cli/findings`; const timeout = AbortSignal.timeout(30_000); const signal = dependencies.signal ? AbortSignal.any([dependencies.signal, timeout]) diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index a2b68dc9f2..c20dd2663d 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -5,6 +5,7 @@ import { createRequire } from "node:module"; import { join } from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { readCloudCredentials } from "./cloud-publish.js"; +import { cloudBaseUrl, DEFAULT_CLOUD_BASE_URL } from "./cloud-endpoint.js"; import { CodexSecurityError } from "./errors.js"; import { codexSecurityStateDirectory, @@ -25,7 +26,6 @@ import type { SourceReportSummary, } from "./external-import-models.js"; -const BASE_URL = "https://chatgpt.com/backend-api/aardvark/external"; const MAX_REQUEST_BYTES = 5 * 1024 * 1024; /** CLI and plugin callers prepare the same immutable request before approval. */ @@ -58,6 +58,7 @@ export interface ExternalPublicationProgress { /** Counts describe acknowledged receipts, even when later transport or readback fails. */ export interface ExternalPublicationResult { + cloudApiUrl: string; status: "complete" | "partial" | "interrupted"; read: number; ready: number; @@ -89,6 +90,7 @@ interface SavedSubmission { } export interface ExternalPublicationPreview extends VendorFindings { + cloudApiUrl: string; accountId: string; destination: ImportRepository; source: FindingImportRequest["source"]; @@ -245,6 +247,8 @@ export async function prepareExternalPublication( } progress({ phase: "discovering", completed: 0 }); const environment = dependencies.environment ?? process.env; + const apiBaseUrl = cloudBaseUrl(environment); + const cloudApiUrl = `${apiBaseUrl}/external`; const credentials = await ( dependencies.credentials ?? (() => readCloudCredentials(environment)) )(); @@ -256,7 +260,7 @@ export async function prepareExternalPublication( // Allow upload and response transport around the server's publication budget. const timeout = AbortSignal.timeout(body ? 60_000 : 30_000); const response = await (dependencies.fetch ?? globalThis.fetch)( - `${BASE_URL}${endpoint}`, + `${cloudApiUrl}${endpoint}`, { method: body ? "POST" : "GET", headers: { @@ -325,6 +329,9 @@ export async function prepareExternalPublication( const key = hash( "sha256", canonicalJson([ + // Keep existing production checkpoints readable; other deployments must + // never share a saved request or receipt with the production default. + ...(apiBaseUrl === DEFAULT_CLOUD_BASE_URL ? [] : [apiBaseUrl]), credentials.account_id, destination.id, destination.repo_connector_id, @@ -481,6 +488,7 @@ export async function prepareExternalPublication( }; const preview: ExternalPublicationPreview = { ...parsed, + cloudApiUrl, accountId: credentials.account_id, destination, source, @@ -537,6 +545,7 @@ export async function prepareExternalPublication( }); } return { + cloudApiUrl, status: parsed.excluded.length || counts.error ? "partial" : "complete", read: parsed.read, diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts index 77459527c5..dde9afb523 100644 --- a/sdk/typescript/src/wiz-findings.ts +++ b/sdk/typescript/src/wiz-findings.ts @@ -1,8 +1,12 @@ import { readFile } from "node:fs/promises"; +import { promisify } from "node:util"; +import { gunzip } from "node:zlib"; import { CodexSecurityError } from "./errors.js"; import { validateExternalEvidence } from "./external-import-contract.js"; import type { ExternalFindingEvidence } from "./external-import-models.js"; +const decompressGzip = promisify(gunzip); + export interface VendorFinding { source_finding_id: string; evidence: ExternalFindingEvidence; @@ -56,6 +60,14 @@ function wizFinding(record: Record): VendorFinding { ); } const assetType = text(asset?.["type"]); + const repositoryAsset = + assetType === "REPOSITORY_BRANCH" || + asset?.["nativeType"] === "github#repositoryBranch"; + // Wiz repository reports prefix root-relative paths with '/'. Workload paths + // are not source locations and must remain only in the original evidence. + const locationPath = text(record["locationPath"]); + const repositoryPath = + repositoryAsset && locationPath ? locationPath.replace(/^\/+/, "") : null; const digest = text(asset?.["imageDigest"]) ?? text(record["imageDigest"]) ?? @@ -74,8 +86,7 @@ function wizFinding(record: Record): VendorFinding { ) { throw new Error("Wiz updatedAt is not a valid timestamp."); } - // Container paths and lastDetectedAt are retained in source_data; neither is a - // repository location or proof that the entire vendor record was updated. + // lastDetectedAt is not proof that the entire vendor record was updated. const evidence = { title: text(record["title"]) ?? @@ -94,14 +105,14 @@ function wizFinding(record: Record): VendorFinding { text(artifact?.["codeLibraryLanguage"]) ?? text(record["codeLibraryLanguage"]), installed_version: version, - manifest_path: null, + manifest_path: repositoryPath, fixed_versions: text(record["fixedVersion"]) ? [record["fixedVersion"]] : [], }, ] : [], - locations: [], + locations: repositoryPath ? [{ path: repositoryPath }] : [], branch: null, code_revision: null, source_scan_id: null, @@ -123,6 +134,13 @@ function records(payload: unknown): unknown[] { throw new Error( "Expected finding objects or a Wiz vulnerabilityFindings response.", ); + if ( + object(envelope["scannersRunStatuses"]) && + object(envelope["codeAnalyzerDetails"]) + ) + throw new Error( + "This is a Wiz Code & Build Scan event, which contains scan metadata rather than vulnerability records. In Vulnerability Findings, filter the repository and selection, then choose Save as → Report with JSON format and Detailed columns. Download that report instead of Raw Event.", + ); if (Array.isArray(envelope["errors"]) && envelope["errors"].length > 0) { throw new Error( "Wiz returned GraphQL errors. Resolve them before publishing this response.", @@ -154,10 +172,15 @@ export async function readVendorFindings( ): Promise { let input: unknown[]; try { + // Wiz's default report export is gzip, sometimes with a .json filename. + // Inspect the bytes rather than requiring users to rename their download. + const bytes = await readFile(path); + const decoded = + bytes[0] === 0x1f && bytes[1] === 0x8b + ? await decompressGzip(bytes) + : bytes; // A replacement character would change vendor identities and retained evidence. - const contents = new TextDecoder("utf-8", { fatal: true }).decode( - await readFile(path), - ); + const contents = new TextDecoder("utf-8", { fatal: true }).decode(decoded); let payload: unknown; try { payload = JSON.parse(contents); diff --git a/sdk/typescript/tests-ts/cloud-publish.test.ts b/sdk/typescript/tests-ts/cloud-publish.test.ts index ba4d2c1a7c..d02b2adcba 100644 --- a/sdk/typescript/tests-ts/cloud-publish.test.ts +++ b/sdk/typescript/tests-ts/cloud-publish.test.ts @@ -512,6 +512,33 @@ describe("Cloud publication", () => { expect(requestedUrl).toBe(publishUrl); }); + test("uses the shared Cloud API base while preserving the publication URL override", async () => { + const { scan, environment } = await fixture(); + const cloudBase = "https://cloud.example.test/pilot/backend-api/aardvark"; + const urls: string[] = []; + const fetch = async (url: string | URL | Request) => { + urls.push(String(url)); + return Response.json(receipt); + }; + await publishScanToCloud(scan, { + environment: { + ...environment, + CODEX_SECURITY_CLOUD_BASE_URL: `${cloudBase}/`, + }, + fetch, + }); + const publishUrl = "https://other-cloud.example.test/publish"; + await publishScanToCloud(scan, { + environment: { + ...environment, + CODEX_SECURITY_CLOUD_BASE_URL: cloudBase, + CODEX_SECURITY_CLOUD_PUBLISH_URL: publishUrl, + }, + fetch, + }); + expect(urls).toEqual([`${cloudBase}/cli/findings`, publishUrl]); + }); + test("reuses the dedicated Codex Security login and honors an explicit logout", async () => { const { scan, environment } = await fixture(); const home = codexSecurityCredentialHome(environment); diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 05078c823a..61dfd8d2c7 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -12,6 +12,7 @@ import { import { tmpdir } from "node:os"; import { join } from "node:path"; import { pathToFileURL } from "node:url"; +import { gzipSync } from "node:zlib"; import { Database } from "bun:sqlite"; import { afterEach, expect, spyOn, test } from "bun:test"; import { @@ -52,6 +53,93 @@ const normalized = (id = "vendor-1", severity = "high") => ({ }, }); +test("Wiz's gzip JSON report retains finding identity, evidence and repository locations", async () => { + const report = { + id: "vendor-report-1", + name: "CVE-2026-0001", + detailedName: "example-package", + version: "1.2.3", + fixedVersion: "1.2.4", + vendorSeverity: "HIGH", + detectionMethod: "LIBRARY", + description: "Reported vulnerable package in the repository lockfile.", + locationPath: "/pnpm-lock.yaml", + lastDetectedAt: "2026-01-01T00:00:00Z", + vulnerableAsset: { + id: "vendor-repository-branch-1", + nativeType: "github#repositoryBranch", + name: "example/project/main", + cloudProviderURL: "https://github.com/example/project/tree/main", + }, + }; + const f = await fixture(); + // Wiz can keep the .json filename while compression is enabled. + await writeFile( + f.file, + gzipSync( + [report, { ...report, id: "vendor-report-2" }] + .map((record) => JSON.stringify(record)) + .join("\n"), + ), + ); + const parsed = await readVendorFindings(f.file); + expect(parsed.read).toBe(2); + expect(parsed.excluded).toEqual([]); + expect(parsed.findings[0]).toMatchObject({ + source_finding_id: report.id, + evidence: { + severity: "high", + packages: [{ name: "example-package", manifest_path: "pnpm-lock.yaml" }], + locations: [{ path: "pnpm-lock.yaml" }], + source_updated_at: null, + code_revision: null, + source_data: report, + }, + }); + const result = await ( + await prepareExternalPublication(f.file, options, f.deps) + ).publish(); + expect(result.counts.created).toBe(2); + expect(result.verified).toBe(2); + + await writeFile( + f.file, + JSON.stringify([ + { + ...report, + vulnerableAsset: { id: "workload-1", type: "CONTAINER_IMAGE" }, + }, + ]), + ); + const workload = await readVendorFindings(f.file); + expect(workload.findings[0]!.evidence.locations).toEqual([]); + expect( + workload.findings[0]!.evidence.packages?.[0]?.manifest_path, + ).toBeNull(); +}); + +test("a Wiz Raw Event export explains the finding-report path before contacting Cloud", async () => { + const f = await fixture({ + id: "scan-event-1", + codeAnalyzerDetails: { commit: { ref: "main" } }, + scannersRunStatuses: { + vulnerabilities: { enabled: true, status: "SUCCESS" }, + }, + analytics: { numVulnerabilityFindings: 3 }, + }); + let contactedCloud = false; + await expect( + prepareExternalPublication(f.file, options, { + ...f.deps, + fetch: async () => { + contactedCloud = true; + throw new Error("Unexpected Cloud request"); + }, + }), + ).rejects.toThrow("scan metadata rather than vulnerability records"); + expect(contactedCloud).toBe(false); +}); + async function fixture(records: unknown = [normalized()]) { const root = await mkdtemp(join(tmpdir(), "external-findings-")); cleanups.push(() => rm(root, { recursive: true, force: true })); @@ -2028,6 +2116,83 @@ test("repository URLs and IDs share the same resumable destination", async () => expect(f.posts[1]).toBe(f.posts[0]); }); +test("Cloud deployment routing covers every request and isolates saved retries", async () => { + const f = await fixture(); + const cloudBase = "https://cloud.example.test/pilot/backend-api/aardvark"; + const requested: string[] = []; + const deps = { + ...f.deps, + environment: { + ...f.deps.environment, + CODEX_SECURITY_CLOUD_BASE_URL: `${cloudBase}///`, + }, + fetch: async (url: string, init: RequestInit) => { + requested.push(url); + expect(url.startsWith(`${cloudBase}/external/`)).toBe(true); + return f.deps.fetch(url, init); + }, + }; + f.state.brokenReadback = true; + const prepared = await prepareExternalPublication(f.file, options, deps); + expect(prepared.preview.cloudApiUrl).toBe(`${cloudBase}/external`); + await expect(prepared.publish()).rejects.toThrow("resume the saved request"); + const otherDeployment = await prepareExternalPublication(f.file, options, { + ...f.deps, + environment: { + ...f.deps.environment, + CODEX_SECURITY_CLOUD_BASE_URL: + "https://other-cloud.example.test/backend-api/aardvark", + }, + }); + expect(otherDeployment.preview.resumed).toBe(false); + const retry = await prepareExternalPublication(f.file, options, deps); + expect(retry.preview.resumed).toBe(true); + expect(retry.preview.requests).toEqual(prepared.preview.requests); + f.state.brokenReadback = false; + const result = await retry.publish(); + expect(result.cloudApiUrl).toBe(`${cloudBase}/external`); + expect(result.verified).toBe(1); + expect(f.posts).toHaveLength(1); + expect(requested.some((url) => url.includes("/repositories?"))).toBe(true); + expect(requested.some((url) => url.includes("/source_reports?"))).toBe(true); + expect(requested.some((url) => url.endsWith("/finding_imports"))).toBe(true); + expect(requested.some((url) => url.includes("/source_reports/"))).toBe(true); +}); + +test.each([ + "", + " ", + "/relative", + "file:///tmp/cloud", + "https://user:password@cloud.example.test", + "https://cloud.example.test?deployment=pilot", + "https://cloud.example.test#pilot", +])( + "invalid Cloud base URL fails before authentication or network: %s", + async (baseUrl) => { + const f = await fixture(); + let calls = 0; + await expect( + prepareExternalPublication(f.file, options, { + ...f.deps, + environment: { + ...f.deps.environment, + CODEX_SECURITY_CLOUD_BASE_URL: baseUrl, + }, + credentials: async () => { + calls++; + throw new Error("Unexpected credential read"); + }, + fetch: async () => { + calls++; + throw new Error("Unexpected network request"); + }, + }), + ).rejects.toThrow("CODEX_SECURITY_CLOUD_BASE_URL"); + expect(calls).toBe(0); + }, +); + test("read lookups and verification run concurrently with bounded requests and ordered input", async () => { const f = await fixture( Array.from({ length: 9 }, (_, i) => normalized(`vendor-${i}`)), From 1cfdeb4b700b1686637f2831d2c0c697be3ffe19 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 9 Oct 2026 01:50:31 -0400 Subject: [PATCH 18/43] fix(package): include Cloud endpoint module in archive checks --- sdk/typescript/scripts/check-package.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/typescript/scripts/check-package.mjs b/sdk/typescript/scripts/check-package.mjs index 117beb156d..f93920828d 100644 --- a/sdk/typescript/scripts/check-package.mjs +++ b/sdk/typescript/scripts/check-package.mjs @@ -157,6 +157,7 @@ const allowedFiles = new Set([ "classify-severity", "classify-scan-severity", "severity-store", + "cloud-endpoint", "cloud-publish", "external-findings-publish", "external-import-contract", From 340e4fd26a405f42009d87d437f8f0386b6fe2bf Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 9 Oct 2026 13:48:13 -0400 Subject: [PATCH 19/43] fix(import): resolve the selected Cloud repository directly --- .../src/external-findings-publish.ts | 2 +- .../external-findings-publish.test.ts | 36 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index c20dd2663d..69c614bfce 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -294,7 +294,7 @@ export async function prepareExternalPublication( do { const result = validateRepositories( await request( - `/repositories?limit=100${page ? `&page=${encodeURIComponent(page)}` : ""}`, + `/repositories?limit=20&repository=${encodeURIComponent(options.repository)}${page ? `&page=${encodeURIComponent(page)}` : ""}`, ), ); destinations.push( diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 61dfd8d2c7..b64842fd22 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -2100,6 +2100,42 @@ test("readback interruption distinguishes acknowledged findings from verified fi } }); +test("repository selection is sent on every discovery page", async () => { + const f = await fixture(); + const repository = `${f.destination().url}.git`; + const queries: URLSearchParams[] = []; + const prepared = await prepareExternalPublication( + f.file, + { ...options, repository }, + { + ...f.deps, + fetch: async (input, init) => { + const url = new URL(input); + if (!url.pathname.endsWith("/repositories")) + return f.deps.fetch(input, init); + queries.push(url.searchParams); + const first = queries.length === 1; + return Response.json({ + object: "page", + data: first ? [] : [f.destination()], + has_more: first, + next: first ? "next-page" : null, + }); + }, + }, + ); + expect(prepared.preview.destination.id).toBe(options.repository); + expect(queries.map((query) => query.get("repository"))).toEqual([ + repository, + repository, + ]); + expect(queries.map((query) => query.get("page"))).toEqual([ + null, + "next-page", + ]); + expect(f.posts).toHaveLength(0); +}); + test("repository URLs and IDs share the same resumable destination", async () => { const f = await fixture(); f.state.loseResponse = true; From 18ab10c6e0cf2d3bf5bfb19ebef4878c578ae9a9 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 9 Oct 2026 20:48:30 -0400 Subject: [PATCH 20/43] fix(cloud): link imports to the selected deployment --- sdk/typescript/docs/cli.md | 9 +++ sdk/typescript/src/cli.ts | 8 +- sdk/typescript/src/cloud-endpoint.ts | 32 ++++++++ .../src/external-findings-publish.ts | 12 ++- .../external-findings-publish.test.ts | 80 ++++++++++++++++++- 5 files changed, 137 insertions(+), 4 deletions(-) diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md index 2c643857f5..36fae6e0c0 100644 --- a/sdk/typescript/docs/cli.md +++ b/sdk/typescript/docs/cli.md @@ -590,6 +590,7 @@ For filesystem and approval behavior, see the | `CODEX_SECURITY_LINEAR_TEAM`, `CODEX_SECURITY_LINEAR_PROJECT` | Default publication destination. | | `CODEX_SECURITY_LINEAR_API_KEY` | Linear personal API key. | | `CODEX_SECURITY_CLOUD_BASE_URL` | Cloud API root for discovery, imports, and scan publication. Defaults to `https://chatgpt.com/backend-api/aardvark`. | +| `CODEX_SECURITY_CLOUD_WEB_URL` | Cloud app URL for repository links when an alternate deployment serves its UI separately. | | `CODEX_SECURITY_EMBEDDINGS_URL` | Local dedupe and findings service embeddings endpoint. | | `GH_HOST` | GitHub Enterprise host for bulk discovery. | | `CODEX_SECURITY_NO_UPDATE_NOTICE`, `NO_UPDATE_NOTIFIER` | Disable interactive update notices. | @@ -833,6 +834,14 @@ With the variable unset, the existing production endpoint remains the default. The older `CODEX_SECURITY_CLOUD_PUBLISH_URL` overrides only the final native scan publication endpoint; it does not route vendor imports. +If that deployment serves its UI separately, set `CODEX_SECURITY_CLOUD_WEB_URL` +to the full Cloud app URL shown in the browser. The import handoff supplies both +deployment settings. The preview and result include a repository-specific +`cloudUrl`, and terminal completion prints the link. This setting only controls +the return link; all API requests still use `CODEX_SECURITY_CLOUD_BASE_URL`. +For custom API destinations without a configured app URL, completion retains +the manual navigation guidance rather than guessing a UI host or port. + Preview the selection and destination: ```bash diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 1a5e4f3f2c..09b4dd03d7 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -574,7 +574,9 @@ function renderExternalPublicationSummary( ? [] : [ "Imported Wiz evidence remains not assessed by Codex.", - "Open the Codex Security Cloud app's Findings view and select this repository. Search indexing can lag an accepted import.", + result.cloudUrl + ? `View findings: ${diagnosticValue(result.cloudUrl)}\nSearch indexing can lag an accepted import.` + : "Open the Codex Security Cloud app's Findings view and select this repository. Search indexing can lag an accepted import.", ]), "", ].join("\n"); @@ -3227,6 +3229,10 @@ export async function main( errorOutput.write( `Cloud API: ${diagnosticValue(preview.cloudApiUrl)}\nAccount: ${diagnosticValue(preview.accountId)}\nDestination: ${diagnosticValue(preview.destination.url)} (${diagnosticValue(preview.destination.id)})\nEnvironment: ${[...new Set(preview.requests.map((request) => request.repository.environment_id))].map(diagnosticValue).join(", ")}\nSource: ${preview.source.provider} / ${diagnosticValue(preview.source.source_key)}\nRead: ${preview.read} Ready: ${preview.findings.length} Excluded: ${preview.excluded.length}${preview.resumed ? "\nResuming the saved submission." : ""}\n`, ); + if (preview.cloudUrl) + errorOutput.write( + `Cloud app: ${diagnosticValue(preview.cloudUrl)}\n`, + ); for (const excluded of preview.excluded) errorOutput.write( `Excluded item ${excluded.position}: ${diagnosticValue(excluded.reason)}\n`, diff --git a/sdk/typescript/src/cloud-endpoint.ts b/sdk/typescript/src/cloud-endpoint.ts index 50dffdff01..4be383533d 100644 --- a/sdk/typescript/src/cloud-endpoint.ts +++ b/sdk/typescript/src/cloud-endpoint.ts @@ -3,6 +3,38 @@ import { CodexSecurityError } from "./errors.js"; export const DEFAULT_CLOUD_BASE_URL = "https://chatgpt.com/backend-api/aardvark"; +const CLOUD_APP_PATH = + "/mcp-app/connector_openai_defense_factory/open_defense_factory"; + +/** Custom deployments can serve their UI separately from the authenticated API. */ +export function cloudFindingsUrl( + environment: NodeJS.ProcessEnv, + repositoryId: string, +): string | undefined { + const configured = environment["CODEX_SECURITY_CLOUD_WEB_URL"]; + const api = new URL(cloudBaseUrl(environment)); + const hosted = api.href === DEFAULT_CLOUD_BASE_URL; + if (configured === undefined && !hosted) return undefined; + let url: URL; + try { + url = new URL(configured?.trim() ?? `${api.origin}${CLOUD_APP_PATH}`); + } catch { + throw new CodexSecurityError( + "CODEX_SECURITY_CLOUD_WEB_URL must be an absolute HTTP(S) Cloud app URL.", + ); + } + if ( + !["https:", "http:"].includes(url.protocol) || + url.username || + url.password + ) + throw new CodexSecurityError( + "CODEX_SECURITY_CLOUD_WEB_URL must be an HTTP(S) Cloud app URL without credentials.", + ); + url.hash = `/repositories/${encodeURIComponent(repositoryId)}/findings`; + return url.href; +} + /** Resolve the deployment once so discovery, writes, and retries share a host. */ export function cloudBaseUrl(environment: NodeJS.ProcessEnv): string { const configured = environment["CODEX_SECURITY_CLOUD_BASE_URL"]; diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 69c614bfce..4408a416d6 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -5,7 +5,11 @@ import { createRequire } from "node:module"; import { join } from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { readCloudCredentials } from "./cloud-publish.js"; -import { cloudBaseUrl, DEFAULT_CLOUD_BASE_URL } from "./cloud-endpoint.js"; +import { + cloudBaseUrl, + cloudFindingsUrl, + DEFAULT_CLOUD_BASE_URL, +} from "./cloud-endpoint.js"; import { CodexSecurityError } from "./errors.js"; import { codexSecurityStateDirectory, @@ -59,6 +63,7 @@ export interface ExternalPublicationProgress { /** Counts describe acknowledged receipts, even when later transport or readback fails. */ export interface ExternalPublicationResult { cloudApiUrl: string; + cloudUrl?: string; status: "complete" | "partial" | "interrupted"; read: number; ready: number; @@ -91,6 +96,7 @@ interface SavedSubmission { export interface ExternalPublicationPreview extends VendorFindings { cloudApiUrl: string; + cloudUrl?: string; accountId: string; destination: ImportRepository; source: FindingImportRequest["source"]; @@ -249,6 +255,8 @@ export async function prepareExternalPublication( const environment = dependencies.environment ?? process.env; const apiBaseUrl = cloudBaseUrl(environment); const cloudApiUrl = `${apiBaseUrl}/external`; + // Validate the configured return destination before authentication or uploads. + cloudFindingsUrl(environment, options.repository); const credentials = await ( dependencies.credentials ?? (() => readCloudCredentials(environment)) )(); @@ -489,6 +497,7 @@ export async function prepareExternalPublication( const preview: ExternalPublicationPreview = { ...parsed, cloudApiUrl, + cloudUrl: cloudFindingsUrl(environment, destination.id), accountId: credentials.account_id, destination, source, @@ -546,6 +555,7 @@ export async function prepareExternalPublication( } return { cloudApiUrl, + cloudUrl: preview.cloudUrl, status: parsed.excluded.length || counts.error ? "partial" : "complete", read: parsed.read, diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index b64842fd22..922b71ef85 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -878,9 +878,13 @@ test.each(["human", "json"])( const result = JSON.parse(cli.stdout.text()); expect(result.status).toBe("partial"); expect(result.counts.created).toBe(1); - expect(result).not.toHaveProperty("cloudUrl"); + expect(result.cloudUrl).toBe( + `https://chatgpt.com/mcp-app/connector_openai_defense_factory/open_defense_factory#/repositories/${options.repository}/findings`, + ); } else { - expect(cli.stdout.text()).toContain("Findings"); + expect(cli.stdout.text()).toContain( + "View findings: https://chatgpt.com/mcp-app/", + ); expect(cli.stdout.text()).not.toContain("/codex/cloud/security/findings"); } }, @@ -2187,6 +2191,7 @@ test("Cloud deployment routing covers every request and isolates saved retries", f.state.brokenReadback = false; const result = await retry.publish(); expect(result.cloudApiUrl).toBe(`${cloudBase}/external`); + expect(result.cloudUrl).toBeUndefined(); expect(result.verified).toBe(1); expect(f.posts).toHaveLength(1); expect(requested.some((url) => url.includes("/repositories?"))).toBe(true); @@ -2195,6 +2200,77 @@ test("Cloud deployment routing covers every request and isolates saved retries", expect(requested.some((url) => url.includes("/source_reports/"))).toBe(true); }); +test("a separate Cloud app URL follows the resolved repository without changing API routing", async () => { + const f = await fixture(); + const requested: string[] = []; + const environment = { + ...f.environment, + CODEX_SECURITY_CLOUD_BASE_URL: + "https://api.example.test/backend-api/aardvark", + CODEX_SECURITY_CLOUD_WEB_URL: + "http://localhost:4200/security?preview=true#/old", + }; + const prepared = await prepareExternalPublication( + f.file, + { ...options, repository: f.destination().url }, + { + ...f.deps, + environment, + fetch: (url, init) => { + requested.push(url); + return f.deps.fetch(url, init); + }, + }, + ); + const link = `http://localhost:4200/security?preview=true#/repositories/${options.repository}/findings`; + expect(prepared.preview.cloudUrl).toBe(link); + expect((await prepared.publish()).cloudUrl).toBe(link); + expect( + requested.every((url) => + url.startsWith(environment.CODEX_SECURITY_CLOUD_BASE_URL), + ), + ).toBe(true); + + const cli = createCliTest(main); + expect( + await cli.runCli( + f.command + .filter((value) => !["--format", "json"].includes(value)) + .concat("--yes"), + { ...f.cliDeps, environment }, + ), + ).toBe(0); + expect(cli.stdout.text()).toContain(`View findings: ${link}`); +}); + +test.each([ + "", + "/relative", + "file:///tmp/cloud", + "https://user:password@example.test", +])( + "invalid Cloud app URL fails before authentication or network: %s", + async (url) => { + const f = await fixture(); + let calls = 0; + await expect( + prepareExternalPublication(f.file, options, { + ...f.deps, + environment: { ...f.environment, CODEX_SECURITY_CLOUD_WEB_URL: url }, + credentials: async () => { + calls++; + throw new Error("Unexpected auth"); + }, + fetch: async () => { + calls++; + throw new Error("Unexpected network"); + }, + }), + ).rejects.toThrow("CODEX_SECURITY_CLOUD_WEB_URL"); + expect(calls).toBe(0); + }, +); + test.each([ "", " ", From 9ebf315778e167e41b987ae0bc26dd0a657408ce Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Fri, 9 Oct 2026 21:04:44 -0400 Subject: [PATCH 21/43] fix(plugin): relay deployment-specific findings links --- plugins/codex-security/skills/publish-findings/SKILL.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/codex-security/skills/publish-findings/SKILL.md b/plugins/codex-security/skills/publish-findings/SKILL.md index 9ae020d12d..c0f38913c2 100644 --- a/plugins/codex-security/skills/publish-findings/SKILL.md +++ b/plugins/codex-security/skills/publish-findings/SKILL.md @@ -13,7 +13,7 @@ Use a stable Wiz tenant/finding namespace as `--source-key`. A project filter is For Wiz UI exports, use **Vulnerability Findings → Save as → Report**, keeping the repository and finding filters. Select **Repository Branch** (remove the default Virtual Machine selection), **JSON**, and **Detailed** columns. Download the completed report; default gzip compression is supported, even with a `.json` filename. **Raw Event** on Code & Build Scans contains scan metadata and cannot supply findings. Do not create a scheduled report or enable notifications or external storage unless requested. -For staging or another Cloud deployment, set `CODEX_SECURITY_CLOUD_BASE_URL` to its API root before both preview and upload. Use that deployment's login and an isolated state directory for tests. Verify `cloudApiUrl` in the preview; it is the API endpoint, not a Cloud UI link. Saved retries are separated by deployment. The native scan setting `CODEX_SECURITY_CLOUD_PUBLISH_URL` does not route vendor imports. Never fall back to production when staging is unavailable. +For staging or another Cloud deployment, set `CODEX_SECURITY_CLOUD_BASE_URL` to its API root before both preview and upload. If the Cloud page or user supplies its app URL, also set `CODEX_SECURITY_CLOUD_WEB_URL` to that full URL so the command can return a repository Findings link. The UI setting does not change API routing or authentication. Use that deployment's login and an isolated state directory for tests. Verify `cloudApiUrl` and any `cloudUrl` in the preview against the intended deployment; the former is the API endpoint and the latter is the UI link. Saved retries are separated by deployment. The native scan setting `CODEX_SECURITY_CLOUD_PUBLISH_URL` does not route vendor imports. Never fall back to production when staging is unavailable. ```sh codex-security publish findings /path/to/selected-wiz-findings.json \ @@ -23,6 +23,6 @@ codex-security publish findings /path/to/selected-wiz-findings.json \ Show a concise preview of the Cloud API endpoint, account, repository, environments, source namespace, severity counts, and exclusions. Identify the saved evidence file and let the user inspect the complete JSON preview; do not paste an entire large report into the conversation. Explain that imported vendor evidence is not a Codex assessment. Obtain explicit confirmation for this upload, including any skipped records, then run the same command without `--dry-run`, adding `--yes`. Existing authorization to inspect or assess findings alone does not authorize sending them to Cloud. A user who already approved the concrete preview need not approve it again. -After a lost response or retryable failure, repeat the same command with unchanged input and destination. The publisher resends only unacknowledged requests, using their saved IDs and bodies. Existing findings keep their original Cloud environment even if the repository’s default changes. After a repository reset, stop and show the error; require fresh discovery and confirmation before a new publication. Report partial failures and retain their explanations. A successful import stores vendor evidence; it does not establish that Codex assessed the finding or that the vulnerability is exploitable. Direct the user to the main Findings view in the Codex Security Cloud app to find the repository’s imported Wiz findings; do not invent a direct Findings URL. +After a lost response or retryable failure, repeat the same command with unchanged input and destination. The publisher resends only unacknowledged requests, using their saved IDs and bodies. Existing findings keep their original Cloud environment even if the repository’s default changes. After a repository reset, stop and show the error; require fresh discovery and confirmation before a new publication. Report partial failures and retain their explanations. A successful import stores vendor evidence; it does not establish that Codex assessed the finding or that the vulnerability is exploitable. Include the returned `cloudUrl` as a clickable Findings link. If it is absent, direct the user to Findings in their intended Cloud app; do not derive a UI URL from the API endpoint or invent a link. Read the result even when the process exits nonzero. `complete` exits 0, `partial` exits 1 for local exclusions or final item failures, and `interrupted` exits 2 while preserving acknowledged counts and receipts. Report `failures` by vendor source ID, distinguish `verified` from `unacknowledged`, and retain `savedSubmission` and the recovery error. A lost response does not mean that no findings reached Cloud. Preparation failures use `failed`; cancellation keeps the normal signal exit code. Terminal progress goes to stderr and does not replace the JSON result. From 837f84c29537c533321f6d8e9f4f031eed4ab019 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 10 Oct 2026 02:29:30 +0000 Subject: [PATCH 22/43] feat(import): map Wiz repository findings --- .../skills/publish-findings/SKILL.md | 10 +- sdk/typescript/docs/cli.md | 121 +++- .../schemas/external-findings.schema.json | 424 ++++++++++++ sdk/typescript/src/cli.ts | 8 +- .../src/external-findings-publish.ts | 10 + .../src/external-import-contract.ts | 47 +- sdk/typescript/src/external-import-models.ts | 104 ++- sdk/typescript/src/wiz-findings.ts | 261 ++++++- sdk/typescript/tests-ts/wiz-findings.test.ts | 650 ++++++++++++++++++ 9 files changed, 1583 insertions(+), 52 deletions(-) create mode 100644 sdk/typescript/tests-ts/wiz-findings.test.ts diff --git a/plugins/codex-security/skills/publish-findings/SKILL.md b/plugins/codex-security/skills/publish-findings/SKILL.md index c0f38913c2..780326f39d 100644 --- a/plugins/codex-security/skills/publish-findings/SKILL.md +++ b/plugins/codex-security/skills/publish-findings/SKILL.md @@ -1,17 +1,21 @@ --- name: publish-findings -description: Import selected Wiz package vulnerabilities to Codex Security Cloud after previewing the destination and evidence. Supports plain or gzip-compressed vulnerability JSON and normalized JSONL; does not import raw SAST, secrets, or IaC findings, run a scan, or assess findings. +description: Import selected Wiz package vulnerabilities, SAST, repository secrets, and IaC findings to Codex Security Cloud after previewing the destination and evidence. Supports plain or gzip-compressed JSON and normalized JSONL; does not import workload secrets or cloud configuration, run a scan, or assess findings. --- # Publish external findings Use the installed `codex-security publish findings` command for both preview and upload. It owns source mapping, Cloud authentication, source versions, and saved request retries. Do not implement another HTTP uploader or put credentials in a command argument. -Resolve the user's destination to its repository URL or Cloud repository ID; the command accepts either. Confirm that the repository is connected in the intended Cloud account and has an authorized environment. Use the existing file-backed ChatGPT login; explain a missing or incompatible login rather than changing credential storage automatically. Read selected Wiz package vulnerabilities through the user's configured connection or supplied export. Save the actual selected JSON records locally, retaining their IDs and evidence. Finish the requested pagination; if the user selected a subset, save that explicit subset as an array. The command does not fetch from Wiz or accept CSV, SAST, secrets, IaC, or external network exports. The publisher also accepts JSONL records with `source_finding_id` and `evidence` as described in the CLI documentation. +Resolve the user's destination to its repository URL or Cloud repository ID; the command accepts either. Confirm that the repository is connected in the intended Cloud account and has an authorized environment. Use the existing file-backed ChatGPT login; explain a missing or incompatible login rather than changing credential storage automatically. Read selected Wiz findings through the user's configured connection or supplied export. Save the actual selected JSON records locally, retaining their IDs and evidence. Finish the requested pagination; if the user selected a subset, save that explicit subset and retain its named collection. The command does not fetch from Wiz or accept CSV, scan events, workload secrets, cloud configuration, or external network exports. The publisher also accepts JSONL records with `source_finding_id` and `evidence` as described in the CLI documentation. Use a stable Wiz tenant/finding namespace as `--source-key`. A project filter is selection context, not a new identity. Confirm the selected assets or builds belong to the destination repository; do not infer a scanner branch or revision from the current checkout. Do not publish vendor credentials, unrelated records, or local assessments as original scanner evidence. -For Wiz UI exports, use **Vulnerability Findings → Save as → Report**, keeping the repository and finding filters. Select **Repository Branch** (remove the default Virtual Machine selection), **JSON**, and **Detailed** columns. Download the completed report; default gzip compression is supported, even with a `.json` filename. **Raw Event** on Code & Build Scans contains scan metadata and cannot supply findings. Do not create a scheduled report or enable notifications or external storage unless requested. +For package vulnerability UI exports, use **Vulnerability Findings → Save as → Report**, keeping the repository and finding filters. Select **Repository Branch** (remove the default Virtual Machine selection), **JSON**, and **Detailed** columns. Download the completed report; default gzip compression is supported, even with a `.json` filename. **Raw Event** on Code & Build Scans contains scan metadata and cannot supply findings. Do not create a scheduled report or enable notifications or external storage unless requested. + +For SAST, repository secrets, and IaC, retain the explicit `data.sastFindings.nodes`, `data.secretInstances.nodes`, or `data.iacFindings.nodes` collection. Bare arrays cannot identify these finding families. Include a verified URL on the source repository metadata or supply `data.versionControlResources.nodes` with matching `repository.id` and `repository.url`. For secrets, the repository is `resource.typedProperties.repository` and resource type must be `REPOSITORY_BRANCH`. Do not map names, inventory node IDs, `providerID`, or Wiz UUIDs directly to Cloud repository IDs. The importer verifies source URLs against the selected destination; resolve a missing or conflicting mapping before uploading. + +Keep raw source records unchanged in `source_data`. New raw occurrence IDs are prefixed `sast:`, `secret:`, or `iac:`; existing package IDs are unchanged. For CSV report conversion, use a reviewed mapping of the actual header names into the documented normalized JSONL profile, set `evidence.details.kind` and the verified `details.repository.url`, and use the same prefixed occurrence ID. The command does not parse CSV directly. Select source paths and line numbers explicitly in secret reports; the standard metadata query lacks them. Preserve rule/CWE, range/snippet, detector validation, and IaC comparison fields in the typed details described in the CLI docs. Preserve empty comparison strings. Do not invent missing IDs, severity, revisions, or locations, retrieve credential values, test credentials, or apply suggested fixes. Vendor status, AI verdict, secret validation, introduced commits, and logical IaC resources are evidence, not Codex assessment or proof of runtime deployment. For staging or another Cloud deployment, set `CODEX_SECURITY_CLOUD_BASE_URL` to its API root before both preview and upload. If the Cloud page or user supplies its app URL, also set `CODEX_SECURITY_CLOUD_WEB_URL` to that full URL so the command can return a repository Findings link. The UI setting does not change API routing or authentication. Use that deployment's login and an isolated state directory for tests. Verify `cloudApiUrl` and any `cloudUrl` in the preview against the intended deployment; the former is the API endpoint and the latter is the UI link. Saved retries are separated by deployment. The native scan setting `CODEX_SECURITY_CLOUD_PUBLISH_URL` does not route vendor imports. Never fall back to production when staging is unavailable. diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md index 36fae6e0c0..8c4b70882c 100644 --- a/sdk/typescript/docs/cli.md +++ b/sdk/typescript/docs/cli.md @@ -793,19 +793,20 @@ again when rerunning. ## Publish findings to Cloud -### Import Wiz package vulnerabilities +### Import Wiz findings -Import selected Wiz package vulnerabilities without creating a scan. Imported -records retain vendor evidence and remain **not assessed by Codex**. Raw SAST, -secret, IaC, and external network findings need separate mappings; this is not -a synchronization of every finding in Wiz. +Import selected Wiz package vulnerabilities, SAST findings, repository secrets, +and IaC findings without creating a scan. Imported records retain vendor evidence +and remain **not assessed by Codex**, including when Wiz marks them resolved or +reports an AI verdict. Workload secrets, cloud configuration findings, and external +network findings are not supported by this repository importer. Before the first import: 1. Select an existing repository in Codex Security Cloud and copy its repository URL (for example, a GitHub URL) or Cloud repository ID. The repository needs an authorized environment, but does not need a completed native scan. -2. In Wiz **Vulnerability Findings**, filter to the intended repository and +2. For **package vulnerabilities**, in Wiz **Vulnerability Findings**, filter to the intended repository and findings. Choose **Save as → Report**, select **Repository Branch** as the resource type (remove the default Virtual Machine selection), **JSON** format, and **Detailed** columns. Retain the repository filter and download the completed @@ -816,7 +817,9 @@ Before the first import: authorized Wiz connection and save their JSON. Include `id`, `name`, `detailedName`, `vendorSeverity` (or `severity`), and `vulnerableAsset.id`; retain available package, image, and source metadata. The command does not - fetch from Wiz. + fetch from Wiz. For SAST, repository secrets, and IaC, use the named API + collections and repository metadata described below; their report formats + differ from package vulnerability reports. 3. Choose a stable source key such as `TENANT_ID/vulnerability-finding`. Reuse it for that Wiz tenant and finding class across exports. Do not create a new key for each project filter or import; that creates different source identities. @@ -867,12 +870,12 @@ exclusions. `--dry-run --format json` retains all normalized findings and comple evidence for inspection. A prompt explicitly includes any records that will be skipped. -Input can be a downloaded Wiz JSON vulnerability report, a vulnerability finding, -an array, a complete -`data.vulnerabilityFindings.nodes` response, or JSONL with one vendor record per -line. Plain and gzip-compressed files are accepted. A response that advertises -another page is rejected: save the explicitly -selected records as an array. Normalized JSONL is also accepted: +Package input can be a downloaded Wiz JSON vulnerability report, a vulnerability +finding, an array, a complete `data.vulnerabilityFindings.nodes` response, or JSONL +with one package record per line. Plain and gzip-compressed files are accepted. +A response that advertises another page is rejected. Complete the requested pages +or save only the explicitly selected records; retain the named collection for +SAST, repository secrets, and IaC. Normalized JSONL is also accepted: ```json { @@ -896,6 +899,98 @@ fetch from Wiz, assess findings, or change vendor or Cloud triage decisions. Findings from external network scans are excluded from this package vulnerability mapping. +For SAST, repository secrets, and IaC, save the documented JSON API collection: + +| Finding family | Collection | Raw occurrence identity | +| ------------------ | ---------------------------- | ----------------------- | +| SAST | `data.sastFindings.nodes` | `sast:` | +| Repository secrets | `data.secretInstances.nodes` | `secret:` | +| IaC | `data.iacFindings.nodes` | `iac:` | + +The collection identifies the mapping. Bare arrays of these raw findings are +excluded because their overlapping fields do not identify a family. An export +may contain multiple named collections. GraphQL errors and unfinished pages stop +the import. Secret resources must have `type: "REPOSITORY_BRANCH"`; a workload's +relationship to source code does not make it a repository secret. + +Each new finding needs a verified source repository URL. Supply `repository.url` +in the finding's repository metadata, or include the matching repository inventory +in `data.versionControlResources.nodes`. For secrets, repository metadata is under +`resource.typedProperties.repository`. The importer joins the inventory's +`repository.id` to that source ID and uses `repository.url`; inventory node IDs, +`providerID`, names, and Wiz UUIDs are not Cloud repository IDs. A supplied URL +must agree with the inventory and with the selected Cloud repository before any +finding is uploaded. For example, this synthetic selection supplies the inventory +alongside its SAST collection: + +```json +{ + "data": { + "sastFindings": { + "nodes": [ + { + "id": "finding-42", + "name": "Untrusted query construction", + "severity": "HIGH", + "repository": { "id": "source-repository-1" }, + "filePath": "src/query.ts", + "startLine": 10, + "endLine": 12 + } + ] + }, + "versionControlResources": { + "nodes": [ + { + "repository": { + "id": "source-repository-1", + "url": "https://github.com/example/project" + } + } + ] + } + } +} +``` + +The adapters preserve each selected raw node exactly in `source_data`. Typed +`evidence.details` carries `kind`, verified `repository`, vendor `source_status`, +and family-specific evidence: + +| Family | Evidence retained in typed fields | +| ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| SAST | CWE IDs, first location's end line and snippet, scanner origin and verdict, remediation instructions | +| Repository secrets | Detector rule, type, confidence, vendor validation status, encryption/management flags, introduced commit when supplied | +| IaC | Rule IDs and name, first location's range and matched content, platform, expected/actual content, separate file link, remediation instructions | + +Vendor status and verdict remain separate from Codex assessment. A secret's +introduced commit is not its scanned revision. Missing source revisions and scan +IDs stay unknown. A logical IaC resource is retained in the original record and +does not establish that it is deployed. Precise original timestamps remain in +`source_data`; only an explicit record-update timestamp maps to the whole-second +`source_updated_at` field. SAST creation time and IaC last-seen time are not update +timestamps. + +Native SAST and secret report downloads can be CSV, including gzip-compressed +CSV. CSV is not parsed directly. Convert a selected report with a reviewed mapping +for its actual column headers into normalized JSONL, preserving each original row +in `source_data`. Set `details.kind` and `details.repository.url` explicitly after +verifying the source repository. Use the same prefixed occurrence ID shown above +so API and report imports update the same finding; normalized IDs are accepted +exactly as supplied. Never substitute a rule ID, CWE, title, token, or file path +for a vendor occurrence ID. Missing IDs or severity must be corrected at the +source rather than invented. + +For repository-secret reports, select `path` and `lineNumber` for source +navigation, detector fields under `rule`, and `vcsDetails.initialCommitHash` only +as the introduced commit. The standard secret metadata API does not supply a +path or line. For IaC reports, explicitly select `filePath`, `startLine`, `endLine`, +`rule.id`, `rule.shortId`, `expectedContent`, `foundContent`, `matchContent`, and +repository/branch metadata. Preserve the original fields; source locations must +be repository-relative, and an end line requires a start line. Keep credential +values out of the metadata selection; the importer does not retrieve or test +credentials or apply remediation instructions. + Inputs must be valid UTF-8. Integer-valued evidence outside JavaScript's safe integer range is rejected; export those values as strings to preserve them exactly. Invalid records appear in `excluded` with their source ID and reason. diff --git a/sdk/typescript/schemas/external-findings.schema.json b/sdk/typescript/schemas/external-findings.schema.json index 9ba52049ef..fc796cb338 100644 --- a/sdk/typescript/schemas/external-findings.schema.json +++ b/sdk/typescript/schemas/external-findings.schema.json @@ -406,6 +406,241 @@ "title": "ImportedAssessment", "type": "object" }, + "ImportedCodeContext": { + "additionalProperties": false, + "description": "Optional range and snippet for the evidence's first source location.", + "properties": { + "end_line": { + "anyOf": [ + { + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "End Line" + }, + "snippet": { + "anyOf": [ + { + "maxLength": 65536, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Snippet" + } + }, + "title": "ImportedCodeContext", + "type": "object" + }, + "ImportedConfigurationDetails": { + "additionalProperties": false, + "properties": { + "platform": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Platform" + }, + "cloud_platform": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Cloud Platform" + }, + "expected": { + "anyOf": [ + { + "maxLength": 65536, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected" + }, + "actual": { + "anyOf": [ + { + "maxLength": 65536, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Actual" + } + }, + "title": "ImportedConfigurationDetails", + "type": "object" + }, + "ImportedFindingDetails": { + "additionalProperties": false, + "description": "Typed repository evidence; vendor state is not a Codex assessment.", + "properties": { + "kind": { + "enum": ["sast", "secret", "iac"], + "title": "Kind", + "type": "string" + }, + "repository": { + "$ref": "#/$defs/ImportedRepositoryReference" + }, + "source_status": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Status" + }, + "scanner_origin": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Scanner Origin" + }, + "scanner_verdict": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Scanner Verdict" + }, + "rule": { + "anyOf": [ + { + "$ref": "#/$defs/ImportedFindingRule" + }, + { + "type": "null" + } + ], + "default": null + }, + "weakness_ids": { + "default": [], + "items": { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + "maxItems": 100, + "title": "Weakness Ids", + "type": "array" + }, + "remediation_instructions": { + "anyOf": [ + { + "maxLength": 65536, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Remediation Instructions" + }, + "code": { + "anyOf": [ + { + "$ref": "#/$defs/ImportedCodeContext" + }, + { + "type": "null" + } + ], + "default": null + }, + "secret": { + "anyOf": [ + { + "$ref": "#/$defs/ImportedSecretDetails" + }, + { + "type": "null" + } + ], + "default": null + }, + "configuration": { + "anyOf": [ + { + "$ref": "#/$defs/ImportedConfigurationDetails" + }, + { + "type": "null" + } + ], + "default": null + }, + "file_url": { + "anyOf": [ + { + "maxLength": 2048, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "File Url" + } + }, + "required": ["kind", "repository"], + "title": "ImportedFindingDetails", + "type": "object" + }, "ImportedFindingEvidence": { "additionalProperties": false, "properties": { @@ -546,12 +781,71 @@ "default": {}, "title": "Source Data", "type": "object" + }, + "details": { + "anyOf": [ + { + "$ref": "#/$defs/ImportedFindingDetails" + }, + { + "type": "null" + } + ] } }, "required": ["title", "severity"], "title": "ImportedFindingEvidence", "type": "object" }, + "ImportedFindingRule": { + "additionalProperties": false, + "properties": { + "id": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Id" + }, + "short_id": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Short Id" + }, + "name": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + } + }, + "title": "ImportedFindingRule", + "type": "object" + }, "ImportedLocation": { "additionalProperties": false, "properties": { @@ -645,6 +939,136 @@ "title": "ImportedPackage", "type": "object" }, + "ImportedRepositoryReference": { + "additionalProperties": false, + "description": "Vendor repository identity, resolved to the full source-control URL.", + "properties": { + "url": { + "maxLength": 2048, + "minLength": 1, + "title": "Url", + "type": "string" + }, + "id": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Id" + }, + "name": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + } + }, + "required": ["url"], + "title": "ImportedRepositoryReference", + "type": "object" + }, + "ImportedSecretDetails": { + "additionalProperties": false, + "properties": { + "type": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Type" + }, + "confidence": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Confidence" + }, + "validation_status": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Validation Status" + }, + "is_encrypted": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Is Encrypted" + }, + "is_managed": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Is Managed" + }, + "introduced_commit": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Introduced Commit" + } + }, + "title": "ImportedSecretDetails", + "type": "object" + }, "LastImport": { "additionalProperties": false, "properties": { diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 09b4dd03d7..2510c9bafd 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -3150,14 +3150,16 @@ export async function main( }); publication.command("findings", { description: - "Preview and import selected Wiz package vulnerabilities to Cloud.", - hint: "Supply a selected Wiz vulnerability JSON report or normalized JSONL; gzip downloads are supported (not CSV, scan events, SAST, secrets, or IaC exports).\nExample: codex-security publish findings selected-wiz.json --to cloud --repository https://github.com/example/project --provider wiz --source-key TENANT_ID/vulnerability-finding --dry-run --format json", + "Preview and import selected Wiz repository findings to Cloud.", + hint: "Supply a package vulnerability JSON report, named SAST/repository-secret/IaC JSON collections with repository metadata, or normalized JSONL. Gzip is supported; CSV, scan events, workload secrets, and cloud configuration exports are not.\nExample: codex-security publish findings selected-wiz.json --to cloud --repository https://github.com/example/project --provider wiz --source-key TENANT_ID/vulnerability-finding --dry-run --format json", destructive: true, mcp: false, args: z.object({ file: z .string() - .describe("Saved Wiz vulnerability findings JSON or normalized JSONL."), + .describe( + "Saved Wiz finding collections, package vulnerability JSON, or normalized JSONL.", + ), }), options: z.object({ to: z.literal("cloud").describe("Publication destination."), diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 4408a416d6..2eedd6eade 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -329,6 +329,16 @@ export async function prepareExternalPublication( : "The destination repository is not available to this Cloud account. Copy its repository URL or ID from Codex Security Cloud and check that you are signed into the intended account.", ); const destination = destinations[0]!; + for (const finding of parsed.findings) { + const sourceRepository = finding.evidence.details?.repository; + if ( + sourceRepository && + repositoryUrl(sourceRepository.url) !== repositoryUrl(destination.url) + ) + throw new CodexSecurityError( + `Finding ${JSON.stringify(finding.source_finding_id)} belongs to ${JSON.stringify(sourceRepository.url)}, which does not match the selected Cloud repository ${JSON.stringify(destination.url)}. Verify the source repository mapping before publishing.`, + ); + } const source = { provider: options.provider, source_key: options.sourceKey }; const state = join( codexSecurityStateDirectory(environment), diff --git a/sdk/typescript/src/external-import-contract.ts b/sdk/typescript/src/external-import-contract.ts index 6e64723503..d49e3136e7 100644 --- a/sdk/typescript/src/external-import-contract.ts +++ b/sdk/typescript/src/external-import-contract.ts @@ -17,6 +17,11 @@ const ajv = new Ajv2020({ }); const STRING_BYTE_LIMITS: Readonly> = { description: 65536, + snippet: 65536, + remediation_instructions: 65536, + expected: 65536, + actual: 65536, + file_url: 2048, path: 2048, url: 2048, manifest_path: 2048, @@ -34,7 +39,10 @@ function validateModelStrings(input: unknown): void { for (const [field, value] of Object.entries(input)) { if (field === "source_data") continue; if (typeof value === "string") { - if (!value.trim() || value.includes("\0")) + if ( + (!value.trim() && !["snippet", "expected", "actual"].includes(field)) || + value.includes("\0") + ) throw new Error( `${field} must be nonempty and contain no NUL characters.`, ); @@ -61,6 +69,25 @@ function validator(name: keyof typeof schema.$defs) { $ref: `#/$defs/${name}`, }); return (input: unknown): T => { + // The backend omits a null details wrapper while retaining all legacy + // defaults. Match that wire representation for saved requests and readback. + const omitNullDetails = (value: unknown): void => { + if (value !== null && typeof value === "object") { + const fields = value as Record; + if (fields["details"] === null) delete fields["details"]; + } + }; + if (name === "ImportedFindingEvidence") omitNullDetails(input); + else if (input && typeof input === "object") { + if (name === "SourceReport" && "evidence" in input) + omitNullDetails(input.evidence); + if ( + name === "FindingImportRequest" && + "items" in input && + Array.isArray(input.items) + ) + for (const item of input.items) omitNullDetails(item?.evidence); + } if (!check(input)) throw new Error(`${name}: ${ajv.errorsText(check.errors)}`); validateModelStrings(input); @@ -93,8 +120,22 @@ export function validateExternalEvidence( ) throw new Error("Locations must use repository-relative paths."); } - if (result.url !== null && result.url !== undefined) { - const url = result.url + const endLine = result.details?.code?.end_line; + if (endLine !== null && endLine !== undefined) { + const firstLine = result.locations?.[0]?.line; + if (firstLine === null || firstLine === undefined || endLine < firstLine) + throw new Error( + "The code end line requires a first source location line and cannot precede it.", + ); + } + const urls = [ + result.url, + result.details?.repository.url, + result.details?.file_url, + ]; + for (const value of urls) { + if (value === null || value === undefined) continue; + const url = value .replace(/[\t\r\n]/gu, "") .replace(/^[\u0000-\u0020]+/u, ""); const authority = /^https?:\/\/([^/?#]+)/iu.exec(url)?.[1]; diff --git a/sdk/typescript/src/external-import-models.ts b/sdk/typescript/src/external-import-models.ts index b2eb4d7897..a5c4a67d1c 100644 --- a/sdk/typescript/src/external-import-models.ts +++ b/sdk/typescript/src/external-import-models.ts @@ -44,12 +44,40 @@ export type SourceScanId = string | null; */ export type ImageDigests = string[]; export type SourceUpdatedAt = number | null; +export type Kind = "sast" | "secret" | "iac"; +export type Url1 = string; +export type Id1 = string | null; +export type Name1 = string | null; +export type SourceStatus = string | null; +export type ScannerOrigin = string | null; +export type ScannerVerdict = string | null; +export type Id2 = string | null; +export type ShortId = string | null; +export type Name2 = string | null; +/** + * @maxItems 100 + */ +export type WeaknessIds = string[]; +export type RemediationInstructions = string | null; +export type EndLine = number | null; +export type Snippet = string | null; +export type Type = string | null; +export type Confidence = string | null; +export type ValidationStatus = string | null; +export type IsEncrypted = boolean | null; +export type IsManaged = boolean | null; +export type IntroducedCommit = string | null; +export type Platform = string | null; +export type CloudPlatform = string | null; +export type Expected = string | null; +export type Actual = string | null; +export type FileUrl = string | null; /** * @minItems 1 * @maxItems 100 */ export type Items = FindingImportItem[]; -export type Id1 = string; +export type Id3 = string; export type Object = "security.finding_import"; export type Actor = string; export type CreatedAt = number; @@ -69,17 +97,17 @@ export type Code = export type Message = string; export type Results = FindingImportResult[]; export type Object1 = "page"; -export type Id2 = string; +export type Id4 = string; export type Object2 = "security.repository"; export type RepoConnectorId1 = string; -export type Url1 = string; +export type Url2 = string; export type DefaultBranch = string | null; export type ResetMarker1 = string; export type ImportEnvironmentId = string | null; export type Data = ImportRepository[]; export type HasMore = boolean; export type Next = string | null; -export type Id3 = string; +export type Id5 = string; export type Object3 = "security.source_report"; export type Origin = "imported"; export type RepoId = string; @@ -90,13 +118,13 @@ export type SourceFindingId1 = string; export type ObservationId1 = string; export type Version1 = number; export type State = "not_assessed"; -export type Id4 = string; +export type Id6 = string; export type Actor1 = string; export type CreatedAt1 = number; export type CreatedAt2 = number; export type UpdatedAt = number; export type Object4 = "page"; -export type Id5 = string; +export type Id7 = string; export type Object5 = "security.source_report_summary"; export type Origin1 = "imported"; export type RepoId1 = string; @@ -162,6 +190,7 @@ export interface ImportedFindingEvidence { image_digests?: ImageDigests; source_updated_at?: SourceUpdatedAt; source_data?: SourceData; + details?: ImportedFindingDetails | null; } export interface ImportedPackage { name: Name; @@ -177,8 +206,59 @@ export interface ImportedLocation { export interface SourceData { [k: string]: unknown; } +/** + * Typed repository evidence; vendor state is not a Codex assessment. + */ +export interface ImportedFindingDetails { + kind: Kind; + repository: ImportedRepositoryReference; + source_status?: SourceStatus; + scanner_origin?: ScannerOrigin; + scanner_verdict?: ScannerVerdict; + rule?: ImportedFindingRule | null; + weakness_ids?: WeaknessIds; + remediation_instructions?: RemediationInstructions; + code?: ImportedCodeContext | null; + secret?: ImportedSecretDetails | null; + configuration?: ImportedConfigurationDetails | null; + file_url?: FileUrl; +} +/** + * Vendor repository identity, resolved to the full source-control URL. + */ +export interface ImportedRepositoryReference { + url: Url1; + id?: Id1; + name?: Name1; +} +export interface ImportedFindingRule { + id?: Id2; + short_id?: ShortId; + name?: Name2; +} +/** + * Optional range and snippet for the evidence's first source location. + */ +export interface ImportedCodeContext { + end_line?: EndLine; + snippet?: Snippet; +} +export interface ImportedSecretDetails { + type?: Type; + confidence?: Confidence; + validation_status?: ValidationStatus; + is_encrypted?: IsEncrypted; + is_managed?: IsManaged; + introduced_commit?: IntroducedCommit; +} +export interface ImportedConfigurationDetails { + platform?: Platform; + cloud_platform?: CloudPlatform; + expected?: Expected; + actual?: Actual; +} export interface FindingImportReceipt { - id: Id1; + id: Id3; object?: Object; repository: ImportDestination; source: ImportSource; @@ -214,16 +294,16 @@ export interface ImportRepositoryPage { next: Next; } export interface ImportRepository { - id: Id2; + id: Id4; object?: Object2; repo_connector_id: RepoConnectorId1; - url: Url1; + url: Url2; default_branch?: DefaultBranch; reset_marker: ResetMarker1; import_environment_id?: ImportEnvironmentId; } export interface SourceReport { - id: Id3; + id: Id5; object?: Object3; origin?: Origin; repo_id: RepoId; @@ -244,7 +324,7 @@ export interface ImportedAssessment { state?: State; } export interface LastImport { - id: Id4; + id: Id6; actor: Actor1; created_at: CreatedAt1; } @@ -255,7 +335,7 @@ export interface SourceReportPage { next: Next1; } export interface SourceReportSummary { - id: Id5; + id: Id7; object?: Object5; origin?: Origin1; repo_id: RepoId1; diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts index dde9afb523..015c4eb1a4 100644 --- a/sdk/typescript/src/wiz-findings.ts +++ b/sdk/typescript/src/wiz-findings.ts @@ -127,12 +127,192 @@ function wizFinding(record: Record): VendorFinding { }; } -function records(payload: unknown): unknown[] { - if (Array.isArray(payload)) return payload; +type RepositoryFindingKind = "sast" | "secret" | "iac"; + +type InputRecord = { + value: unknown; + kind?: "sca" | RepositoryFindingKind | "cloud_configuration"; +}; + +type RepositoryMetadata = { + id: string | null; + name: string | null; + url: string; +}; + +type FindingInput = { + records: InputRecord[]; + repositories: Map; +}; + +function repositoryUrl(value: string): string { + return value.replace(/\/$/u, "").replace(/\.git$/u, ""); +} + +function sourceRepository( + repository: Record | undefined, + repositories: FindingInput["repositories"], +): RepositoryMetadata { + const id = text(repository?.["id"]); + const inventory = id ? repositories.get(id) : undefined; + const suppliedUrl = text(repository?.["url"]); + if ( + suppliedUrl && + inventory && + repositoryUrl(suppliedUrl) !== repositoryUrl(inventory.url) + ) + throw new Error( + "The finding and repository inventory have conflicting repository URLs.", + ); + const url = suppliedUrl ?? inventory?.url; + if (!url) + throw new Error( + "A verified source repository URL is required. Include repository.url or the matching repository.id and repository.url from data.versionControlResources.nodes; names and Wiz IDs are not Cloud repository IDs.", + ); + return { + id, + name: text(repository?.["name"]) ?? inventory?.name ?? null, + url, + }; +} + +function sourceBranch( + branch: Record | undefined, + repositoryName: string | null, +): string | null { + const name = text(branch?.["name"]); + const prefix = repositoryName ? `${repositoryName}/` : null; + return name && prefix && name.startsWith(prefix) + ? name.slice(prefix.length) || null + : name; +} + +function updatedAt(value: unknown, field: string): number | null { + const timestamp = text(value); + if (!timestamp) return null; + const seconds = Date.parse(timestamp) / 1000; + if (!Number.isFinite(seconds) || seconds < 0) + throw new Error(`Wiz ${field} is not a valid timestamp.`); + return Math.floor(seconds); +} + +// Empty code and configuration strings are meaningful vendor evidence. +function content(value: unknown): string | null { + return typeof value === "string" ? value : null; +} + +/** Map only explicitly identified repository finding collections. */ +function repositoryFinding( + kind: RepositoryFindingKind, + record: Record, + repositories: FindingInput["repositories"], +): VendorFinding { + const id = sourceFindingId(record["id"]); + const resource = object(record["resource"]); + if (kind === "secret" && resource?.["type"] !== "REPOSITORY_BRANCH") + throw new Error( + "Only repository-branch secret findings can be published to a repository. Workload secrets require a resource-scoped import and are not supported.", + ); + const repository = sourceRepository( + kind === "secret" + ? object(object(resource?.["typedProperties"])?.["repository"]) + : object(record["repository"]), + repositories, + ); + const branch = + kind === "secret" + ? resource + : object(record[kind === "sast" ? "repositoryBranch" : "branch"]); + const rule = object(record["rule"]); + const path = text(record[kind === "secret" ? "path" : "filePath"]); + const line = record[kind === "secret" ? "lineNumber" : "startLine"]; + const details = { + kind, + repository, + source_status: text(record["status"]), + scanner_origin: kind === "sast" ? text(record["origin"]) : null, + scanner_verdict: + kind === "sast" ? text(object(record["aiAnalysis"])?.["verdict"]) : null, + rule: rule + ? { + id: text(rule["id"]), + short_id: text(rule["shortId"]), + name: text(rule["name"]), + } + : null, + weakness_ids: + kind === "sast" && Array.isArray(record["weaknesses"]) + ? record["weaknesses"].map((weakness) => object(weakness)?.["id"]) + : [], + remediation_instructions: + text(record["remediationInstructions"]) ?? + text(rule?.["remediationInstructions"]), + code: + path && kind !== "secret" + ? { + end_line: record["endLine"] ?? null, + snippet: content( + record[kind === "sast" ? "snippet" : "matchContent"], + ), + } + : null, + secret: + kind === "secret" + ? { + type: text(record["type"]), + confidence: text(record["confidence"]), + validation_status: text(record["validationStatus"]), + is_encrypted: record["isEncrypted"] ?? null, + is_managed: record["isManaged"] ?? null, + introduced_commit: text( + object(record["vcsDetails"])?.["initialCommitHash"], + ), + } + : null, + configuration: + kind === "iac" + ? { + platform: text(record["platform"]), + cloud_platform: text(record["cloudPlatform"]), + expected: content(record["expectedContent"]), + actual: content(record["foundContent"]), + } + : null, + file_url: text(record["fileURL"]), + }; + return { + // Existing package occurrence IDs remain unchanged. New families cannot + // collide with one another when the same tenant source key is used. + source_finding_id: sourceFindingId(`${kind}:${id}`), + evidence: validateExternalEvidence({ + title: text(record["name"]) ?? text(rule?.["name"]), + severity: text(record["severity"])?.toLowerCase(), + description: text(record["description"]) ?? text(rule?.["description"]), + url: text(record["wizUrl"]) ?? text(record["portalUrl"]), + locations: path ? [{ path, line: line ?? null }] : [], + branch: sourceBranch(branch, repository.name), + code_revision: null, + source_scan_id: null, + source_updated_at: + kind === "secret" + ? updatedAt(record["lastUpdatedAt"], "lastUpdatedAt") + : kind === "iac" + ? updatedAt(record["updatedAt"], "updatedAt") + : null, + source_data: record, + details, + }), + }; +} + +function records(payload: unknown): FindingInput { + const repositories: FindingInput["repositories"] = new Map(); + if (Array.isArray(payload)) + return { records: payload.map((value) => ({ value })), repositories }; const envelope = object(payload); if (!envelope) throw new Error( - "Expected finding objects or a Wiz vulnerabilityFindings response.", + "Expected finding objects or a named Wiz finding collection.", ); if ( object(envelope["scannersRunStatuses"]) && @@ -141,36 +321,66 @@ function records(payload: unknown): unknown[] { throw new Error( "This is a Wiz Code & Build Scan event, which contains scan metadata rather than vulnerability records. In Vulnerability Findings, filter the repository and selection, then choose Save as → Report with JSON format and Detailed columns. Download that report instead of Raw Event.", ); - if (Array.isArray(envelope["errors"]) && envelope["errors"].length > 0) { + if (Array.isArray(envelope["errors"]) && envelope["errors"].length > 0) throw new Error( "Wiz returned GraphQL errors. Resolve them before publishing this response.", ); - } if ( envelope["source_finding_id"] !== undefined || envelope["id"] !== undefined ) - return [envelope]; + return { records: [{ value: envelope }], repositories }; const data = object(envelope["data"]) ?? envelope; - const findings = object(data["vulnerabilityFindings"]); - if (findings && Array.isArray(findings["nodes"])) { - const page = object(findings["pageInfo"]); - if (page?.["hasNextPage"] === true) { + const inventory = object(data["versionControlResources"]); + if (object(inventory?.["pageInfo"])?.["hasNextPage"] === true) + throw new Error( + "Wiz repository inventory has another page. Finish the selected inventory before publishing.", + ); + if (Array.isArray(inventory?.["nodes"])) { + for (const entry of inventory["nodes"]) { + const repository = object(object(entry)?.["repository"]); + const id = text(repository?.["id"]); + const url = text(repository?.["url"]); + if (!id || !url) continue; + const previous = repositories.get(id); + if (previous && repositoryUrl(previous.url) !== repositoryUrl(url)) + throw new Error( + `Repository inventory contains conflicting URLs for ${JSON.stringify(id)}.`, + ); + repositories.set(id, { id, name: text(repository?.["name"]), url }); + } + } + const collections = { + vulnerabilityFindings: "sca", + sastFindings: "sast", + secretInstances: "secret", + iacFindings: "iac", + configurationFindings: "cloud_configuration", + } as const; + const selected: InputRecord[] = []; + let found = false; + for (const [root, kind] of Object.entries(collections)) { + const collection = object(data[root]); + if (!collection) continue; + found = true; + if (!Array.isArray(collection["nodes"])) + throw new Error(`Wiz ${root}.nodes must be an array.`); + if (object(collection["pageInfo"])?.["hasNextPage"] === true) throw new Error( - "Wiz response has another page. Save the explicitly selected records as an array before publishing.", + "Wiz response has another page. Complete the requested pages or save the explicitly selected records in their named collection before publishing.", ); - } - return findings["nodes"]; + selected.push(...collection["nodes"].map((value) => ({ value, kind }))); } + if (found) return { records: selected, repositories }; throw new Error( - "Unsupported Wiz export. Supply vulnerability finding objects or normalized JSONL.", + "Unsupported Wiz export. Supply vulnerability finding objects, named sastFindings/secretInstances/iacFindings collections, or normalized JSONL. CSV reports must be converted to the documented normalized JSONL profile.", ); } export async function readVendorFindings( path: string, ): Promise { - let input: unknown[]; + let input: FindingInput; try { // Wiz's default report export is gzip, sometimes with a .json filename. // Inspect the bytes rather than requiring users to rename their download. @@ -198,13 +408,13 @@ export async function readVendorFindings( ); } const result: VendorFindings = { - read: input.length, + read: input.records.length, findings: [], excluded: [], }; const identities = new Set(); - for (const [index, item] of input.entries()) { - const record = object(item); + for (const [index, item] of input.records.entries()) { + const record = object(item.value); try { if (!record) throw new Error("Finding must be an object."); let finding: VendorFinding; @@ -222,7 +432,22 @@ export async function readVendorFindings( source_finding_id: sourceFindingId(record["source_finding_id"]), evidence: validateExternalEvidence(record["evidence"]), }; + } else if (item.kind === "cloud_configuration") { + throw new Error( + "Cloud configuration findings require a resource-scoped import and cannot be published to a repository.", + ); + } else if (item.kind && item.kind !== "sca") { + finding = repositoryFinding(item.kind, record, input.repositories); } else { + if ( + !item.kind && + (object(record["repository"]) || + object(record["repositoryBranch"]) || + object(record["resource"])) + ) + throw new Error( + "Raw SAST, secret, and IaC records require their named sastFindings, secretInstances, or iacFindings collection. Bare arrays do not identify the finding family; use normalized JSONL for an explicit mapping.", + ); finding = wizFinding(record); } if (identities.has(finding.source_finding_id)) { diff --git a/sdk/typescript/tests-ts/wiz-findings.test.ts b/sdk/typescript/tests-ts/wiz-findings.test.ts new file mode 100644 index 0000000000..6b6542dad7 --- /dev/null +++ b/sdk/typescript/tests-ts/wiz-findings.test.ts @@ -0,0 +1,650 @@ +import { hash } from "node:crypto"; +import { mkdtemp, mkdir, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { gzipSync } from "node:zlib"; +import { afterEach, expect, test } from "bun:test"; +import { main } from "../src/cli.js"; +import { prepareExternalPublication } from "../src/external-findings-publish.js"; +import { validateExternalEvidence } from "../src/external-import-contract.js"; +import type { + FindingImportRequest, + SourceReport, +} from "../src/external-import-models.js"; +import { readVendorFindings } from "../src/wiz-findings.js"; +import { dependencies } from "./cli-fixtures.js"; +import { createCliTest } from "./support/cli-run.js"; + +const directories: string[] = []; +afterEach(async () => { + await Promise.all( + directories + .splice(0) + .map((path) => rm(path, { recursive: true, force: true })), + ); +}); +const repository = { id: "wiz-repository", name: "example/project" }; +const repositoryUrl = "https://github.com/example/project"; +const inventory = { + nodes: [ + { + id: "wiz-branch-inventory", + platform: "GITHUB", + providerID: "github.com##example/project##feature/parser", + type: "REPOSITORY_BRANCH", + repository: { ...repository, url: repositoryUrl }, + }, + ], + pageInfo: { hasNextPage: false }, +}; +const sast = { + id: "occurrence-1", + name: "Untrusted query construction", + description: "Synthetic source finding", + severity: "HIGH", + status: "OPEN", + createdAt: "2026-10-01T10:00:00.123Z", + filePath: "src/query.ts", + startLine: 10, + endLine: 12, + snippet: "query(input)", + repository, + repositoryBranch: { + id: "wiz-branch", + name: "example/project/feature/parser", + }, + weaknesses: [{ id: "CWE-89", name: "SQL injection" }], + origin: "THIRD_PARTY_SCANNER", + remediationInstructions: "Use a parameterized query.", + aiAnalysis: { verdict: "FALSE_POSITIVE" }, +}; +const secret = { + id: "occurrence-1", + name: "Example credential metadata", + severity: "MEDIUM", + type: "API_KEY", + confidence: "HIGH", + validationStatus: "NOT_VALIDATED", + isEncrypted: false, + isManaged: true, + status: "RESOLVED", + lastUpdatedAt: "2026-10-01T10:00:00.987Z", + resolvedAt: "2026-10-01T10:00:00.987Z", + rule: { id: "example-secret-rule", name: "Example API key", type: "API_KEY" }, + resource: { + id: "wiz-branch", + type: "REPOSITORY_BRANCH", + name: "example/project/feature/parser", + typedProperties: { repository }, + }, + secretDataEntities: [ + { id: "metadata-only", name: "Example detector metadata", type: "API_KEY" }, + ], +}; +const iac = { + id: "occurrence-1", + name: "Example infrastructure setting", + severity: "LOW", + status: "OPEN", + expectedContent: "", + foundContent: " ", + matchContent: "enabled = true", + filePath: "infra/main.tf", + startLine: 20, + endLine: 22, + repository, + branch: { id: "wiz-branch", name: "feature/parser" }, + platform: "TERRAFORM", + cloudPlatform: "AWS", + fileURL: "https://github.com/example/project/blob/main/infra/main.tf", + wizUrl: "https://vendor.example.test/findings/occurrence-1", + firstSeenAt: "2026-10-01T10:00:00Z", + lastSeenAt: "2026-10-02T10:00:00Z", + rule: { id: "example-rule", shortId: "EXAMPLE-1", name: "Example setting" }, + resourceGraphEntity: { + id: "logical-resource", + type: "BUCKET", + properties: { deployed: false }, + }, + fileRemediation: { filePath: "infra/main.tf" }, +}; + +function envelope(root: string, nodes: unknown[], withInventory = true) { + return { + data: { + [root]: { nodes, pageInfo: { hasNextPage: false } }, + ...(withInventory ? { versionControlResources: inventory } : {}), + }, + }; +} + +async function fixture(payload: unknown, compressed = false) { + const root = await mkdtemp(join(tmpdir(), "wiz-repository-findings-")); + directories.push(root); + const file = join(root, "selected.json"); + const contents = JSON.stringify(payload); + await writeFile(file, compressed ? gzipSync(contents) : contents); + return { root, file }; +} + +async function parse(payload: unknown) { + const { file } = await fixture(payload); + return readVendorFindings(file); +} + +test("SAST maps source range, CWE, scanner verdict, and repository inventory without changing raw evidence", async () => { + const { file } = await fixture(envelope("sastFindings", [sast]), true); + const parsed = await readVendorFindings(file); + expect(parsed.excluded).toEqual([]); + expect(parsed.findings[0]).toMatchObject({ + source_finding_id: "sast:occurrence-1", + evidence: { + severity: "high", + locations: [{ path: "src/query.ts", line: 10 }], + branch: "feature/parser", + code_revision: null, + source_scan_id: null, + source_updated_at: null, + advisory_ids: [], + packages: [], + source_data: sast, + details: { + kind: "sast", + repository: { ...repository, url: repositoryUrl }, + source_status: "OPEN", + scanner_origin: "THIRD_PARTY_SCANNER", + scanner_verdict: "FALSE_POSITIVE", + weakness_ids: ["CWE-89"], + remediation_instructions: sast.remediationInstructions, + code: { end_line: 12, snippet: "query(input)" }, + }, + }, + }); + expect(parsed.findings[0]!.evidence).not.toHaveProperty("assessment"); +}); + +test("repository secrets preserve detector and vendor validation metadata without inventing a source location or scanned revision", async () => { + const detailed = { + ...secret, + id: "detailed", + path: "config/example.env", + lineNumber: 3, + vcsDetails: { initialCommitHash: "a".repeat(40) }, + }; + const parsed = await parse(envelope("secretInstances", [secret, detailed])); + expect(parsed.excluded).toEqual([]); + expect(parsed.findings[0]!.evidence).toMatchObject({ + severity: "medium", + locations: [], + source_updated_at: 1790848800, + code_revision: null, + source_data: secret, + details: { + kind: "secret", + source_status: "RESOLVED", + rule: { id: "example-secret-rule", name: "Example API key" }, + secret: { + type: "API_KEY", + confidence: "HIGH", + validation_status: "NOT_VALIDATED", + is_encrypted: false, + is_managed: true, + introduced_commit: null, + }, + }, + }); + expect(parsed.findings[1]!.evidence).toMatchObject({ + locations: [{ path: detailed.path, line: 3 }], + code_revision: null, + details: { secret: { introduced_commit: "a".repeat(40) } }, + source_data: detailed, + }); +}); + +test("IaC keeps exact empty comparison text, the rule, file link, and logical resource separately", async () => { + const parsed = await parse(envelope("iacFindings", [iac])); + expect(parsed.excluded).toEqual([]); + expect(parsed.findings[0]!.evidence).toMatchObject({ + url: iac.wizUrl, + source_updated_at: null, + code_revision: null, + source_data: iac, + details: { + kind: "iac", + rule: { + id: "example-rule", + short_id: "EXAMPLE-1", + name: "Example setting", + }, + code: { end_line: 22, snippet: "enabled = true" }, + configuration: { + platform: "TERRAFORM", + cloud_platform: "AWS", + expected: "", + actual: " ", + }, + file_url: iac.fileURL, + }, + }); +}); + +test("explicit finding families namespace reused vendor occurrence IDs", async () => { + const parsed = await parse({ + data: { + versionControlResources: inventory, + sastFindings: { nodes: [sast] }, + secretInstances: { nodes: [secret] }, + iacFindings: { nodes: [iac] }, + }, + }); + expect(parsed.excluded).toEqual([]); + expect(parsed.findings.map((finding) => finding.source_finding_id)).toEqual([ + "sast:occurrence-1", + "secret:occurrence-1", + "iac:occurrence-1", + ]); +}); + +test("bare arrays cannot guess a new finding family even when package fields also exist", async () => { + const ambiguous = { + ...sast, + detailedName: "example-package", + vulnerableAsset: { id: "asset" }, + }; + const parsed = await parse([sast, secret, iac, ambiguous]); + expect(parsed.findings).toEqual([]); + expect(parsed.excluded).toHaveLength(4); + expect(parsed.excluded.every((item) => item.reason.includes("named"))).toBe( + true, + ); + expect( + (await parse(envelope("sastFindings", [ambiguous]))).findings[0]!.evidence + .source_data, + ).toEqual(ambiguous); +}); + +test("repository mapping joins only repository.id, accepts supplied URLs, and rejects conflicts", async () => { + const wrongJoin = { + ...inventory, + nodes: [ + { + ...inventory.nodes[0], + id: repository.id, + repository: { + id: "different-repository", + name: repository.name, + url: repositoryUrl, + }, + }, + ], + }; + const missing = await parse({ + data: { + sastFindings: { nodes: [sast] }, + versionControlResources: wrongJoin, + }, + }); + expect(missing.findings).toEqual([]); + expect(missing.excluded[0]!.reason).toContain( + "verified source repository URL", + ); + const enriched = { + ...sast, + repository: { ...repository, url: repositoryUrl }, + }; + expect( + (await parse(envelope("sastFindings", [enriched], false))).excluded, + ).toEqual([]); + const conflict = { + ...sast, + repository: { ...repository, url: "https://github.com/example/other" }, + }; + expect( + (await parse(envelope("sastFindings", [conflict]))).excluded[0]!.reason, + ).toContain("conflicting"); +}); + +test("workload secrets and cloud configuration remain visible exclusions in a mixed explicit export", async () => { + const workload = { + ...secret, + id: "workload", + resource: { ...secret.resource, type: "VIRTUAL_MACHINE" }, + }; + const parsed = await parse({ + data: { + versionControlResources: inventory, + secretInstances: { nodes: [secret, workload] }, + configurationFindings: { + nodes: [{ id: "configuration", name: "Example check" }], + }, + }, + }); + expect(parsed.findings).toHaveLength(1); + expect(parsed.excluded).toEqual([ + expect.objectContaining({ + source_finding_id: "workload", + reason: expect.stringContaining("Workload secrets"), + }), + expect.objectContaining({ + source_finding_id: "configuration", + reason: expect.stringContaining("Cloud configuration"), + }), + ]); +}); + +test("GraphQL errors, unfinished finding pages, and unfinished inventory stop parsing", async () => { + await expect( + parse({ + ...envelope("sastFindings", [sast]), + errors: [{ message: "partial" }], + }), + ).rejects.toThrow("GraphQL errors"); + await expect( + parse({ + data: { + sastFindings: { nodes: [sast], pageInfo: { hasNextPage: true } }, + }, + }), + ).rejects.toThrow("another page"); + await expect( + parse({ + data: { + sastFindings: { nodes: [sast] }, + versionControlResources: { + ...inventory, + pageInfo: { hasNextPage: true }, + }, + }, + }), + ).rejects.toThrow("inventory has another page"); +}); + +test("new adapters do not fabricate missing occurrence IDs, severity, or line ranges", async () => { + const invalid = [ + { ...sast, id: null }, + { ...sast, severity: null }, + { ...sast, startLine: null }, + { ...sast, endLine: 9 }, + { ...sast, filePath: "../outside.ts" }, + ]; + const parsed = await parse(envelope("sastFindings", invalid)); + expect(parsed.findings).toEqual([]); + expect(parsed.excluded).toHaveLength(invalid.length); +}); + +test("legacy evidence defaults retain their saved-submission bytes, including explicit null details", () => { + const input = { + title: "Legacy evidence", + severity: "high", + locations: [{ path: "package.json" }], + packages: [{ name: "example-package" }], + }; + const legacy = validateExternalEvidence(structuredClone(input)); + expect(hash("sha256", JSON.stringify(legacy))).toBe( + "88f2d96656a5ad322d5f16fce7704366e359a8101d035b311a1917d4f6e31caf", + ); + const withNull = validateExternalEvidence({ + ...structuredClone(input), + details: null, + }); + expect(withNull).toEqual(legacy); + expect(withNull).not.toHaveProperty("details"); +}); + +test("normalized JSONL retains explicit IDs, original evidence, and empty source text", async () => { + const raw = { details: null, example: "raw evidence stays exact" }; + const payload = { + source_finding_id: "explicit-identity", + evidence: { + title: "Explicit finding", + severity: "high", + source_data: raw, + details: { + kind: "sast", + repository: { url: repositoryUrl }, + code: { snippet: " " }, + }, + }, + }; + const parsed = await parse([payload]); + expect(parsed.excluded).toEqual([]); + expect(parsed.findings[0]!.source_finding_id).toBe("explicit-identity"); + expect(parsed.findings[0]!.evidence.source_data).toEqual(raw); + expect(parsed.findings[0]!.evidence.details?.code?.snippet).toBe(" "); +}); + +test("new detail links and long source text follow the same Cloud constraints", async () => { + const good = { ...sast, snippet: "é".repeat(32768) }; + const tooLong = { ...sast, id: "long", snippet: "é".repeat(32769) }; + const unsafeUrl = { + ...sast, + id: "unsafe", + repository: { ...repository, url: "https://user@example.test/project" }, + }; + const parsed = await parse(envelope("sastFindings", [good, tooLong])); + expect(parsed.findings).toHaveLength(1); + expect(parsed.excluded[0]!.reason).toContain("65536 UTF-8 bytes"); + expect( + (await parse(envelope("sastFindings", [unsafeUrl], false))).excluded[0]! + .reason, + ).toContain("without credentials"); +}); + +async function cloudFixture(payload: unknown) { + const f = await fixture(payload); + const environment = { + ...process.env, + CODEX_HOME: join(f.root, "login"), + CODEX_SECURITY_STATE_DIR: join(f.root, "state"), + }; + await mkdir(environment.CODEX_HOME); + await writeFile( + join(environment.CODEX_HOME, "config.toml"), + 'cli_auth_credentials_store = "file"\n', + ); + await writeFile( + join(environment.CODEX_HOME, "auth.json"), + JSON.stringify({ + tokens: { + access_token: "synthetic-token", + account_id: "synthetic-account", + }, + }), + ); + const destination = { + id: "cloud-repository", + object: "security.repository", + repo_connector_id: "github", + url: repositoryUrl, + default_branch: "main", + reset_marker: "generation-1", + import_environment_id: "environment-1", + }; + const posts: FindingImportRequest[] = []; + const calls: string[] = []; + const reports = new Map(); + const transport = async ( + url: string, + init: RequestInit, + ): Promise => { + calls.push(`${init.method} ${url}`); + const target = new URL(url); + const json = (body: unknown) => + new Response(JSON.stringify(body), { + headers: { "Content-Type": "application/json" }, + }); + if (target.pathname.endsWith("/repositories")) + return json({ data: [destination], has_more: false, next: null }); + if (target.pathname.endsWith("/source_reports")) { + const report = reports.get(target.searchParams.get("source_finding_id")!); + const summaries = report + ? [ + { + ...report, + object: "security.source_report_summary", + evidence: undefined, + last_import: undefined, + title: report.evidence.title, + severity: report.evidence.severity, + source_updated_at: report.evidence.source_updated_at, + }, + ] + : []; + return json({ data: summaries, has_more: false, next: null }); + } + if (target.pathname.includes("/source_reports/")) + return json( + [...reports.values()].find( + (report) => report.id === target.pathname.split("/").at(-1), + ), + ); + if ( + target.pathname.endsWith("/finding_imports") && + init.method === "POST" + ) { + const request = JSON.parse(String(init.body)) as FindingImportRequest; + posts.push(request); + const counts = { created: 0, updated: 0, unchanged: 0, error: 0 }; + const results = request.items.map((item) => { + const previous = reports.get(item.source_finding_id); + const outcome = previous + ? JSON.stringify(previous.evidence) === JSON.stringify(item.evidence) + ? "unchanged" + : "updated" + : "created"; + counts[outcome]++; + const version = + (previous?.version ?? 0) + (outcome === "unchanged" ? 0 : 1); + const identity = hash("sha256", item.source_finding_id); + const report: SourceReport = { + id: `aif_${identity}`, + repo_id: destination.id, + repo_connector_id: destination.repo_connector_id, + environment_id: "environment-1", + canonical_finding_id: + previous?.canonical_finding_id ?? `acf_${identity.slice(0, 32)}`, + source: request.source, + source_finding_id: item.source_finding_id, + observation_id: `aio_${hash("sha256", `${identity}:${version}`)}`, + version, + evidence: item.evidence, + assessment: { state: "not_assessed" }, + created_at: 1, + updated_at: 1, + }; + if (previous) report.id = previous.id; + reports.set(item.source_finding_id, report); + return { + client_id: item.client_id, + outcome, + source_report_id: report.id, + observation_id: report.observation_id, + canonical_finding_id: report.canonical_finding_id, + version, + }; + }); + return json({ + id: request.request_id, + repository: request.repository, + source: request.source, + actor: "synthetic-user", + created_at: 1, + item_count: request.items.length, + counts, + results, + }); + } + throw new Error(`Unexpected injected Cloud request ${init.method} ${url}`); + }; + const options = { + provider: "wiz" as const, + sourceKey: "synthetic-tenant/wiz", + repository: repositoryUrl, + }; + const command = [ + "publish", + "findings", + f.file, + "--to", + "cloud", + "--provider", + "wiz", + "--repository", + repositoryUrl, + "--source-key", + options.sourceKey, + "--format", + "json", + ]; + return { + ...f, + posts, + calls, + reports, + options, + command, + environment, + deps: { environment, fetch: transport }, + cliDeps: { ...dependencies(), environment, cloudFetch: transport }, + }; +} + +for (const [root, record, kind] of [ + ["sastFindings", sast, "sast"], + ["secretInstances", secret, "secret"], + ["iacFindings", iac, "iac"], +] as const) { + test(`CLI previews, publishes, and reimports ${kind} through injected transport without changing Codex assessment`, async () => { + const f = await cloudFixture(envelope(root, [record])); + const cli = createCliTest(main); + expect(await cli.runCli([...f.command, "--dry-run"], f.cliDeps)).toBe(0); + const preview = JSON.parse(cli.stdout.text()); + expect(preview.findings[0].evidence.details.kind).toBe(kind); + expect(f.posts).toHaveLength(0); + const upload = createCliTest(main); + const exitCode = await upload.runCli([...f.command, "--yes"], f.cliDeps); + expect(exitCode, upload.stdout.text() + upload.stderr.text()).toBe(0); + expect(f.posts).toHaveLength(1); + const first = f.reports.get(`${kind}:occurrence-1`)!; + expect(first.assessment).toEqual({ state: "not_assessed" }); + expect(first.evidence.source_data).toEqual(record); + const replay = await prepareExternalPublication(f.file, f.options, f.deps); + expect((await replay.publish()).counts.unchanged).toBe(1); + expect(f.reports.get(`${kind}:occurrence-1`)!.canonical_finding_id).toBe( + first.canonical_finding_id, + ); + const changed = { ...record, status: "RESOLVED" }; + await writeFile(f.file, JSON.stringify(envelope(root, [changed]))); + await ( + await prepareExternalPublication(f.file, f.options, f.deps) + ).publish(); + expect(f.reports.get(`${kind}:occurrence-1`)!.assessment).toEqual({ + state: "not_assessed", + }); + expect(f.reports.get(`${kind}:occurrence-1`)!.evidence.source_data).toEqual( + changed, + ); + }); +} + +test("a wrong source repository stops raw and normalized imports before source lookup or upload", async () => { + const other = { + ...sast, + repository: { ...repository, url: "https://github.com/example/other" }, + }; + const f = await cloudFixture(envelope("sastFindings", [other], false)); + await expect( + prepareExternalPublication(f.file, f.options, f.deps), + ).rejects.toThrow("does not match the selected Cloud repository"); + expect(f.calls).toHaveLength(1); + expect(f.posts).toEqual([]); + const normalized = (await readVendorFindings(f.file)).findings; + await writeFile(f.file, JSON.stringify(normalized)); + await expect( + prepareExternalPublication(f.file, f.options, f.deps), + ).rejects.toThrow("does not match"); + expect(f.posts).toEqual([]); + expect( + await readdir(join(f.root, "state")).catch(() => undefined), + ).toBeUndefined(); +}); From e593fcfa918753970138fb13012880d967c2bf8a Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 10 Oct 2026 02:40:51 +0000 Subject: [PATCH 23/43] fix(import): preserve normalized evidence and saved retries --- .../schemas/external-findings.schema.json | 2 +- .../src/external-findings-publish.ts | 20 +-- .../src/external-import-contract.ts | 67 +++++-- sdk/typescript/src/external-import-models.ts | 4 +- .../fixtures/wiz-details-roundtrip.json | 167 ++++++++++++++++++ sdk/typescript/tests-ts/wiz-findings.test.ts | 80 ++++++++- 6 files changed, 312 insertions(+), 28 deletions(-) create mode 100644 sdk/typescript/tests-ts/fixtures/wiz-details-roundtrip.json diff --git a/sdk/typescript/schemas/external-findings.schema.json b/sdk/typescript/schemas/external-findings.schema.json index fc796cb338..d2cfb967a7 100644 --- a/sdk/typescript/schemas/external-findings.schema.json +++ b/sdk/typescript/schemas/external-findings.schema.json @@ -1011,7 +1011,7 @@ } ], "default": null, - "title": "Confidence" + "title": "Secret scanner confidence" }, "validation_status": { "anyOf": [ diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 2eedd6eade..7e4ba65206 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -329,16 +329,6 @@ export async function prepareExternalPublication( : "The destination repository is not available to this Cloud account. Copy its repository URL or ID from Codex Security Cloud and check that you are signed into the intended account.", ); const destination = destinations[0]!; - for (const finding of parsed.findings) { - const sourceRepository = finding.evidence.details?.repository; - if ( - sourceRepository && - repositoryUrl(sourceRepository.url) !== repositoryUrl(destination.url) - ) - throw new CodexSecurityError( - `Finding ${JSON.stringify(finding.source_finding_id)} belongs to ${JSON.stringify(sourceRepository.url)}, which does not match the selected Cloud repository ${JSON.stringify(destination.url)}. Verify the source repository mapping before publishing.`, - ); - } const source = { provider: options.provider, source_key: options.sourceKey }; const state = join( codexSecurityStateDirectory(environment), @@ -419,6 +409,16 @@ export async function prepareExternalPublication( } let requests = saved?.requests; if (!requests) { + for (const finding of parsed.findings) { + const sourceRepository = finding.evidence.details?.repository; + if ( + sourceRepository && + repositoryUrl(sourceRepository.url) !== repositoryUrl(destination.url) + ) + throw new CodexSecurityError( + `Finding ${JSON.stringify(finding.source_finding_id)} belongs to ${JSON.stringify(sourceRepository.url)}, which does not match the selected Cloud repository ${JSON.stringify(destination.url)}. Verify the source repository mapping before publishing.`, + ); + } let preparedCount = 0; progress({ phase: "preparing", diff --git a/sdk/typescript/src/external-import-contract.ts b/sdk/typescript/src/external-import-contract.ts index d49e3136e7..bab0d38388 100644 --- a/sdk/typescript/src/external-import-contract.ts +++ b/sdk/typescript/src/external-import-contract.ts @@ -63,30 +63,75 @@ function validateModelStrings(input: unknown): void { } } +type ModelSchema = { + $ref?: string; + anyOf?: ModelSchema[]; + properties?: Record; + [key: string]: unknown; +}; +const modelSchemas: Record = schema.$defs; +const detailValidators = new Map>(); + +/** AJV does not apply defaults through nullable anyOf references. Validate the + * present detail objects directly against their generated model definitions so + * sparse normalized inputs match the backend's serialized evidence. */ +function materializeDetailDefaults( + value: unknown, + fieldSchema: ModelSchema, +): void { + if (value === null || typeof value !== "object" || Array.isArray(value)) + return; + const reference = + fieldSchema.$ref ?? fieldSchema.anyOf?.find((branch) => branch.$ref)?.$ref; + if (!reference) return; + const name = reference.split("/").at(-1)!; + const definition = modelSchemas[name]!; + let check = detailValidators.get(reference); + if (!check) { + check = ajv.compile({ $defs: schema.$defs, $ref: reference }); + detailValidators.set(reference, check); + } + if (!check(value)) + throw new Error(`${name}: ${ajv.errorsText(check.errors)}`); + for (const [field, childSchema] of Object.entries( + definition.properties ?? {}, + )) + materializeDetailDefaults( + (value as Record)[field], + childSchema, + ); +} + +function normalizeEvidenceDetails(value: unknown): void { + if (value !== null && typeof value === "object") { + const fields = value as Record; + // Only this new wrapper is omitted for null; retain every legacy default. + if (fields["details"] === null) delete fields["details"]; + else + materializeDetailDefaults( + fields["details"], + modelSchemas["ImportedFindingEvidence"]!.properties!["details"]!, + ); + } +} + function validator(name: keyof typeof schema.$defs) { const check = ajv.compile({ $defs: schema.$defs, $ref: `#/$defs/${name}`, }); return (input: unknown): T => { - // The backend omits a null details wrapper while retaining all legacy - // defaults. Match that wire representation for saved requests and readback. - const omitNullDetails = (value: unknown): void => { - if (value !== null && typeof value === "object") { - const fields = value as Record; - if (fields["details"] === null) delete fields["details"]; - } - }; - if (name === "ImportedFindingEvidence") omitNullDetails(input); + if (name === "ImportedFindingEvidence") normalizeEvidenceDetails(input); else if (input && typeof input === "object") { if (name === "SourceReport" && "evidence" in input) - omitNullDetails(input.evidence); + normalizeEvidenceDetails(input.evidence); if ( name === "FindingImportRequest" && "items" in input && Array.isArray(input.items) ) - for (const item of input.items) omitNullDetails(item?.evidence); + for (const item of input.items) + normalizeEvidenceDetails(item?.evidence); } if (!check(input)) throw new Error(`${name}: ${ajv.errorsText(check.errors)}`); diff --git a/sdk/typescript/src/external-import-models.ts b/sdk/typescript/src/external-import-models.ts index a5c4a67d1c..18ac5a9e87 100644 --- a/sdk/typescript/src/external-import-models.ts +++ b/sdk/typescript/src/external-import-models.ts @@ -62,7 +62,7 @@ export type RemediationInstructions = string | null; export type EndLine = number | null; export type Snippet = string | null; export type Type = string | null; -export type Confidence = string | null; +export type SecretScannerConfidence = string | null; export type ValidationStatus = string | null; export type IsEncrypted = boolean | null; export type IsManaged = boolean | null; @@ -245,7 +245,7 @@ export interface ImportedCodeContext { } export interface ImportedSecretDetails { type?: Type; - confidence?: Confidence; + confidence?: SecretScannerConfidence; validation_status?: ValidationStatus; is_encrypted?: IsEncrypted; is_managed?: IsManaged; diff --git a/sdk/typescript/tests-ts/fixtures/wiz-details-roundtrip.json b/sdk/typescript/tests-ts/fixtures/wiz-details-roundtrip.json new file mode 100644 index 0000000000..fd8814a9da --- /dev/null +++ b/sdk/typescript/tests-ts/fixtures/wiz-details-roundtrip.json @@ -0,0 +1,167 @@ +[ + { + "name": "sast-sparse-code-rule", + "input": { + "kind": "sast", + "repository": { + "url": "https://github.com/example/project" + }, + "code": { + "snippet": " " + }, + "rule": { + "id": "example-rule" + } + }, + "expected": { + "kind": "sast", + "repository": { + "url": "https://github.com/example/project", + "id": null, + "name": null + }, + "source_status": null, + "scanner_origin": null, + "scanner_verdict": null, + "rule": { + "id": "example-rule", + "short_id": null, + "name": null + }, + "weakness_ids": [], + "remediation_instructions": null, + "code": { + "end_line": null, + "snippet": " " + }, + "secret": null, + "configuration": null, + "file_url": null + } + }, + { + "name": "secret-empty-models", + "input": { + "kind": "secret", + "repository": { + "url": "https://github.com/example/project" + }, + "secret": {}, + "rule": {} + }, + "expected": { + "kind": "secret", + "repository": { + "url": "https://github.com/example/project", + "id": null, + "name": null + }, + "source_status": null, + "scanner_origin": null, + "scanner_verdict": null, + "rule": { + "id": null, + "short_id": null, + "name": null + }, + "weakness_ids": [], + "remediation_instructions": null, + "code": null, + "secret": { + "type": null, + "confidence": null, + "validation_status": null, + "is_encrypted": null, + "is_managed": null, + "introduced_commit": null + }, + "configuration": null, + "file_url": null + } + }, + { + "name": "iac-empty-comparison", + "input": { + "kind": "iac", + "repository": { + "url": "https://github.com/example/project" + }, + "configuration": { + "expected": "", + "actual": " " + }, + "rule": { + "short_id": "EXAMPLE-1" + } + }, + "expected": { + "kind": "iac", + "repository": { + "url": "https://github.com/example/project", + "id": null, + "name": null + }, + "source_status": null, + "scanner_origin": null, + "scanner_verdict": null, + "rule": { + "id": null, + "short_id": "EXAMPLE-1", + "name": null + }, + "weakness_ids": [], + "remediation_instructions": null, + "code": null, + "secret": null, + "configuration": { + "platform": null, + "cloud_platform": null, + "expected": "", + "actual": " " + }, + "file_url": null + } + }, + { + "name": "iac-empty-models", + "input": { + "kind": "iac", + "repository": { + "url": "https://github.com/example/project" + }, + "configuration": {}, + "rule": {}, + "code": {} + }, + "expected": { + "kind": "iac", + "repository": { + "url": "https://github.com/example/project", + "id": null, + "name": null + }, + "source_status": null, + "scanner_origin": null, + "scanner_verdict": null, + "rule": { + "id": null, + "short_id": null, + "name": null + }, + "weakness_ids": [], + "remediation_instructions": null, + "code": { + "end_line": null, + "snippet": null + }, + "secret": null, + "configuration": { + "platform": null, + "cloud_platform": null, + "expected": null, + "actual": null + }, + "file_url": null + } + } +] diff --git a/sdk/typescript/tests-ts/wiz-findings.test.ts b/sdk/typescript/tests-ts/wiz-findings.test.ts index 6b6542dad7..f7e955a991 100644 --- a/sdk/typescript/tests-ts/wiz-findings.test.ts +++ b/sdk/typescript/tests-ts/wiz-findings.test.ts @@ -8,12 +8,14 @@ import { main } from "../src/cli.js"; import { prepareExternalPublication } from "../src/external-findings-publish.js"; import { validateExternalEvidence } from "../src/external-import-contract.js"; import type { + ExternalFindingEvidence, FindingImportRequest, SourceReport, } from "../src/external-import-models.js"; import { readVendorFindings } from "../src/wiz-findings.js"; import { dependencies } from "./cli-fixtures.js"; import { createCliTest } from "./support/cli-run.js"; +import roundtripFixtures from "./fixtures/wiz-details-roundtrip.json" with { type: "json" }; const directories: string[] = []; afterEach(async () => { @@ -463,6 +465,10 @@ async function cloudFixture(payload: unknown) { const posts: FindingImportRequest[] = []; const calls: string[] = []; const reports = new Map(); + const state: { + failReadback: boolean; + readbackDetails?: ExternalFindingEvidence["details"]; + } = { failReadback: false }; const transport = async ( url: string, init: RequestInit, @@ -492,12 +498,24 @@ async function cloudFixture(payload: unknown) { : []; return json({ data: summaries, has_more: false, next: null }); } - if (target.pathname.includes("/source_reports/")) + if (target.pathname.includes("/source_reports/")) { + if (state.failReadback) + return new Response( + JSON.stringify({ error: { message: "Readback unavailable" } }), + { status: 503 }, + ); + const report = [...reports.values()].find( + (item) => item.id === target.pathname.split("/").at(-1), + ); return json( - [...reports.values()].find( - (report) => report.id === target.pathname.split("/").at(-1), - ), + report && state.readbackDetails + ? { + ...report, + evidence: { ...report.evidence, details: state.readbackDetails }, + } + : report, ); + } if ( target.pathname.endsWith("/finding_imports") && init.method === "POST" @@ -581,6 +599,8 @@ async function cloudFixture(payload: unknown) { posts, calls, reports, + state, + destination, options, command, environment, @@ -648,3 +668,55 @@ test("a wrong source repository stops raw and normalized imports before source l await readdir(join(f.root, "state")).catch(() => undefined), ).toBeUndefined(); }); + +test("saved typed requests resume by repository ID after a repository rename", async () => { + const f = await cloudFixture(envelope("sastFindings", [sast])); + const options = { ...f.options, repository: f.destination.id }; + f.state.failReadback = true; + await expect( + (await prepareExternalPublication(f.file, options, f.deps)).publish(), + ).rejects.toThrow("Readback unavailable"); + expect(f.posts).toHaveLength(1); + f.destination.url = "https://github.com/example/renamed"; + f.state.failReadback = false; + const resumed = await prepareExternalPublication(f.file, options, f.deps); + expect(resumed.preview.resumed).toBe(true); + expect(resumed.preview.destination.url).toBe(f.destination.url); + expect((await resumed.publish()).verified).toBe(1); + expect(f.posts).toHaveLength(1); + expect( + f.reports.get("sast:occurrence-1")!.evidence.details!.repository.url, + ).toBe(repositoryUrl); +}); + +// Expected details were serialized by the canonical Cloud validation model. +// Keep that readback independent of the CLI validator so nullable defaults cannot +// be hidden by a transport that merely echoes the submitted evidence. +for (const golden of roundtripFixtures) { + test(`sparse ${golden.name} evidence matches canonical Cloud readback`, async () => { + const evidence = { + title: "Synthetic normalized finding", + severity: "high", + details: golden.input, + source_data: { id: golden.name, details: null }, + }; + const f = await cloudFixture([ + { source_finding_id: golden.name, evidence }, + ]); + f.state.readbackDetails = structuredClone(golden.expected) as NonNullable< + ExternalFindingEvidence["details"] + >; + const prepared = await prepareExternalPublication( + f.file, + f.options, + f.deps, + ); + expect(prepared.preview.findings[0]!.evidence.details).toEqual( + f.state.readbackDetails, + ); + expect((await prepared.publish()).verified).toBe(1); + expect(f.posts[0]!.items[0]!.evidence.source_data).toEqual( + evidence.source_data, + ); + }); +} From bf38913b48346db73b9ef7e9a58308b32ebda38d Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 10 Oct 2026 02:50:41 +0000 Subject: [PATCH 24/43] fix(import): keep secret confidence schema distinct --- sdk/typescript/schemas/external-findings.schema.json | 12 +++++++----- sdk/typescript/src/external-import-models.ts | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/sdk/typescript/schemas/external-findings.schema.json b/sdk/typescript/schemas/external-findings.schema.json index d2cfb967a7..bd0f50e102 100644 --- a/sdk/typescript/schemas/external-findings.schema.json +++ b/sdk/typescript/schemas/external-findings.schema.json @@ -982,6 +982,11 @@ "title": "ImportedRepositoryReference", "type": "object" }, + "ImportedSecretConfidence": { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, "ImportedSecretDetails": { "additionalProperties": false, "properties": { @@ -1002,16 +1007,13 @@ "confidence": { "anyOf": [ { - "maxLength": 512, - "minLength": 1, - "type": "string" + "$ref": "#/$defs/ImportedSecretConfidence" }, { "type": "null" } ], - "default": null, - "title": "Secret scanner confidence" + "default": null }, "validation_status": { "anyOf": [ diff --git a/sdk/typescript/src/external-import-models.ts b/sdk/typescript/src/external-import-models.ts index 18ac5a9e87..efc87b42f7 100644 --- a/sdk/typescript/src/external-import-models.ts +++ b/sdk/typescript/src/external-import-models.ts @@ -62,7 +62,7 @@ export type RemediationInstructions = string | null; export type EndLine = number | null; export type Snippet = string | null; export type Type = string | null; -export type SecretScannerConfidence = string | null; +export type ImportedSecretConfidence = string; export type ValidationStatus = string | null; export type IsEncrypted = boolean | null; export type IsManaged = boolean | null; @@ -245,7 +245,7 @@ export interface ImportedCodeContext { } export interface ImportedSecretDetails { type?: Type; - confidence?: SecretScannerConfidence; + confidence?: ImportedSecretConfidence | null; validation_status?: ValidationStatus; is_encrypted?: IsEncrypted; is_managed?: IsManaged; From b749aa999c98e106f47119caa324670ec6be2dc1 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 10 Oct 2026 03:00:46 +0000 Subject: [PATCH 25/43] fix(import): match GitHub repository URL casing --- .../src/external-findings-publish.ts | 10 +- .../src/external-import-contract.ts | 16 +++ sdk/typescript/src/wiz-findings.ts | 13 +- sdk/typescript/tests-ts/wiz-findings.test.ts | 113 ++++++++++++++++++ 4 files changed, 139 insertions(+), 13 deletions(-) diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 7e4ba65206..d95ddb8e29 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -17,6 +17,7 @@ import { } from "./runtime.js"; import { readVendorFindings, type VendorFindings } from "./wiz-findings.js"; import { + repositoryUrlKey, validateImportRequest, validateImportReceipt, validateRepositories, @@ -128,10 +129,6 @@ async function readInBatches( return output; } -function repositoryUrl(value: string): string { - return value.replace(/\/$/u, "").replace(/\.git$/u, ""); -} - function canonicalJson(value: unknown): string { return JSON.stringify(value, (_key, child: unknown) => { if (child !== null && typeof child === "object" && !Array.isArray(child)) { @@ -309,7 +306,7 @@ export async function prepareExternalPublication( ...result.data.filter( (item) => item.id === options.repository || - repositoryUrl(item.url) === repositoryUrl(options.repository), + repositoryUrlKey(item.url) === repositoryUrlKey(options.repository), ), ); page = result.next; @@ -413,7 +410,8 @@ export async function prepareExternalPublication( const sourceRepository = finding.evidence.details?.repository; if ( sourceRepository && - repositoryUrl(sourceRepository.url) !== repositoryUrl(destination.url) + repositoryUrlKey(sourceRepository.url) !== + repositoryUrlKey(destination.url) ) throw new CodexSecurityError( `Finding ${JSON.stringify(finding.source_finding_id)} belongs to ${JSON.stringify(sourceRepository.url)}, which does not match the selected Cloud repository ${JSON.stringify(destination.url)}. Verify the source repository mapping before publishing.`, diff --git a/sdk/typescript/src/external-import-contract.ts b/sdk/typescript/src/external-import-contract.ts index bab0d38388..32d9f84703 100644 --- a/sdk/typescript/src/external-import-contract.ts +++ b/sdk/typescript/src/external-import-contract.ts @@ -29,6 +29,22 @@ const STRING_BYTE_LIMITS: Readonly> = { next: 4096, }; +/** Compare repository URLs without changing the retained vendor evidence. + * GitHub repository paths are case-insensitive; unknown VCS hosts stay strict. + * Keep protocols and explicit ports distinct, matching the Cloud URL contract. */ +export function repositoryUrlKey(value: string): string { + const normalized = value.replace(/\/$/u, "").replace(/\.git$/u, ""); + const parts = /^(https?):\/\/([^/?#]+)([^?#]*)(.*)$/iu.exec(normalized); + if (!parts) return normalized; + const authority = parts[2]!.toLowerCase(); + const hostname = authority.replace(/:\d+$/u, ""); + const path = + hostname === "github.com" || hostname.endsWith(".ghe.com") + ? parts[3]!.toLowerCase() + : parts[3]!; + return `${parts[1]!.toLowerCase()}://${authority}${path}${parts[4]!}`; +} + // These model constraints are not represented by JSON Schema character limits. function validateModelStrings(input: unknown): void { if (input === null || typeof input !== "object") return; diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts index 015c4eb1a4..969c4662a0 100644 --- a/sdk/typescript/src/wiz-findings.ts +++ b/sdk/typescript/src/wiz-findings.ts @@ -2,7 +2,10 @@ import { readFile } from "node:fs/promises"; import { promisify } from "node:util"; import { gunzip } from "node:zlib"; import { CodexSecurityError } from "./errors.js"; -import { validateExternalEvidence } from "./external-import-contract.js"; +import { + repositoryUrlKey, + validateExternalEvidence, +} from "./external-import-contract.js"; import type { ExternalFindingEvidence } from "./external-import-models.js"; const decompressGzip = promisify(gunzip); @@ -145,10 +148,6 @@ type FindingInput = { repositories: Map; }; -function repositoryUrl(value: string): string { - return value.replace(/\/$/u, "").replace(/\.git$/u, ""); -} - function sourceRepository( repository: Record | undefined, repositories: FindingInput["repositories"], @@ -159,7 +158,7 @@ function sourceRepository( if ( suppliedUrl && inventory && - repositoryUrl(suppliedUrl) !== repositoryUrl(inventory.url) + repositoryUrlKey(suppliedUrl) !== repositoryUrlKey(inventory.url) ) throw new Error( "The finding and repository inventory have conflicting repository URLs.", @@ -343,7 +342,7 @@ function records(payload: unknown): FindingInput { const url = text(repository?.["url"]); if (!id || !url) continue; const previous = repositories.get(id); - if (previous && repositoryUrl(previous.url) !== repositoryUrl(url)) + if (previous && repositoryUrlKey(previous.url) !== repositoryUrlKey(url)) throw new Error( `Repository inventory contains conflicting URLs for ${JSON.stringify(id)}.`, ); diff --git a/sdk/typescript/tests-ts/wiz-findings.test.ts b/sdk/typescript/tests-ts/wiz-findings.test.ts index f7e955a991..30c674a7b3 100644 --- a/sdk/typescript/tests-ts/wiz-findings.test.ts +++ b/sdk/typescript/tests-ts/wiz-findings.test.ts @@ -720,3 +720,116 @@ for (const golden of roundtripFixtures) { ); }); } + +test("GitHub repository metadata accepts case variants without changing source URLs", async () => { + const supplied = { + ...sast, + repository: { + ...repository, + url: "HTTPS://GITHUB.COM/Example/PROJECT.git/", + }, + }; + const mixedInventory = { + ...inventory, + nodes: [ + ...inventory.nodes, + { + ...inventory.nodes[0]!, + repository: { + ...repository, + url: "https://github.com/EXAMPLE/Project.git/", + }, + }, + ], + }; + const parsed = await parse({ + data: { + sastFindings: { nodes: [supplied] }, + versionControlResources: mixedInventory, + }, + }); + expect(parsed.excluded).toEqual([]); + expect(parsed.findings[0]!.evidence.source_data).toEqual(supplied); + expect(parsed.findings[0]!.evidence.details!.repository.url).toBe( + supplied.repository.url, + ); +}); + +test.each([ + [ + "https://github.com/Example/Project.git/", + "https://github.com/example/project", + true, + ], + [ + "https://EXAMPLE.ghe.com/Team/Project.git/", + "https://example.ghe.com/team/project", + true, + ], + [ + "https://github.com/Example/Other.git/", + "https://github.com/example/project", + false, + ], + [ + "https://other.ghe.com/team/project", + "https://example.ghe.com/team/project", + false, + ], + [ + "https://code.example.test/Team/Project", + "https://code.example.test/team/project", + false, + ], + [ + "http://github.com/example/project", + "https://github.com/example/project", + false, + ], + [ + "https://github.com:443/example/project", + "https://github.com/example/project", + false, + ], + [ + "https://example.ghe.com/Team/Project.GIT/", + "https://example.ghe.com/team/project", + false, + ], +])( + "repository URL binding preserves identity: %s to %s", + async (sourceUrl, destinationUrl, matches) => { + const record = { ...sast, repository: { ...repository, url: sourceUrl } }; + const f = await cloudFixture(envelope("sastFindings", [record], false)); + f.destination.url = destinationUrl; + const options = { ...f.options, repository: f.destination.id }; + if (matches) { + const publication = await prepareExternalPublication( + f.file, + options, + f.deps, + ); + expect((await publication.publish()).verified).toBe(1); + expect(f.posts[0]!.items[0]!.evidence.source_data).toEqual(record); + } else { + await expect( + prepareExternalPublication(f.file, options, f.deps), + ).rejects.toThrow("does not match"); + expect(f.posts).toEqual([]); + } + }, +); + +test("Cloud repository selection accepts a GitHub URL case variant", async () => { + const f = await cloudFixture(envelope("sastFindings", [sast])); + const prepared = await prepareExternalPublication( + f.file, + { + ...f.options, + repository: "HTTPS://GITHUB.COM/Example/PROJECT.git/", + }, + f.deps, + ); + expect(prepared.preview.destination.id).toBe(f.destination.id); + expect(f.posts).toEqual([]); +}); From 772a6366aa07f019172f3bc7ead0d7e5d17b5fd8 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 10 Oct 2026 03:26:54 +0000 Subject: [PATCH 26/43] fix(import): preserve large exports and failure output --- sdk/typescript/docs/cli.md | 6 + sdk/typescript/src/cli.ts | 53 ++++- .../src/external-findings-publish.ts | 36 ++-- sdk/typescript/src/wiz-findings.ts | 2 +- sdk/typescript/tests-ts/wiz-findings.test.ts | 190 +++++++++++++++++- 5 files changed, 263 insertions(+), 24 deletions(-) diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md index 8c4b70882c..26f0e844f3 100644 --- a/sdk/typescript/docs/cli.md +++ b/sdk/typescript/docs/cli.md @@ -1015,6 +1015,12 @@ Results distinguish completion from partial or interrupted work: | `interrupted` | 2 | Upload, verification, or receipt persistence stopped; acknowledged work is included in the result. | | `failed` | 2 | Preparation failed before uploading; structured output includes the error. | +For finding imports, `--full-output` requires `--format json` or `--format jsonl`. +Failed and interrupted imports set the envelope's `ok` to `false`, with +`error.code: "IMPORT_FAILED"`, while retaining available results, receipts, and +counts under `data`. Other full-output format combinations are rejected before +reading the input or contacting Cloud; omit `--full-output` to use those formats. + Ctrl-C and SIGTERM retain exit codes 130 and 143. Interrupted JSON output preserves `receipts`, `counts`, `failures` with vendor source IDs, `excluded`, `verified`, and `unacknowledged`. A lost response can mean unacknowledged findings were accepted; diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 2510c9bafd..4c711358c4 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -1776,15 +1776,20 @@ export async function main( let renderedPatch: string | undefined; let patchStructuredError = false; let scanStructuredError = false; - let incompleteScanOutput: - { format: "json" | "jsonl"; message: string } | undefined; + let publicationStructuredError = false; + let failedFullOutput: + { format: "json" | "jsonl"; code: string; message: string } | undefined; const recordIncompleteScanOutput = (outcome: ScanOutcome, format: string) => { if ( outcome.coverageError !== undefined && (format === "json" || format === "jsonl") && argv.includes("--full-output") ) { - incompleteScanOutput = { format, message: outcome.coverageError }; + failedFullOutput = { + format, + code: "SCAN_FAILED", + message: outcome.coverageError, + }; } }; let filteredScanFailure: @@ -3151,7 +3156,7 @@ export async function main( publication.command("findings", { description: "Preview and import selected Wiz repository findings to Cloud.", - hint: "Supply a package vulnerability JSON report, named SAST/repository-secret/IaC JSON collections with repository metadata, or normalized JSONL. Gzip is supported; CSV, scan events, workload secrets, and cloud configuration exports are not.\nExample: codex-security publish findings selected-wiz.json --to cloud --repository https://github.com/example/project --provider wiz --source-key TENANT_ID/vulnerability-finding --dry-run --format json", + hint: "Supply a package vulnerability JSON report, named SAST/repository-secret/IaC JSON collections with repository metadata, or normalized JSONL. Gzip is supported; CSV, scan events, workload secrets, and cloud configuration exports are not. --full-output requires --format json or --format jsonl.\nExample: codex-security publish findings selected-wiz.json --to cloud --repository https://github.com/example/project --provider wiz --source-key TENANT_ID/vulnerability-finding --dry-run --format json", destructive: true, mcp: false, args: z.object({ @@ -3186,7 +3191,21 @@ export async function main( ), }), output: z.record(z.string(), z.unknown()).optional(), - async run({ args, options, format, formatExplicit }) { + async run({ args, options, format, formatExplicit, error: incurError }) { + if ( + argv.includes("--full-output") && + format !== "json" && + format !== "jsonl" + ) { + publicationStructuredError = true; + exitCode = 2; + return incurError({ + code: "INVALID_ARGUMENTS", + message: + "Finding imports support --full-output with --format json or --format jsonl. Choose one of those formats or omit --full-output.", + exitCode, + }); + } const controller = new AbortController(); const removeSignals = listenForAbort(dependencies, controller); const structured = formatExplicit && format !== "toon"; @@ -3297,6 +3316,15 @@ export async function main( errorOutput.write(renderExternalPublicationSummary(result)); return { ...result }; } catch (error) { + if ( + argv.includes("--full-output") && + (format === "json" || format === "jsonl") + ) + failedFullOutput = { + format, + code: "IMPORT_FAILED", + message: diagnosticValue(error), + }; reportPublicationError( diagnosticValue(error), controller.signal.aborted ? controller.signal.reason : undefined, @@ -6330,7 +6358,12 @@ export async function main( let frameworkOutput = frameworkCapture.text(); if (notice !== undefined) errorOutput.write(formatUpdateNotice(notice)); if (frameworkExit !== undefined) { - if (policyFullOutput || patchStructuredError || scanStructuredError) { + if ( + policyFullOutput || + patchStructuredError || + scanStructuredError || + publicationStructuredError + ) { if (exitCode === 0) exitCode = 2; } else { if (exitCode !== 0) return exitCode; @@ -6355,7 +6388,7 @@ export async function main( streamedLogs, frameworkOutput ? JSON.parse(frameworkOutput).cta : undefined, ); - if (incompleteScanOutput !== undefined) { + if (failedFullOutput !== undefined) { const envelope: JsonValue = JSON.parse(frameworkOutput); // Token-count output is a number, not a full-output envelope. if (isJsonObject(envelope) && envelope["ok"] === true) { @@ -6364,12 +6397,12 @@ export async function main( ...envelope, ok: false, error: { - code: "SCAN_FAILED", - message: incompleteScanOutput.message, + code: failedFullOutput.code, + message: failedFullOutput.message, }, }, null, - incompleteScanOutput.format === "json" ? 2 : undefined, + failedFullOutput.format === "json" ? 2 : undefined, )}\n`; } } diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index d95ddb8e29..ecdfe2511e 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -129,11 +129,17 @@ async function readInBatches( return output; } -function canonicalJson(value: unknown): string { +function canonicalJson( + value: unknown, + compareKeys?: (left: string, right: string) => number, +): string { return JSON.stringify(value, (_key, child: unknown) => { if (child !== null && typeof child === "object" && !Array.isArray(child)) { + const record = child as Record; return Object.fromEntries( - Object.entries(child).sort(([a], [b]) => a.localeCompare(b)), + Object.keys(record) + .sort(compareKeys) + .map((key) => [key, record[key]]), ); } return child; @@ -333,16 +339,22 @@ export async function prepareExternalPublication( ); const key = hash( "sha256", - canonicalJson([ - // Keep existing production checkpoints readable; other deployments must - // never share a saved request or receipt with the production default. - ...(apiBaseUrl === DEFAULT_CLOUD_BASE_URL ? [] : [apiBaseUrl]), - credentials.account_id, - destination.id, - destination.repo_connector_id, - source, - parsed.findings, - ]), + // Retain the original ordering only for checkpoint filenames so existing + // saved requests remain discoverable. Evidence comparisons use a total, + // locale-independent key order, including collating-equivalent Unicode keys. + canonicalJson( + [ + // Keep existing production checkpoints readable; other deployments must + // never share a saved request or receipt with the production default. + ...(apiBaseUrl === DEFAULT_CLOUD_BASE_URL ? [] : [apiBaseUrl]), + credentials.account_id, + destination.id, + destination.repo_connector_id, + source, + parsed.findings, + ], + (left, right) => left.localeCompare(right), + ), ); const pendingPath = join(state, `${key}.pending.json`); let saved: SavedSubmission | undefined; diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts index 969c4662a0..a8744338b5 100644 --- a/sdk/typescript/src/wiz-findings.ts +++ b/sdk/typescript/src/wiz-findings.ts @@ -368,7 +368,7 @@ function records(payload: unknown): FindingInput { throw new Error( "Wiz response has another page. Complete the requested pages or save the explicitly selected records in their named collection before publishing.", ); - selected.push(...collection["nodes"].map((value) => ({ value, kind }))); + for (const value of collection["nodes"]) selected.push({ value, kind }); } if (found) return { records: selected, repositories }; throw new Error( diff --git a/sdk/typescript/tests-ts/wiz-findings.test.ts b/sdk/typescript/tests-ts/wiz-findings.test.ts index 30c674a7b3..a53bf4846c 100644 --- a/sdk/typescript/tests-ts/wiz-findings.test.ts +++ b/sdk/typescript/tests-ts/wiz-findings.test.ts @@ -5,7 +5,11 @@ import { join } from "node:path"; import { gzipSync } from "node:zlib"; import { afterEach, expect, test } from "bun:test"; import { main } from "../src/cli.js"; -import { prepareExternalPublication } from "../src/external-findings-publish.js"; +import { + ExternalPublicationError, + prepareExternalPublication, +} from "../src/external-findings-publish.js"; +import { DEFAULT_CLOUD_BASE_URL } from "../src/cloud-endpoint.js"; import { validateExternalEvidence } from "../src/external-import-contract.js"; import type { ExternalFindingEvidence, @@ -468,6 +472,7 @@ async function cloudFixture(payload: unknown) { const state: { failReadback: boolean; readbackDetails?: ExternalFindingEvidence["details"]; + readbackEvidence?: ExternalFindingEvidence; } = { failReadback: false }; const transport = async ( url: string, @@ -507,6 +512,8 @@ async function cloudFixture(payload: unknown) { const report = [...reports.values()].find( (item) => item.id === target.pathname.split("/").at(-1), ); + if (report && state.readbackEvidence) + return json({ ...report, evidence: state.readbackEvidence }); return json( report && state.readbackDetails ? { @@ -833,3 +840,184 @@ test("Cloud repository selection accepts a GitHub URL case variant", async () => expect(prepared.preview.destination.id).toBe(f.destination.id); expect(f.posts).toEqual([]); }); + +test("Node reads a large complete named collection without a function argument limit", async () => { + const count = 150_000; + const records = Array.from({ length: count }, (_, index) => ({ + id: `vendor-${index}`, + name: "CVE-2099-0001", + detailedName: "example-package", + severity: "HIGH", + vulnerableAsset: { id: "synthetic-asset" }, + })); + const f = await fixture(envelope("vulnerabilityFindings", records, false)); + // Run the actual parser under Node; Bun does not have the same argument limit. + const bundle = await mkdtemp(join(import.meta.dir, "..", ".wiz-parser-")); + directories.push(bundle); + const runner = join(bundle, "large-input.mts"); + await writeFile( + runner, + ` + import { readVendorFindings } from "../src/wiz-findings.js"; + const parsed = await readVendorFindings(process.argv[2]!); + console.log(JSON.stringify({ + read: parsed.read, ready: parsed.findings.length, excluded: parsed.excluded.length, + first: parsed.findings[0]?.source_finding_id, + last: parsed.findings.at(-1)?.source_finding_id, + })); + `, + ); + const built = await Bun.build({ + entrypoints: [runner], + outdir: bundle, + target: "node", + format: "esm", + packages: "external", + }); + expect(built.success).toBe(true); + const child = Bun.spawn( + [Bun.which("node")!, built.outputs[0]!.path, f.file], + { stdout: "pipe", stderr: "pipe" }, + ); + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]); + expect(exitCode, stderr).toBe(0); + expect(JSON.parse(stdout)).toEqual({ + read: count, + ready: count, + excluded: 0, + first: "vendor-0", + last: `vendor-${count - 1}`, + }); +}); + +test("Unicode member reordering preserves readback equality and historical checkpoint filenames", async () => { + const input = { + source_finding_id: "unicode-evidence", + evidence: { + title: "Unicode vendor metadata", + severity: "high", + source_data: { é: "precomposed", "e\u0301": "combining", Z: 1, a: 2 }, + }, + }; + const f = await cloudFixture([input]); + const deps = { + ...f.deps, + environment: { + ...f.environment, + CODEX_SECURITY_CLOUD_BASE_URL: DEFAULT_CLOUD_BASE_URL, + }, + }; + const prepared = await prepareExternalPublication(f.file, f.options, deps); + f.state.failReadback = true; + let failure: ExternalPublicationError | undefined; + try { + await prepared.publish(); + } catch (error) { + expect(error).toBeInstanceOf(ExternalPublicationError); + failure = error as ExternalPublicationError; + } + // Frozen from the original publisher's production checkpoint format. + expect(failure?.result.savedSubmission).toBe( + join( + f.environment.CODEX_SECURITY_STATE_DIR, + "external-finding-publications", + "ddeb8ec0be8765f89ef5aeb759dde7753d7ff72f8b8bb25f89a591b045ad40ae.pending.json", + ), + ); + const evidence = structuredClone(prepared.preview.findings[0]!.evidence); + evidence.source_data = Object.fromEntries( + Object.entries(evidence.source_data!).reverse(), + ); + f.state.readbackEvidence = evidence; + f.state.failReadback = false; + const retry = await prepareExternalPublication(f.file, f.options, deps); + expect(retry.preview.resumed).toBe(true); + expect((await retry.publish()).verified).toBe(1); + expect(f.posts).toHaveLength(1); + expect(f.posts[0]!.items[0]!.evidence.source_data).toEqual( + input.evidence.source_data, + ); +}); + +for (const format of ["json", "jsonl"] as const) { + test(`failed ${format} full-output imports emit failure envelopes`, async () => { + const f = await cloudFixture(envelope("sastFindings", [sast])); + await writeFile(f.file, "{"); + const cli = createCliTest(main); + expect( + await cli.runCli( + [ + ...f.command.slice(0, -2), + "--format", + format, + "--full-output", + "--yes", + ], + f.cliDeps, + ), + ).toBe(2); + const output = JSON.parse(cli.stdout.text()); + expect(output.ok).toBe(false); + expect(output.error.code).toBe("IMPORT_FAILED"); + expect(output.data.status).toBe("failed"); + expect(output.meta.command).toBe("publish findings"); + expect(f.calls).toEqual([]); + }); + + test(`interrupted ${format} full-output preserves acknowledged receipts and resumes`, async () => { + const f = await cloudFixture(envelope("sastFindings", [sast])); + f.state.failReadback = true; + const command = [ + ...f.command.slice(0, -2), + "--format", + format, + "--full-output", + "--yes", + ]; + const cli = createCliTest(main); + expect(await cli.runCli(command, f.cliDeps)).toBe(2); + const output = JSON.parse(cli.stdout.text()); + expect(output.ok).toBe(false); + expect(output.error.code).toBe("IMPORT_FAILED"); + expect(output.data).toMatchObject({ + status: "interrupted", + counts: { created: 1 }, + verified: 0, + }); + expect(output.data.receipts).toHaveLength(1); + expect(output.data.savedSubmission).toBeString(); + f.state.failReadback = false; + const retry = createCliTest(main); + expect(await retry.runCli(command, f.cliDeps)).toBe(0); + expect(JSON.parse(retry.stdout.text())).toMatchObject({ + ok: true, + data: { status: "complete", verified: 1 }, + }); + expect(f.posts).toHaveLength(1); + }); +} + +test.each([undefined, "toon", "yaml", "md"])( + "unsupported full-output format %s is rejected before reading or contacting Cloud", + async (format) => { + const f = await cloudFixture(envelope("sastFindings", [sast])); + await rm(f.file); + const cli = createCliTest(main); + const command = [ + ...f.command.slice(0, -2), + ...(format ? ["--format", format] : []), + "--full-output", + "--yes", + ]; + expect(await cli.runCli(command, f.cliDeps)).toBe(2); + expect( + (cli.stdout.text() + cli.stderr.text()).replace(/\s+/g, " "), + ).toContain("--format json or --format jsonl"); + expect(f.calls).toEqual([]); + expect(f.posts).toEqual([]); + }, +); From 5a44d4c13320a636347c6d278ffd736b73845d39 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 10 Oct 2026 04:18:39 +0000 Subject: [PATCH 27/43] fix(import): derive branch qualifiers from repository URLs --- .../src/external-import-contract.ts | 4 +- sdk/typescript/src/wiz-findings.ts | 27 +++++++-- sdk/typescript/tests-ts/wiz-findings.test.ts | 56 +++++++++++++++++++ 3 files changed, 80 insertions(+), 7 deletions(-) diff --git a/sdk/typescript/src/external-import-contract.ts b/sdk/typescript/src/external-import-contract.ts index 32d9f84703..7da1c18fc8 100644 --- a/sdk/typescript/src/external-import-contract.ts +++ b/sdk/typescript/src/external-import-contract.ts @@ -45,7 +45,9 @@ export function repositoryUrlKey(value: string): string { return `${parts[1]!.toLowerCase()}://${authority}${path}${parts[4]!}`; } -// These model constraints are not represented by JSON Schema character limits. +// Cloud ImportModel applies these constraints to both requests and responses. +// Its UTF-8 limits are not exported as JSON Schema character limits: even a +// plain string field such as a receipt error message inherits the 512-byte bound. function validateModelStrings(input: unknown): void { if (input === null || typeof input !== "object") return; if (Array.isArray(input)) { diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts index a8744338b5..e6b358b981 100644 --- a/sdk/typescript/src/wiz-findings.ts +++ b/sdk/typescript/src/wiz-findings.ts @@ -177,13 +177,28 @@ function sourceRepository( function sourceBranch( branch: Record | undefined, - repositoryName: string | null, + repository: RepositoryMetadata, ): string | null { const name = text(branch?.["name"]); - const prefix = repositoryName ? `${repositoryName}/` : null; - return name && prefix && name.startsWith(prefix) - ? name.slice(prefix.length) || null - : name; + if (!name) return null; + const prefix = repository.name ? repository.name + "/" : null; + if (prefix && name.startsWith(prefix)) + return name.slice(prefix.length) || null; + + // URL-only metadata still identifies Wiz's repository-qualified branch name. + // Compare only the qualifier, preserving the branch suffix's case and slashes. + const repositoryKey = repositoryUrlKey(repository.url); + const url = /^(https?:\/\/[^/?#]+)\/([^?#]+)$/iu.exec(repositoryKey); + if (!url) return repository.name ? name : null; + const prefixParts = url[2]!.split("/").length; + const parts = name.split("/"); + if ( + parts.length > prefixParts && + repositoryUrlKey(url[1] + "/" + parts.slice(0, prefixParts).join("/")) === + repositoryKey + ) + return parts.slice(prefixParts).join("/") || null; + return name; } function updatedAt(value: unknown, field: string): number | null { @@ -289,7 +304,7 @@ function repositoryFinding( description: text(record["description"]) ?? text(rule?.["description"]), url: text(record["wizUrl"]) ?? text(record["portalUrl"]), locations: path ? [{ path, line: line ?? null }] : [], - branch: sourceBranch(branch, repository.name), + branch: sourceBranch(branch, repository), code_revision: null, source_scan_id: null, source_updated_at: diff --git a/sdk/typescript/tests-ts/wiz-findings.test.ts b/sdk/typescript/tests-ts/wiz-findings.test.ts index a53bf4846c..c031b420da 100644 --- a/sdk/typescript/tests-ts/wiz-findings.test.ts +++ b/sdk/typescript/tests-ts/wiz-findings.test.ts @@ -1021,3 +1021,59 @@ test.each([undefined, "toon", "yaml", "md"])( expect(f.posts).toEqual([]); }, ); + +for (const kind of ["sast", "secret", "iac"] as const) { + test.each([ + [repositoryUrl, "example/project/feature/parser", "feature/parser"], + [ + "https://GitHub.com/Example/Project.git/", + "EXAMPLE/PROJECT/Feature/Parser/More", + "Feature/Parser/More", + ], + [repositoryUrl, "Feature/Parser", "Feature/Parser"], + [repositoryUrl, "fix/example/project/Parser", "fix/example/project/Parser"], + [repositoryUrl, "example/project", "example/project"], + ["https://github.com/", "example/project/feature/parser", null], + ])( + kind + " URL-only repository metadata preserves the branch from %s: %s", + async (url, branch, expected) => { + const repository = { url }; + const record = + kind === "sast" + ? { + ...sast, + repository, + repositoryBranch: { id: "wiz-branch", name: branch }, + } + : kind === "secret" + ? { + ...secret, + resource: { + ...secret.resource, + name: branch, + typedProperties: { repository }, + }, + } + : { + ...iac, + repository, + branch: { id: "wiz-branch", name: branch }, + }; + const root = + kind === "sast" + ? "sastFindings" + : kind === "secret" + ? "secretInstances" + : "iacFindings"; + const parsed = await parse(envelope(root, [record], false)); + expect(parsed.excluded).toEqual([]); + expect(parsed.findings[0]!.evidence.branch).toBe(expected); + expect(parsed.findings[0]!.evidence.details!.repository).toEqual({ + url, + name: null, + id: null, + }); + expect(parsed.findings[0]!.evidence.source_data).toEqual(record); + }, + ); +} From a431663b5e166ca86424525cb98456338ca86f19 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 10 Oct 2026 04:50:04 +0000 Subject: [PATCH 28/43] fix(import): match custom GitHub Enterprise repository URLs --- .../skills/publish-findings/SKILL.md | 2 +- sdk/typescript/docs/cli.md | 8 +- .../src/external-findings-publish.ts | 13 ++- sdk/typescript/tests-ts/wiz-findings.test.ts | 84 ++++++++++++++++--- 4 files changed, 89 insertions(+), 18 deletions(-) diff --git a/plugins/codex-security/skills/publish-findings/SKILL.md b/plugins/codex-security/skills/publish-findings/SKILL.md index 780326f39d..688144a710 100644 --- a/plugins/codex-security/skills/publish-findings/SKILL.md +++ b/plugins/codex-security/skills/publish-findings/SKILL.md @@ -7,7 +7,7 @@ description: Import selected Wiz package vulnerabilities, SAST, repository secre Use the installed `codex-security publish findings` command for both preview and upload. It owns source mapping, Cloud authentication, source versions, and saved request retries. Do not implement another HTTP uploader or put credentials in a command argument. -Resolve the user's destination to its repository URL or Cloud repository ID; the command accepts either. Confirm that the repository is connected in the intended Cloud account and has an authorized environment. Use the existing file-backed ChatGPT login; explain a missing or incompatible login rather than changing credential storage automatically. Read selected Wiz findings through the user's configured connection or supplied export. Save the actual selected JSON records locally, retaining their IDs and evidence. Finish the requested pagination; if the user selected a subset, save that explicit subset and retain its named collection. The command does not fetch from Wiz or accept CSV, scan events, workload secrets, cloud configuration, or external network exports. The publisher also accepts JSONL records with `source_finding_id` and `evidence` as described in the CLI documentation. +Cloud destinations currently support connected GitHub and GitHub Enterprise repositories, including custom Enterprise hostnames. Resolve the user's destination to its repository URL or Cloud repository ID; the command accepts either. Confirm that the repository is connected in the intended Cloud account and has an authorized environment. Use the existing file-backed ChatGPT login; explain a missing or incompatible login rather than changing credential storage automatically. Read selected Wiz findings through the user's configured connection or supplied export. Save the actual selected JSON records locally, retaining their IDs and evidence. Finish the requested pagination; if the user selected a subset, save that explicit subset and retain its named collection. The command does not fetch from Wiz or accept CSV, scan events, workload secrets, cloud configuration, or external network exports. The publisher also accepts JSONL records with `source_finding_id` and `evidence` as described in the CLI documentation. Use a stable Wiz tenant/finding namespace as `--source-key`. A project filter is selection context, not a new identity. Confirm the selected assets or builds belong to the destination repository; do not infer a scanner branch or revision from the current checkout. Do not publish vendor credentials, unrelated records, or local assessments as original scanner evidence. diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md index 26f0e844f3..858ba71b62 100644 --- a/sdk/typescript/docs/cli.md +++ b/sdk/typescript/docs/cli.md @@ -799,12 +799,14 @@ Import selected Wiz package vulnerabilities, SAST findings, repository secrets, and IaC findings without creating a scan. Imported records retain vendor evidence and remain **not assessed by Codex**, including when Wiz marks them resolved or reports an AI verdict. Workload secrets, cloud configuration findings, and external -network findings are not supported by this repository importer. +network findings are not supported by this repository importer. Cloud destinations +currently support connected GitHub and GitHub Enterprise repositories, including +custom Enterprise hostnames. Named Wiz exports do not add other VCS destinations. Before the first import: -1. Select an existing repository in Codex Security Cloud and copy its repository - URL (for example, a GitHub URL) or Cloud repository ID. The repository needs an +1. Select an existing repository in Codex Security Cloud and copy its URL or + Cloud repository ID. The repository needs an authorized environment, but does not need a completed native scan. 2. For **package vulnerabilities**, in Wiz **Vulnerability Findings**, filter to the intended repository and findings. Choose **Save as → Report**, select **Repository Branch** as the diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index ecdfe2511e..29d1da5ec0 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -33,6 +33,12 @@ import type { const MAX_REQUEST_BYTES = 5 * 1024 * 1024; +// Cloud's import catalog contains GitHub repositories, including custom Enterprise +// hosts. Match its case-insensitive URL identity without merging schemes or ports. +function cloudRepositoryUrlKey(value: string): string { + return repositoryUrlKey(value).toLowerCase(); +} + /** CLI and plugin callers prepare the same immutable request before approval. */ export interface ExternalPublicationOptions { repository: string; @@ -312,7 +318,8 @@ export async function prepareExternalPublication( ...result.data.filter( (item) => item.id === options.repository || - repositoryUrlKey(item.url) === repositoryUrlKey(options.repository), + cloudRepositoryUrlKey(item.url) === + cloudRepositoryUrlKey(options.repository), ), ); page = result.next; @@ -422,8 +429,8 @@ export async function prepareExternalPublication( const sourceRepository = finding.evidence.details?.repository; if ( sourceRepository && - repositoryUrlKey(sourceRepository.url) !== - repositoryUrlKey(destination.url) + cloudRepositoryUrlKey(sourceRepository.url) !== + cloudRepositoryUrlKey(destination.url) ) throw new CodexSecurityError( `Finding ${JSON.stringify(finding.source_finding_id)} belongs to ${JSON.stringify(sourceRepository.url)}, which does not match the selected Cloud repository ${JSON.stringify(destination.url)}. Verify the source repository mapping before publishing.`, diff --git a/sdk/typescript/tests-ts/wiz-findings.test.ts b/sdk/typescript/tests-ts/wiz-findings.test.ts index c031b420da..54137994dc 100644 --- a/sdk/typescript/tests-ts/wiz-findings.test.ts +++ b/sdk/typescript/tests-ts/wiz-findings.test.ts @@ -784,9 +784,9 @@ test.each([ false, ], [ - "https://code.example.test/Team/Project", - "https://code.example.test/team/project", - false, + "https://github.example.test/Team/Project.git/", + "https://github.example.test/team/project", + true, ], [ "http://github.com/example/project", @@ -827,18 +827,80 @@ test.each([ }, ); -test("Cloud repository selection accepts a GitHub URL case variant", async () => { - const f = await cloudFixture(envelope("sastFindings", [sast])); +test.each(["github.com", "example.ghe.com", "github.example.test"])( + "Cloud GitHub repository selection accepts a URL case variant on %s", + async (host) => { + const url = "https://" + host + "/example/project"; + const record = { ...sast, repository: { ...repository, url } }; + const f = await cloudFixture(envelope("sastFindings", [record], false)); + f.destination.url = url; + const prepared = await prepareExternalPublication( + f.file, + { + ...f.options, + repository: "HTTPS://" + host.toUpperCase() + "/Example/PROJECT.git/", + }, + f.deps, + ); + expect(prepared.preview.destination.id).toBe(f.destination.id); + expect(f.posts).toEqual([]); + }, +); + +test("custom GitHub Enterprise binding preserves branch case and exact source evidence", async () => { + const record = { + ...sast, + repository: { + ...repository, + name: "Team/Project", + url: "https://github.example.test/Team/Project.git/", + }, + repositoryBranch: { + id: "wiz-branch", + name: "Team/Project/Feature/Parser/More", + }, + }; + const f = await cloudFixture(envelope("sastFindings", [record], false)); + f.destination.url = "https://github.example.test/team/project"; + f.destination.repo_connector_id = "custom-github"; const prepared = await prepareExternalPublication( f.file, - { - ...f.options, - repository: "HTTPS://GITHUB.COM/Example/PROJECT.git/", - }, + { ...f.options, repository: f.destination.id }, f.deps, ); - expect(prepared.preview.destination.id).toBe(f.destination.id); - expect(f.posts).toEqual([]); + expect((await prepared.publish()).verified).toBe(1); + const evidence = f.posts[0]!.items[0]!.evidence; + expect(evidence.branch).toBe("Feature/Parser/More"); + expect(evidence.details!.repository.url).toBe(record.repository.url); + expect(evidence.source_data).toEqual(record); +}); + +test("unidentified source hosts still require exact supplied and inventory URL paths", async () => { + const record = { + ...sast, + repository: { + ...repository, + url: "https://code.example.test/Team/Project", + }, + }; + const parsed = await parse({ + data: { + sastFindings: { nodes: [record] }, + versionControlResources: { + nodes: [ + { + id: "unidentified-branch", + repository: { + ...repository, + url: "https://code.example.test/team/project", + }, + }, + ], + }, + }, + }); + expect(parsed.findings).toEqual([]); + expect(parsed.excluded[0]!.reason).toContain("conflicting repository URLs"); }); test("Node reads a large complete named collection without a function argument limit", async () => { From 2e6daad6780bc6571f4c3d7cd9cad32dfb50215d Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 10 Oct 2026 05:25:03 +0000 Subject: [PATCH 29/43] fix(import): use Wiz inventory platform for repository matching --- .../skills/publish-findings/SKILL.md | 2 +- sdk/typescript/docs/cli.md | 5 +- sdk/typescript/src/wiz-findings.ts | 75 +++++-- sdk/typescript/tests-ts/wiz-findings.test.ts | 193 ++++++++++++++++++ 4 files changed, 260 insertions(+), 15 deletions(-) diff --git a/plugins/codex-security/skills/publish-findings/SKILL.md b/plugins/codex-security/skills/publish-findings/SKILL.md index 688144a710..3ea68bf03c 100644 --- a/plugins/codex-security/skills/publish-findings/SKILL.md +++ b/plugins/codex-security/skills/publish-findings/SKILL.md @@ -13,7 +13,7 @@ Use a stable Wiz tenant/finding namespace as `--source-key`. A project filter is For package vulnerability UI exports, use **Vulnerability Findings → Save as → Report**, keeping the repository and finding filters. Select **Repository Branch** (remove the default Virtual Machine selection), **JSON**, and **Detailed** columns. Download the completed report; default gzip compression is supported, even with a `.json` filename. **Raw Event** on Code & Build Scans contains scan metadata and cannot supply findings. Do not create a scheduled report or enable notifications or external storage unless requested. -For SAST, repository secrets, and IaC, retain the explicit `data.sastFindings.nodes`, `data.secretInstances.nodes`, or `data.iacFindings.nodes` collection. Bare arrays cannot identify these finding families. Include a verified URL on the source repository metadata or supply `data.versionControlResources.nodes` with matching `repository.id` and `repository.url`. For secrets, the repository is `resource.typedProperties.repository` and resource type must be `REPOSITORY_BRANCH`. Do not map names, inventory node IDs, `providerID`, or Wiz UUIDs directly to Cloud repository IDs. The importer verifies source URLs against the selected destination; resolve a missing or conflicting mapping before uploading. +For SAST, repository secrets, and IaC, retain the explicit `data.sastFindings.nodes`, `data.secretInstances.nodes`, or `data.iacFindings.nodes` collection. Bare arrays cannot identify these finding families. Include a verified URL on the source repository metadata or supply `data.versionControlResources.nodes` with matching `repository.id` and `repository.url`. Retain the documented inventory `platform` field; explicit `GITHUB` establishes case-insensitive matching on custom GitHub Enterprise hosts. Resolve conflicting platform or URL metadata for a repository ID before publishing. For secrets, the repository is `resource.typedProperties.repository` and resource type must be `REPOSITORY_BRANCH`. Do not map names, inventory node IDs, `providerID`, or Wiz UUIDs directly to Cloud repository IDs. The importer verifies source URLs against the selected destination; resolve a missing or conflicting mapping before uploading. Keep raw source records unchanged in `source_data`. New raw occurrence IDs are prefixed `sast:`, `secret:`, or `iac:`; existing package IDs are unchanged. For CSV report conversion, use a reviewed mapping of the actual header names into the documented normalized JSONL profile, set `evidence.details.kind` and the verified `details.repository.url`, and use the same prefixed occurrence ID. The command does not parse CSV directly. Select source paths and line numbers explicitly in secret reports; the standard metadata query lacks them. Preserve rule/CWE, range/snippet, detector validation, and IaC comparison fields in the typed details described in the CLI docs. Preserve empty comparison strings. Do not invent missing IDs, severity, revisions, or locations, retrieve credential values, test credentials, or apply suggested fixes. Vendor status, AI verdict, secret validation, introduced commits, and logical IaC resources are evidence, not Codex assessment or proof of runtime deployment. diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md index 858ba71b62..f77bfe449a 100644 --- a/sdk/typescript/docs/cli.md +++ b/sdk/typescript/docs/cli.md @@ -922,7 +922,10 @@ in `data.versionControlResources.nodes`. For secrets, repository metadata is und `repository.id` to that source ID and uses `repository.url`; inventory node IDs, `providerID`, names, and Wiz UUIDs are not Cloud repository IDs. A supplied URL must agree with the inventory and with the selected Cloud repository before any -finding is uploaded. For example, this synthetic selection supplies the inventory +finding is uploaded. Keep each inventory node's `platform` field: `GITHUB` +establishes case-insensitive repository and branch-prefix matching on custom +GitHub Enterprise hosts. Duplicate entries for one repository ID must agree on +their platform and repository URL. For example, this synthetic selection supplies the inventory alongside its SAST collection: ```json diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts index e6b358b981..60c224b989 100644 --- a/sdk/typescript/src/wiz-findings.ts +++ b/sdk/typescript/src/wiz-findings.ts @@ -145,9 +145,19 @@ type RepositoryMetadata = { type FindingInput = { records: InputRecord[]; - repositories: Map; + repositories: Map; }; +// Wiz's documented inventory platform establishes custom GitHub host semantics. +// Keep this context separate from the repository metadata published as evidence. +function inventoryRepositoryUrlKey( + value: string, + platform: string | null | undefined, +): string { + const key = repositoryUrlKey(value); + return platform === "GITHUB" ? key.toLowerCase() : key; +} + function sourceRepository( repository: Record | undefined, repositories: FindingInput["repositories"], @@ -158,7 +168,8 @@ function sourceRepository( if ( suppliedUrl && inventory && - repositoryUrlKey(suppliedUrl) !== repositoryUrlKey(inventory.url) + inventoryRepositoryUrlKey(suppliedUrl, inventory.platform) !== + inventoryRepositoryUrlKey(inventory.url, inventory.platform) ) throw new Error( "The finding and repository inventory have conflicting repository URLs.", @@ -178,6 +189,7 @@ function sourceRepository( function sourceBranch( branch: Record | undefined, repository: RepositoryMetadata, + platform: string | null | undefined, ): string | null { const name = text(branch?.["name"]); if (!name) return null; @@ -187,15 +199,17 @@ function sourceBranch( // URL-only metadata still identifies Wiz's repository-qualified branch name. // Compare only the qualifier, preserving the branch suffix's case and slashes. - const repositoryKey = repositoryUrlKey(repository.url); + const repositoryKey = inventoryRepositoryUrlKey(repository.url, platform); const url = /^(https?:\/\/[^/?#]+)\/([^?#]+)$/iu.exec(repositoryKey); if (!url) return repository.name ? name : null; const prefixParts = url[2]!.split("/").length; const parts = name.split("/"); if ( parts.length > prefixParts && - repositoryUrlKey(url[1] + "/" + parts.slice(0, prefixParts).join("/")) === - repositoryKey + inventoryRepositoryUrlKey( + url[1] + "/" + parts.slice(0, prefixParts).join("/"), + platform, + ) === repositoryKey ) return parts.slice(prefixParts).join("/") || null; return name; @@ -304,7 +318,11 @@ function repositoryFinding( description: text(record["description"]) ?? text(rule?.["description"]), url: text(record["wizUrl"]) ?? text(record["portalUrl"]), locations: path ? [{ path, line: line ?? null }] : [], - branch: sourceBranch(branch, repository), + branch: sourceBranch( + branch, + repository, + repository.id ? repositories.get(repository.id)?.platform : undefined, + ), code_revision: null, source_scan_id: null, source_updated_at: @@ -351,17 +369,48 @@ function records(payload: unknown): FindingInput { "Wiz repository inventory has another page. Finish the selected inventory before publishing.", ); if (Array.isArray(inventory?.["nodes"])) { - for (const entry of inventory["nodes"]) { - const repository = object(object(entry)?.["repository"]); + const entries = inventory["nodes"].flatMap((entry) => { + const node = object(entry); + const repository = object(node?.["repository"]); const id = text(repository?.["id"]); const url = text(repository?.["url"]); - if (!id || !url) continue; - const previous = repositories.get(id); - if (previous && repositoryUrlKey(previous.url) !== repositoryUrlKey(url)) + return id && url + ? [ + { + id, + name: text(repository?.["name"]), + url, + platform: text(node?.["platform"]), + }, + ] + : []; + }); + // Resolve provider hints before comparing URLs, so duplicate row order cannot + // discard a known platform or reject a case variant before its GITHUB hint. + for (const entry of entries) { + const previous = repositories.get(entry.id); + if ( + previous?.platform && + entry.platform && + previous.platform !== entry.platform + ) + throw new Error( + `Repository inventory contains conflicting platforms for ${JSON.stringify(entry.id)}.`, + ); + repositories.set(entry.id, { + ...entry, + platform: entry.platform ?? previous?.platform ?? null, + }); + } + for (const entry of entries) { + const resolved = repositories.get(entry.id)!; + if ( + inventoryRepositoryUrlKey(entry.url, resolved.platform) !== + inventoryRepositoryUrlKey(resolved.url, resolved.platform) + ) throw new Error( - `Repository inventory contains conflicting URLs for ${JSON.stringify(id)}.`, + `Repository inventory contains conflicting URLs for ${JSON.stringify(entry.id)}.`, ); - repositories.set(id, { id, name: text(repository?.["name"]), url }); } } const collections = { diff --git a/sdk/typescript/tests-ts/wiz-findings.test.ts b/sdk/typescript/tests-ts/wiz-findings.test.ts index 54137994dc..72064d9665 100644 --- a/sdk/typescript/tests-ts/wiz-findings.test.ts +++ b/sdk/typescript/tests-ts/wiz-findings.test.ts @@ -1139,3 +1139,196 @@ for (const kind of ["sast", "secret", "iac"] as const) { }, ); } + +// Synthetic nodes follow Wiz's documented VersionControlResources platform field. +const enterpriseUrl = "https://github.example.test/example/project"; +function inventoryNode(platform: string | undefined, url = enterpriseUrl) { + return { + id: "wiz-inventory-node", + type: "REPOSITORY_BRANCH", + platform, + repository: { id: repository.id, url }, + }; +} + +for (const kind of ["sast", "secret", "iac"] as const) { + test.each([false, true])( + kind + + " GITHUB inventory maps custom-host URLs and branches (supplied URL: %j)", + async (withSourceUrl) => { + const sourceUrl = "https://GITHUB.EXAMPLE.TEST/Example/PROJECT.git/"; + const sourceRepository = { + id: repository.id, + ...(withSourceUrl ? { url: sourceUrl } : {}), + }; + const branch = { + id: "wiz-branch", + name: "EXAMPLE/PROJECT/Feature/Parser/More", + }; + const record = + kind === "secret" + ? { + ...secret, + resource: { + ...secret.resource, + name: branch.name, + typedProperties: { repository: sourceRepository }, + }, + } + : { + ...(kind === "sast" ? sast : iac), + repository: sourceRepository, + [kind === "sast" ? "repositoryBranch" : "branch"]: branch, + }; + const root = { + sast: "sastFindings", + secret: "secretInstances", + iac: "iacFindings", + }[kind]; + const f = await cloudFixture({ + data: { + [root]: { nodes: [record] }, + versionControlResources: { + nodes: [ + { + ...inventoryNode("GITHUB"), + type: withSourceUrl ? "REPOSITORY_BRANCH" : "CI_WORKFLOW", + }, + ], + }, + }, + }); + f.destination.url = enterpriseUrl; + const publication = await prepareExternalPublication( + f.file, + { ...f.options, repository: f.destination.id }, + f.deps, + ); + expect((await publication.publish()).verified).toBe(1); + const evidence = f.posts[0]!.items[0]!.evidence; + expect(evidence.branch).toBe("Feature/Parser/More"); + expect(evidence.details!.repository).toEqual({ + id: repository.id, + name: null, + url: withSourceUrl ? sourceUrl : enterpriseUrl, + }); + expect(evidence.source_data).toEqual(record); + }, + ); +} + +test.each(["GITLAB", "github"])( + "inventory platform %s does not imply GitHub custom-host matching", + async (platform) => { + const record = { + ...sast, + repository: { + ...repository, + url: "https://github.example.test/Example/Project", + }, + }; + const parsed = await parse({ + data: { + sastFindings: { nodes: [record] }, + versionControlResources: { nodes: [inventoryNode(platform)] }, + }, + }); + expect(parsed.findings).toEqual([]); + expect(parsed.excluded[0]!.reason).toContain("conflicting repository URLs"); + }, +); + +test.each([0, 1, 2])( + "GITHUB inventory duplicate matching is independent of hint position %j", + async (githubPosition) => { + const urls = [ + "https://github.example.test/Example/Project.git/", + "https://github.example.test/EXAMPLE/PROJECT", + enterpriseUrl, + ]; + const nodes = urls.map((url, index) => + inventoryNode(index === githubPosition ? "GITHUB" : undefined, url), + ); + const record = { + ...sast, + repository: { id: repository.id }, + repositoryBranch: { + id: "wiz-branch", + name: "EXAMPLE/PROJECT/Feature/Parser", + }, + }; + const parsed = await parse({ + data: { + sastFindings: { nodes: [record] }, + versionControlResources: { nodes }, + }, + }); + expect(parsed.excluded).toEqual([]); + expect(parsed.findings[0]!.evidence.branch).toBe("Feature/Parser"); + expect(parsed.findings[0]!.evidence.details!.repository).toEqual({ + id: repository.id, + name: null, + url: enterpriseUrl, + }); + expect(parsed.findings[0]!.evidence.source_data).toEqual(record); + }, +); + +test.each([ + ["GITHUB", "GITLAB"], + ["GITLAB", "GITHUB"], +])("inventory platform conflicts reject %s before %s", async (first, last) => { + await expect( + parse({ + data: { + sastFindings: { nodes: [sast] }, + versionControlResources: { + nodes: [first, undefined, last].map((platform) => + inventoryNode(platform), + ), + }, + }, + }), + ).rejects.toThrow("conflicting platforms"); +}); + +test.each([undefined, "GITLAB"])( + "inventory duplicate URLs stay strict without GITHUB provenance: %j", + async (platform) => { + await expect( + parse({ + data: { + sastFindings: { nodes: [sast] }, + versionControlResources: { + nodes: [ + inventoryNode( + platform, + "https://github.example.test/Example/Project", + ), + inventoryNode(platform), + ], + }, + }, + }), + ).rejects.toThrow("conflicting URLs"); + }, +); + +test("GITHUB inventory provenance does not permit a different repository URL", async () => { + await expect( + parse({ + data: { + sastFindings: { nodes: [sast] }, + versionControlResources: { + nodes: [ + inventoryNode("GITHUB"), + inventoryNode( + "GITHUB", + "https://github.example.test/example/other", + ), + ], + }, + }, + }), + ).rejects.toThrow("conflicting URLs"); +}); From 2fdc1f3d58e0900a4f304432200aa59aee6e2b82 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 10 Oct 2026 05:39:19 +0000 Subject: [PATCH 30/43] Preserve branch suffixes across GitHub repository case variants --- sdk/typescript/src/wiz-findings.ts | 10 +++-- sdk/typescript/tests-ts/wiz-findings.test.ts | 46 ++++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts index e6b358b981..a9f827e71c 100644 --- a/sdk/typescript/src/wiz-findings.ts +++ b/sdk/typescript/src/wiz-findings.ts @@ -186,16 +186,18 @@ function sourceBranch( return name.slice(prefix.length) || null; // URL-only metadata still identifies Wiz's repository-qualified branch name. - // Compare only the qualifier, preserving the branch suffix's case and slashes. - const repositoryKey = repositoryUrlKey(repository.url); + // Supported GitHub destinations use case-insensitive repository qualifiers. + // Preserve the branch suffix's case and slashes and the original evidence. + const repositoryKey = repositoryUrlKey(repository.url).toLowerCase(); const url = /^(https?:\/\/[^/?#]+)\/([^?#]+)$/iu.exec(repositoryKey); if (!url) return repository.name ? name : null; const prefixParts = url[2]!.split("/").length; const parts = name.split("/"); if ( parts.length > prefixParts && - repositoryUrlKey(url[1] + "/" + parts.slice(0, prefixParts).join("/")) === - repositoryKey + repositoryUrlKey( + url[1] + "/" + parts.slice(0, prefixParts).join("/"), + ).toLowerCase() === repositoryKey ) return parts.slice(prefixParts).join("/") || null; return name; diff --git a/sdk/typescript/tests-ts/wiz-findings.test.ts b/sdk/typescript/tests-ts/wiz-findings.test.ts index 54137994dc..e58ef83120 100644 --- a/sdk/typescript/tests-ts/wiz-findings.test.ts +++ b/sdk/typescript/tests-ts/wiz-findings.test.ts @@ -875,6 +875,52 @@ test("custom GitHub Enterprise binding preserves branch case and exact source ev expect(evidence.source_data).toEqual(record); }); +test.each([ + ["sastFindings", "Team/Project", "team/project"], + ["sastFindings", "team/project", "Team/Project"], + ["secretInstances", "Team/Project", "team/project"], + ["secretInstances", "team/project", "Team/Project"], + ["iacFindings", "Team/Project", "team/project"], + ["iacFindings", "team/project", "Team/Project"], +])( + "%s custom GitHub URL-only branch qualifiers preserve suffix case (%s, %s)", + async (collection, repositoryPath, qualifier) => { + const sourceRepository = { + id: "wiz-repository", + url: `https://github.example.test/${repositoryPath}`, + }; + const branch = { + id: "wiz-branch", + name: `${qualifier}/Feature/Parser/More`, + }; + const record = + collection === "secretInstances" + ? { + ...secret, + resource: { + ...secret.resource, + name: branch.name, + typedProperties: { repository: sourceRepository }, + }, + } + : collection === "iacFindings" + ? { ...iac, repository: sourceRepository, branch } + : { ...sast, repository: sourceRepository, repositoryBranch: branch }; + const f = await cloudFixture(envelope(collection!, [record], false)); + f.destination.url = "https://github.example.test/team/project"; + const prepared = await prepareExternalPublication( + f.file, + { ...f.options, repository: f.destination.id }, + f.deps, + ); + expect((await prepared.publish()).verified).toBe(1); + const evidence = f.posts[0]!.items[0]!.evidence; + expect(evidence.branch).toBe("Feature/Parser/More"); + expect(evidence.details!.repository.url).toBe(sourceRepository.url); + expect(evidence.source_data).toEqual(record); + }, +); + test("unidentified source hosts still require exact supplied and inventory URL paths", async () => { const record = { ...sast, From 74f2266e0e4d60e6646ad0b46756cf8ec940cbe4 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 10 Oct 2026 06:45:38 +0000 Subject: [PATCH 31/43] Keep publication checkpoints stable across inventory order and locales --- .../src/external-findings-publish.ts | 61 +++-- sdk/typescript/src/wiz-findings.ts | 7 + sdk/typescript/tests-ts/wiz-findings.test.ts | 253 +++++++++++++++--- 3 files changed, 262 insertions(+), 59 deletions(-) diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 29d1da5ec0..4182e04b70 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -344,26 +344,53 @@ export async function prepareExternalPublication( codexSecurityStateDirectory(environment), "external-finding-publications", ); - const key = hash( + const checkpointIdentity = [ + // Keep existing production checkpoints readable; other deployments must + // never share a saved request or receipt with the production default. + ...(apiBaseUrl === DEFAULT_CLOUD_BASE_URL ? [] : [apiBaseUrl]), + credentials.account_id, + destination.id, + destination.repo_connector_id, + source, + parsed.findings, + ]; + const key = hash("sha256", canonicalJson(checkpointIdentity)); + const pendingPath = join(state, `${key}.pending.json`); + const legacyKey = hash( "sha256", - // Retain the original ordering only for checkpoint filenames so existing - // saved requests remain discoverable. Evidence comparisons use a total, - // locale-independent key order, including collating-equivalent Unicode keys. - canonicalJson( - [ - // Keep existing production checkpoints readable; other deployments must - // never share a saved request or receipt with the production default. - ...(apiBaseUrl === DEFAULT_CLOUD_BASE_URL ? [] : [apiBaseUrl]), - credentials.account_id, - destination.id, - destination.repo_connector_id, - source, - parsed.findings, - ], - (left, right) => left.localeCompare(right), + canonicalJson(checkpointIdentity, (left, right) => + left.localeCompare(right), ), ); - const pendingPath = join(state, `${key}.pending.json`); + if (legacyKey !== key) { + const legacyPath = join(state, `${legacyKey}.pending.json`); + const legacy = await lstat(legacyPath).catch( + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return undefined; + throw error; + }, + ); + if (legacy) { + // Preserve requests staged by the earlier locale-based filename format. + // Hold both publication locks while moving the original checkpoint. + await withImportLock(state, key, dependencies.signal, async () => + withImportLock(state, legacyKey, dependencies.signal, async () => { + const current = await lstat(pendingPath).catch( + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return undefined; + throw error; + }, + ); + if (!current) + await rename(legacyPath, pendingPath).catch( + (error: NodeJS.ErrnoException) => { + if (error.code !== "ENOENT") throw error; + }, + ); + }), + ); + } + } let saved: SavedSubmission | undefined; try { const serialized = await readFile(pendingPath, "utf8"); diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts index 37a8dc1e96..1924df29fa 100644 --- a/sdk/typescript/src/wiz-findings.ts +++ b/sdk/typescript/src/wiz-findings.ts @@ -394,6 +394,13 @@ function records(payload: unknown): FindingInput { ); repositories.set(entry.id, { ...entry, + // Equivalent inventory aliases must produce the same evidence when the + // export reorders its rows. Keep a stable supplied URL and nonempty name. + url: previous && previous.url < entry.url ? previous.url : entry.url, + name: + previous?.name && (!entry.name || previous.name < entry.name) + ? previous.name + : entry.name, platform: entry.platform ?? previous?.platform ?? null, }); } diff --git a/sdk/typescript/tests-ts/wiz-findings.test.ts b/sdk/typescript/tests-ts/wiz-findings.test.ts index 47170708d5..20e92a7f22 100644 --- a/sdk/typescript/tests-ts/wiz-findings.test.ts +++ b/sdk/typescript/tests-ts/wiz-findings.test.ts @@ -1,5 +1,13 @@ import { hash } from "node:crypto"; -import { mkdtemp, mkdir, readdir, rm, writeFile } from "node:fs/promises"; +import { + mkdtemp, + mkdir, + readFile, + readdir, + rename, + rm, + writeFile, +} from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { gzipSync } from "node:zlib"; @@ -1002,53 +1010,172 @@ test("Node reads a large complete named collection without a function argument l }); }); -test("Unicode member reordering preserves readback equality and historical checkpoint filenames", async () => { +test.each([false, true])( + "Unicode readback resumes historical checkpoints (canonical present: %j)", + async (keepCanonical) => { + const input = { + source_finding_id: "unicode-evidence", + evidence: { + title: "Unicode vendor metadata", + severity: "high", + source_data: { é: "precomposed", "e\u0301": "combining", Z: 1, a: 2 }, + }, + }; + const f = await cloudFixture([input]); + const deps = { + ...f.deps, + environment: { + ...f.environment, + CODEX_SECURITY_CLOUD_BASE_URL: DEFAULT_CLOUD_BASE_URL, + }, + }; + const prepared = await prepareExternalPublication(f.file, f.options, deps); + f.state.failReadback = true; + let failure: ExternalPublicationError | undefined; + try { + await prepared.publish(); + } catch (error) { + expect(error).toBeInstanceOf(ExternalPublicationError); + failure = error as ExternalPublicationError; + } + const pendingPath = failure!.result.savedSubmission!; + const saved = await readFile(pendingPath, "utf8"); + // Frozen from the original publisher's production checkpoint format. + const historicalPath = join( + f.environment.CODEX_SECURITY_STATE_DIR, + "external-finding-publications", + "ddeb8ec0be8765f89ef5aeb759dde7753d7ff72f8b8bb25f89a591b045ad40ae.pending.json", + ); + expect(pendingPath).not.toBe(historicalPath); + if (keepCanonical) { + const legacy = JSON.parse(saved); + legacy.requests[0].request_id = "synthetic-older-request"; + legacy.receipts = []; + await writeFile(historicalPath, JSON.stringify(legacy)); + } else { + await rename(pendingPath, historicalPath); + } + const evidence = structuredClone(prepared.preview.findings[0]!.evidence); + evidence.source_data = Object.fromEntries( + Object.entries(evidence.source_data!).reverse(), + ); + f.state.readbackEvidence = evidence; + f.state.failReadback = false; + const retry = await prepareExternalPublication(f.file, f.options, deps); + expect(retry.preview.resumed).toBe(true); + expect(retry.preview.requests).toEqual(prepared.preview.requests); + expect(await readFile(pendingPath, "utf8")).toBe(saved); + expect( + ( + await readdir( + join( + f.environment.CODEX_SECURITY_STATE_DIR, + "external-finding-publications", + ), + ) + ).includes(historicalPath.split(/[\\/]/u).at(-1)!), + ).toBe(keepCanonical); + expect((await retry.publish()).verified).toBe(1); + expect(f.posts).toHaveLength(1); + expect(f.posts[0]!.items[0]!.evidence.source_data).toEqual( + input.evidence.source_data, + ); + }, +); + +test("Node resumes the same uncertain publication across locales and Unicode key order", async () => { const input = { - source_finding_id: "unicode-evidence", + source_finding_id: "locale-evidence", evidence: { - title: "Unicode vendor metadata", + title: "Synthetic Unicode metadata", severity: "high", - source_data: { é: "precomposed", "e\u0301": "combining", Z: 1, a: 2 }, + source_data: { z: 1, ä: 2, é: "precomposed", "e\u0301": "combining" }, }, }; const f = await cloudFixture([input]); - const deps = { - ...f.deps, - environment: { - ...f.environment, - CODEX_SECURITY_CLOUD_BASE_URL: DEFAULT_CLOUD_BASE_URL, - }, - }; - const prepared = await prepareExternalPublication(f.file, f.options, deps); - f.state.failReadback = true; - let failure: ExternalPublicationError | undefined; - try { - await prepared.publish(); - } catch (error) { - expect(error).toBeInstanceOf(ExternalPublicationError); - failure = error as ExternalPublicationError; - } - // Frozen from the original publisher's production checkpoint format. - expect(failure?.result.savedSubmission).toBe( - join( - f.environment.CODEX_SECURITY_STATE_DIR, - "external-finding-publications", - "ddeb8ec0be8765f89ef5aeb759dde7753d7ff72f8b8bb25f89a591b045ad40ae.pending.json", - ), - ); - const evidence = structuredClone(prepared.preview.findings[0]!.evidence); - evidence.source_data = Object.fromEntries( - Object.entries(evidence.source_data!).reverse(), - ); - f.state.readbackEvidence = evidence; - f.state.failReadback = false; - const retry = await prepareExternalPublication(f.file, f.options, deps); - expect(retry.preview.resumed).toBe(true); - expect((await retry.publish()).verified).toBe(1); - expect(f.posts).toHaveLength(1); - expect(f.posts[0]!.items[0]!.evidence.source_data).toEqual( - input.evidence.source_data, + const bundle = await mkdtemp(join(import.meta.dir, "..", ".wiz-locale-")); + directories.push(bundle); + const runner = join(bundle, "publication.mts"); + await writeFile( + runner, + ` + import { prepareExternalPublication } from "../src/external-findings-publish.js"; + // Select ICU collation explicitly so this also exercises both locales on + // Windows hosts whose process default does not follow LANG. + const collation = new Intl.Collator(process.argv[2]); + String.prototype.localeCompare = function(other) { + return collation.compare(String(this), other); + }; + const prepared = await prepareExternalPublication( + ${JSON.stringify(f.file)}, ${JSON.stringify(f.options)}, { + environment: ${JSON.stringify({ + CODEX_HOME: f.environment.CODEX_HOME, + CODEX_SECURITY_STATE_DIR: f.environment.CODEX_SECURITY_STATE_DIR, + })}, + credentials: async () => ({ access_token: "synthetic-token", account_id: "synthetic-account" }), + fetch: async (url, options) => { + if (options.method === "POST") throw new Error("Synthetic uncertain upload"); + const body = new URL(url).pathname.endsWith("/repositories") + ? { data: [${JSON.stringify(f.destination)}], has_more: false, next: null } + : { data: [], has_more: false, next: null }; + return new Response(JSON.stringify(body)); + }, + }, + ); + try { await prepared.publish(); } catch (error) { + console.log(JSON.stringify({ + locale: collation.resolvedOptions().locale, + resumed: prepared.preview.resumed, + requestId: prepared.preview.requests[0].request_id, + savedSubmission: error.result.savedSubmission, + })); + } + `, ); + const built = await Bun.build({ + entrypoints: [runner], + outdir: bundle, + target: "node", + format: "esm", + packages: "external", + }); + expect(built.success).toBe(true); + const run = async (locale: string) => { + const child = Bun.spawn( + [Bun.which("node")!, built.outputs[0]!.path, locale], + { + stdout: "pipe", + stderr: "pipe", + }, + ); + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]); + expect(exitCode, stderr).toBe(0); + return JSON.parse(stdout) as { + locale: string; + resumed: boolean; + requestId: string; + savedSubmission: string; + }; + }; + const first = await run("en-US"); + expect(first.resumed).toBe(false); + const second = await run("sv-SE"); + expect(second.locale).not.toBe(first.locale); + expect(second.resumed).toBe(true); + expect(second.requestId).toBe(first.requestId); + expect(second.savedSubmission).toBe(first.savedSubmission); + input.evidence.source_data = Object.fromEntries( + Object.entries(input.evidence.source_data).reverse(), + ) as typeof input.evidence.source_data; + await writeFile(f.file, JSON.stringify([input])); + const reordered = await run("en-US"); + expect(reordered.resumed).toBe(true); + expect(reordered.requestId).toBe(first.requestId); + expect(reordered.savedSubmission).toBe(first.savedSubmission); }); for (const format of ["json", "jsonl"] as const) { @@ -1314,12 +1441,54 @@ test.each([0, 1, 2])( expect(parsed.findings[0]!.evidence.details!.repository).toEqual({ id: repository.id, name: null, - url: enterpriseUrl, + url: urls[1]!, }); expect(parsed.findings[0]!.evidence.source_data).toEqual(record); }, ); +test("equivalent inventory rows preserve publication identity when reordered", async () => { + const record = { ...sast, repository: { id: repository.id } }; + const nodes = [ + { + ...inventoryNode("GITHUB"), + repository: { + ...repository, + name: "Example/Project", + url: "https://github.com/Example/Project.git", + }, + }, + { + ...inventoryNode(undefined), + repository: { ...repository, url: repositoryUrl }, + }, + ]; + const payload = { + data: { + sastFindings: { nodes: [record] }, + versionControlResources: { nodes }, + }, + }; + const f = await cloudFixture(payload); + const prepared = await prepareExternalPublication(f.file, f.options, f.deps); + f.state.failReadback = true; + await expect(prepared.publish()).rejects.toBeInstanceOf( + ExternalPublicationError, + ); + nodes.reverse(); + await writeFile(f.file, JSON.stringify(payload)); + const retry = await prepareExternalPublication(f.file, f.options, f.deps); + expect(retry.preview.findings).toEqual(prepared.preview.findings); + expect(retry.preview.findings[0]!.evidence.source_data).toEqual(record); + expect(retry.preview.resumed).toBe(true); + expect(retry.preview.requests[0]!.request_id).toBe( + prepared.preview.requests[0]!.request_id, + ); + f.state.failReadback = false; + expect((await retry.publish()).verified).toBe(1); + expect(f.posts).toHaveLength(1); +}); + test.each([ ["GITHUB", "GITLAB"], ["GITLAB", "GITHUB"], From 77c62081a9b5574931046fa9139b36f24c188cff Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 10 Oct 2026 06:59:31 +0000 Subject: [PATCH 32/43] Preserve staged publication evidence across inventory normalization --- .../src/external-findings-publish.ts | 131 +++++++++++------ sdk/typescript/src/wiz-findings.ts | 43 +++++- sdk/typescript/tests-ts/wiz-findings.test.ts | 133 ++++++++++++++++++ 3 files changed, 264 insertions(+), 43 deletions(-) diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 4182e04b70..7181b47b9c 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -15,7 +15,10 @@ import { codexSecurityStateDirectory, validatePreparedOutputDir, } from "./runtime.js"; -import { readVendorFindings, type VendorFindings } from "./wiz-findings.js"; +import { + readVendorFindingsForPublication, + type VendorFindings, +} from "./wiz-findings.js"; import { repositoryUrlKey, validateImportRequest, @@ -99,6 +102,7 @@ interface SavedSubmission { accountId: string; requests: FindingImportRequest[]; receipts?: FindingImportReceipt[]; + inputProjection?: { inputDigest: string; savedEvidenceDigest: string }; } export interface ExternalPublicationPreview extends VendorFindings { @@ -254,7 +258,8 @@ export async function prepareExternalPublication( ); } progress({ phase: "reading", completed: 0 }); - const parsed = await readVendorFindings(path); + const { current: parsed, legacy: legacyParsed } = + await readVendorFindingsForPublication(path); if (parsed.findings.length === 0) { throw new CodexSecurityError( `No supported findings are ready to publish. ${parsed.excluded.map((item) => `Item ${item.position}: ${item.reason}`).join(" ")}`, @@ -352,14 +357,57 @@ export async function prepareExternalPublication( destination.id, destination.repo_connector_id, source, - parsed.findings, ]; - const key = hash("sha256", canonicalJson(checkpointIdentity)); + const inputDigest = hash("sha256", canonicalJson(parsed.findings)); + function validateSaved(serialized: string, legacy = false): SavedSubmission { + const content = JSON.parse(serialized) as SavedSubmission; + if ( + content.accountId !== credentials.account_id || + !Array.isArray(content.requests) || + content.requests.length === 0 + ) + throw new Error("Saved import account or requests do not match."); + content.requests = content.requests.map(validateImportRequest); + const savedEvidence = canonicalJson( + content.requests.flatMap((batch) => + batch.items.map(({ source_finding_id, evidence }) => ({ + source_finding_id, + evidence, + })), + ), + ); + const matches = + !legacy && content.inputProjection !== undefined + ? content.inputProjection?.inputDigest === inputDigest && + content.inputProjection?.savedEvidenceDigest === + hash("sha256", savedEvidence) + : savedEvidence === + canonicalJson(legacy ? legacyParsed.findings : parsed.findings); + if (!matches) + throw new Error( + "Saved import evidence does not match the selected input.", + ); + for (const submission of content.requests) { + if ( + submission.repository.id !== destination.id || + submission.repository.repo_connector_id !== + destination.repo_connector_id || + canonicalJson(submission.source) !== canonicalJson(source) + ) + throw new Error("Saved import destination does not match."); + } + return content; + } + const key = hash( + "sha256", + canonicalJson([...checkpointIdentity, parsed.findings]), + ); const pendingPath = join(state, `${key}.pending.json`); const legacyKey = hash( "sha256", - canonicalJson(checkpointIdentity, (left, right) => - left.localeCompare(right), + canonicalJson( + [...checkpointIdentity, legacyParsed.findings], + (left, right) => left.localeCompare(right), ), ); if (legacyKey !== key) { @@ -381,12 +429,40 @@ export async function prepareExternalPublication( throw error; }, ); - if (!current) - await rename(legacyPath, pendingPath).catch( - (error: NodeJS.ErrnoException) => { - if (error.code !== "ENOENT") throw error; - }, + if (current) return; + const serialized = await readFile(legacyPath, "utf8").catch( + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return undefined; + throw error; + }, + ); + if (serialized === undefined) return; + const content = validateSaved(serialized, true); + if ( + content.requests.some( + (submission) => + submission.repository.reset_marker !== destination.reset_marker, + ) + ) { + await rm(legacyPath); + throw new CloudImportError( + 409, + "The repository was reset after this submission. The saved request was retired without uploading. Review the destination and run the command again to approve a fresh publication.", ); + } + const savedEvidenceDigest = hash( + "sha256", + canonicalJson(legacyParsed.findings), + ); + if (savedEvidenceDigest === inputDigest) { + await rename(legacyPath, pendingPath); + } else { + // Bind the new input projection to the original immutable evidence, + // so later inventory reordering still resumes these exact requests. + content.inputProjection = { inputDigest, savedEvidenceDigest }; + await writeAtomicJson(pendingPath, content); + await rm(legacyPath); + } }), ); } @@ -394,36 +470,8 @@ export async function prepareExternalPublication( let saved: SavedSubmission | undefined; try { const serialized = await readFile(pendingPath, "utf8"); - const content = JSON.parse(serialized) as SavedSubmission; - if ( - content.accountId !== credentials.account_id || - !Array.isArray(content.requests) || - content.requests.length === 0 - ) - throw new Error("Saved import account or requests do not match."); - content.requests = content.requests.map(validateImportRequest); - if ( - canonicalJson( - content.requests.flatMap((batch) => - batch.items.map(({ source_finding_id, evidence }) => ({ - source_finding_id, - evidence, - })), - ), - ) !== canonicalJson(parsed.findings) - ) { - throw new Error( - "Saved import evidence does not match the selected input.", - ); - } + const content = validateSaved(serialized); for (const submission of content.requests) { - if ( - submission.repository.id !== destination.id || - submission.repository.repo_connector_id !== - destination.repo_connector_id || - canonicalJson(submission.source) !== canonicalJson(source) - ) - throw new Error("Saved import destination does not match."); if (submission.repository.reset_marker !== destination.reset_marker) { // The old request is never sent after reset. A subsequent explicit // invocation prepares a fresh submission and asks for approval again. @@ -547,6 +595,9 @@ export async function prepareExternalPublication( accountId: credentials.account_id, requests, receipts: saved?.receipts, + ...(saved?.inputProjection + ? { inputProjection: saved.inputProjection } + : {}), }; const preview: ExternalPublicationPreview = { ...parsed, diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts index 1924df29fa..6ef2a4913c 100644 --- a/sdk/typescript/src/wiz-findings.ts +++ b/sdk/typescript/src/wiz-findings.ts @@ -146,6 +146,7 @@ type RepositoryMetadata = { type FindingInput = { records: InputRecord[]; repositories: Map; + legacyRepositories?: FindingInput["repositories"]; }; // Wiz's documented inventory platform establishes custom GitHub host semantics. @@ -334,6 +335,7 @@ function repositoryFinding( function records(payload: unknown): FindingInput { const repositories: FindingInput["repositories"] = new Map(); + const legacyRepositories: FindingInput["repositories"] = new Map(); if (Array.isArray(payload)) return { records: payload.map((value) => ({ value })), repositories }; const envelope = object(payload); @@ -403,6 +405,10 @@ function records(payload: unknown): FindingInput { : entry.name, platform: entry.platform ?? previous?.platform ?? null, }); + legacyRepositories.set(entry.id, { + ...entry, + platform: entry.platform ?? previous?.platform ?? null, + }); } for (const entry of entries) { const resolved = repositories.get(entry.id)!; @@ -436,7 +442,17 @@ function records(payload: unknown): FindingInput { ); for (const value of collection["nodes"]) selected.push({ value, kind }); } - if (found) return { records: selected, repositories }; + if (found) + return { + records: selected, + repositories, + ...([...repositories].some(([id, current]) => { + const legacy = legacyRepositories.get(id)!; + return current.url !== legacy.url || current.name !== legacy.name; + }) + ? { legacyRepositories } + : {}), + }; throw new Error( "Unsupported Wiz export. Supply vulnerability finding objects, named sastFindings/secretInstances/iacFindings collections, or normalized JSONL. CSV reports must be converted to the documented normalized JSONL profile.", ); @@ -445,7 +461,25 @@ function records(payload: unknown): FindingInput { export async function readVendorFindings( path: string, ): Promise { - let input: FindingInput; + return normalizeFindings(await readFindingInput(path)); +} + +/** @internal Preserve older inventory projections only for saved publication replay. */ +export async function readVendorFindingsForPublication(path: string): Promise<{ + current: VendorFindings; + legacy: VendorFindings; +}> { + const input = await readFindingInput(path); + const current = normalizeFindings(input); + return { + current, + legacy: input.legacyRepositories + ? normalizeFindings({ ...input, repositories: input.legacyRepositories }) + : current, + }; +} + +async function readFindingInput(path: string): Promise { try { // Wiz's default report export is gzip, sometimes with a .json filename. // Inspect the bytes rather than requiring users to rename their download. @@ -465,13 +499,16 @@ export async function readVendorFindings( .filter((line) => line.trim()) .map((line) => JSON.parse(line)); } - input = records(payload); + return records(payload); } catch (cause) { throw new CodexSecurityError( `Could not read vendor findings as UTF-8 JSON or JSONL: ${cause instanceof Error ? cause.message : String(cause)}`, { cause }, ); } +} + +function normalizeFindings(input: FindingInput): VendorFindings { const result: VendorFindings = { read: input.records.length, findings: [], diff --git a/sdk/typescript/tests-ts/wiz-findings.test.ts b/sdk/typescript/tests-ts/wiz-findings.test.ts index 20e92a7f22..3ceb104992 100644 --- a/sdk/typescript/tests-ts/wiz-findings.test.ts +++ b/sdk/typescript/tests-ts/wiz-findings.test.ts @@ -1489,6 +1489,139 @@ test("equivalent inventory rows preserve publication identity when reordered", a expect(f.posts).toHaveLength(1); }); +test.each([false, true])( + "inventory normalization upgrades saved immutable requests and receipts (reset: %j)", + async (reset) => { + const record = { ...sast, repository: { id: repository.id } }; + const nodes = [ + { + ...inventoryNode("GITHUB"), + repository: { + ...repository, + name: "Example/Project", + url: "https://github.com/Example/Project.git", + }, + }, + { + ...inventoryNode("GITHUB"), + repository: { ...repository, url: repositoryUrl }, + }, + ]; + const payload = { + data: { + sastFindings: { nodes: [record] }, + versionControlResources: { nodes }, + }, + }; + // The earlier mapper used the last inventory row. This single-row export + // stages exactly that historical evidence with real publication receipts. + const f = await cloudFixture({ + data: { ...payload.data, versionControlResources: { nodes: [nodes[1]] } }, + }); + const deps = { + ...f.deps, + environment: { + ...f.environment, + CODEX_SECURITY_CLOUD_BASE_URL: DEFAULT_CLOUD_BASE_URL, + }, + }; + const original = await prepareExternalPublication(f.file, f.options, deps); + f.state.failReadback = true; + let originalPath = ""; + try { + await original.publish(); + } catch (error) { + expect(error).toBeInstanceOf(ExternalPublicationError); + originalPath = (error as ExternalPublicationError).result + .savedSubmission!; + } + const saved = JSON.parse(await readFile(originalPath, "utf8")); + expect(saved.receipts).toHaveLength(1); + const identity = [ + original.preview.accountId, + original.preview.destination.id, + original.preview.destination.repo_connector_id, + original.preview.source, + original.preview.findings, + ]; + // Reproduce the previous checkpoint encoding, before code-unit key ordering. + const oldKey = hash( + "sha256", + JSON.stringify(identity, (_key, child) => + child !== null && typeof child === "object" && !Array.isArray(child) + ? Object.fromEntries( + Object.keys(child) + .sort((left, right) => left.localeCompare(right)) + .map((key) => [key, child[key]]), + ) + : child, + ), + ); + const oldPath = join( + f.environment.CODEX_SECURITY_STATE_DIR, + "external-finding-publications", + `${oldKey}.pending.json`, + ); + if (oldPath !== originalPath) await rename(originalPath, oldPath); + await writeFile(f.file, JSON.stringify(payload)); + if (reset) { + f.destination.reset_marker = "generation-2"; + await expect( + prepareExternalPublication(f.file, f.options, deps), + ).rejects.toThrow("saved request was retired"); + await expect(readFile(oldPath, "utf8")).rejects.toMatchObject({ + code: "ENOENT", + }); + expect(f.posts).toHaveLength(1); + return; + } + const upgraded = await prepareExternalPublication(f.file, f.options, deps); + expect(upgraded.preview.resumed).toBe(true); + expect(upgraded.preview.requests).toEqual(original.preview.requests); + expect(upgraded.preview.findings).not.toEqual(original.preview.findings); + let migratedPath = ""; + try { + await upgraded.publish(); + } catch (error) { + expect(error).toBeInstanceOf(ExternalPublicationError); + migratedPath = (error as ExternalPublicationError).result + .savedSubmission!; + } + const migrated = await readFile(migratedPath, "utf8"); + const migratedData = JSON.parse(migrated); + expect(migratedData.requests).toEqual(saved.requests); + expect(migratedData.receipts).toEqual(saved.receipts); + nodes.reverse(); + await writeFile(f.file, JSON.stringify(payload)); + const reordered = await prepareExternalPublication(f.file, f.options, deps); + expect(reordered.preview.resumed).toBe(true); + expect(reordered.preview.requests).toEqual(original.preview.requests); + migratedData.requests[0].items[0].evidence.description = + "Modified saved evidence"; + await writeFile(migratedPath, JSON.stringify(migratedData)); + await expect( + prepareExternalPublication(f.file, f.options, deps), + ).rejects.toThrow("Saved import evidence"); + await writeFile(migratedPath, migrated); + const incomplete = JSON.parse(migrated); + delete incomplete.inputProjection.savedEvidenceDigest; + await writeFile(migratedPath, JSON.stringify(incomplete)); + await expect( + prepareExternalPublication(f.file, f.options, deps), + ).rejects.toThrow("Saved import evidence"); + await writeFile(migratedPath, migrated); + const newer = f.reports.get("sast:occurrence-1")!; + newer.version += 1; + newer.evidence.description = "Newer Cloud evidence"; + f.state.failReadback = false; + expect((await reordered.publish()).verified).toBe(1); + expect(f.posts).toHaveLength(1); + expect(f.reports.get("sast:occurrence-1")!.evidence.description).toBe( + "Newer Cloud evidence", + ); + }, +); + test.each([ ["GITHUB", "GITLAB"], ["GITLAB", "GITHUB"], From e3d9054925e3da9147c50a9b2b44bfbf75c7d30f Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 10 Oct 2026 07:14:22 +0000 Subject: [PATCH 33/43] Keep migrated publication retries bound to their saved selection --- .../src/external-findings-publish.ts | 88 +++++--- sdk/typescript/tests-ts/wiz-findings.test.ts | 195 +++++++++++++++++- 2 files changed, 247 insertions(+), 36 deletions(-) diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 7181b47b9c..5226b9b3da 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -166,6 +166,19 @@ function sameSubmission( ); } +function sameFindingSelection( + left: VendorFindings["findings"], + right: VendorFindings["findings"], +): boolean { + return ( + left.length === right.length && + left.every( + (finding, index) => + finding.source_finding_id === right[index]!.source_finding_id, + ) + ); +} + async function writeAtomicJson(path: string, value: unknown): Promise { const temporary = `${path}.${randomUUID()}.tmp`; try { @@ -368,21 +381,22 @@ export async function prepareExternalPublication( ) throw new Error("Saved import account or requests do not match."); content.requests = content.requests.map(validateImportRequest); - const savedEvidence = canonicalJson( - content.requests.flatMap((batch) => - batch.items.map(({ source_finding_id, evidence }) => ({ - source_finding_id, - evidence, - })), - ), + const savedFindings = content.requests.flatMap((batch) => + batch.items.map(({ source_finding_id, evidence }) => ({ + source_finding_id, + evidence, + })), ); + const savedEvidence = canonicalJson(savedFindings); + const expected = legacy ? legacyParsed.findings : parsed.findings; const matches = - !legacy && content.inputProjection !== undefined - ? content.inputProjection?.inputDigest === inputDigest && + content.inputProjection !== undefined + ? sameFindingSelection(savedFindings, expected) && + content.inputProjection?.inputDigest === + hash("sha256", canonicalJson(expected)) && content.inputProjection?.savedEvidenceDigest === hash("sha256", savedEvidence) - : savedEvidence === - canonicalJson(legacy ? legacyParsed.findings : parsed.findings); + : savedEvidence === canonicalJson(expected); if (!matches) throw new Error( "Saved import evidence does not match the selected input.", @@ -410,56 +424,70 @@ export async function prepareExternalPublication( (left, right) => left.localeCompare(right), ), ); + let recovered: SavedSubmission | undefined; if (legacyKey !== key) { const legacyPath = join(state, `${legacyKey}.pending.json`); - const legacy = await lstat(legacyPath).catch( + const captured = await readFile(legacyPath, "utf8").catch( (error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return undefined; throw error; }, ); - if (legacy) { + if (captured !== undefined) { // Preserve requests staged by the earlier locale-based filename format. // Hold both publication locks while moving the original checkpoint. await withImportLock(state, key, dependencies.signal, async () => withImportLock(state, legacyKey, dependencies.signal, async () => { - const current = await lstat(pendingPath).catch( + const current = await readFile(pendingPath, "utf8").catch( (error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return undefined; throw error; }, ); - if (current) return; + if (current !== undefined) { + recovered = validateSaved(current); + return; + } const serialized = await readFile(legacyPath, "utf8").catch( (error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return undefined; throw error; }, ); - if (serialized === undefined) return; - const content = validateSaved(serialized, true); + // A competing resumer may have completed and removed both paths while + // this caller waited. Retain its observed immutable request and receipts. + const content = validateSaved(serialized ?? captured, true); + if (!sameFindingSelection(legacyParsed.findings, parsed.findings)) + throw new CodexSecurityError( + "The saved import selects different findings from this input. Resume the original selection before publishing this selection.", + ); if ( content.requests.some( (submission) => submission.repository.reset_marker !== destination.reset_marker, ) ) { - await rm(legacyPath); + if (serialized !== undefined) await rm(legacyPath); throw new CloudImportError( 409, "The repository was reset after this submission. The saved request was retired without uploading. Review the destination and run the command again to approve a fresh publication.", ); } - const savedEvidenceDigest = hash( - "sha256", - canonicalJson(legacyParsed.findings), - ); - if (savedEvidenceDigest === inputDigest) { + const savedEvidenceDigest = + content.inputProjection?.savedEvidenceDigest ?? + hash("sha256", canonicalJson(legacyParsed.findings)); + const rebindProjection = + content.inputProjection !== undefined || + savedEvidenceDigest !== inputDigest; + if (rebindProjection) + content.inputProjection = { inputDigest, savedEvidenceDigest }; + recovered = content; + if (serialized === undefined) return; + if (!rebindProjection) { await rename(legacyPath, pendingPath); } else { // Bind the new input projection to the original immutable evidence, // so later inventory reordering still resumes these exact requests. - content.inputProjection = { inputDigest, savedEvidenceDigest }; await writeAtomicJson(pendingPath, content); await rm(legacyPath); } @@ -467,10 +495,15 @@ export async function prepareExternalPublication( ); } } - let saved: SavedSubmission | undefined; + let saved: SavedSubmission | undefined = recovered; try { const serialized = await readFile(pendingPath, "utf8"); - const content = validateSaved(serialized); + saved = validateSaved(serialized); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (saved) { + const content = saved; for (const submission of content.requests) { if (submission.repository.reset_marker !== destination.reset_marker) { // The old request is never sent after reset. A subsequent explicit @@ -494,9 +527,6 @@ export async function prepareExternalPublication( ); } } - saved = content; - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } let requests = saved?.requests; if (!requests) { diff --git a/sdk/typescript/tests-ts/wiz-findings.test.ts b/sdk/typescript/tests-ts/wiz-findings.test.ts index 3ceb104992..e305292363 100644 --- a/sdk/typescript/tests-ts/wiz-findings.test.ts +++ b/sdk/typescript/tests-ts/wiz-findings.test.ts @@ -1489,10 +1489,17 @@ test("equivalent inventory rows preserve publication identity when reordered", a expect(f.posts).toHaveLength(1); }); -test.each([false, true])( - "inventory normalization upgrades saved immutable requests and receipts (reset: %j)", - async (reset) => { +test.each(["resume", "reset", "coverage", "rebind"])( + "inventory normalization upgrades saved immutable requests and receipts (%s)", + async (scenario) => { const record = { ...sast, repository: { id: repository.id } }; + const selected = + scenario === "coverage" + ? [ + { ...record, id: "already-ready", repository }, + { ...record, id: "newly-ready" }, + ] + : [record]; const nodes = [ { ...inventoryNode("GITHUB"), @@ -1504,12 +1511,16 @@ test.each([false, true])( }, { ...inventoryNode("GITHUB"), - repository: { ...repository, url: repositoryUrl }, + repository: { + ...repository, + name: scenario === "coverage" ? "z".repeat(513) : repository.name, + url: repositoryUrl, + }, }, ]; const payload = { data: { - sastFindings: { nodes: [record] }, + sastFindings: { nodes: selected }, versionControlResources: { nodes }, }, }; @@ -1564,7 +1575,16 @@ test.each([false, true])( ); if (oldPath !== originalPath) await rename(originalPath, oldPath); await writeFile(f.file, JSON.stringify(payload)); - if (reset) { + if (scenario === "coverage") { + expect((await readVendorFindings(f.file)).findings).toHaveLength(2); + await expect( + prepareExternalPublication(f.file, f.options, deps), + ).rejects.toThrow("selects different findings"); + expect(JSON.parse(await readFile(oldPath, "utf8"))).toEqual(saved); + expect(f.posts).toHaveLength(1); + return; + } + if (scenario === "reset") { f.destination.reset_marker = "generation-2"; await expect( prepareExternalPublication(f.file, f.options, deps), @@ -1610,11 +1630,25 @@ test.each([false, true])( prepareExternalPublication(f.file, f.options, deps), ).rejects.toThrow("Saved import evidence"); await writeFile(migratedPath, migrated); + let replay = reordered; + if (scenario === "rebind") { + nodes.unshift({ + ...nodes[1]!, + repository: { ...nodes[1]!.repository, name: "EXAMPLE/PROJECT" }, + }); + await writeFile(f.file, JSON.stringify(payload)); + replay = await prepareExternalPublication(f.file, f.options, deps); + expect(replay.preview.resumed).toBe(true); + expect(replay.preview.requests).toEqual(original.preview.requests); + expect( + replay.preview.findings[0]!.evidence.details!.repository.name, + ).toBe("EXAMPLE/PROJECT"); + } const newer = f.reports.get("sast:occurrence-1")!; newer.version += 1; newer.evidence.description = "Newer Cloud evidence"; f.state.failReadback = false; - expect((await reordered.publish()).verified).toBe(1); + expect((await replay.publish()).verified).toBe(1); expect(f.posts).toHaveLength(1); expect(f.reports.get("sast:occurrence-1")!.evidence.description).toBe( "Newer Cloud evidence", @@ -1680,3 +1714,150 @@ test("GITHUB inventory provenance does not permit a different repository URL", a }), ).rejects.toThrow("conflicting URLs"); }); +test("concurrent legacy inventory resumers retain the acknowledged request after migration finishes", async () => { + const name = + "concurrent legacy inventory resumers retain the acknowledged request after migration finishes"; + const { runTestInSubprocess } = await import("./support/test-subprocess.js"); + if (runTestInSubprocess(import.meta.filename, name)) return; + const fs = await import("node:fs/promises"); + const { spyOn } = await import("bun:test"); + const record = { ...sast, repository: { id: repository.id } }; + const nodes = [ + { + ...inventoryNode("GITHUB"), + repository: { + ...repository, + name: "Example/Project", + url: "https://github.com/Example/Project.git", + }, + }, + { + ...inventoryNode("GITHUB"), + repository: { ...repository, url: repositoryUrl }, + }, + ]; + const payload = { + data: { + sastFindings: { nodes: [record] }, + versionControlResources: { nodes }, + }, + }; + const f = await cloudFixture({ + data: { ...payload.data, versionControlResources: { nodes: [nodes[1]] } }, + }); + const deps = { + ...f.deps, + environment: { + ...f.environment, + CODEX_SECURITY_CLOUD_BASE_URL: DEFAULT_CLOUD_BASE_URL, + }, + }; + const original = await prepareExternalPublication(f.file, f.options, deps); + f.state.failReadback = true; + let originalPath = ""; + try { + await original.publish(); + } catch (error) { + expect(error).toBeInstanceOf(ExternalPublicationError); + originalPath = (error as ExternalPublicationError).result.savedSubmission!; + } + const saved = JSON.parse(await readFile(originalPath, "utf8")); + expect(saved.receipts).toHaveLength(1); + const identity = [ + original.preview.accountId, + original.preview.destination.id, + original.preview.destination.repo_connector_id, + original.preview.source, + ]; + const encode = (value: unknown, legacy: boolean) => + JSON.stringify(value, (_key, child) => + child !== null && typeof child === "object" && !Array.isArray(child) + ? Object.fromEntries( + Object.keys(child) + .sort( + legacy ? (left, right) => left.localeCompare(right) : undefined, + ) + .map((key) => [key, child[key]]), + ) + : child, + ); + const directory = join( + f.environment.CODEX_SECURITY_STATE_DIR, + "external-finding-publications", + ); + const oldKey = hash( + "sha256", + encode([...identity, original.preview.findings], true), + ); + const oldPath = join(directory, `${oldKey}.pending.json`); + if (oldPath !== originalPath) await rename(originalPath, oldPath); + await writeFile(f.file, JSON.stringify(payload)); + const current = await readVendorFindings(f.file); + const key = hash("sha256", encode([...identity, current.findings], false)); + const pendingPath = join(directory, `${key}.pending.json`); + const lockPath = join(directory, `${key}.lock.sqlite`); + expect(pendingPath).not.toBe(oldPath); + const firstAtLock = Promise.withResolvers(); + const secondAtLock = Promise.withResolvers(); + const releaseSecond = Promise.withResolvers(); + const originalLstat = fs.lstat; + let lockEntries = 0; + const metadata = spyOn(fs, "lstat").mockImplementation((async ( + ...args: Parameters + ) => { + if (String(args[0]) === lockPath) { + lockEntries += 1; + if (lockEntries === 1) { + firstAtLock.resolve(); + await secondAtLock.promise; + } else if (lockEntries === 2) { + secondAtLock.resolve(); + await releaseSecond.promise; + } + } + return await originalLstat(...args); + }) as typeof fs.lstat); + let first: ReturnType | undefined; + let second: ReturnType | undefined; + try { + first = prepareExternalPublication(f.file, f.options, deps); + await firstAtLock.promise; + second = prepareExternalPublication(f.file, f.options, deps); + await secondAtLock.promise; + const leading = await first; + expect(leading.preview.resumed).toBe(true); + expect(leading.preview.requests).toEqual(saved.requests); + expect(JSON.parse(await readFile(pendingPath, "utf8")).receipts).toEqual( + saved.receipts, + ); + f.state.failReadback = false; + expect((await leading.publish()).verified).toBe(1); + await expect(readFile(pendingPath, "utf8")).rejects.toMatchObject({ + code: "ENOENT", + }); + await expect(readFile(oldPath, "utf8")).rejects.toMatchObject({ + code: "ENOENT", + }); + const newer = structuredClone(f.reports.get("sast:occurrence-1")!); + newer.version += 1; + newer.evidence.description = + "Newer Cloud evidence after the first resumer completed"; + f.reports.set("sast:occurrence-1", newer); + releaseSecond.resolve(); + const trailing = await second; + const result = await trailing.publish(); + expect(f.posts).toHaveLength(1); + expect(trailing.preview.resumed).toBe(true); + expect(trailing.preview.requests).toEqual(saved.requests); + expect(result.receipts).toEqual(saved.receipts); + expect(result.verified).toBe(1); + expect(f.reports.get("sast:occurrence-1")).toEqual(newer); + } finally { + secondAtLock.resolve(); + releaseSecond.resolve(); + await Promise.allSettled( + [first, second].filter((value) => value !== undefined), + ); + metadata.mockRestore(); + } +}); From 17e95913789272f559e52e02b7544084e1d7b874 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 10 Oct 2026 07:46:51 +0000 Subject: [PATCH 34/43] fix(import): stabilize outstanding publication identity --- .../src/external-findings-publish.ts | 75 ++++++++++---- sdk/typescript/src/wiz-findings.ts | 45 ++++++++- sdk/typescript/tests-ts/wiz-findings.test.ts | 97 ++++++++++++++++--- 3 files changed, 176 insertions(+), 41 deletions(-) diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 5226b9b3da..d25570c379 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -166,6 +166,17 @@ function sameSubmission( ); } +function submissionFindings( + requests: readonly FindingImportRequest[], +): VendorFindings["findings"] { + return requests.flatMap((batch) => + batch.items.map(({ source_finding_id, evidence }) => ({ + source_finding_id, + evidence, + })), + ); +} + function sameFindingSelection( left: VendorFindings["findings"], right: VendorFindings["findings"], @@ -271,8 +282,12 @@ export async function prepareExternalPublication( ); } progress({ phase: "reading", completed: 0 }); - const { current: parsed, legacy: legacyParsed } = - await readVendorFindingsForPublication(path); + const { + current: parsed, + legacy: legacyParsed, + currentIdentity, + legacyIdentity, + } = await readVendorFindingsForPublication(path); if (parsed.findings.length === 0) { throw new CodexSecurityError( `No supported findings are ready to publish. ${parsed.excluded.map((item) => `Item ${item.position}: ${item.reason}`).join(" ")}`, @@ -371,7 +386,8 @@ export async function prepareExternalPublication( destination.repo_connector_id, source, ]; - const inputDigest = hash("sha256", canonicalJson(parsed.findings)); + const inputDigest = hash("sha256", canonicalJson(currentIdentity)); + const legacyInputDigest = hash("sha256", canonicalJson(legacyIdentity)); function validateSaved(serialized: string, legacy = false): SavedSubmission { const content = JSON.parse(serialized) as SavedSubmission; if ( @@ -381,19 +397,14 @@ export async function prepareExternalPublication( ) throw new Error("Saved import account or requests do not match."); content.requests = content.requests.map(validateImportRequest); - const savedFindings = content.requests.flatMap((batch) => - batch.items.map(({ source_finding_id, evidence }) => ({ - source_finding_id, - evidence, - })), - ); + const savedFindings = submissionFindings(content.requests); const savedEvidence = canonicalJson(savedFindings); const expected = legacy ? legacyParsed.findings : parsed.findings; const matches = content.inputProjection !== undefined ? sameFindingSelection(savedFindings, expected) && content.inputProjection?.inputDigest === - hash("sha256", canonicalJson(expected)) && + (legacy ? legacyInputDigest : inputDigest) && content.inputProjection?.savedEvidenceDigest === hash("sha256", savedEvidence) : savedEvidence === canonicalJson(expected); @@ -414,10 +425,10 @@ export async function prepareExternalPublication( } const key = hash( "sha256", - canonicalJson([...checkpointIdentity, parsed.findings]), + canonicalJson([...checkpointIdentity, currentIdentity]), ); const pendingPath = join(state, `${key}.pending.json`); - const legacyKey = hash( + const historicalKey = hash( "sha256", canonicalJson( [...checkpointIdentity, legacyParsed.findings], @@ -425,7 +436,11 @@ export async function prepareExternalPublication( ), ); let recovered: SavedSubmission | undefined; - if (legacyKey !== key) { + for (const legacyKey of new Set([ + hash("sha256", canonicalJson([...checkpointIdentity, legacyIdentity])), + historicalKey, + ])) { + if (legacyKey === key) continue; const legacyPath = join(state, `${legacyKey}.pending.json`); const captured = await readFile(legacyPath, "utf8").catch( (error: NodeJS.ErrnoException) => { @@ -444,16 +459,32 @@ export async function prepareExternalPublication( throw error; }, ); - if (current !== undefined) { - recovered = validateSaved(current); - return; - } const serialized = await readFile(legacyPath, "utf8").catch( (error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return undefined; throw error; }, ); + if (current !== undefined) { + recovered = validateSaved(current); + if (serialized !== undefined) { + // A crash may leave both copies after installing the canonical file. + // Retire only an identical immutable submission, never another request. + let duplicate: SavedSubmission | null = null; + try { + duplicate = JSON.parse(serialized) as SavedSubmission | null; + } catch (error) { + if (!(error instanceof SyntaxError)) throw error; + } + if ( + duplicate !== null && + typeof duplicate === "object" && + sameSubmission(duplicate, recovered) + ) + await rm(legacyPath); + } + return; + } // A competing resumer may have completed and removed both paths while // this caller waited. Retain its observed immutable request and receipts. const content = validateSaved(serialized ?? captured, true); @@ -625,9 +656,13 @@ export async function prepareExternalPublication( accountId: credentials.account_id, requests, receipts: saved?.receipts, - ...(saved?.inputProjection - ? { inputProjection: saved.inputProjection } - : {}), + inputProjection: saved?.inputProjection ?? { + inputDigest, + savedEvidenceDigest: hash( + "sha256", + canonicalJson(submissionFindings(requests)), + ), + }, }; const preview: ExternalPublicationPreview = { ...parsed, diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts index 6ef2a4913c..9b51a21621 100644 --- a/sdk/typescript/src/wiz-findings.ts +++ b/sdk/typescript/src/wiz-findings.ts @@ -468,14 +468,45 @@ export async function readVendorFindings( export async function readVendorFindingsForPublication(path: string): Promise<{ current: VendorFindings; legacy: VendorFindings; + currentIdentity: unknown[]; + legacyIdentity: unknown[]; }> { const input = await readFindingInput(path); - const current = normalizeFindings(input); + const normalized = new WeakSet(); + const current = normalizeFindings(input, normalized); + const legacy = input.legacyRepositories + ? normalizeFindings( + { ...input, repositories: input.legacyRepositories }, + normalized, + ) + : current; + const identity = (parsed: VendorFindings) => + parsed.findings.map((finding) => { + if (normalized.has(finding)) return finding; + const repository = finding.evidence.details?.repository; + return { + source_finding_id: finding.source_finding_id, + evidence: { + // Raw vendor records retain every original branch/ref and source field. + // Inventory display aliases must not change an outstanding request's identity. + source_data: finding.evidence.source_data, + repository: repository + ? { + id: repository.id, + url: inventoryRepositoryUrlKey( + repository.url, + input.repositories.get(repository.id ?? "")?.platform, + ), + } + : null, + }, + }; + }); return { current, - legacy: input.legacyRepositories - ? normalizeFindings({ ...input, repositories: input.legacyRepositories }) - : current, + legacy, + currentIdentity: identity(current), + legacyIdentity: identity(legacy), }; } @@ -508,7 +539,10 @@ async function readFindingInput(path: string): Promise { } } -function normalizeFindings(input: FindingInput): VendorFindings { +function normalizeFindings( + input: FindingInput, + normalized?: WeakSet, +): VendorFindings { const result: VendorFindings = { read: input.records.length, findings: [], @@ -559,6 +593,7 @@ function normalizeFindings(input: FindingInput): VendorFindings { } identities.add(finding.source_finding_id); result.findings.push(finding); + if ("source_finding_id" in record) normalized?.add(finding); } catch (error) { if (error instanceof CodexSecurityError) throw error; result.excluded.push({ diff --git a/sdk/typescript/tests-ts/wiz-findings.test.ts b/sdk/typescript/tests-ts/wiz-findings.test.ts index e305292363..267589d9c6 100644 --- a/sdk/typescript/tests-ts/wiz-findings.test.ts +++ b/sdk/typescript/tests-ts/wiz-findings.test.ts @@ -24,7 +24,10 @@ import type { FindingImportRequest, SourceReport, } from "../src/external-import-models.js"; -import { readVendorFindings } from "../src/wiz-findings.js"; +import { + readVendorFindings, + readVendorFindingsForPublication, +} from "../src/wiz-findings.js"; import { dependencies } from "./cli-fixtures.js"; import { createCliTest } from "./support/cli-run.js"; import roundtripFixtures from "./fixtures/wiz-details-roundtrip.json" with { type: "json" }; @@ -1010,9 +1013,9 @@ test("Node reads a large complete named collection without a function argument l }); }); -test.each([false, true])( - "Unicode readback resumes historical checkpoints (canonical present: %j)", - async (keepCanonical) => { +test.each(["migrate", "distinct", "duplicate"])( + "Unicode readback resumes historical checkpoints (%s)", + async (scenario) => { const input = { source_finding_id: "unicode-evidence", evidence: { @@ -1039,7 +1042,10 @@ test.each([false, true])( failure = error as ExternalPublicationError; } const pendingPath = failure!.result.savedSubmission!; - const saved = await readFile(pendingPath, "utf8"); + const historical = JSON.parse(await readFile(pendingPath, "utf8")); + delete historical.inputProjection; + const saved = JSON.stringify(historical); + await writeFile(pendingPath, saved); // Frozen from the original publisher's production checkpoint format. const historicalPath = join( f.environment.CODEX_SECURITY_STATE_DIR, @@ -1047,10 +1053,12 @@ test.each([false, true])( "ddeb8ec0be8765f89ef5aeb759dde7753d7ff72f8b8bb25f89a591b045ad40ae.pending.json", ); expect(pendingPath).not.toBe(historicalPath); - if (keepCanonical) { + if (scenario !== "migrate") { const legacy = JSON.parse(saved); - legacy.requests[0].request_id = "synthetic-older-request"; - legacy.receipts = []; + if (scenario === "distinct") { + legacy.requests[0].request_id = "synthetic-older-request"; + legacy.receipts = []; + } await writeFile(historicalPath, JSON.stringify(legacy)); } else { await rename(pendingPath, historicalPath); @@ -1074,12 +1082,17 @@ test.each([false, true])( ), ) ).includes(historicalPath.split(/[\\/]/u).at(-1)!), - ).toBe(keepCanonical); + ).toBe(scenario === "distinct"); expect((await retry.publish()).verified).toBe(1); expect(f.posts).toHaveLength(1); expect(f.posts[0]!.items[0]!.evidence.source_data).toEqual( input.evidence.source_data, ); + if (scenario === "duplicate") + expect( + (await prepareExternalPublication(f.file, f.options, deps)).preview + .resumed, + ).toBe(false); }, ); @@ -1489,17 +1502,67 @@ test("equivalent inventory rows preserve publication identity when reordered", a expect(f.posts).toHaveLength(1); }); -test.each(["resume", "reset", "coverage", "rebind"])( +test.each(["raw", "normalized"])( + "publication identity retains explicit branch changes (%s)", + async (kind) => { + const record = structuredClone(sast); + const normalized = { + source_finding_id: "explicit-branch", + evidence: { + title: "Synthetic finding", + severity: "high", + branch: "main", + source_data: { id: "unchanged-source" }, + }, + }; + const input = + kind === "raw" ? envelope("sastFindings", [record]) : [normalized]; + const f = await cloudFixture(input); + const original = await prepareExternalPublication( + f.file, + f.options, + f.deps, + ); + f.state.failReadback = true; + await expect(original.publish()).rejects.toBeInstanceOf( + ExternalPublicationError, + ); + if (kind === "raw") + record.repositoryBranch.name = "example/project/changed"; + else normalized.evidence.branch = "changed"; + await writeFile(f.file, JSON.stringify(input)); + const changed = await prepareExternalPublication(f.file, f.options, f.deps); + expect(changed.preview.resumed).toBe(false); + expect(changed.preview.requests[0]!.request_id).not.toBe( + original.preview.requests[0]!.request_id, + ); + expect(changed.preview.findings[0]!.evidence.branch).toBe("changed"); + }, +); + +test.each(["resume", "reset", "coverage", "rebind", "prepend", "excluded"])( "inventory normalization upgrades saved immutable requests and receipts (%s)", async (scenario) => { - const record = { ...sast, repository: { id: repository.id } }; + const record = { + ...sast, + vendorMetadata: { z: 1, ä: 2 }, + repository: { id: repository.id }, + }; const selected = scenario === "coverage" ? [ { ...record, id: "already-ready", repository }, { ...record, id: "newly-ready" }, ] - : [record]; + : scenario === "excluded" + ? [ + record, + { + source_finding_id: "sast:occurrence-1", + evidence: { title: "Excluded input", severity: "invalid" }, + }, + ] + : [record]; const nodes = [ { ...inventoryNode("GITHUB"), @@ -1547,6 +1610,8 @@ test.each(["resume", "reset", "coverage", "rebind"])( .savedSubmission!; } const saved = JSON.parse(await readFile(originalPath, "utf8")); + delete saved.inputProjection; + await writeFile(originalPath, JSON.stringify(saved)); expect(saved.receipts).toHaveLength(1); const identity = [ original.preview.accountId, @@ -1611,7 +1676,7 @@ test.each(["resume", "reset", "coverage", "rebind"])( const migratedData = JSON.parse(migrated); expect(migratedData.requests).toEqual(saved.requests); expect(migratedData.receipts).toEqual(saved.receipts); - nodes.reverse(); + if (scenario !== "prepend" && scenario !== "excluded") nodes.reverse(); await writeFile(f.file, JSON.stringify(payload)); const reordered = await prepareExternalPublication(f.file, f.options, deps); expect(reordered.preview.resumed).toBe(true); @@ -1631,7 +1696,7 @@ test.each(["resume", "reset", "coverage", "rebind"])( ).rejects.toThrow("Saved import evidence"); await writeFile(migratedPath, migrated); let replay = reordered; - if (scenario === "rebind") { + if (["rebind", "prepend", "excluded"].includes(scenario)) { nodes.unshift({ ...nodes[1]!, repository: { ...nodes[1]!.repository, name: "EXAMPLE/PROJECT" }, @@ -1792,8 +1857,8 @@ test("concurrent legacy inventory resumers retain the acknowledged request after const oldPath = join(directory, `${oldKey}.pending.json`); if (oldPath !== originalPath) await rename(originalPath, oldPath); await writeFile(f.file, JSON.stringify(payload)); - const current = await readVendorFindings(f.file); - const key = hash("sha256", encode([...identity, current.findings], false)); + const { currentIdentity } = await readVendorFindingsForPublication(f.file); + const key = hash("sha256", encode([...identity, currentIdentity], false)); const pendingPath = join(directory, `${key}.pending.json`); const lockPath = join(directory, `${key}.lock.sqlite`); expect(pendingPath).not.toBe(oldPath); From cbb9f7769ef46621fe6a6c4dfca6d9fb01765f48 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sat, 10 Oct 2026 07:57:59 +0000 Subject: [PATCH 35/43] fix(import): preserve inventory qualifiers during replay --- sdk/typescript/src/wiz-findings.ts | 53 ++++++-- sdk/typescript/tests-ts/wiz-findings.test.ts | 131 ++++++++++++++++++- 2 files changed, 166 insertions(+), 18 deletions(-) diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts index 9b51a21621..0de8543db9 100644 --- a/sdk/typescript/src/wiz-findings.ts +++ b/sdk/typescript/src/wiz-findings.ts @@ -145,7 +145,10 @@ type RepositoryMetadata = { type FindingInput = { records: InputRecord[]; - repositories: Map; + repositories: Map< + string, + RepositoryMetadata & { platform: string | null; names?: string[] } + >; legacyRepositories?: FindingInput["repositories"]; }; @@ -190,12 +193,20 @@ function sourceRepository( function sourceBranch( branch: Record | undefined, repository: RepositoryMetadata, + inventoryNames: readonly string[] = [], ): string | null { const name = text(branch?.["name"]); if (!name) return null; - const prefix = repository.name ? repository.name + "/" : null; - if (prefix && name.startsWith(prefix)) - return name.slice(prefix.length) || null; + const qualifiers = [ + ...new Set([ + ...(repository.name ? [repository.name] : []), + ...inventoryNames, + ]), + ].sort((left, right) => right.length - left.length); + for (const qualifier of qualifiers) { + const prefix = qualifier + "/"; + if (name.startsWith(prefix)) return name.slice(prefix.length) || null; + } // URL-only metadata still identifies Wiz's repository-qualified branch name. // Supported GitHub destinations use case-insensitive repository qualifiers. @@ -241,12 +252,11 @@ function repositoryFinding( throw new Error( "Only repository-branch secret findings can be published to a repository. Workload secrets require a resource-scoped import and are not supported.", ); - const repository = sourceRepository( + const suppliedRepository = kind === "secret" ? object(object(resource?.["typedProperties"])?.["repository"]) - : object(record["repository"]), - repositories, - ); + : object(record["repository"]); + const repository = sourceRepository(suppliedRepository, repositories); const branch = kind === "secret" ? resource @@ -318,7 +328,13 @@ function repositoryFinding( description: text(record["description"]) ?? text(rule?.["description"]), url: text(record["wizUrl"]) ?? text(record["portalUrl"]), locations: path ? [{ path, line: line ?? null }] : [], - branch: sourceBranch(branch, repository), + branch: sourceBranch( + branch, + repository, + text(suppliedRepository?.["name"]) + ? [] + : repositories.get(repository.id ?? "")?.names, + ), code_revision: null, source_scan_id: null, source_updated_at: @@ -396,6 +412,12 @@ function records(payload: unknown): FindingInput { ); repositories.set(entry.id, { ...entry, + names: [ + ...new Set([ + ...(previous?.names ?? []), + ...(entry.name ? [entry.name] : []), + ]), + ], // Equivalent inventory aliases must produce the same evidence when the // export reorders its rows. Keep a stable supplied URL and nonempty name. url: previous && previous.url < entry.url ? previous.url : entry.url, @@ -448,7 +470,11 @@ function records(payload: unknown): FindingInput { repositories, ...([...repositories].some(([id, current]) => { const legacy = legacyRepositories.get(id)!; - return current.url !== legacy.url || current.name !== legacy.name; + return ( + current.url !== legacy.url || + current.name !== legacy.name || + current.names?.some((name) => name !== legacy.name) + ); }) ? { legacyRepositories } : {}), @@ -493,10 +519,9 @@ export async function readVendorFindingsForPublication(path: string): Promise<{ repository: repository ? { id: repository.id, - url: inventoryRepositoryUrlKey( - repository.url, - input.repositories.get(repository.id ?? "")?.platform, - ), + // Publication targets use Cloud's GitHub URL semantics. Input + // joins above still require provenance for custom-host folding. + url: repositoryUrlKey(repository.url).toLowerCase(), } : null, }, diff --git a/sdk/typescript/tests-ts/wiz-findings.test.ts b/sdk/typescript/tests-ts/wiz-findings.test.ts index 267589d9c6..fac663ee23 100644 --- a/sdk/typescript/tests-ts/wiz-findings.test.ts +++ b/sdk/typescript/tests-ts/wiz-findings.test.ts @@ -1337,6 +1337,104 @@ function inventoryNode(platform: string | undefined, url = enterpriseUrl) { }; } +test.each([false, true])( + "publication checkpoints survive optional inventory platform changes (%j)", + async (withHint) => { + const inventory = inventoryNode( + withHint ? "GITHUB" : undefined, + "https://github.example.test/Example/Project", + ); + const payload = { + data: { + sastFindings: { + nodes: [{ ...sast, repository: { id: repository.id } }], + }, + versionControlResources: { nodes: [inventory] }, + }, + }; + const f = await cloudFixture(payload); + f.destination.url = enterpriseUrl; + const options = { ...f.options, repository: f.destination.id }; + const original = await prepareExternalPublication(f.file, options, f.deps); + f.state.failReadback = true; + await expect(original.publish()).rejects.toBeInstanceOf( + ExternalPublicationError, + ); + inventory.platform = withHint ? undefined : "GITHUB"; + await writeFile(f.file, JSON.stringify(payload)); + const retry = await prepareExternalPublication(f.file, options, f.deps); + expect(retry.preview.findings).toEqual(original.preview.findings); + expect(retry.preview.resumed).toBe(true); + expect(retry.preview.requests).toEqual(original.preview.requests); + const newer = f.reports.get("sast:occurrence-1")!; + newer.version += 1; + newer.evidence.description = "Newer Cloud evidence"; + f.state.failReadback = false; + expect((await retry.publish()).verified).toBe(1); + expect(f.posts).toHaveLength(1); + expect(newer.evidence.description).toBe("Newer Cloud evidence"); + }, +); + +test.each(["project", "project/component"])( + "inventory aliases preserve branch qualifier %s in either row order", + async (alias) => { + const record = { + ...sast, + repository: { id: repository.id }, + repositoryBranch: { + ...sast.repositoryBranch, + name: `${alias}/Feature/Parser`, + }, + }; + const nodes = ["example/project", "project", alias].map((name) => ({ + ...inventoryNode("GITHUB", repositoryUrl), + repository: { ...repository, url: repositoryUrl, name }, + })); + for (const selected of [nodes, [...nodes].reverse()]) { + const parsed = await parse({ + data: { + sastFindings: { nodes: [record] }, + versionControlResources: { nodes: selected }, + }, + }); + expect(parsed.excluded).toEqual([]); + expect(parsed.findings[0]!.evidence.branch).toBe("Feature/Parser"); + expect(parsed.findings[0]!.evidence.source_data).toEqual(record); + } + }, +); + +test("an explicit source repository name takes precedence over inventory aliases", async () => { + const record = { + ...sast, + repository: { id: repository.id, name: "project" }, + repositoryBranch: { + ...sast.repositoryBranch, + name: "project/component/Feature", + }, + }; + const parsed = await parse({ + data: { + sastFindings: { nodes: [record] }, + versionControlResources: { + nodes: [ + { + ...inventoryNode("GITHUB", repositoryUrl), + repository: { + ...repository, + url: repositoryUrl, + name: "project/component", + }, + }, + ], + }, + }, + }); + expect(parsed.excluded).toEqual([]); + expect(parsed.findings[0]!.evidence.branch).toBe("component/Feature"); +}); + for (const kind of ["sast", "secret", "iac"] as const) { test.each([false, true])( kind + @@ -1540,13 +1638,29 @@ test.each(["raw", "normalized"])( }, ); -test.each(["resume", "reset", "coverage", "rebind", "prepend", "excluded"])( +test.each([ + "resume", + "reset", + "coverage", + "rebind", + "prepend", + "excluded", + "qualifier", +])( "inventory normalization upgrades saved immutable requests and receipts (%s)", async (scenario) => { const record = { ...sast, vendorMetadata: { z: 1, ä: 2 }, repository: { id: repository.id }, + ...(scenario === "qualifier" + ? { + repositoryBranch: { + ...sast.repositoryBranch, + name: "project/component/Feature", + }, + } + : {}), }; const selected = scenario === "coverage" @@ -1568,15 +1682,24 @@ test.each(["resume", "reset", "coverage", "rebind", "prepend", "excluded"])( ...inventoryNode("GITHUB"), repository: { ...repository, - name: "Example/Project", - url: "https://github.com/Example/Project.git", + name: + scenario === "qualifier" ? "project/component" : "Example/Project", + url: + scenario === "qualifier" + ? repositoryUrl + : "https://github.com/Example/Project.git", }, }, { ...inventoryNode("GITHUB"), repository: { ...repository, - name: scenario === "coverage" ? "z".repeat(513) : repository.name, + name: + scenario === "coverage" + ? "z".repeat(513) + : scenario === "qualifier" + ? "project" + : repository.name, url: repositoryUrl, }, }, From 7027a6a77312eadac49290be1005566a335572e3 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sun, 11 Oct 2026 18:09:17 +0000 Subject: [PATCH 36/43] refactor(import): share checkpoint reads and timestamp parsing --- .../src/external-findings-publish.ts | 52 ++++++------------- sdk/typescript/src/wiz-findings.ts | 11 +--- 2 files changed, 16 insertions(+), 47 deletions(-) diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index d25570c379..d70ee826ac 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -190,6 +190,13 @@ function sameFindingSelection( ); } +async function readOptionalFile(path: string): Promise { + return readFile(path, "utf8").catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return undefined; + throw error; + }); +} + async function writeAtomicJson(path: string, value: unknown): Promise { const temporary = `${path}.${randomUUID()}.tmp`; try { @@ -442,29 +449,14 @@ export async function prepareExternalPublication( ])) { if (legacyKey === key) continue; const legacyPath = join(state, `${legacyKey}.pending.json`); - const captured = await readFile(legacyPath, "utf8").catch( - (error: NodeJS.ErrnoException) => { - if (error.code === "ENOENT") return undefined; - throw error; - }, - ); + const captured = await readOptionalFile(legacyPath); if (captured !== undefined) { // Preserve requests staged by the earlier locale-based filename format. // Hold both publication locks while moving the original checkpoint. await withImportLock(state, key, dependencies.signal, async () => withImportLock(state, legacyKey, dependencies.signal, async () => { - const current = await readFile(pendingPath, "utf8").catch( - (error: NodeJS.ErrnoException) => { - if (error.code === "ENOENT") return undefined; - throw error; - }, - ); - const serialized = await readFile(legacyPath, "utf8").catch( - (error: NodeJS.ErrnoException) => { - if (error.code === "ENOENT") return undefined; - throw error; - }, - ); + const current = await readOptionalFile(pendingPath); + const serialized = await readOptionalFile(legacyPath); if (current !== undefined) { recovered = validateSaved(current); if (serialized !== undefined) { @@ -526,13 +518,9 @@ export async function prepareExternalPublication( ); } } - let saved: SavedSubmission | undefined = recovered; - try { - const serialized = await readFile(pendingPath, "utf8"); - saved = validateSaved(serialized); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - } + const serialized = await readOptionalFile(pendingPath); + const saved = + serialized === undefined ? recovered : validateSaved(serialized); if (saved) { const content = saved; for (const submission of content.requests) { @@ -540,12 +528,7 @@ export async function prepareExternalPublication( // The old request is never sent after reset. A subsequent explicit // invocation prepares a fresh submission and asks for approval again. await withImportLock(state, key, dependencies.signal, async () => { - const current = await readFile(pendingPath, "utf8").catch( - (error: NodeJS.ErrnoException) => { - if (error.code === "ENOENT") return undefined; - throw error; - }, - ); + const current = await readOptionalFile(pendingPath); if ( current !== undefined && sameSubmission(JSON.parse(current) as SavedSubmission, content) @@ -680,12 +663,7 @@ export async function prepareExternalPublication( progress({ phase: "waiting", completed: 0 }); return await withImportLock(state, key, dependencies.signal, async () => { // Re-read under the lock to keep checkpoints from another publisher. - const pending = await readFile(pendingPath, "utf8").catch( - (error: NodeJS.ErrnoException) => { - if (error.code === "ENOENT") return undefined; - throw error; - }, - ); + const pending = await readOptionalFile(pendingPath); const current = pending === undefined ? submission diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts index 0de8543db9..eef894894a 100644 --- a/sdk/typescript/src/wiz-findings.ts +++ b/sdk/typescript/src/wiz-findings.ts @@ -81,14 +81,6 @@ function wizFinding(record: Record): VendorFinding { text(asset?.["containerImageId"]) ?? text(asset?.["ImageExternalId"])) : null); - const updated = text(record["updatedAt"]); - const updatedSeconds = updated ? Date.parse(updated) / 1000 : null; - if ( - updatedSeconds !== null && - (!Number.isFinite(updatedSeconds) || updatedSeconds < 0) - ) { - throw new Error("Wiz updatedAt is not a valid timestamp."); - } // lastDetectedAt is not proof that the entire vendor record was updated. const evidence = { title: @@ -120,8 +112,7 @@ function wizFinding(record: Record): VendorFinding { code_revision: null, source_scan_id: null, image_digests: digest ? [digest] : [], - source_updated_at: - updatedSeconds === null ? null : Math.floor(updatedSeconds), + source_updated_at: updatedAt(record["updatedAt"], "updatedAt"), source_data: record, }; return { From 6b183868d322d43532f7cd2cd1a1b4fadd9568d7 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sun, 11 Oct 2026 20:25:21 +0000 Subject: [PATCH 37/43] fix(imports): select complete repository branch qualifiers --- sdk/typescript/src/wiz-findings.ts | 46 ++++++++-------- sdk/typescript/tests-ts/wiz-findings.test.ts | 56 ++++++++++++++++---- 2 files changed, 68 insertions(+), 34 deletions(-) diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts index eef894894a..d789daf844 100644 --- a/sdk/typescript/src/wiz-findings.ts +++ b/sdk/typescript/src/wiz-findings.ts @@ -188,33 +188,31 @@ function sourceBranch( ): string | null { const name = text(branch?.["name"]); if (!name) return null; - const qualifiers = [ - ...new Set([ - ...(repository.name ? [repository.name] : []), - ...inventoryNames, - ]), - ].sort((left, right) => right.length - left.length); - for (const qualifier of qualifiers) { - const prefix = qualifier + "/"; - if (name.startsWith(prefix)) return name.slice(prefix.length) || null; - } + const qualifiers = [...inventoryNames]; + if (repository.name) qualifiers.push(repository.name); - // URL-only metadata still identifies Wiz's repository-qualified branch name. - // Supported GitHub destinations use case-insensitive repository qualifiers. - // Preserve the branch suffix's case and slashes and the original evidence. + // Include the URL qualifier before selecting the longest repository prefix. + // Keep URL normalization and the branch suffix's original case and slashes. const repositoryKey = repositoryUrlKey(repository.url).toLowerCase(); const url = /^(https?:\/\/[^/?#]+)\/([^?#]+)$/iu.exec(repositoryKey); - if (!url) return repository.name ? name : null; - const prefixParts = url[2]!.split("/").length; - const parts = name.split("/"); - if ( - parts.length > prefixParts && - repositoryUrlKey( - url[1] + "/" + parts.slice(0, prefixParts).join("/"), - ).toLowerCase() === repositoryKey - ) - return parts.slice(prefixParts).join("/") || null; - return name; + if (url) { + const prefixParts = url[2]!.split("/").length; + const parts = name.split("/"); + const qualifier = parts.slice(0, prefixParts).join("/"); + if ( + parts.length > prefixParts && + repositoryUrlKey(url[1] + "/" + qualifier).toLowerCase() === repositoryKey + ) + qualifiers.push(qualifier); + } + for (const qualifier of qualifiers.sort( + (left, right) => right.length - left.length, + )) { + const prefix = qualifier + "/"; + if (name.toLowerCase().startsWith(prefix.toLowerCase())) + return name.slice(prefix.length) || null; + } + return url || repository.name ? name : null; } function updatedAt(value: unknown, field: string): number | null { diff --git a/sdk/typescript/tests-ts/wiz-findings.test.ts b/sdk/typescript/tests-ts/wiz-findings.test.ts index fac663ee23..6ef848ef25 100644 --- a/sdk/typescript/tests-ts/wiz-findings.test.ts +++ b/sdk/typescript/tests-ts/wiz-findings.test.ts @@ -1272,20 +1272,56 @@ test.each([undefined, "toon", "yaml", "md"])( for (const kind of ["sast", "secret", "iac"] as const) { test.each([ - [repositoryUrl, "example/project/feature/parser", "feature/parser"], + [repositoryUrl, "example/project/feature/parser", "feature/parser", null], [ "https://GitHub.com/Example/Project.git/", "EXAMPLE/PROJECT/Feature/Parser/More", "Feature/Parser/More", + null, ], - [repositoryUrl, "Feature/Parser", "Feature/Parser"], - [repositoryUrl, "fix/example/project/Parser", "fix/example/project/Parser"], - [repositoryUrl, "example/project", "example/project"], - ["https://github.com/", "example/project/feature/parser", null], + [ + "https://GitHub.com/Example/Project.git/", + "EXAMPLE/PROJECT.git/Feature/Parser/More", + "Feature/Parser/More", + null, + ], + [ + repositoryUrl, + "example/project.git/Feature/Parser", + "Feature/Parser", + null, + ], + [ + "https://github.com/demo/demo", + "demo/demo/Feature/Parser/More", + "Feature/Parser/More", + "demo", + ], + [ + "https://github.com/Demo/Demo.git/", + "DEMO/DEMO.git/Feature/Parser/More", + "Feature/Parser/More", + "demo", + ], + [ + "https://github.example.test/team/project", + "PROJECT/Feature/Parser", + "Feature/Parser", + "project", + ], + [repositoryUrl, "Feature/Parser", "Feature/Parser", null], + [ + repositoryUrl, + "fix/example/project/Parser", + "fix/example/project/Parser", + null, + ], + [repositoryUrl, "example/project", "example/project", null], + ["https://github.com/", "example/project/feature/parser", null, null], ])( - kind + " URL-only repository metadata preserves the branch from %s: %s", - async (url, branch, expected) => { - const repository = { url }; + kind + " repository qualifiers preserve the branch from %s: %s", + async (url, branch, expected, name) => { + const repository = { url, ...(name ? { name } : {}) }; const record = kind === "sast" ? { @@ -1318,7 +1354,7 @@ for (const kind of ["sast", "secret", "iac"] as const) { expect(parsed.findings[0]!.evidence.branch).toBe(expected); expect(parsed.findings[0]!.evidence.details!.repository).toEqual({ url, - name: null, + name, id: null, }); expect(parsed.findings[0]!.evidence.source_data).toEqual(record); @@ -1384,7 +1420,7 @@ test.each(["project", "project/component"])( repository: { id: repository.id }, repositoryBranch: { ...sast.repositoryBranch, - name: `${alias}/Feature/Parser`, + name: `${alias.toUpperCase()}/Feature/Parser`, }, }; const nodes = ["example/project", "project", alias].map((name) => ({ From 24815eff6f594d51ff22a1a20e7f62b750584a84 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Sun, 11 Oct 2026 21:08:23 +0000 Subject: [PATCH 38/43] test(package): include the external finding schema in archive fixtures --- sdk/typescript/tests-ts/package-tar-listing.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/typescript/tests-ts/package-tar-listing.test.ts b/sdk/typescript/tests-ts/package-tar-listing.test.ts index 11ba4ef2ad..f5cfd6460e 100644 --- a/sdk/typescript/tests-ts/package-tar-listing.test.ts +++ b/sdk/typescript/tests-ts/package-tar-listing.test.ts @@ -77,6 +77,7 @@ function packageTar({ "package/docs/findings-service.md", "package/docs/dedupe-records.md", "package/schemas/project-config.schema.json", + "package/schemas/external-findings.schema.json", "package/LICENSE", ...executablePaths, ...packageDistFiles, From a561d24ce5bdeab57b90796d20159eea76d24066 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Tue, 13 Oct 2026 21:08:16 -0700 Subject: [PATCH 39/43] feat: expose normalized external finding publication in the SDK --- sdk/typescript/README.md | 7 ++ .../scripts/fixtures/package-consumer.ts | 17 +++++ sdk/typescript/scripts/package-dist-files.mjs | 1 + sdk/typescript/scripts/smoke-package.mjs | 2 +- .../src/external-findings-publish.ts | 65 ++++++++++++++++--- sdk/typescript/src/external-findings.ts | 25 +++++++ sdk/typescript/src/index.ts | 21 ++++++ sdk/typescript/src/wiz-findings.ts | 22 +++---- .../external-findings-publish.test.ts | 29 +++++++++ 9 files changed, 164 insertions(+), 25 deletions(-) create mode 100644 sdk/typescript/src/external-findings.ts diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index 7196c9f52d..90ec6b851d 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -488,6 +488,13 @@ grant Cloud access. See [Cloud publication](docs/cli.md#publish-findings-to-clou for setup and review steps, or [Linear publication](docs/cli.md#publish-completed-scans-to-linear) for issue creation. +The SDK also exposes `prepareExternalPublication(file, options)` for saved Wiz +exports and `prepareNormalizedExternalPublication(findings, options)` for +normalized external evidence. Both resolve the Cloud repository and return a +`preview` plus a separate `publish()` operation. Inspect the preview before +calling `publish()`. Imported evidence remains not assessed by Codex, and saved +requests and receipts support retries after interruption. + ## Findings storage and deduplication Saved-scan deduplication uses local SQLite directly: diff --git a/sdk/typescript/scripts/fixtures/package-consumer.ts b/sdk/typescript/scripts/fixtures/package-consumer.ts index 28a771c4b0..4e3e3f5819 100644 --- a/sdk/typescript/scripts/fixtures/package-consumer.ts +++ b/sdk/typescript/scripts/fixtures/package-consumer.ts @@ -1,5 +1,10 @@ import { CodexSecurity, + prepareExternalPublication, + prepareNormalizedExternalPublication, + type ExternalFindings, + type ExternalPublicationOptions, + type ExternalPublicationResult, DiffTarget, classifySeverity, classifyScanSeverity, @@ -241,3 +246,15 @@ export async function dedupeRecords( }; return await deduplicateRecords(input, { reviewRunner, signal }); } + +export async function externalPublication( + file: string, + input: ExternalFindings, + options: ExternalPublicationOptions, +): Promise { + const fromFile = await prepareExternalPublication(file, options); + fromFile.preview.requests[0]?.source.provider satisfies + "wiz" | "endor" | "snyk" | undefined; + const prepared = await prepareNormalizedExternalPublication(input, options); + return prepared.publish(); +} diff --git a/sdk/typescript/scripts/package-dist-files.mjs b/sdk/typescript/scripts/package-dist-files.mjs index fce07aa36a..5a85e246ca 100644 --- a/sdk/typescript/scripts/package-dist-files.mjs +++ b/sdk/typescript/scripts/package-dist-files.mjs @@ -12,6 +12,7 @@ const distModules = [ "severity-store", "cloud-endpoint", "cloud-publish", + "external-findings", "external-findings-publish", "external-import-contract", "external-import-models", diff --git a/sdk/typescript/scripts/smoke-package.mjs b/sdk/typescript/scripts/smoke-package.mjs index 4edb9d308c..8c77781c07 100644 --- a/sdk/typescript/scripts/smoke-package.mjs +++ b/sdk/typescript/scripts/smoke-package.mjs @@ -395,7 +395,7 @@ try { "--input-type=module", "--eval", `const sdk = await import(${JSON.stringify(packageManifest.name)}); - for (const name of ["CodexSecurity", "publishScan", "publishScanToCustom", "checkScanPublication", "deduplicateScan", "deduplicateRecords", "classifySeverity", "classifyScanSeverity", "classifyScanDirectorySeverity", "matchScanFindings", "securityPolicyDiff", "loadProjectConfig", "resolveProjectConfig"]) { + for (const name of ["CodexSecurity", "publishScan", "publishScanToCustom", "checkScanPublication", "deduplicateScan", "deduplicateRecords", "classifySeverity", "classifyScanSeverity", "classifyScanDirectorySeverity", "matchScanFindings", "securityPolicyDiff", "loadProjectConfig", "resolveProjectConfig", "prepareExternalPublication", "prepareNormalizedExternalPublication"]) { if (typeof sdk[name] !== "function") { throw new Error("The installed package does not export " + name + "."); } diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index d70ee826ac..a6863bb589 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -15,13 +15,15 @@ import { codexSecurityStateDirectory, validatePreparedOutputDir, } from "./runtime.js"; -import { - readVendorFindingsForPublication, - type VendorFindings, -} from "./wiz-findings.js"; +import { readVendorFindingsForPublication } from "./wiz-findings.js"; +import type { + ExternalFindings, + ExternalPublicationInput, +} from "./external-findings.js"; import { repositoryUrlKey, validateImportRequest, + validateExternalEvidence, validateImportReceipt, validateRepositories, validateSourceReports, @@ -77,7 +79,7 @@ export interface ExternalPublicationResult { status: "complete" | "partial" | "interrupted"; read: number; ready: number; - excluded: VendorFindings["excluded"]; + excluded: ExternalFindings["excluded"]; receipts: FindingImportReceipt[]; counts: Required; failures: { source_finding_id: string; code: string; message: string }[]; @@ -105,7 +107,7 @@ interface SavedSubmission { inputProjection?: { inputDigest: string; savedEvidenceDigest: string }; } -export interface ExternalPublicationPreview extends VendorFindings { +export interface ExternalPublicationPreview extends ExternalFindings { cloudApiUrl: string; cloudUrl?: string; accountId: string; @@ -168,7 +170,7 @@ function sameSubmission( function submissionFindings( requests: readonly FindingImportRequest[], -): VendorFindings["findings"] { +): ExternalFindings["findings"] { return requests.flatMap((batch) => batch.items.map(({ source_finding_id, evidence }) => ({ source_finding_id, @@ -178,8 +180,8 @@ function submissionFindings( } function sameFindingSelection( - left: VendorFindings["findings"], - right: VendorFindings["findings"], + left: ExternalFindings["findings"], + right: ExternalFindings["findings"], ): boolean { return ( left.length === right.length && @@ -275,6 +277,49 @@ export async function prepareExternalPublication( path: string, options: ExternalPublicationOptions, dependencies: ExternalPublicationDependencies = {}, +): Promise { + const input = await readVendorFindingsForPublication(path); + return preparePublicationInput(input, options, dependencies); +} + +/** Prepare normalized provider evidence without reading a vendor export. */ +export async function prepareNormalizedExternalPublication( + input: ExternalFindings, + options: ExternalPublicationOptions, + dependencies: ExternalPublicationDependencies = {}, +): Promise { + // Validation adds Cloud defaults; leave the caller's objects unchanged. + const current = structuredClone(input); + const identities = new Set(); + for (const finding of current.findings) { + const id = finding.source_finding_id; + if (!id.trim() || id.includes("\0") || Buffer.byteLength(id) > 512) + throw new CodexSecurityError( + "Source finding id must contain 1–512 UTF-8 bytes and no NUL characters.", + ); + if (identities.has(id)) + throw new CodexSecurityError( + `Duplicate source finding id ${JSON.stringify(id)}.`, + ); + identities.add(id); + finding.evidence = validateExternalEvidence(finding.evidence); + } + return preparePublicationInput( + { + current, + legacy: current, + currentIdentity: current.findings, + legacyIdentity: current.findings, + }, + options, + dependencies, + ); +} + +async function preparePublicationInput( + input: ExternalPublicationInput, + options: ExternalPublicationOptions, + dependencies: ExternalPublicationDependencies, ): Promise { const progress = (event: ExternalPublicationProgress): void => { try { @@ -294,7 +339,7 @@ export async function prepareExternalPublication( legacy: legacyParsed, currentIdentity, legacyIdentity, - } = await readVendorFindingsForPublication(path); + } = input; if (parsed.findings.length === 0) { throw new CodexSecurityError( `No supported findings are ready to publish. ${parsed.excluded.map((item) => `Item ${item.position}: ${item.reason}`).join(" ")}`, diff --git a/sdk/typescript/src/external-findings.ts b/sdk/typescript/src/external-findings.ts new file mode 100644 index 0000000000..3c4a0436ca --- /dev/null +++ b/sdk/typescript/src/external-findings.ts @@ -0,0 +1,25 @@ +import type { ExternalFindingEvidence } from "./external-import-models.js"; + +/** A provider occurrence and its evidence, before Cloud assigns a source report. */ +export interface ExternalFinding { + source_finding_id: string; + evidence: ExternalFindingEvidence; +} + +export interface ExternalFindings { + read: number; + findings: ExternalFinding[]; + excluded: { + position: number; + source_finding_id: string | null; + reason: string; + }[]; +} + +/** @internal Keep earlier provider projections only for immutable request replay. */ +export interface ExternalPublicationInput { + current: ExternalFindings; + legacy: ExternalFindings; + currentIdentity: unknown; + legacyIdentity: unknown; +} diff --git a/sdk/typescript/src/index.ts b/sdk/typescript/src/index.ts index f9e8dd83b3..312fb99ca9 100644 --- a/sdk/typescript/src/index.ts +++ b/sdk/typescript/src/index.ts @@ -163,6 +163,27 @@ export type { DeduplicationDiagnosticObserver, } from "./deduplication/diagnostics.js"; export type { FindingEmbedder } from "./server/embeddings.js"; +export { + ExternalPublicationError, + prepareExternalPublication, + prepareNormalizedExternalPublication, +} from "./external-findings-publish.js"; +export type { + ExternalPublicationOptions, + ExternalPublicationDependencies, + ExternalPublicationPreview, + ExternalPublicationProgress, + ExternalPublicationResult, + PreparedExternalPublication, +} from "./external-findings-publish.js"; +export type { ExternalFinding, ExternalFindings } from "./external-findings.js"; +export type { + ExternalFindingEvidence, + FindingImportRequest, + FindingImportReceipt, + ImportRepository, + SourceReport, +} from "./external-import-models.js"; export { importGitHubCodeScanningAlerts } from "./github.js"; export type { GitHubCodeScanningImportOptions, diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts index d789daf844..cd15c4fecf 100644 --- a/sdk/typescript/src/wiz-findings.ts +++ b/sdk/typescript/src/wiz-findings.ts @@ -6,24 +6,18 @@ import { repositoryUrlKey, validateExternalEvidence, } from "./external-import-contract.js"; -import type { ExternalFindingEvidence } from "./external-import-models.js"; const decompressGzip = promisify(gunzip); -export interface VendorFinding { - source_finding_id: string; - evidence: ExternalFindingEvidence; -} +import type { + ExternalFinding as VendorFinding, + ExternalFindings as VendorFindings, +} from "./external-findings.js"; -export interface VendorFindings { - read: number; - findings: VendorFinding[]; - excluded: { - position: number; - source_finding_id: string | null; - reason: string; - }[]; -} +export type { + ExternalFinding as VendorFinding, + ExternalFindings as VendorFindings, +} from "./external-findings.js"; function object(value: unknown): Record | undefined { return value !== null && typeof value === "object" && !Array.isArray(value) diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 922b71ef85..b1749c54f1 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -18,6 +18,7 @@ import { afterEach, expect, spyOn, test } from "bun:test"; import { ExternalPublicationError, prepareExternalPublication, + prepareNormalizedExternalPublication, type ExternalPublicationProgress, } from "../src/external-findings-publish.js"; import { readVendorFindings } from "../src/wiz-findings.js"; @@ -2385,3 +2386,31 @@ test("human previews stay compact while JSON previews retain complete evidence", ).toBe("complete-vendor-evidence-5"); expect(f.posts).toHaveLength(0); }); + +test("normalized SDK publication preserves caller evidence and exclusion accounting", async () => { + const f = await fixture(); + const input = { + read: 2, + findings: [normalized()], + excluded: [ + { + position: 2, + source_finding_id: null, + reason: "Unsupported source record.", + }, + ], + }; + const original = structuredClone(input); + const prepared = await prepareNormalizedExternalPublication( + input as Parameters[0], + options, + f.deps, + ); + expect(input).toEqual(original); + expect(prepared.preview.read).toBe(2); + const result = await prepared.publish(); + expect(result.status).toBe("partial"); + expect(result.counts.created).toBe(1); + expect(result.excluded).toEqual(input.excluded); + expect(result.verified).toBe(1); +}); From 0eb55e16c481a4e4d7d222fac2c3ec9fc5fa9a2c Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Tue, 13 Oct 2026 21:23:17 -0700 Subject: [PATCH 40/43] feat: import GitHub code-scanning alerts and security advisories to Cloud --- .../skills/publish-findings/SKILL.md | 32 +- .../publish-findings/agents/openai.yaml | 2 +- sdk/typescript/README.md | 8 +- sdk/typescript/docs/cli.md | 99 ++++++ .../schemas/external-findings.schema.json | 70 +++- .../scripts/fixtures/package-consumer.ts | 38 ++- sdk/typescript/scripts/package-dist-files.mjs | 2 + sdk/typescript/scripts/smoke-package.mjs | 2 +- sdk/typescript/src/cli.ts | 107 +++++- .../src/external-findings-publish.ts | 13 +- .../src/external-import-contract.ts | 24 +- sdk/typescript/src/external-import-models.ts | 21 +- sdk/typescript/src/github-advisories.ts | 218 ++++++++++++ sdk/typescript/src/github-findings.ts | 315 ++++++++++++++++++ sdk/typescript/src/index.ts | 10 + sdk/typescript/tests-ts/cli-fixtures.ts | 4 + .../external-findings-publish.test.ts | 147 +++++++- .../tests-ts/github-advisories.test.ts | 199 +++++++++++ .../tests-ts/github-findings.test.ts | 213 ++++++++++++ .../support/github-import-fixtures.ts | 82 +++++ sdk/typescript/tests-ts/wiz-findings.test.ts | 10 +- 21 files changed, 1577 insertions(+), 39 deletions(-) create mode 100644 sdk/typescript/src/github-advisories.ts create mode 100644 sdk/typescript/src/github-findings.ts create mode 100644 sdk/typescript/tests-ts/github-advisories.test.ts create mode 100644 sdk/typescript/tests-ts/github-findings.test.ts create mode 100644 sdk/typescript/tests-ts/support/github-import-fixtures.ts diff --git a/plugins/codex-security/skills/publish-findings/SKILL.md b/plugins/codex-security/skills/publish-findings/SKILL.md index 3ea68bf03c..489f4f43ce 100644 --- a/plugins/codex-security/skills/publish-findings/SKILL.md +++ b/plugins/codex-security/skills/publish-findings/SKILL.md @@ -1,13 +1,17 @@ --- name: publish-findings -description: Import selected Wiz package vulnerabilities, SAST, repository secrets, and IaC findings to Codex Security Cloud after previewing the destination and evidence. Supports plain or gzip-compressed JSON and normalized JSONL; does not import workload secrets or cloud configuration, run a scan, or assess findings. +description: Import selected Wiz findings, GitHub code-scanning alerts, or GHSA security advisories to Codex Security Cloud after previewing the destination and evidence. Supports saved JSON/JSONL and gzip; does not run a scan, assess applicability, or synchronize triage decisions. --- # Publish external findings Use the installed `codex-security publish findings` command for both preview and upload. It owns source mapping, Cloud authentication, source versions, and saved request retries. Do not implement another HTTP uploader or put credentials in a command argument. -Cloud destinations currently support connected GitHub and GitHub Enterprise repositories, including custom Enterprise hostnames. Resolve the user's destination to its repository URL or Cloud repository ID; the command accepts either. Confirm that the repository is connected in the intended Cloud account and has an authorized environment. Use the existing file-backed ChatGPT login; explain a missing or incompatible login rather than changing credential storage automatically. Read selected Wiz findings through the user's configured connection or supplied export. Save the actual selected JSON records locally, retaining their IDs and evidence. Finish the requested pagination; if the user selected a subset, save that explicit subset and retain its named collection. The command does not fetch from Wiz or accept CSV, scan events, workload secrets, cloud configuration, or external network exports. The publisher also accepts JSONL records with `source_finding_id` and `evidence` as described in the CLI documentation. +Cloud destinations currently support connected GitHub and GitHub Enterprise repositories, including custom Enterprise hostnames. Resolve the user's destination to its repository URL or Cloud repository ID; the command accepts either. Confirm that the repository is connected in the intended Cloud account and has an authorized environment. Use the existing file-backed ChatGPT login; explain a missing or incompatible login rather than changing credential storage automatically. + +## Wiz findings + +Read selected Wiz findings through the user's configured connection or supplied export. Save the actual selected JSON records locally, retaining their IDs and evidence. Finish the requested pagination; if the user selected a subset, save that explicit subset and retain its named collection. The command does not fetch from Wiz or accept CSV, scan events, workload secrets, cloud configuration, or external network exports. The publisher also accepts JSONL records with `source_finding_id` and `evidence` as described in the CLI documentation. Use a stable Wiz tenant/finding namespace as `--source-key`. A project filter is selection context, not a new identity. Confirm the selected assets or builds belong to the destination repository; do not infer a scanner branch or revision from the current checkout. Do not publish vendor credentials, unrelated records, or local assessments as original scanner evidence. @@ -17,6 +21,30 @@ For SAST, repository secrets, and IaC, retain the explicit `data.sastFindings.no Keep raw source records unchanged in `source_data`. New raw occurrence IDs are prefixed `sast:`, `secret:`, or `iac:`; existing package IDs are unchanged. For CSV report conversion, use a reviewed mapping of the actual header names into the documented normalized JSONL profile, set `evidence.details.kind` and the verified `details.repository.url`, and use the same prefixed occurrence ID. The command does not parse CSV directly. Select source paths and line numbers explicitly in secret reports; the standard metadata query lacks them. Preserve rule/CWE, range/snippet, detector validation, and IaC comparison fields in the typed details described in the CLI docs. Preserve empty comparison strings. Do not invent missing IDs, severity, revisions, or locations, retrieve credential values, test credentials, or apply suggested fixes. Vendor status, AI verdict, secret validation, introduced commits, and logical IaC resources are evidence, not Codex assessment or proof of runtime deployment. +## GitHub alerts and advisories + +Keep code-scanning alerts and GHSA advisories as separate source families. Read code-scanning alerts with the existing `codex-security import github OWNER/REPO --format json` command. It defaults to open alerts on the default branch; use `--github-alert NUMBER` repeatedly for exact alerts or `--github-ref REF` for a selected ref. Save the complete output without field or token filters. The mapper retains the selected instance's ref, revision, status, rule, scanner, location and original evidence. Missing security severity remains `unknown`; do not turn SARIF error/warning/note into vulnerability ratings. + +Read advisories with `codex-security import github-advisories OWNER/REPO --format json`. It defaults to published repository advisories. `--github-advisory GHSA-xxxx-xxxx-xxxx` selects exact repository advisory IDs and may include private drafts only when the user selects them. To list private repository advisories, require the user's explicit state selection using `--github-state draft|triage|closed`. Do not combine a nondefault state with exact IDs. For public database records, use `--github-advisory-source global` with one or more explicit `--github-advisory` IDs; never enumerate or mirror the global database. Both readers are read-only and use the user's GitHub CLI authentication and `GH_HOST` when configured. Stop on a read failure instead of publishing an incomplete selection. + +Publish the saved reader output with `--provider github`. Use a stable namespace such as `HOST/OWNER/REPO/code-scanning` or `HOST/OWNER/REPO/security-advisories` as `--source-key`, with separate keys for the two families. Code-scanning identities include the ref; GHSA records retain their advisory identity. Code-scanning and repository advisories must match the selected Cloud repository. Global advisory evidence omits source-repository provenance; the explicit Cloud destination selects where to investigate it. Preserve affected package ranges and patched versions without claiming an installed vulnerable dependency or inventing source locations. The original GHSA/CVE IDs remain evidence, not cross-provider source identities. GitHub dismissal, resolution and advisory withdrawal are not Codex assessments. Do not write back to GitHub or include Dependabot alerts without a separately supported ingestion path. + +```sh +codex-security import github example/project --format json > /path/to/alerts.json +codex-security publish findings /path/to/alerts.json \ + --to cloud --repository https://github.com/example/project --provider github \ + --source-key github.com/example/project/code-scanning --dry-run --format json + +codex-security import github-advisories example/project \ + --github-advisory-source global --github-advisory GHSA-2345-6789-cfgh \ + --format json > /path/to/advisories.json +codex-security publish findings /path/to/advisories.json \ + --to cloud --repository https://github.com/example/project --provider github \ + --source-key github.com/example/project/security-advisories --dry-run --format json +``` + +## Review and publish + For staging or another Cloud deployment, set `CODEX_SECURITY_CLOUD_BASE_URL` to its API root before both preview and upload. If the Cloud page or user supplies its app URL, also set `CODEX_SECURITY_CLOUD_WEB_URL` to that full URL so the command can return a repository Findings link. The UI setting does not change API routing or authentication. Use that deployment's login and an isolated state directory for tests. Verify `cloudApiUrl` and any `cloudUrl` in the preview against the intended deployment; the former is the API endpoint and the latter is the UI link. Saved retries are separated by deployment. The native scan setting `CODEX_SECURITY_CLOUD_PUBLISH_URL` does not route vendor imports. Never fall back to production when staging is unavailable. ```sh diff --git a/plugins/codex-security/skills/publish-findings/agents/openai.yaml b/plugins/codex-security/skills/publish-findings/agents/openai.yaml index c6d611bc38..624f9f5a08 100644 --- a/plugins/codex-security/skills/publish-findings/agents/openai.yaml +++ b/plugins/codex-security/skills/publish-findings/agents/openai.yaml @@ -1,6 +1,6 @@ interface: display_name: "Publish Findings" short_description: "Preview and publish vendor findings to Cloud" - default_prompt: "Use $publish-findings to preview selected Wiz findings for my Cloud repository." + default_prompt: "Use $publish-findings to preview selected Wiz findings, GitHub code-scanning alerts, or GHSA advisories for my Cloud repository." policy: allow_implicit_invocation: true diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index 90ec6b851d..6275696850 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -488,12 +488,14 @@ grant Cloud access. See [Cloud publication](docs/cli.md#publish-findings-to-clou for setup and review steps, or [Linear publication](docs/cli.md#publish-completed-scans-to-linear) for issue creation. -The SDK also exposes `prepareExternalPublication(file, options)` for saved Wiz -exports and `prepareNormalizedExternalPublication(findings, options)` for +The SDK also exposes `prepareExternalPublication(file, options)` for saved Wiz or GitHub +imports and `prepareNormalizedExternalPublication(findings, options)` for normalized external evidence. Both resolve the Cloud repository and return a `preview` plus a separate `publish()` operation. Inspect the preview before calling `publish()`. Imported evidence remains not assessed by Codex, and saved -requests and receipts support retries after interruption. +requests and receipts support retries after interruption. GitHub code-scanning +alerts and GHSA advisories have [separate readers and mappings](docs/cli.md#import-github-findings-to-cloud); +advisory imports preserve affected ranges without claiming an installed dependency. ## Findings storage and deduplication diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md index 63c60e5324..ddb2d12302 100644 --- a/sdk/typescript/docs/cli.md +++ b/sdk/typescript/docs/cli.md @@ -1667,6 +1667,105 @@ record absolute Node/CLI paths. If those change, back up and remove the verified generated hook, then reinstall with the same threshold. Restore the backup if installation fails. The installer respects `core.hooksPath`. +## Import GitHub findings to Cloud + +Save code-scanning alerts with the existing read-only importer, then inspect the +Cloud publication preview: + +```bash +codex-security import github example/project --format json > /path/outside/repository/alerts.json +codex-security publish findings /path/outside/repository/alerts.json \ + --to cloud --provider github --repository https://github.com/example/project \ + --source-key github.com/example/project/code-scanning --dry-run --format json +``` + +After reviewing the destination, source, exclusions and evidence, repeat the +publication command without `--dry-run` and confirm it, or add `--yes` for a +previously reviewed input. Authentication, batching, saved receipts, partial +results and interrupted-upload recovery follow the [Wiz publication flow](#import-wiz-findings). +JSON, JSONL and gzip are supported. Save complete import output without field or +token filters. + +Use a stable source key for the GitHub host, repository and finding family. +Code-scanning identities also include the selected ref, so an import from another +branch cannot replace an existing branch's evidence. GitHub's source state and +full original alert stay attached to the source report; dismissal or a fixed +state does not create a Codex assessment. Alerts without a security severity are +reported as `unknown`; SARIF error/warning/note levels are not vulnerability ratings. + +The SDK offers the same preview and publication boundary: + +```ts +import { + importGitHubCodeScanningAlerts, + normalizeGitHubCodeScanningAlerts, + prepareNormalizedExternalPublication, +} from "@openai/codex-security"; + +const alerts = await importGitHubCodeScanningAlerts({ + repository: "example/project", +}); +const prepared = await prepareNormalizedExternalPublication( + normalizeGitHubCodeScanningAlerts(alerts), + { + provider: "github", + repository: "https://github.com/example/project", + sourceKey: "github.com/example/project/code-scanning", + }, +); +console.log(prepared.preview); +// After reviewing the preview: +const receipt = await prepared.publish(); +``` + +### Import GHSA advisories separately + +Repository advisories describe disclosures maintained by a repository. Global +advisories describe affected software in GitHub's public advisory database; they +do not establish that your repository installs that software. + +```bash +# Read only published advisories for this repository. +codex-security import github-advisories example/project --format json > /path/outside/repository/advisories.json + +# Select individual public database records without enumerating the database. +codex-security import github-advisories example/project \ + --github-advisory-source global --github-advisory GHSA-2345-6789-cfgh \ + --format json > /path/outside/repository/advisories.json + +codex-security publish findings /path/outside/repository/advisories.json \ + --to cloud --provider github --repository https://github.com/example/project \ + --source-key github.com/example/project/security-advisories --dry-run --format json +``` + +`--github-advisory-source` defaults to `repository`. Repeat `--github-advisory` +to select exact IDs from that API, regardless of advisory state. Repository +lists default to `--github-state published`; `draft`, `triage`, and `closed` +require explicit selection. A nondefault state cannot be combined with exact +IDs. Global selection requires at least one ID and never lists the advisory +database. Both readers use `GH_HOST` and GitHub CLI authentication by default; +the SDK also accepts `githubHost` and `githubToken`. + +Review the saved evidence before publication, including any selected private +advisories. Import failure returns no partial selection. Publication reports +invalid records as exclusions and retains acknowledged receipts if a later +batch or readback fails. + +Advisory evidence retains GHSA/CVE IDs, affected version ranges, available +patched versions, source status and original JSON. It does not infer an installed +version or source location. Global advisories omit a source repository; the +explicit Cloud destination binds the import for investigation. Repository +advisories retain and check their source repository. Withdrawal remains vendor +state, and all imported evidence remains not assessed by Codex. Use a separate +source key from code-scanning alerts; shared advisory IDs do not combine source +records from different providers. + +In the SDK, call `importGitHubAdvisories()` and `normalizeGitHubAdvisories()` +with `prepareNormalizedExternalPublication()` as shown above. The former defaults +to `{ source: "repository", state: "published" }`; use +`{ source: "global", advisoryIds: ["GHSA-2345-6789-cfgh"] }` for selected database +records. Both forms require `repository: "OWNER/REPO"` as source or target context. + ## Import alerts from the CLI ```bash diff --git a/sdk/typescript/schemas/external-findings.schema.json b/sdk/typescript/schemas/external-findings.schema.json index bd0f50e102..c10ad248ee 100644 --- a/sdk/typescript/schemas/external-findings.schema.json +++ b/sdk/typescript/schemas/external-findings.schema.json @@ -378,7 +378,7 @@ "additionalProperties": false, "properties": { "provider": { - "enum": ["wiz", "endor", "snyk"], + "enum": ["wiz", "endor", "snyk", "github"], "title": "Provider", "type": "string" }, @@ -393,6 +393,20 @@ "title": "ImportSource", "type": "object" }, + "ImportedAdvisoryContext": { + "additionalProperties": false, + "description": "Where the advisory was read; applicability is assessed separately.", + "properties": { + "source": { + "enum": ["repository", "global"], + "title": "Source", + "type": "string" + } + }, + "required": ["source"], + "title": "ImportedAdvisoryContext", + "type": "object" + }, "ImportedAssessment": { "additionalProperties": false, "properties": { @@ -506,12 +520,29 @@ "description": "Typed repository evidence; vendor state is not a Codex assessment.", "properties": { "kind": { - "enum": ["sast", "secret", "iac"], + "enum": ["sast", "secret", "iac", "advisory"], "title": "Kind", "type": "string" }, "repository": { - "$ref": "#/$defs/ImportedRepositoryReference" + "anyOf": [ + { + "$ref": "#/$defs/ImportedRepositoryReference" + }, + { + "type": "null" + } + ] + }, + "advisory": { + "anyOf": [ + { + "$ref": "#/$defs/ImportedAdvisoryContext" + }, + { + "type": "null" + } + ] }, "source_status": { "anyOf": [ @@ -637,7 +668,7 @@ "title": "File Url" } }, - "required": ["kind", "repository"], + "required": ["kind"], "title": "ImportedFindingDetails", "type": "object" }, @@ -651,7 +682,14 @@ "type": "string" }, "severity": { - "enum": ["critical", "high", "medium", "low", "informational"], + "enum": [ + "critical", + "high", + "medium", + "low", + "informational", + "unknown" + ], "title": "Severity", "type": "string" }, @@ -910,6 +948,19 @@ "default": null, "title": "Installed Version" }, + "vulnerable_version_range": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Vulnerable Version Range" + }, "manifest_path": { "anyOf": [ { @@ -1285,7 +1336,14 @@ "type": "string" }, "severity": { - "enum": ["critical", "high", "medium", "low", "informational"], + "enum": [ + "critical", + "high", + "medium", + "low", + "informational", + "unknown" + ], "title": "Severity", "type": "string" }, diff --git a/sdk/typescript/scripts/fixtures/package-consumer.ts b/sdk/typescript/scripts/fixtures/package-consumer.ts index 4e3e3f5819..805b87632c 100644 --- a/sdk/typescript/scripts/fixtures/package-consumer.ts +++ b/sdk/typescript/scripts/fixtures/package-consumer.ts @@ -1,5 +1,9 @@ import { CodexSecurity, + importGitHubCodeScanningAlerts, + importGitHubAdvisories, + normalizeGitHubAdvisories, + normalizeGitHubCodeScanningAlerts, prepareExternalPublication, prepareNormalizedExternalPublication, type ExternalFindings, @@ -254,7 +258,39 @@ export async function externalPublication( ): Promise { const fromFile = await prepareExternalPublication(file, options); fromFile.preview.requests[0]?.source.provider satisfies - "wiz" | "endor" | "snyk" | undefined; + "wiz" | "endor" | "snyk" | "github" | undefined; const prepared = await prepareNormalizedExternalPublication(input, options); return prepared.publish(); } + +export async function publishGitHubAlerts(): Promise { + const imported = await importGitHubCodeScanningAlerts({ + repository: "example/project", + }); + const prepared = await prepareNormalizedExternalPublication( + normalizeGitHubCodeScanningAlerts(imported), + { + provider: "github", + sourceKey: "github.com/example/project/code-scanning", + repository: "https://github.com/example/project", + }, + ); + return prepared.publish(); +} + +export async function publishGitHubAdvisories(): Promise { + const imported = await importGitHubAdvisories({ + repository: "example/project", + source: "global", + advisoryIds: ["GHSA-2345-6789-cfgh"], + }); + const prepared = await prepareNormalizedExternalPublication( + normalizeGitHubAdvisories(imported), + { + provider: "github", + sourceKey: "github.com/example/project/security-advisories", + repository: "https://github.com/example/project", + }, + ); + return prepared.publish(); +} diff --git a/sdk/typescript/scripts/package-dist-files.mjs b/sdk/typescript/scripts/package-dist-files.mjs index 5a85e246ca..8179438bcf 100644 --- a/sdk/typescript/scripts/package-dist-files.mjs +++ b/sdk/typescript/scripts/package-dist-files.mjs @@ -44,6 +44,8 @@ const distModules = [ "finding-catalogue", "findings-import", "github", + "github-advisories", + "github-findings", "index", "import-scan", "knowledge-base", diff --git a/sdk/typescript/scripts/smoke-package.mjs b/sdk/typescript/scripts/smoke-package.mjs index 8c77781c07..a5f2e7ab4e 100644 --- a/sdk/typescript/scripts/smoke-package.mjs +++ b/sdk/typescript/scripts/smoke-package.mjs @@ -395,7 +395,7 @@ try { "--input-type=module", "--eval", `const sdk = await import(${JSON.stringify(packageManifest.name)}); - for (const name of ["CodexSecurity", "publishScan", "publishScanToCustom", "checkScanPublication", "deduplicateScan", "deduplicateRecords", "classifySeverity", "classifyScanSeverity", "classifyScanDirectorySeverity", "matchScanFindings", "securityPolicyDiff", "loadProjectConfig", "resolveProjectConfig", "prepareExternalPublication", "prepareNormalizedExternalPublication"]) { + for (const name of ["CodexSecurity", "publishScan", "publishScanToCustom", "checkScanPublication", "deduplicateScan", "deduplicateRecords", "classifySeverity", "classifyScanSeverity", "classifyScanDirectorySeverity", "matchScanFindings", "securityPolicyDiff", "loadProjectConfig", "resolveProjectConfig", "prepareExternalPublication", "prepareNormalizedExternalPublication", "importGitHubCodeScanningAlerts", "normalizeGitHubCodeScanningAlerts", "readGitHubFindings", "importGitHubAdvisories", "normalizeGitHubAdvisories"]) { if (typeof sdk[name] !== "function") { throw new Error("The installed package does not export " + name + "."); } diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index a88a3da1bc..3a6614c3a6 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -173,6 +173,10 @@ import { ScanCostLimitExceededError, ScanInterruptedError, } from "./errors.js"; +import { + GITHUB_ADVISORY_STATES, + importGitHubAdvisories, +} from "./github-advisories.js"; import { GITHUB_ALERT_STATES, importGitHubCodeScanningAlerts, @@ -582,7 +586,7 @@ function renderExternalPublicationSummary( ...(result.status === "interrupted" ? [] : [ - "Imported Wiz evidence remains not assessed by Codex.", + "Imported evidence remains not assessed by Codex.", result.cloudUrl ? `View findings: ${diagnosticValue(result.cloudUrl)}\nSearch indexing can lag an accepted import.` : "Open the Codex Security Cloud app's Findings view and select this repository. Search indexing can lag an accepted import.", @@ -1207,6 +1211,7 @@ interface CliDependencies { planComponents?: typeof planComponents; linearClient?: LinearClientFactory; importGitHubAlerts?: typeof importGitHubCodeScanningAlerts; + importGitHubAdvisories?: typeof importGitHubAdvisories; importScan?: typeof importScan; runWorkbench( args: readonly string[], @@ -3246,16 +3251,15 @@ export async function main( }, }); publication.command("findings", { - description: - "Preview and import selected Wiz repository findings to Cloud.", - hint: "Supply a package vulnerability JSON report, named SAST/repository-secret/IaC JSON collections with repository metadata, or normalized JSONL. Gzip is supported; CSV, scan events, workload secrets, and cloud configuration exports are not. --full-output requires --format json or --format jsonl.\nExample: codex-security publish findings selected-wiz.json --to cloud --repository https://github.com/example/project --provider wiz --source-key TENANT_ID/vulnerability-finding --dry-run --format json", + description: "Preview and import selected Wiz or GitHub findings to Cloud.", + hint: "For GitHub, supply unfiltered JSON/JSONL saved by import github or import github-advisories. For Wiz, supply a package vulnerability JSON report, named SAST/repository-secret/IaC JSON collections with repository metadata, or normalized JSONL. Gzip is supported; CSV, scan events, workload secrets, and cloud configuration exports are not. --full-output requires --format json or --format jsonl.\nExample: codex-security publish findings selected-wiz.json --to cloud --repository https://github.com/example/project --provider wiz --source-key TENANT_ID/vulnerability-finding --dry-run --format json", destructive: true, mcp: false, args: z.object({ file: z .string() .describe( - "Saved Wiz finding collections, package vulnerability JSON, or normalized JSONL.", + "Saved GitHub import output, Wiz finding collections, or normalized Wiz JSONL.", ), }), options: z.object({ @@ -3264,8 +3268,8 @@ export async function main( "Authorized Cloud repository ID or repository URL copied from Cloud.", ), provider: z - .literal("wiz") - .describe("Vendor that produced these findings."), + .enum(["wiz", "github"]) + .describe("Provider that produced these findings."), sourceKey: optionValue("--source-key").describe( "Stable vendor tenant and finding namespace; not a project selection filter.", ), @@ -3466,7 +3470,7 @@ export async function main( }, }); const imports = Cli.create("import", { - description: "Read GitHub code scanning alerts.", + description: "Read GitHub code scanning alerts and security advisories.", }).command("github", { description: "Read GitHub code scanning alerts without changing GitHub.", hint: "To save CSV or JSON findings as a local scan, use codex-security scan import --help.", @@ -3547,6 +3551,93 @@ export async function main( } }, }); + imports.command("github-advisories", { + description: "Read GitHub security advisories without changing GitHub.", + hint: "Defaults to published repository advisories. Use --github-advisory-source global with explicit --github-advisory IDs to select public database entries. Advisories do not establish that the selected repository has an installed vulnerable dependency.", + destructive: false, + mcp: false, + args: z.object({ + repository: z + .string() + .min(1) + .describe( + "Source repository, or target context for global IDs, in OWNER/REPO form.", + ), + }), + options: z.object({ + githubAdvisory: z + .array( + optionValue("--github-advisory").regex( + /^GHSA-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}$/iu, + "GitHub advisory IDs must use GHSA-xxxx-xxxx-xxxx form.", + ), + ) + .default([]) + .describe( + "Exact GHSA ID; repeat to select a subset. Required for global advisories.", + ), + githubAdvisorySource: z + .enum(["repository", "global"]) + .default("repository") + .describe( + "Advisory API to read; global requires exact IDs and never lists the database.", + ), + githubState: z + .enum(GITHUB_ADVISORY_STATES) + .default("published") + .describe( + "Repository advisory list state; omit when selecting exact GHSA IDs.", + ), + }), + output: z + .array( + z.object({ + source: z.literal("github-security-advisory"), + selection: z.enum(["repository", "global"]), + repository: z.string(), + githubHost: z.string(), + ghsaId: z.string(), + url: z.string(), + advisory: z.record(z.string(), z.unknown()), + }), + ) + .optional(), + async run({ args, options }) { + const controller = new AbortController(); + const removeSignals = listenForAbort(dependencies, controller); + try { + return await ( + dependencies.importGitHubAdvisories ?? importGitHubAdvisories + )( + { + repository: args.repository, + advisoryIds: options.githubAdvisory, + source: options.githubAdvisorySource, + state: options.githubState, + signal: controller.signal, + }, + { + environment: dependencies.environment, + currentDirectory: dependencies.currentDirectory(), + }, + ); + } catch (error) { + const signal = controller.signal.reason; + if (signal === "SIGINT" || signal === "SIGTERM") { + exitCode = signal === "SIGINT" ? 130 : 143; + errorOutput.write( + `codex-security: GitHub advisory import ${signal === "SIGINT" ? "canceled" : "terminated"}.\n`, + ); + } else { + exitCode = 2; + errorOutput.write(`codex-security: ${diagnosticValue(error)}\n`); + } + return undefined; + } finally { + removeSignals(); + } + }, + }); const infoOutput = z.object({ sdkVersion: z.string(), bundledPluginVersion: z.string(), diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index a6863bb589..283b7b7fc9 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -16,6 +16,7 @@ import { validatePreparedOutputDir, } from "./runtime.js"; import { readVendorFindingsForPublication } from "./wiz-findings.js"; +import { readGitHubFindings } from "./github-findings.js"; import type { ExternalFindings, ExternalPublicationInput, @@ -47,7 +48,7 @@ function cloudRepositoryUrlKey(value: string): string { /** CLI and plugin callers prepare the same immutable request before approval. */ export interface ExternalPublicationOptions { repository: string; - provider: "wiz"; + provider: "wiz" | "github"; sourceKey: string; } @@ -278,6 +279,12 @@ export async function prepareExternalPublication( options: ExternalPublicationOptions, dependencies: ExternalPublicationDependencies = {}, ): Promise { + if (options.provider === "github") + return prepareNormalizedExternalPublication( + await readGitHubFindings(path), + options, + dependencies, + ); const input = await readVendorFindingsForPublication(path); return preparePublicationInput(input, options, dependencies); } @@ -330,7 +337,9 @@ async function preparePublicationInput( }; if (!options.sourceKey.trim() || Buffer.byteLength(options.sourceKey) > 512) { throw new CodexSecurityError( - "The source key must contain 1–512 UTF-8 bytes. Use a stable Wiz tenant/finding namespace, such as TENANT_ID/vulnerability-finding, across exports.", + options.provider === "wiz" + ? "The source key must contain 1–512 UTF-8 bytes. Use a stable Wiz tenant/finding namespace, such as TENANT_ID/vulnerability-finding, across exports." + : "The source key must contain 1–512 UTF-8 bytes. Use a stable GitHub host/repository/finding namespace across imports.", ); } progress({ phase: "reading", completed: 0 }); diff --git a/sdk/typescript/src/external-import-contract.ts b/sdk/typescript/src/external-import-contract.ts index 7da1c18fc8..e76b13b731 100644 --- a/sdk/typescript/src/external-import-contract.ts +++ b/sdk/typescript/src/external-import-contract.ts @@ -183,6 +183,28 @@ export function validateExternalEvidence( ) throw new Error("Locations must use repository-relative paths."); } + const details = result.details; + if (details) { + if (details.kind === "advisory") { + if (!details.advisory) + throw new Error("Advisory evidence requires its source context."); + if (details.advisory.source === "global" && details.repository) + throw new Error( + "Global advisory evidence cannot claim a source repository.", + ); + if (details.advisory.source === "repository" && !details.repository) + throw new Error( + "Repository advisory evidence requires its source repository.", + ); + } else { + if (!details.repository) + throw new Error( + "Repository finding evidence requires its source repository.", + ); + if (details.advisory) + throw new Error("Advisory source context requires advisory evidence."); + } + } const endLine = result.details?.code?.end_line; if (endLine !== null && endLine !== undefined) { const firstLine = result.locations?.[0]?.line; @@ -193,7 +215,7 @@ export function validateExternalEvidence( } const urls = [ result.url, - result.details?.repository.url, + result.details?.repository?.url, result.details?.file_url, ]; for (const value of urls) { diff --git a/sdk/typescript/src/external-import-models.ts b/sdk/typescript/src/external-import-models.ts index efc87b42f7..d588d2f64b 100644 --- a/sdk/typescript/src/external-import-models.ts +++ b/sdk/typescript/src/external-import-models.ts @@ -5,13 +5,14 @@ export type Id = string; export type RepoConnectorId = string; export type EnvironmentId = string; export type ResetMarker = string; -export type Provider = "wiz" | "endor" | "snyk"; +export type Provider = "wiz" | "endor" | "snyk" | "github"; export type SourceKey = string; export type ClientId = string; export type SourceFindingId = string; export type ExpectedVersion = number; export type Title = string; -export type Severity = "critical" | "high" | "medium" | "low" | "informational"; +export type Severity = + "critical" | "high" | "medium" | "low" | "informational" | "unknown"; export type Description = string | null; export type Url = string | null; /** @@ -21,6 +22,7 @@ export type AdvisoryIds = string[]; export type Name = string; export type Ecosystem = string | null; export type InstalledVersion = string | null; +export type VulnerableVersionRange = string | null; export type ManifestPath = string | null; /** * @maxItems 100 @@ -44,10 +46,11 @@ export type SourceScanId = string | null; */ export type ImageDigests = string[]; export type SourceUpdatedAt = number | null; -export type Kind = "sast" | "secret" | "iac"; +export type Kind = "sast" | "secret" | "iac" | "advisory"; export type Url1 = string; export type Id1 = string | null; export type Name1 = string | null; +export type Source = "repository" | "global"; export type SourceStatus = string | null; export type ScannerOrigin = string | null; export type ScannerVerdict = string | null; @@ -136,7 +139,7 @@ export type ObservationId2 = string; export type Version2 = number; export type Title1 = string; export type Severity1 = - "critical" | "high" | "medium" | "low" | "informational"; + "critical" | "high" | "medium" | "low" | "informational" | "unknown"; export type SourceUpdatedAt1 = number | null; export type CreatedAt3 = number; export type UpdatedAt1 = number; @@ -196,6 +199,7 @@ export interface ImportedPackage { name: Name; ecosystem?: Ecosystem; installed_version?: InstalledVersion; + vulnerable_version_range?: VulnerableVersionRange; manifest_path?: ManifestPath; fixed_versions?: FixedVersions; } @@ -211,7 +215,8 @@ export interface SourceData { */ export interface ImportedFindingDetails { kind: Kind; - repository: ImportedRepositoryReference; + repository?: ImportedRepositoryReference | null; + advisory?: ImportedAdvisoryContext | null; source_status?: SourceStatus; scanner_origin?: ScannerOrigin; scanner_verdict?: ScannerVerdict; @@ -231,6 +236,12 @@ export interface ImportedRepositoryReference { id?: Id1; name?: Name1; } +/** + * Where the advisory was read; applicability is assessed separately. + */ +export interface ImportedAdvisoryContext { + source: Source; +} export interface ImportedFindingRule { id?: Id2; short_id?: ShortId; diff --git a/sdk/typescript/src/github-advisories.ts b/sdk/typescript/src/github-advisories.ts new file mode 100644 index 0000000000..403d5bd71e --- /dev/null +++ b/sdk/typescript/src/github-advisories.ts @@ -0,0 +1,218 @@ +import type { Octokit } from "@octokit/core"; +import { CodexSecurityError, errorMessage } from "./errors.js"; +import { createAuthenticatedGitHub } from "./github.js"; + +export const GITHUB_ADVISORY_STATES = [ + "published", + "draft", + "triage", + "closed", +] as const; + +export interface GitHubAdvisoryImportOptions { + /** Source repository, or the selected target for explicit global advisory IDs. */ + repository: string; + /** Exact GHSA IDs. Required for the global advisory API. */ + advisoryIds?: readonly string[]; + /** Repository advisories by default; global never enumerates the database. */ + source?: "repository" | "global"; + /** Repository list state (default: published). Exact IDs ignore state. */ + state?: (typeof GITHUB_ADVISORY_STATES)[number]; + /** Defaults to GH_HOST or github.com. */ + githubHost?: string; + /** Omit to use the authenticated GitHub CLI. */ + githubToken?: string; + signal?: AbortSignal; +} + +type RepositoryAdvisory = Awaited>; +type GlobalAdvisory = Awaited>; + +export interface ImportedGitHubAdvisory { + source: "github-security-advisory"; + selection: "repository" | "global"; + repository: string; + githubHost: string; + ghsaId: string; + url: string; + /** Upstream disclosure, not proof that the selected repository is affected. */ + advisory: RepositoryAdvisory | GlobalAdvisory; +} + +interface GitHubAdvisoryImportDependencies { + environment?: NodeJS.ProcessEnv; + currentDirectory?: string; + createGitHub?: (host: string, signal?: AbortSignal) => Promise; +} + +/** Read repository disclosures or explicitly selected public GHSA records. */ +export async function importGitHubAdvisories( + options: GitHubAdvisoryImportOptions, + dependencies: GitHubAdvisoryImportDependencies = {}, +): Promise { + options.signal?.throwIfAborted(); + const repository = options.repository.trim(); + const parts = /^([a-z0-9_.-]+)\/([a-z0-9_.-]+)$/iu.exec(repository); + if (!parts || parts.slice(1).some((part) => /^\.{1,2}$/u.test(part))) + throw new CodexSecurityError("GitHub repository must be OWNER/REPO."); + const owner = parts[1]!; + const repo = parts[2]!; + const ids = [ + ...new Set( + (options.advisoryIds ?? []).map((id) => + id + .trim() + .toLowerCase() + .replace(/^ghsa-/u, "GHSA-"), + ), + ), + ]; + if (ids.some((id) => !/^GHSA-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}$/u.test(id))) + throw new CodexSecurityError( + "GitHub advisory IDs must use GHSA-xxxx-xxxx-xxxx form.", + ); + const source = options.source ?? "repository"; + const state = options.state ?? "published"; + if (source !== "repository" && source !== "global") + throw new CodexSecurityError( + "GitHub advisory source must be repository or global.", + ); + if (!GITHUB_ADVISORY_STATES.includes(state)) + throw new CodexSecurityError("GitHub advisory state is invalid."); + if (source === "global" && ids.length === 0) + throw new CodexSecurityError( + "Select explicit GHSA IDs for global advisories; database-wide import is not supported.", + ); + if (ids.length > 0 && state !== "published") + throw new CodexSecurityError( + "--github-state only filters repository lists; omit it when selecting exact GHSA IDs.", + ); + const environment = dependencies.environment ?? process.env; + const host = + options.githubHost?.trim() ?? + (environment["GH_HOST"]?.trim() || "github.com"); + if (!host) throw new CodexSecurityError("GitHub host must not be empty."); + + try { + const github = dependencies.createGitHub + ? await dependencies.createGitHub(host, options.signal) + : await createAuthenticatedGitHub(host, { + token: options.githubToken, + environment, + currentDirectory: dependencies.currentDirectory, + signal: options.signal, + }); + const advisories: (RepositoryAdvisory | GlobalAdvisory)[] = []; + if (ids.length > 0) { + for (const id of ids) { + options.signal?.throwIfAborted(); + advisories.push( + source === "global" + ? await fetchGlobalAdvisory(github, id, options.signal) + : await fetchRepositoryAdvisory( + github, + owner, + repo, + id, + options.signal, + ), + ); + } + } else { + let after: string | undefined; + const cursors = new Set(); + do { + options.signal?.throwIfAborted(); + const response = await github.request( + "GET /repos/{owner}/{repo}/security-advisories", + { + owner, + repo, + state, + per_page: 100, + after, + request: { signal: options.signal, redirect: "error" }, + }, + ); + advisories.push(...response.data); + const next = response.headers.link + ?.split(",") + .find((link) => /;\s*rel="next"/u.test(link)); + const nextUrl = next?.match(/<([^>]+)>/u)?.[1]; + // Keep requests on the authenticated route; only carry over the cursor. + after = nextUrl + ? (new URL(nextUrl).searchParams.get("after") ?? undefined) + : undefined; + if (next && (!after || cursors.has(after))) + throw new CodexSecurityError( + "GitHub returned invalid advisory pagination.", + ); + if (after) cursors.add(after); + } while (after !== undefined); + } + return advisories.map((advisory) => ({ + source: "github-security-advisory", + selection: source, + repository, + githubHost: host, + ghsaId: advisory.ghsa_id, + url: advisory.html_url, + advisory, + })); + } catch (error) { + options.signal?.throwIfAborted(); + if (error instanceof CodexSecurityError) throw error; + const status = + typeof error === "object" && error !== null && "status" in error + ? error.status + : undefined; + if (status === 401) + throw new CodexSecurityError("GitHub authentication failed."); + if (status === 403) + throw new CodexSecurityError( + "GitHub denied security advisory access. Check repository advisory permissions and API rate limits.", + ); + if (status === 404) + throw new CodexSecurityError( + "GitHub repository or security advisory was not found or is not accessible.", + ); + if (status === 429) + throw new CodexSecurityError( + "GitHub request was rate limited. Wait and retry.", + ); + throw new CodexSecurityError( + `GitHub security advisory request failed${typeof status === "number" ? ` (HTTP ${status})` : ""}: ${errorMessage(error)}`, + ); + } +} + +async function fetchRepositoryAdvisory( + github: Octokit, + owner: string, + repo: string, + id: string, + signal?: AbortSignal, +) { + const response = await github.request( + "GET /repos/{owner}/{repo}/security-advisories/{ghsa_id}", + { + owner, + repo, + ghsa_id: id, + request: { signal, redirect: "error" }, + }, + ); + return response.data; +} + +async function fetchGlobalAdvisory( + github: Octokit, + id: string, + signal?: AbortSignal, +) { + const response = await github.request("GET /advisories/{ghsa_id}", { + ghsa_id: id, + request: { signal, redirect: "error" }, + }); + return response.data; +} diff --git a/sdk/typescript/src/github-findings.ts b/sdk/typescript/src/github-findings.ts new file mode 100644 index 0000000000..6a482785ad --- /dev/null +++ b/sdk/typescript/src/github-findings.ts @@ -0,0 +1,315 @@ +import { hash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { promisify } from "node:util"; +import { gunzip } from "node:zlib"; +import { CodexSecurityError } from "./errors.js"; +import { validateExternalEvidence } from "./external-import-contract.js"; +import type { ExternalFinding, ExternalFindings } from "./external-findings.js"; +import type { ImportedGitHubCodeScanningAlert } from "./github.js"; +import type { ImportedGitHubAdvisory } from "./github-advisories.js"; + +const decompressGzip = promisify(gunzip); + +function object(value: unknown): Record | undefined { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +function text(value: unknown): string | null { + return typeof value === "string" && value.trim() ? value : null; +} + +function repositoryName(value: unknown): string { + const repository = text(value); + const parts = /^([a-z0-9_.-]+)\/([a-z0-9_.-]+)$/iu.exec(repository ?? ""); + if (!parts || parts.slice(1).some((part) => /^\.{1,2}$/u.test(part))) + throw new Error("GitHub repository must be OWNER/REPO."); + return repository!; +} + +function sourceTimestamp(value: unknown): number | null { + if (value === undefined || value === null) return null; + const seconds = typeof value === "string" ? Date.parse(value) / 1000 : NaN; + if (!Number.isFinite(seconds) || seconds < 0) + throw new Error("GitHub updated_at is not a valid timestamp."); + return Math.floor(seconds); +} + +function severity(value: unknown): string { + // SARIF error/warning/note are not vulnerability severity ratings. + return typeof value === "string" && + ["critical", "high", "medium", "low", "informational"].includes(value) + ? value + : "unknown"; +} + +function codeScanningFinding(record: Record): ExternalFinding { + const repository = repositoryName(record["repository"]); + const alert = object(record["alert"]); + const number = record["number"]; + if ( + !alert || + typeof number !== "number" || + !Number.isSafeInteger(number) || + number < 1 || + alert["number"] !== number + ) + throw new Error( + "Expected a GitHub code-scanning import with matching alert numbers.", + ); + const url = text(record["url"]); + if (!url || url !== alert["html_url"]) + throw new Error("GitHub alert URL must match the original alert."); + const parsedUrl = new URL(url); + if ( + parsedUrl.pathname.toLowerCase() !== + `/${repository}/security/code-scanning/${number}`.toLowerCase() + ) + throw new Error( + "GitHub alert URL does not match the selected repository and alert.", + ); + const rule = object(alert["rule"]); + const instance = object(alert["most_recent_instance"]); + const ref = text(instance?.["ref"]); + if (!ref) + throw new Error( + "GitHub code-scanning evidence requires the selected instance ref.", + ); + const location = object(instance?.["location"]); + const tool = object(alert["tool"]); + const message = text(object(instance?.["message"])?.["text"]); + const description = + text(rule?.["full_description"]) ?? text(rule?.["description"]); + const tags = Array.isArray(rule?.["tags"]) ? rule["tags"] : []; + const weaknessIds = [ + ...new Set( + tags.flatMap((tag: unknown) => { + const match = + typeof tag === "string" + ? /^external\/cwe\/cwe-(\d+)$/iu.exec(tag) + : null; + return match ? [`CWE-${Number(match[1])}`] : []; + }), + ), + ]; + const path = text(location?.["path"]); + const sourceId = `${parsedUrl.host.toLowerCase()}/${repository.toLowerCase()}/code-scanning/${number}/${hash("sha256", ref)}`; + return { + source_finding_id: sourceId, + evidence: validateExternalEvidence({ + title: + text(rule?.["name"]) ?? + text(rule?.["description"]) ?? + `GitHub code-scanning alert #${number}`, + severity: severity(rule?.["security_severity_level"]), + description: + [...new Set([message, description].filter(Boolean))].join("\n\n") || + null, + url, + locations: path ? [{ path, line: location?.["start_line"] ?? null }] : [], + branch: ref, + code_revision: text(instance?.["commit_sha"]), + source_updated_at: sourceTimestamp(alert["updated_at"]), + source_data: record, + details: { + kind: "sast", + repository: { + url: `${parsedUrl.origin}/${repository}`, + name: repository, + }, + source_status: text(instance?.["state"]) ?? text(alert["state"]), + scanner_origin: text(tool?.["name"]), + rule: { id: text(rule?.["id"]), name: text(rule?.["name"]) }, + weakness_ids: weaknessIds, + remediation_instructions: text(rule?.["help"]), + code: path ? { end_line: location?.["end_line"] ?? null } : null, + }, + }), + }; +} + +function advisoryFinding(record: Record): ExternalFinding { + const repository = repositoryName(record["repository"]); + const advisory = object(record["advisory"]); + const id = text(record["ghsaId"]); + const selection = record["selection"]; + if ( + !advisory || + !id || + !/^GHSA-[A-Z0-9]{4}-[A-Z0-9]{4}-[A-Z0-9]{4}$/iu.test(id) || + text(advisory["ghsa_id"])?.toUpperCase() !== id.toUpperCase() + ) + throw new Error( + "Expected a GitHub advisory import with matching GHSA IDs.", + ); + if (selection !== "repository" && selection !== "global") + throw new Error("GitHub advisory selection must be repository or global."); + const url = text(record["url"]); + if (!url || url !== advisory["html_url"]) + throw new Error("GitHub advisory URL must match the original advisory."); + const parsedUrl = new URL(url); + const host = text(record["githubHost"]); + const expectedPath = + selection === "repository" + ? `/${repository}/security/advisories/${id}` + : `/advisories/${id}`; + if ( + parsedUrl.host.toLowerCase() !== host?.toLowerCase() || + parsedUrl.pathname.toLowerCase() !== expectedPath.toLowerCase() + ) + throw new Error( + "GitHub advisory URL does not match the selected source and GHSA ID.", + ); + const vulnerabilities = + advisory["vulnerabilities"] === null ? [] : advisory["vulnerabilities"]; + if (!Array.isArray(vulnerabilities)) + throw new Error("GitHub advisory vulnerabilities must be an array."); + const identifiers = Array.isArray(advisory["identifiers"]) + ? advisory["identifiers"] + : []; + const advisoryIds = [ + id, + text(advisory["cve_id"]), + ...identifiers.map((identifier) => text(object(identifier)?.["value"])), + ].filter((value): value is string => value !== null); + const packages = vulnerabilities.flatMap((value) => { + const vulnerability = object(value); + const pkg = object(vulnerability?.["package"]); + const name = text(pkg?.["name"]); + // GitHub permits incomplete package metadata in advisory disclosures. + // Preserve it in source_data without inventing a typed package identity. + if (!name) return []; + const fixedVersion = + text(vulnerability?.["first_patched_version"]) ?? + text(vulnerability?.["patched_versions"]); + return [ + { + name, + ecosystem: text(pkg?.["ecosystem"]), + ...(text(vulnerability?.["vulnerable_version_range"]) + ? { + vulnerable_version_range: + vulnerability!["vulnerable_version_range"], + } + : {}), + fixed_versions: fixedVersion ? [fixedVersion] : [], + }, + ]; + }); + const cwes = Array.isArray(advisory["cwes"]) ? advisory["cwes"] : []; + return { + source_finding_id: `${parsedUrl.host.toLowerCase()}/${selection === "repository" ? repository.toLowerCase() : "global"}/security-advisory/${id.toUpperCase()}`, + evidence: validateExternalEvidence({ + title: text(advisory["summary"]) ?? id, + severity: severity(advisory["severity"]), + description: text(advisory["description"]), + url, + advisory_ids: [...new Set(advisoryIds)], + packages, + source_updated_at: sourceTimestamp(advisory["updated_at"]), + source_data: record, + details: { + kind: "advisory", + advisory: { source: selection }, + ...(selection === "repository" + ? { + repository: { + url: `${parsedUrl.origin}/${repository}`, + name: repository, + }, + } + : {}), + source_status: text(advisory["withdrawn_at"]) + ? "withdrawn" + : (text(advisory["state"]) ?? + (selection === "global" ? "published" : null)), + scanner_origin: "GitHub Security Advisories", + weakness_ids: cwes + .map((value) => text(object(value)?.["cwe_id"])) + .filter((value): value is string => value !== null), + }, + }), + }; +} + +function normalizeRecords(records: readonly unknown[]): ExternalFindings { + const result: ExternalFindings = { + read: records.length, + findings: [], + excluded: [], + }; + const identities = new Set(); + for (const [index, value] of records.entries()) { + const record = object(value); + try { + if (!record) throw new Error("GitHub finding must be an object."); + let finding: ExternalFinding; + if (record["source"] === "github-code-scanning") + finding = codeScanningFinding(record); + else if (record["source"] === "github-security-advisory") + finding = advisoryFinding(record); + else + throw new Error( + "Expected saved output from import github or import github-advisories; raw or incomplete API responses need an explicit mapping.", + ); + if (identities.has(finding.source_finding_id)) + throw new CodexSecurityError( + `Duplicate source finding id ${JSON.stringify(finding.source_finding_id)}. Select each GitHub occurrence once.`, + ); + identities.add(finding.source_finding_id); + result.findings.push(finding); + } catch (error) { + if (error instanceof CodexSecurityError) throw error; + result.excluded.push({ + position: index + 1, + source_finding_id: text(record?.["url"]), + reason: error instanceof Error ? error.message : String(error), + }); + } + } + return result; +} + +/** Map selected code-scanning instances while retaining complete upstream evidence. */ +export function normalizeGitHubCodeScanningAlerts( + alerts: readonly ImportedGitHubCodeScanningAlert[], +): ExternalFindings { + return normalizeRecords(alerts); +} + +/** Map advisory disclosures without asserting an installed vulnerable dependency. */ +export function normalizeGitHubAdvisories( + advisories: readonly ImportedGitHubAdvisory[], +): ExternalFindings { + return normalizeRecords(advisories); +} + +/** Read unfiltered JSON/JSONL output saved by the read-only GitHub import commands. */ +export async function readGitHubFindings( + path: string, +): Promise { + try { + const bytes = await readFile(path); + const decoded = + bytes[0] === 0x1f && bytes[1] === 0x8b + ? await decompressGzip(bytes) + : bytes; + const contents = new TextDecoder("utf-8", { fatal: true }).decode(decoded); + let payload: unknown; + try { + payload = JSON.parse(contents); + } catch { + payload = contents + .split(/\r?\n/u) + .filter((line) => line.trim()) + .map((line) => JSON.parse(line)); + } + return normalizeRecords(Array.isArray(payload) ? payload : [payload]); + } catch (cause) { + throw new CodexSecurityError( + `Could not read GitHub findings as UTF-8 JSON or JSONL: ${cause instanceof Error ? cause.message : String(cause)}`, + { cause }, + ); + } +} diff --git a/sdk/typescript/src/index.ts b/sdk/typescript/src/index.ts index 312fb99ca9..83bc349acf 100644 --- a/sdk/typescript/src/index.ts +++ b/sdk/typescript/src/index.ts @@ -184,6 +184,16 @@ export type { ImportRepository, SourceReport, } from "./external-import-models.js"; +export { + normalizeGitHubCodeScanningAlerts, + normalizeGitHubAdvisories, + readGitHubFindings, +} from "./github-findings.js"; +export { importGitHubAdvisories } from "./github-advisories.js"; +export type { + GitHubAdvisoryImportOptions, + ImportedGitHubAdvisory, +} from "./github-advisories.js"; export { importGitHubCodeScanningAlerts } from "./github.js"; export type { GitHubCodeScanningImportOptions, diff --git a/sdk/typescript/tests-ts/cli-fixtures.ts b/sdk/typescript/tests-ts/cli-fixtures.ts index d9822ec3e9..f0cd76630d 100644 --- a/sdk/typescript/tests-ts/cli-fixtures.ts +++ b/sdk/typescript/tests-ts/cli-fixtures.ts @@ -211,6 +211,7 @@ export function dependencies( onCodex?: OnCodex; linearClient?: MainDependencies["linearClient"]; importGitHubAlerts?: MainDependencies["importGitHubAlerts"]; + importGitHubAdvisories?: MainDependencies["importGitHubAdvisories"]; onRepositoryCommand?: ( ...arguments_: Parameters ) => string | Promise; @@ -290,6 +291,9 @@ export function dependencies( ...(options.linearClient === undefined ? {} : { linearClient: options.linearClient }), + ...(options.importGitHubAdvisories === undefined + ? {} + : { importGitHubAdvisories: options.importGitHubAdvisories }), ...(options.importGitHubAlerts === undefined ? {} : { importGitHubAlerts: options.importGitHubAlerts }), diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index b1749c54f1..9d83a812fa 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -21,6 +21,7 @@ import { prepareNormalizedExternalPublication, type ExternalPublicationProgress, } from "../src/external-findings-publish.js"; +import { readGitHubFindings } from "../src/github-findings.js"; import { readVendorFindings } from "../src/wiz-findings.js"; import { validateExternalEvidence, @@ -34,6 +35,10 @@ import type { } from "../src/external-import-models.js"; import { main } from "../src/cli.js"; import { dependencies } from "./cli-fixtures.js"; +import { + importedAlert, + importedAdvisory, +} from "./support/github-import-fixtures.js"; import { createCliTest } from "./support/cli-run.js"; const cleanups: (() => Promise)[] = []; @@ -574,7 +579,7 @@ test("Wiz network-scan findings are excluded without requiring a detection metho }); test("unsupported records are visible exclusions; duplicate identities and unfinished pages stop publication", async () => { - const f = await fixture([normalized(), normalized("unknown", "unknown")]); + const f = await fixture([normalized(), normalized("unknown", "unsupported")]); const parsed = await readVendorFindings(f.file); expect(parsed.findings).toHaveLength(1); expect(parsed.excluded).toHaveLength(1); @@ -841,7 +846,10 @@ test.each(["human", "json"])( source_data: { id: "vendor-selected", note: "Original vendor detail" }, }, }; - const f = await fixture([selected, normalized("unsupported", "unknown")]); + const f = await fixture([ + selected, + normalized("unsupported", "unsupported"), + ]); const cli = createCliTest(main); let prompted = false; expect( @@ -2017,7 +2025,7 @@ test("unsafe evidence integers are rejected while exact strings and safe integer test("mixed valid and rejected input reports partial completion with exit code 1", async () => { const f = await fixture([ - normalized("invalid", "unknown"), + normalized("invalid", "unsupported"), normalized("valid"), ]); const cli = createCliTest(main); @@ -2035,7 +2043,7 @@ test("mixed valid and rejected input reports partial completion with exit code 1 test("server item failures identify the vendor record after local exclusions", async () => { const f = await fixture([ - normalized("invalid", "unknown"), + normalized("invalid", "unsupported"), normalized("rejected-vendor"), ]); f.state.finalError = true; @@ -2414,3 +2422,134 @@ test("normalized SDK publication preserves caller evidence and exclusion account expect(result.excluded).toEqual(input.excluded); expect(result.verified).toBe(1); }); + +test("GitHub publication previews, imports, re-imports, updates and resumes selected-ref evidence", async () => { + const imported = importedAlert(); + const f = await fixture([imported]); + const githubOptions = { + ...options, + provider: "github" as const, + sourceKey: "github.com/example/project/code-scanning", + }; + const command = f.command.map((arg) => + arg === "wiz" + ? "github" + : arg === options.sourceKey + ? githubOptions.sourceKey + : arg, + ); + const cli = createCliTest(main); + expect( + await cli.runCli([...command, "--dry-run", "--format", "json"], f.cliDeps), + ).toBe(0); + expect(f.posts).toHaveLength(0); + expect(JSON.parse(cli.stdout.text()).source.provider).toBe("github"); + const first = await ( + await prepareExternalPublication(f.file, githubOptions, f.deps) + ).publish(); + expect(first.counts.created).toBe(1); + expect(first.verified).toBe(1); + const repeated = await ( + await prepareExternalPublication(f.file, githubOptions, f.deps) + ).publish(); + expect(repeated.counts.unchanged).toBe(1); + imported.alert.most_recent_instance.state = "fixed"; + await writeFile(f.file, JSON.stringify([imported])); + f.state.loseResponse = true; + await expect( + (await prepareExternalPublication(f.file, githubOptions, f.deps)).publish(), + ).rejects.toBeInstanceOf(ExternalPublicationError); + const prepared = await prepareExternalPublication( + f.file, + githubOptions, + f.deps, + ); + expect(prepared.preview.resumed).toBe(true); + const resumed = await prepared.publish(); + expect(resumed.counts.updated).toBe(1); + expect(resumed.verified).toBe(1); + expect([...f.reports.values()][0]).toMatchObject({ + source: { provider: "github" }, + evidence: { details: { source_status: "fixed" }, source_data: imported }, + assessment: { state: "not_assessed" }, + }); + await writeFile(f.file, JSON.stringify([importedAlert("refs/heads/review")])); + const differentRef = await ( + await prepareExternalPublication(f.file, githubOptions, f.deps) + ).publish(); + expect(differentRef.counts.created).toBe(1); + expect(f.reports.size).toBe(2); +}); + +test("GitHub imports reject a different Cloud destination before uploading", async () => { + const imported = importedAlert("refs/heads/main", "git.example.test"); + const f = await fixture([imported]); + await expect( + prepareExternalPublication( + f.file, + { ...options, provider: "github" }, + f.deps, + ), + ).rejects.toThrow("does not match the selected Cloud repository"); + expect(f.posts).toHaveLength(0); +}); + +test("global GHSA publication retains unassessed applicability through repeat import and withdrawal", async () => { + const imported = importedAdvisory("global"); + const f = await fixture([imported]); + const selected = { + ...options, + provider: "github" as const, + sourceKey: "github.com/example/project/security-advisories", + }; + expect( + ( + await ( + await prepareExternalPublication(f.file, selected, f.deps) + ).publish() + ).counts.created, + ).toBe(1); + expect( + ( + await ( + await prepareExternalPublication(f.file, selected, f.deps) + ).publish() + ).counts.unchanged, + ).toBe(1); + const withdrawn = { + ...imported, + advisory: { ...imported.advisory, withdrawn_at: "2026-01-03T00:00:00Z" }, + }; + await writeFile(f.file, JSON.stringify([withdrawn])); + const result = await ( + await prepareExternalPublication(f.file, selected, f.deps) + ).publish(); + expect(result.counts.updated).toBe(1); + expect(result.verified).toBe(1); + const report = [...f.reports.values()][0]!; + expect(report.evidence.details).toMatchObject({ + kind: "advisory", + advisory: { source: "global" }, + source_status: "withdrawn", + }); + expect(report.evidence.details).not.toHaveProperty("repository"); + expect(report.assessment?.state).toBe("not_assessed"); + expect(report.evidence.packages![0]!.installed_version).toBeNull(); +}); + +test("saved GitHub JSONL and gzip preserve evidence while malformed UTF-8 cannot change identities", async () => { + const imported = [importedAlert(), importedAdvisory("global")]; + const f = await fixture(imported); + const jsonl = imported.map((record) => JSON.stringify(record)).join("\n"); + for (const bytes of [Buffer.from(jsonl), gzipSync(jsonl)]) { + await writeFile(f.file, bytes); + const parsed = await readGitHubFindings(f.file); + expect(parsed.read).toBe(2); + expect(parsed.excluded).toEqual([]); + expect( + parsed.findings.map((finding) => finding.evidence.source_data) as unknown, + ).toEqual(imported); + } + await writeFile(f.file, Buffer.from([0xff])); + await expect(readGitHubFindings(f.file)).rejects.toThrow("UTF-8"); +}); diff --git a/sdk/typescript/tests-ts/github-advisories.test.ts b/sdk/typescript/tests-ts/github-advisories.test.ts new file mode 100644 index 0000000000..c4f5b42d91 --- /dev/null +++ b/sdk/typescript/tests-ts/github-advisories.test.ts @@ -0,0 +1,199 @@ +import { Octokit } from "@octokit/core"; +import { expect, mock, test } from "bun:test"; +import { + importGitHubAdvisories, + type GitHubAdvisoryImportOptions, +} from "../src/github-advisories.js"; +import { main } from "../src/cli.js"; +import { dependencies, FakeSignals } from "./cli-fixtures.js"; +import { captureCli, runCapturedCli } from "./support/cli-run.js"; +import { importedAdvisory } from "./support/github-import-fixtures.js"; + +function client(respond: (url: URL) => Response): Octokit { + return new Octokit({ + auth: "SYNTHETIC_ADVISORY_TOKEN", + request: { + fetch: async ( + resource: Parameters[0], + init?: Parameters[1], + ) => { + expect(init?.method).toBe("GET"); + expect(init?.redirect).toBe("error"); + return respond( + new URL(resource instanceof Request ? resource.url : resource), + ); + }, + }, + }); +} + +test("repository advisory lists default to published and paginate cursors with full evidence", async () => { + const paths: string[] = []; + const github = client((url) => { + paths.push(url.toString()); + expect(url.pathname).toBe("/repos/example/project/security-advisories"); + expect(url.searchParams.get("state")).toBe("published"); + expect(url.searchParams.get("per_page")).toBe("100"); + return Response.json( + [importedAdvisory().advisory], + url.searchParams.has("after") + ? undefined + : { + headers: { + link: '; rel="next"', + }, + }, + ); + }); + const imported = await importGitHubAdvisories( + { repository: "example/project" }, + { createGitHub: async () => github }, + ); + expect(paths).toHaveLength(2); + expect(new URL(paths[1]!).searchParams.get("after")).toBe("cursor-2"); + expect(imported[0]).toEqual(importedAdvisory()); + expect(JSON.stringify(imported)).not.toContain("SYNTHETIC_ADVISORY_TOKEN"); +}); + +test("global advisory selection uses only explicit deduplicated IDs and preserves the source family", async () => { + const selected = importedAdvisory("global"); + const paths: string[] = []; + const github = client((url) => { + paths.push(url.pathname); + return Response.json(selected.advisory); + }); + const imported = await importGitHubAdvisories( + { + repository: "example/project", + source: "global", + advisoryIds: [selected.ghsaId.toLowerCase(), selected.ghsaId], + }, + { createGitHub: async () => github }, + ); + expect(paths).toEqual([`/advisories/${selected.ghsaId}`]); + expect(imported).toEqual([selected]); +}); + +test("explicit repository IDs allow private drafts without listing other advisories", async () => { + const selected = importedAdvisory(); + const draft = { ...selected.advisory, state: "draft" as const }; + const github = client((url) => { + expect(url.pathname).toBe( + `/repos/example/project/security-advisories/${selected.ghsaId}`, + ); + expect(url.searchParams.has("state")).toBe(false); + return Response.json(draft); + }); + const imported = await importGitHubAdvisories( + { repository: "example/project", advisoryIds: [selected.ghsaId] }, + { createGitHub: async () => github }, + ); + expect(imported[0]!.advisory).toEqual(draft); +}); + +test("global crawl and misleading exact-ID state filters fail before GitHub authentication", async () => { + const createGitHub = mock(async () => client(() => Response.json([]))); + await expect( + importGitHubAdvisories( + { repository: "example/project", source: "global" }, + { createGitHub }, + ), + ).rejects.toThrow("explicit GHSA IDs"); + await expect( + importGitHubAdvisories( + { + repository: "example/project", + advisoryIds: [importedAdvisory().ghsaId], + state: "draft", + }, + { createGitHub }, + ), + ).rejects.toThrow("only filters repository lists"); + expect(createGitHub).not.toHaveBeenCalled(); +}); + +test("a later advisory page failure returns no incomplete selection", async () => { + const github = client((url) => + url.searchParams.has("after") + ? Response.json({ message: "Rate limit" }, { status: 429 }) + : Response.json([importedAdvisory().advisory], { + headers: { + link: '; rel="next"', + }, + }), + ); + await expect( + importGitHubAdvisories( + { repository: "example/project" }, + { createGitHub: async () => github }, + ), + ).rejects.toThrow("rate limited"); +}); + +test("GitHub advisory CLI exposes separate selection defaults and exact public IDs", async () => { + let options: GitHubAdvisoryImportOptions | undefined; + const output = captureCli(main, "stdout"); + expect( + await output.run( + [ + "import", + "github-advisories", + "example/project", + "--github-advisory", + importedAdvisory().ghsaId, + "--github-advisory-source", + "global", + "--format", + "json", + ], + dependencies({ + importGitHubAdvisories: async (value) => { + options = value; + return [importedAdvisory("global")]; + }, + }), + ), + ).toBe(0); + expect(options).toMatchObject({ + repository: "example/project", + source: "global", + state: "published", + advisoryIds: [importedAdvisory().ghsaId], + }); + expect(JSON.parse(output.text())).toEqual([importedAdvisory("global")]); + const schema = captureCli(main, "stdout"); + expect( + await schema.run( + ["import", "github-advisories", "--schema", "--format", "json"], + dependencies(), + ), + ).toBe(0); + expect(JSON.parse(schema.text())).toMatchObject({ + options: { + properties: { + githubAdvisorySource: { default: "repository" }, + githubState: { default: "published" }, + }, + }, + }); +}); + +test("advisory CLI cancellation removes handlers and preserves its exit code", async () => { + const signals = new FakeSignals(); + expect( + await runCapturedCli( + main, + ["import", "github-advisories", "example/project"], + dependencies({ + signals, + importGitHubAdvisories: async ({ signal }) => { + signals.emit("SIGINT"); + signal!.throwIfAborted(); + return []; + }, + }), + ), + ).toBe(130); + expect(signals.listeners.get("SIGINT")?.size).toBe(0); + expect(signals.listeners.get("SIGTERM")?.size).toBe(0); +}); diff --git a/sdk/typescript/tests-ts/github-findings.test.ts b/sdk/typescript/tests-ts/github-findings.test.ts new file mode 100644 index 0000000000..2dc917ace1 --- /dev/null +++ b/sdk/typescript/tests-ts/github-findings.test.ts @@ -0,0 +1,213 @@ +import { expect, test } from "bun:test"; +import { validateExternalEvidence } from "../src/external-import-contract.js"; +import { + normalizeGitHubCodeScanningAlerts, + normalizeGitHubAdvisories, +} from "../src/github-findings.js"; +import type { ImportedGitHubCodeScanningAlert } from "../src/github.js"; + +import { + importedAlert, + importedAdvisory, +} from "./support/github-import-fixtures.js"; + +test("code-scanning mapping preserves full evidence and selected instance status without assessing it", () => { + const imported = importedAlert("refs/heads/review"); + const result = normalizeGitHubCodeScanningAlerts([imported]); + expect(result.excluded).toEqual([]); + expect(result.findings[0]).toMatchObject({ + evidence: { + title: "Untrusted query input", + severity: "high", + branch: "refs/heads/review", + code_revision: "a".repeat(40), + source_updated_at: 1767312000, + source_data: imported, + locations: [{ path: "src/query.ts", line: 12 }], + details: { + kind: "sast", + repository: { url: "https://github.com/example/project" }, + source_status: "open", + scanner_origin: "Synthetic Scanner", + weakness_ids: ["CWE-89"], + code: { end_line: 14 }, + }, + }, + }); + expect(result.findings[0]!.evidence).not.toHaveProperty("assessment"); +}); + +test("code-scanning identities isolate hosts, repositories and selected refs but not updated evidence", () => { + const first = importedAlert(); + const changed = structuredClone(first); + changed.alert.most_recent_instance.commit_sha = "b".repeat(40); + changed.alert.most_recent_instance.state = "fixed"; + const identity = (input: ImportedGitHubCodeScanningAlert) => + normalizeGitHubCodeScanningAlerts([input]).findings[0]!.source_finding_id; + expect(identity(changed)).toBe(identity(first)); + expect(identity(importedAlert("refs/heads/review"))).not.toBe( + identity(first), + ); + expect( + identity(importedAlert("refs/heads/main", "git.example.test")), + ).not.toBe(identity(first)); + const other = JSON.parse( + JSON.stringify(first).replaceAll("example/project", "example/other"), + ) as ImportedGitHubCodeScanningAlert; + expect(identity(other)).not.toBe(identity(first)); + expect(() => normalizeGitHubCodeScanningAlerts([first, changed])).toThrow( + "Duplicate source finding id", + ); +}); + +test("SARIF levels do not fabricate security severity and malformed siblings remain explicit exclusions", () => { + const noRating = importedAlert(); + delete noRating.alert.rule.security_severity_level; + const badRepository = importedAlert("refs/heads/other"); + badRepository.repository = "example/wrong"; + const result = normalizeGitHubCodeScanningAlerts([badRepository, noRating]); + expect(result.read).toBe(2); + expect(result.findings).toHaveLength(1); + expect(result.findings[0]!.evidence.severity).toBe("unknown"); + expect(result.excluded).toEqual([ + { + position: 1, + source_finding_id: badRepository.url, + reason: + "GitHub alert URL does not match the selected repository and alert.", + }, + ]); +}); + +test.each(["repository", "global"] as const)( + "%s advisories preserve package ranges without claiming an installed dependency", + (selection) => { + const imported = importedAdvisory(selection); + const result = normalizeGitHubAdvisories([imported]); + expect(result.excluded).toEqual([]); + expect(result.findings[0]!.evidence).toMatchObject({ + advisory_ids: [imported.ghsaId, "CVE-2026-0001"], + packages: [ + { + name: "example-package", + ecosystem: "npm", + vulnerable_version_range: ">= 1.0.0, < 1.2.3", + installed_version: null, + manifest_path: null, + fixed_versions: ["1.2.3"], + }, + ], + locations: [], + source_data: imported, + details: { + kind: "advisory", + advisory: { source: selection }, + source_status: "published", + }, + }); + if (selection === "repository") + expect(result.findings[0]!.evidence.details!.repository!.url).toBe( + "https://github.com/example/project", + ); + else + expect(result.findings[0]!.evidence.details).not.toHaveProperty( + "repository", + ); + }, +); + +test("advisory withdrawal and unknown severity remain source evidence with stable identity", () => { + const imported = importedAdvisory("global"); + const before = normalizeGitHubAdvisories([imported]).findings[0]!; + const changed = { + ...imported, + advisory: { + ...imported.advisory, + withdrawn_at: "2026-01-03T00:00:00Z", + severity: null, + }, + }; + const after = normalizeGitHubAdvisories([changed as typeof imported]) + .findings[0]!; + expect(after.source_finding_id).toBe(before.source_finding_id); + expect(after.evidence.severity).toBe("unknown"); + expect(after.evidence.details!.source_status).toBe("withdrawn"); + expect(after.evidence).not.toHaveProperty("assessment"); +}); + +test("repository/global advisory identities stay separate and source URL conflicts are explicit exclusions", () => { + const repository = importedAdvisory(); + const global = importedAdvisory("global"); + expect(normalizeGitHubAdvisories([repository, global]).findings).toHaveLength( + 2, + ); + repository.repository = "example/other"; + const result = normalizeGitHubAdvisories([repository, global]); + expect(result.findings).toHaveLength(1); + expect(result.excluded[0]!.reason).toContain( + "does not match the selected source", + ); +}); + +test.each(["repository", "global"] as const)( + "%s advisories retain disclosures with nullable package metadata", + (selection) => { + const imported = importedAdvisory(selection); + const known = imported.advisory.vulnerabilities![0]!; + for (const vulnerabilities of [ + null, + [ + { ...known, package: null }, + { ...known, package: { ecosystem: "npm", name: null } }, + known, + ], + ]) { + const input = { + ...imported, + advisory: { ...imported.advisory, vulnerabilities }, + } as unknown as typeof imported; + const result = normalizeGitHubAdvisories([input]); + expect(result.excluded).toEqual([]); + expect(result.findings).toHaveLength(1); + expect(result.findings[0]!.evidence.source_data as unknown).toEqual( + input, + ); + expect(result.findings[0]!.evidence.packages).toHaveLength( + vulnerabilities === null ? 0 : 1, + ); + } + }, +); + +test("normalized global advisory evidence rejects invented source repository provenance", () => { + const evidence = normalizeGitHubAdvisories([importedAdvisory("global")]) + .findings[0]!.evidence; + evidence.details!.repository = { url: "https://github.com/example/project" }; + expect(() => validateExternalEvidence(evidence)).toThrow( + "Global advisory evidence cannot claim a source repository", + ); +}); + +test.each([ + { kind: "sast" }, + { kind: "secret" }, + { kind: "iac" }, + { kind: "advisory" }, + { kind: "advisory", advisory: { source: "repository" } }, + { + kind: "sast", + repository: { url: "https://github.com/example/project" }, + advisory: { source: "repository" }, + }, +])( + "external evidence enforces repository and advisory source context: %j", + (details) => { + expect(() => + validateExternalEvidence({ + title: "Synthetic evidence", + severity: "unknown", + details, + }), + ).toThrow(); + }, +); diff --git a/sdk/typescript/tests-ts/support/github-import-fixtures.ts b/sdk/typescript/tests-ts/support/github-import-fixtures.ts new file mode 100644 index 0000000000..4d8335b4d0 --- /dev/null +++ b/sdk/typescript/tests-ts/support/github-import-fixtures.ts @@ -0,0 +1,82 @@ +import type { ImportedGitHubCodeScanningAlert } from "../../src/github.js"; + +export function importedAlert( + ref = "refs/heads/main", + host = "github.com", +): ImportedGitHubCodeScanningAlert { + const url = `https://${host}/example/project/security/code-scanning/12`; + return { + source: "github-code-scanning", + repository: "example/project", + number: 12, + url, + alert: { + number: 12, + html_url: url, + state: "dismissed", + dismissed_reason: "false positive", + dismissed_comment: "Synthetic earlier review.", + updated_at: "2026-01-02T00:00:00Z", + rule: { + id: "synthetic/sql-query", + name: "Untrusted query input", + security_severity_level: "high", + severity: "error", + description: "Query input reaches the database.", + help: "Use query parameters.", + tags: ["external/cwe/cwe-089"], + }, + tool: { name: "Synthetic Scanner", version: "1.0" }, + most_recent_instance: { + ref, + commit_sha: "a".repeat(40), + state: "open", + message: { text: "Untrusted input at the query." }, + location: { path: "src/query.ts", start_line: 12, end_line: 14 }, + }, + } as ImportedGitHubCodeScanningAlert["alert"], + }; +} + +export function importedAdvisory( + selection: "repository" | "global" = "repository", +) { + const ghsaId = "GHSA-2345-6789-cfgh"; + const url = + selection === "repository" + ? `https://github.com/example/project/security/advisories/${ghsaId}` + : `https://github.com/advisories/${ghsaId}`; + return { + source: "github-security-advisory" as const, + selection, + repository: "example/project", + githubHost: "github.com", + ghsaId, + url, + advisory: { + ghsa_id: ghsaId, + cve_id: "CVE-2026-0001", + html_url: url, + summary: "Synthetic dependency advisory", + description: "A published report for an example package.", + severity: "high", + state: "published", + withdrawn_at: null, + updated_at: "2026-01-02T00:00:00Z", + identifiers: [ + { type: "GHSA", value: ghsaId }, + { type: "CVE", value: "CVE-2026-0001" }, + ], + vulnerabilities: [ + { + package: { name: "example-package", ecosystem: "npm" }, + vulnerable_version_range: ">= 1.0.0, < 1.2.3", + ...(selection === "repository" + ? { patched_versions: "1.2.3" } + : { first_patched_version: "1.2.3" }), + }, + ], + cwes: [{ cwe_id: "CWE-79", name: "Synthetic weakness" }], + }, + } as unknown as import("../../src/github-advisories.js").ImportedGitHubAdvisory; +} diff --git a/sdk/typescript/tests-ts/wiz-findings.test.ts b/sdk/typescript/tests-ts/wiz-findings.test.ts index 6ef848ef25..4329ebfb67 100644 --- a/sdk/typescript/tests-ts/wiz-findings.test.ts +++ b/sdk/typescript/tests-ts/wiz-findings.test.ts @@ -703,7 +703,7 @@ test("saved typed requests resume by repository ID after a repository rename", a expect((await resumed.publish()).verified).toBe(1); expect(f.posts).toHaveLength(1); expect( - f.reports.get("sast:occurrence-1")!.evidence.details!.repository.url, + f.reports.get("sast:occurrence-1")!.evidence.details!.repository!.url, ).toBe(repositoryUrl); }); @@ -768,7 +768,7 @@ test("GitHub repository metadata accepts case variants without changing source U }); expect(parsed.excluded).toEqual([]); expect(parsed.findings[0]!.evidence.source_data).toEqual(supplied); - expect(parsed.findings[0]!.evidence.details!.repository.url).toBe( + expect(parsed.findings[0]!.evidence.details!.repository!.url).toBe( supplied.repository.url, ); }); @@ -882,7 +882,7 @@ test("custom GitHub Enterprise binding preserves branch case and exact source ev expect((await prepared.publish()).verified).toBe(1); const evidence = f.posts[0]!.items[0]!.evidence; expect(evidence.branch).toBe("Feature/Parser/More"); - expect(evidence.details!.repository.url).toBe(record.repository.url); + expect(evidence.details!.repository!.url).toBe(record.repository.url); expect(evidence.source_data).toEqual(record); }); @@ -927,7 +927,7 @@ test.each([ expect((await prepared.publish()).verified).toBe(1); const evidence = f.posts[0]!.items[0]!.evidence; expect(evidence.branch).toBe("Feature/Parser/More"); - expect(evidence.details!.repository.url).toBe(sourceRepository.url); + expect(evidence.details!.repository!.url).toBe(sourceRepository.url); expect(evidence.source_data).toEqual(record); }, ); @@ -1865,7 +1865,7 @@ test.each([ expect(replay.preview.resumed).toBe(true); expect(replay.preview.requests).toEqual(original.preview.requests); expect( - replay.preview.findings[0]!.evidence.details!.repository.name, + replay.preview.findings[0]!.evidence.details!.repository!.name, ).toBe("EXAMPLE/PROJECT"); } const newer = f.reports.get("sast:occurrence-1")!; From 382c68205b9ff666be08dddfeec120a2d9a11eaa Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Tue, 13 Oct 2026 21:26:01 -0700 Subject: [PATCH 41/43] fix: retain publication dependency types in the installed SDK --- sdk/typescript/src/external-findings-publish.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 283b7b7fc9..5873fa88aa 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -52,7 +52,7 @@ export interface ExternalPublicationOptions { sourceKey: string; } -/** @internal Injectable transport and account for offline integration tests. */ +/** Optional transport, account, cancellation, and progress hooks. */ export interface ExternalPublicationDependencies { environment?: NodeJS.ProcessEnv; fetch?: (url: string, options: RequestInit) => Promise; From 88bf5687e8c83b0d18b1cf1f476592695e4ecede Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Tue, 13 Oct 2026 21:47:41 -0700 Subject: [PATCH 42/43] fix: preserve GitHub source metadata and canonical import evidence --- .../skills/publish-findings/SKILL.md | 2 +- sdk/typescript/docs/cli.md | 7 ++- .../src/external-import-contract.ts | 20 +++++- sdk/typescript/src/github-findings.ts | 16 +++-- .../external-findings-publish.test.ts | 62 +++++++++++++++++++ .../tests-ts/github-findings.test.ts | 34 ++++++++++ .../support/github-import-fixtures.ts | 6 +- 7 files changed, 134 insertions(+), 13 deletions(-) diff --git a/plugins/codex-security/skills/publish-findings/SKILL.md b/plugins/codex-security/skills/publish-findings/SKILL.md index 489f4f43ce..de2e3fb8eb 100644 --- a/plugins/codex-security/skills/publish-findings/SKILL.md +++ b/plugins/codex-security/skills/publish-findings/SKILL.md @@ -23,7 +23,7 @@ Keep raw source records unchanged in `source_data`. New raw occurrence IDs are p ## GitHub alerts and advisories -Keep code-scanning alerts and GHSA advisories as separate source families. Read code-scanning alerts with the existing `codex-security import github OWNER/REPO --format json` command. It defaults to open alerts on the default branch; use `--github-alert NUMBER` repeatedly for exact alerts or `--github-ref REF` for a selected ref. Save the complete output without field or token filters. The mapper retains the selected instance's ref, revision, status, rule, scanner, location and original evidence. Missing security severity remains `unknown`; do not turn SARIF error/warning/note into vulnerability ratings. +Keep code-scanning alerts and GHSA advisories as separate source families. Read code-scanning alerts with the existing `codex-security import github OWNER/REPO --format json` command. It defaults to open alerts on the default branch; use `--github-alert NUMBER` repeatedly for exact alerts or `--github-ref REF` for a selected ref. Save the complete output without field or token filters. The mapper retains the selected instance's ref, revision, detection state, rule, scanner, location and original evidence. Typed source status honors an alert-wide dismissal; otherwise it uses the selected instance's open/fixed state. Dismissal is not a Codex assessment. Missing security severity remains `unknown`; do not turn SARIF error/warning/note into vulnerability ratings. Read advisories with `codex-security import github-advisories OWNER/REPO --format json`. It defaults to published repository advisories. `--github-advisory GHSA-xxxx-xxxx-xxxx` selects exact repository advisory IDs and may include private drafts only when the user selects them. To list private repository advisories, require the user's explicit state selection using `--github-state draft|triage|closed`. Do not combine a nondefault state with exact IDs. For public database records, use `--github-advisory-source global` with one or more explicit `--github-advisory` IDs; never enumerate or mirror the global database. Both readers are read-only and use the user's GitHub CLI authentication and `GH_HOST` when configured. Stop on a read failure instead of publishing an incomplete selection. diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md index ddb2d12302..dce0f6631e 100644 --- a/sdk/typescript/docs/cli.md +++ b/sdk/typescript/docs/cli.md @@ -1689,9 +1689,10 @@ token filters. Use a stable source key for the GitHub host, repository and finding family. Code-scanning identities also include the selected ref, so an import from another branch cannot replace an existing branch's evidence. GitHub's source state and -full original alert stay attached to the source report; dismissal or a fixed -state does not create a Codex assessment. Alerts without a security severity are -reported as `unknown`; SARIF error/warning/note levels are not vulnerability ratings. +full original alert stay attached to the source report. An alert-wide dismissal +applies to every branch; otherwise the selected instance supplies the open/fixed +state. Dismissal or a fixed state does not create a Codex assessment. Alerts without +a security severity are reported as `unknown`; SARIF error/warning/note levels are not vulnerability ratings. The SDK offers the same preview and publication boundary: diff --git a/sdk/typescript/src/external-import-contract.ts b/sdk/typescript/src/external-import-contract.ts index e76b13b731..b289e74251 100644 --- a/sdk/typescript/src/external-import-contract.ts +++ b/sdk/typescript/src/external-import-contract.ts @@ -123,13 +123,29 @@ function materializeDetailDefaults( function normalizeEvidenceDetails(value: unknown): void { if (value !== null && typeof value === "object") { const fields = value as Record; - // Only this new wrapper is omitted for null; retain every legacy default. + // Cloud omits these new optional fields when null. Preserve every legacy + // default so saved Wiz requests still replay the same evidence bytes. + if (Array.isArray(fields["packages"])) + for (const value of fields["packages"]) { + if ( + value && + typeof value === "object" && + value["vulnerable_version_range"] === null + ) + delete value["vulnerable_version_range"]; + } if (fields["details"] === null) delete fields["details"]; - else + else { + const details = fields["details"]; + if (details && typeof details === "object" && !Array.isArray(details)) + for (const field of ["repository", "advisory"]) + if ((details as Record)[field] === null) + delete (details as Record)[field]; materializeDetailDefaults( fields["details"], modelSchemas["ImportedFindingEvidence"]!.properties!["details"]!, ); + } } } diff --git a/sdk/typescript/src/github-findings.ts b/sdk/typescript/src/github-findings.ts index 6a482785ad..08c3919c55 100644 --- a/sdk/typescript/src/github-findings.ts +++ b/sdk/typescript/src/github-findings.ts @@ -118,7 +118,11 @@ function codeScanningFinding(record: Record): ExternalFinding { url: `${parsedUrl.origin}/${repository}`, name: repository, }, - source_status: text(instance?.["state"]) ?? text(alert["state"]), + // Dismissal covers every branch; detection state belongs to the selected ref. + source_status: + alert["state"] === "dismissed" + ? "dismissed" + : (text(instance?.["state"]) ?? text(alert["state"])), scanner_origin: text(tool?.["name"]), rule: { id: text(rule?.["id"]), name: text(rule?.["name"]) }, weakness_ids: weaknessIds, @@ -149,13 +153,17 @@ function advisoryFinding(record: Record): ExternalFinding { if (!url || url !== advisory["html_url"]) throw new Error("GitHub advisory URL must match the original advisory."); const parsedUrl = new URL(url); - const host = text(record["githubHost"]); + const host = text(record["githubHost"])?.toLowerCase(); + // Enterprise APIs may return the public advisory database's canonical link. + const canonicalGlobalLink = + selection === "global" && parsedUrl.host === "github.com"; const expectedPath = selection === "repository" ? `/${repository}/security/advisories/${id}` : `/advisories/${id}`; if ( - parsedUrl.host.toLowerCase() !== host?.toLowerCase() || + !host || + (parsedUrl.host.toLowerCase() !== host && !canonicalGlobalLink) || parsedUrl.pathname.toLowerCase() !== expectedPath.toLowerCase() ) throw new Error( @@ -199,7 +207,7 @@ function advisoryFinding(record: Record): ExternalFinding { }); const cwes = Array.isArray(advisory["cwes"]) ? advisory["cwes"] : []; return { - source_finding_id: `${parsedUrl.host.toLowerCase()}/${selection === "repository" ? repository.toLowerCase() : "global"}/security-advisory/${id.toUpperCase()}`, + source_finding_id: `${host}/${selection === "repository" ? repository.toLowerCase() : "global"}/security-advisory/${id.toUpperCase()}`, evidence: validateExternalEvidence({ title: text(advisory["summary"]) ?? id, severity: severity(advisory["severity"]), diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 9d83a812fa..edbd3f7d66 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -2553,3 +2553,65 @@ test("saved GitHub JSONL and gzip preserve evidence while malformed UTF-8 cannot await writeFile(f.file, Buffer.from([0xff])); await expect(readGitHubFindings(f.file)).rejects.toThrow("UTF-8"); }); + +test("normalized publication omits only new null fields to match canonical Cloud readback", async () => { + const f = await fixture(); + const evidence = { + title: "Synthetic advisory without known package range", + severity: "unknown" as const, + packages: [ + { + name: "example-package", + vulnerable_version_range: null, + installed_version: null, + }, + ], + details: { + kind: "advisory" as const, + repository: null, + advisory: { source: "global" as const }, + }, + }; + const prepared = await prepareNormalizedExternalPublication( + { + read: 1, + findings: [{ source_finding_id: "synthetic-global-advisory", evidence }], + excluded: [], + }, + { ...options, provider: "github" }, + f.deps, + ); + const sent = prepared.preview.findings[0]!.evidence; + expect(sent.packages![0]).not.toHaveProperty("vulnerable_version_range"); + expect(sent.packages![0]!.installed_version).toBeNull(); + expect(sent.details).not.toHaveProperty("repository"); + expect(evidence.details.repository).toBeNull(); + expect((await prepared.publish()).verified).toBe(1); + + await writeFile( + f.file, + JSON.stringify([ + { + source_finding_id: "synthetic-sast", + evidence: { + title: "Synthetic SAST evidence", + severity: "high", + packages: [ + { name: "example-package", vulnerable_version_range: null }, + ], + details: { + kind: "sast", + repository: { url: "https://github.com/example/project" }, + advisory: null, + }, + }, + }, + ]), + ); + const wiz = await prepareExternalPublication(f.file, options, f.deps); + expect(wiz.preview.findings[0]!.evidence.details).not.toHaveProperty( + "advisory", + ); + expect(wiz.preview.findings[0]!.evidence.details!.scanner_origin).toBeNull(); + expect((await wiz.publish()).verified).toBe(1); +}); diff --git a/sdk/typescript/tests-ts/github-findings.test.ts b/sdk/typescript/tests-ts/github-findings.test.ts index 2dc917ace1..f210d0105e 100644 --- a/sdk/typescript/tests-ts/github-findings.test.ts +++ b/sdk/typescript/tests-ts/github-findings.test.ts @@ -211,3 +211,37 @@ test.each([ ).toThrow(); }, ); + +test("global advisories accept canonical public links from an Enterprise API without changing source host identity", () => { + const imported = importedAdvisory("global"); + imported.githubHost = "git.example.test"; + const result = normalizeGitHubAdvisories([imported]); + expect(result.excluded).toEqual([]); + expect(result.findings[0]!.source_finding_id).toStartWith( + "git.example.test/global/security-advisory/", + ); + expect(result.findings[0]!.evidence.url).toBe(imported.url); + expect(result.findings[0]!.evidence.source_data as unknown).toEqual(imported); + expect(result.findings[0]!.evidence.details).not.toHaveProperty("repository"); + const wrongRepositoryHost = importedAdvisory("repository"); + wrongRepositoryHost.githubHost = "git.example.test"; + expect( + normalizeGitHubAdvisories([wrongRepositoryHost]).excluded, + ).toHaveLength(1); +}); + +test("repository-wide dismissal takes precedence over detection state while selected refs keep their own open/fixed state", () => { + const imported = importedAlert("refs/heads/review"); + imported.alert.state = "dismissed"; + imported.alert.dismissed_reason = "false positive"; + expect( + normalizeGitHubCodeScanningAlerts([imported]).findings[0]!.evidence.details! + .source_status, + ).toBe("dismissed"); + imported.alert.state = "fixed"; + imported.alert.dismissed_reason = null; + expect( + normalizeGitHubCodeScanningAlerts([imported]).findings[0]!.evidence.details! + .source_status, + ).toBe("open"); +}); diff --git a/sdk/typescript/tests-ts/support/github-import-fixtures.ts b/sdk/typescript/tests-ts/support/github-import-fixtures.ts index 4d8335b4d0..f0be018430 100644 --- a/sdk/typescript/tests-ts/support/github-import-fixtures.ts +++ b/sdk/typescript/tests-ts/support/github-import-fixtures.ts @@ -13,9 +13,9 @@ export function importedAlert( alert: { number: 12, html_url: url, - state: "dismissed", - dismissed_reason: "false positive", - dismissed_comment: "Synthetic earlier review.", + state: "open", + dismissed_reason: null, + dismissed_comment: null, updated_at: "2026-01-02T00:00:00Z", rule: { id: "synthetic/sql-query", From 9351529f6dfa9e61ae86fcbe8d678d041e2e3932 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Tue, 13 Oct 2026 22:08:57 -0700 Subject: [PATCH 43/43] fix: retain GitHub URL identities and unmodified raw evidence --- .../src/external-findings-publish.ts | 7 ++ sdk/typescript/src/github-findings.ts | 31 ++++-- .../external-findings-publish.test.ts | 103 ++++++++++++++++++ .../tests-ts/github-findings.test.ts | 97 +++++++++++++++++ 4 files changed, 230 insertions(+), 8 deletions(-) diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 5873fa88aa..aa6f0fe179 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -297,6 +297,13 @@ export async function prepareNormalizedExternalPublication( ): Promise { // Validation adds Cloud defaults; leave the caller's objects unchanged. const current = structuredClone(input); + // structuredClone preserves aliases. Detach every raw record before adding + // typed defaults, including aliases shared with another finding's evidence. + for (const finding of current.findings) + if (finding.evidence.source_data !== undefined) + finding.evidence.source_data = structuredClone( + finding.evidence.source_data, + ); const identities = new Set(); for (const finding of current.findings) { const id = finding.source_finding_id; diff --git a/sdk/typescript/src/github-findings.ts b/sdk/typescript/src/github-findings.ts index 08c3919c55..659e20125f 100644 --- a/sdk/typescript/src/github-findings.ts +++ b/sdk/typescript/src/github-findings.ts @@ -28,6 +28,18 @@ function repositoryName(value: unknown): string { return repository!; } +function sourceUrl(value: string): { + parsed: URL; + host: string; + origin: string; +} { + const parsed = new URL(value); + const origin = /^(https?:\/\/([^/?#]+))/iu.exec(value); + if (!origin) throw new Error("GitHub evidence URL must be an HTTP(S) link."); + // URL.origin/host remove explicit default ports; Cloud keeps those distinct. + return { parsed, host: origin[2]!.toLowerCase(), origin: origin[1]! }; +} + function sourceTimestamp(value: unknown): number | null { if (value === undefined || value === null) return null; const seconds = typeof value === "string" ? Date.parse(value) / 1000 : NaN; @@ -61,10 +73,13 @@ function codeScanningFinding(record: Record): ExternalFinding { const url = text(record["url"]); if (!url || url !== alert["html_url"]) throw new Error("GitHub alert URL must match the original alert."); - const parsedUrl = new URL(url); + const { parsed: parsedUrl, host, origin } = sourceUrl(url); if ( - parsedUrl.pathname.toLowerCase() !== - `/${repository}/security/code-scanning/${number}`.toLowerCase() + !["security/code-scanning", "code-scanning"].some( + (route) => + parsedUrl.pathname.toLowerCase() === + `/${repository}/${route}/${number}`.toLowerCase(), + ) ) throw new Error( "GitHub alert URL does not match the selected repository and alert.", @@ -94,7 +109,7 @@ function codeScanningFinding(record: Record): ExternalFinding { ), ]; const path = text(location?.["path"]); - const sourceId = `${parsedUrl.host.toLowerCase()}/${repository.toLowerCase()}/code-scanning/${number}/${hash("sha256", ref)}`; + const sourceId = `${host}/${repository.toLowerCase()}/code-scanning/${number}/${hash("sha256", ref)}`; return { source_finding_id: sourceId, evidence: validateExternalEvidence({ @@ -115,7 +130,7 @@ function codeScanningFinding(record: Record): ExternalFinding { details: { kind: "sast", repository: { - url: `${parsedUrl.origin}/${repository}`, + url: `${origin}/${repository}`, name: repository, }, // Dismissal covers every branch; detection state belongs to the selected ref. @@ -152,7 +167,7 @@ function advisoryFinding(record: Record): ExternalFinding { const url = text(record["url"]); if (!url || url !== advisory["html_url"]) throw new Error("GitHub advisory URL must match the original advisory."); - const parsedUrl = new URL(url); + const { parsed: parsedUrl, host: linkHost, origin } = sourceUrl(url); const host = text(record["githubHost"])?.toLowerCase(); // Enterprise APIs may return the public advisory database's canonical link. const canonicalGlobalLink = @@ -163,7 +178,7 @@ function advisoryFinding(record: Record): ExternalFinding { : `/advisories/${id}`; if ( !host || - (parsedUrl.host.toLowerCase() !== host && !canonicalGlobalLink) || + (linkHost !== host && !canonicalGlobalLink) || parsedUrl.pathname.toLowerCase() !== expectedPath.toLowerCase() ) throw new Error( @@ -223,7 +238,7 @@ function advisoryFinding(record: Record): ExternalFinding { ...(selection === "repository" ? { repository: { - url: `${parsedUrl.origin}/${repository}`, + url: `${origin}/${repository}`, name: repository, }, } diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index edbd3f7d66..f98239250e 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -2615,3 +2615,106 @@ test("normalized publication omits only new null fields to match canonical Cloud expect(wiz.preview.findings[0]!.evidence.details!.scanner_origin).toBeNull(); expect((await wiz.publish()).verified).toBe(1); }); + +test("normalized SDK publication detaches raw evidence aliases before validating every finding", async () => { + const f = await fixture(); + const sharedPackage = { name: "example-package" }; + const input = { + read: 2, + findings: [ + { + source_finding_id: "shared-first", + evidence: { + title: "First report", + severity: "high" as const, + packages: [sharedPackage], + source_data: { package: sharedPackage }, + }, + }, + { + source_finding_id: "shared-second", + evidence: { + title: "Second report", + severity: "high" as const, + source_data: { earlier_package: sharedPackage }, + }, + }, + ], + excluded: [], + }; + const original = JSON.stringify(input); + const prepared = await prepareNormalizedExternalPublication( + input, + options, + f.deps, + ); + expect(JSON.stringify(input)).toBe(original); + expect(prepared.preview.findings[0]!.evidence.source_data).toEqual({ + package: { name: "example-package" }, + }); + expect(prepared.preview.findings[1]!.evidence.source_data).toEqual({ + earlier_package: { name: "example-package" }, + }); + await writeFile(f.file, JSON.stringify(input.findings)); + const fromFile = await prepareExternalPublication(f.file, options, f.deps); + expect(prepared.preview.findings).toEqual(fromFile.preview.findings); +}); + +test.each(["code-scanning", "advisory"])( + "%s imports bind an explicit default port to the matching Cloud repository", + async (family) => { + const alert = importedAlert("refs/heads/main", "github.com:443"); + const advisory = importedAdvisory(); + advisory.githubHost = "github.com:443"; + advisory.url = advisory.advisory.html_url = advisory.url.replace( + "github.com", + advisory.githubHost, + ); + const f = await fixture([family === "code-scanning" ? alert : advisory]); + const selected = { ...options, provider: "github" as const }; + await expect( + prepareExternalPublication(f.file, selected, f.deps), + ).rejects.toThrow("does not match the selected Cloud repository"); + expect(f.posts).toHaveLength(0); + const matching = { + ...f.deps, + fetch: async (url: string, init: RequestInit) => { + const response = await f.deps.fetch(url, init); + if (new URL(url).pathname.endsWith("/repositories")) { + const body = await response.json(); + body.data[0].url = "https://github.com:443/example/project"; + return Response.json(body); + } + return response; + }, + }; + const result = await ( + await prepareExternalPublication(f.file, selected, matching) + ).publish(); + expect(result.counts.created).toBe(1); + expect(result.verified).toBe(1); + }, +); + +test("normalized SDK publication rejects nonfinite raw evidence without JSON coercion", async () => { + const f = await fixture(); + const input = { + read: 1, + findings: [ + { + source_finding_id: "nonfinite-raw-evidence", + evidence: { + title: "Synthetic evidence", + severity: "high" as const, + source_data: { score: Infinity }, + }, + }, + ], + excluded: [], + }; + await expect( + prepareNormalizedExternalPublication(input, options, f.deps), + ).rejects.toThrow("finite JSON numbers"); + expect(input.findings[0]!.evidence.source_data.score).toBe(Infinity); + expect(f.posts).toHaveLength(0); +}); diff --git a/sdk/typescript/tests-ts/github-findings.test.ts b/sdk/typescript/tests-ts/github-findings.test.ts index f210d0105e..7901ed4779 100644 --- a/sdk/typescript/tests-ts/github-findings.test.ts +++ b/sdk/typescript/tests-ts/github-findings.test.ts @@ -1,4 +1,7 @@ import { expect, test } from "bun:test"; +import { Octokit } from "@octokit/core"; +import { importGitHubCodeScanningAlerts } from "../src/github.js"; +import { importGitHubAdvisories } from "../src/github-advisories.js"; import { validateExternalEvidence } from "../src/external-import-contract.js"; import { normalizeGitHubCodeScanningAlerts, @@ -245,3 +248,97 @@ test("repository-wide dismissal takes precedence over detection state while sele .source_status, ).toBe("open"); }); + +test.each(["security/code-scanning", "code-scanning"])( + "code-scanning accepts the %s HTML route without changing identity or repository binding", + (route) => { + const original = importedAlert(); + const imported = structuredClone(original); + imported.url = `https://github.com/example/project/${route}/12`; + imported.alert.html_url = imported.url; + const result = normalizeGitHubCodeScanningAlerts([imported]); + expect(result.excluded).toEqual([]); + expect(result.findings[0]!.source_finding_id).toBe( + normalizeGitHubCodeScanningAlerts([original]).findings[0]! + .source_finding_id, + ); + expect(result.findings[0]!.evidence.source_data as unknown).toEqual( + imported, + ); + imported.repository = "example/other"; + expect(normalizeGitHubCodeScanningAlerts([imported]).excluded).toHaveLength( + 1, + ); + imported.repository = original.repository; + imported.url = `https://github.com/example/project/${route}/13`; + imported.alert.html_url = imported.url; + expect(normalizeGitHubCodeScanningAlerts([imported]).excluded).toHaveLength( + 1, + ); + }, +); + +test("real GitHub readers and mappings preserve explicit ports in repository evidence and identities", async () => { + const codeIds = new Set(); + const advisoryIds = new Set(); + for (const port of ["", ":443", ":8443"]) { + const host = `git.example.test${port}`; + const alert = importedAlert("refs/heads/main", host); + const advisory = importedAdvisory(); + advisory.githubHost = host; + advisory.url = advisory.advisory.html_url = advisory.url.replace( + "github.com", + host, + ); + const github = new Octokit({ + request: { + fetch: async (resource: Parameters[0]) => { + const url = new URL( + resource instanceof Request ? resource.url : resource, + ); + return Response.json( + url.pathname.includes("code-scanning") + ? alert.alert + : advisory.advisory, + ); + }, + }, + }); + const dependency = { + createGitHub: async (selectedHost: string) => { + expect(selectedHost).toBe(host); + return github; + }, + }; + const importedAlerts = await importGitHubCodeScanningAlerts( + { repository: alert.repository, alertNumbers: [12], githubHost: host }, + dependency, + ); + const code = normalizeGitHubCodeScanningAlerts(importedAlerts); + const importedAdvisories = await importGitHubAdvisories( + { + repository: advisory.repository, + advisoryIds: [advisory.ghsaId], + githubHost: host, + }, + dependency, + ); + const disclosed = normalizeGitHubAdvisories(importedAdvisories); + expect(code.excluded).toEqual([]); + expect(disclosed.excluded).toEqual([]); + for (const result of [code, disclosed]) + expect(result.findings[0]!.evidence.details!.repository!.url).toBe( + `https://${host}/example/project`, + ); + expect(code.findings[0]!.evidence.source_data as unknown).toEqual( + importedAlerts[0], + ); + expect(disclosed.findings[0]!.evidence.source_data as unknown).toEqual( + importedAdvisories[0], + ); + codeIds.add(code.findings[0]!.source_finding_id); + advisoryIds.add(disclosed.findings[0]!.source_finding_id); + } + expect(codeIds.size).toBe(3); + expect(advisoryIds.size).toBe(3); +});