From 750810162aab670aedb8940acc7b77dea6048d11 Mon Sep 17 00:00:00 2001 From: Pengfei Hu Date: Mon, 19 Oct 2026 02:18:19 +0800 Subject: [PATCH 1/4] fix(baseline): verify each baseline against its own audit rows (#1030) Every baseline in a directory appends to one baseline-audit.log, and verify_baseline compared the file with the log's last row whichever baseline wrote it. Saving baseline-prod.json turned an untouched baseline.json into a critical SHIP-BASELINE-INTEGRITY-MISMATCH, while a copy of the baseline saved last, or a renamed baseline, verified clean. Verification now uses the latest row recorded for that baseline's file name (the final component of baseline_path, either separator, which is how released versions recorded it). A log with rows but none for the baseline is missing_audit_log, titled with the file, what the log records and the baseline save that records it. Released audit logs verify unchanged. The hash reads CRLF and lone CR as LF, as it always has; the docstring that claimed line endings were detected is corrected, and the normalisation is explicit and reads through the static-input snapshot. --- docs/baseline.md | 8 + docs/checks.json | 4 +- docs/checks.md | 5 +- docs/checks/baseline.yaml | 4 +- llms-full.txt | 5 +- .../checks/baseline_integrity.py | 5 +- src/agents_shipgate/core/baseline.py | 52 +++- src/agents_shipgate/core/baseline_audit.py | 118 +++++++-- tests/test_baseline_integrity.py | 226 ++++++++++++++++++ 9 files changed, 393 insertions(+), 34 deletions(-) diff --git a/docs/baseline.md b/docs/baseline.md index 795a09e17..1a3e3521d 100644 --- a/docs/baseline.md +++ b/docs/baseline.md @@ -28,6 +28,14 @@ previously-set value, and preserves each entry's original JSON — hand-edits trip the integrity hash (`SHIP-BASELINE-INTEGRITY-MISMATCH`). +Several baselines can share a directory (for example `baseline.json` and +`baseline-prod.json`). They append to its one `baseline-audit.log`, and each +verifies against the rows its own saves wrote, so saving one never changes +whether another verifies. A baseline renamed or copied by hand has no audit +history under its new name; save it under that name to record it. Line-ending +conversions (a Windows checkout's CRLF) are not edits: the hash reads them as +LF. + Severity values in the baseline are the report-visible severities after `checks.severity_overrides` are applied. Matching still uses fingerprints, not severity, so changing an override does not create a new baseline identity. diff --git a/docs/checks.json b/docs/checks.json index 657dd5e33..18e68f084 100644 --- a/docs/checks.json +++ b/docs/checks.json @@ -1290,9 +1290,11 @@ "computed_hash", "audit_log_path", "latest_audit_run_id", + "latest_audit_baseline_path", + "recorded_baselines", "error" ], - "fires_when": "The baseline file's SHA-256 differs from the latest audit log entry's hash_after, the audit log is missing, empty, or malformed, an entry's provenance.run_id is not in the audit log, or an entry pre-dates v0.5 and lacks provenance entirely.", + "fires_when": "The baseline file's SHA-256 differs from the hash_after of the latest audit log entry recorded for that baseline file, the audit log is missing, empty, or malformed, the audit log has no entry for that baseline file, an entry's provenance.run_id is not in the audit log, or an entry pre-dates v0.5 and lacks provenance entirely.", "floor_severity": null, "id": "SHIP-BASELINE-INTEGRITY-MISMATCH", "mvp_tier": "lifecycle", diff --git a/docs/checks.md b/docs/checks.md index 4281dcabb..1acb3b80d 100644 --- a/docs/checks.md +++ b/docs/checks.md @@ -1052,8 +1052,9 @@ unused write/admin scopes are `high`; other unused scopes are `medium`. Baseline file integrity check failed. Emitted when the baseline JSON has been edited outside `agents-shipgate baseline save` (hash mismatch against the -audit log), when the audit log is missing or empty for a non-empty baseline, -when the audit log is malformed, when an entry's `provenance.run_id` is not +latest audit log entry recorded for that baseline file), when the audit log is +missing or empty for a non-empty baseline or has no entry for that baseline +file, when the audit log is malformed, when an entry's `provenance.run_id` is not present in the audit log, or when an entry pre-dates the v0.5 provenance contract. In `baseline.integrity_mode: strict` the finding carries `blocks_release=true` diff --git a/docs/checks/baseline.yaml b/docs/checks/baseline.yaml index 7af988738..f813386b4 100644 --- a/docs/checks/baseline.yaml +++ b/docs/checks/baseline.yaml @@ -54,7 +54,7 @@ checks: rationale: |- The baseline JSON has been edited outside `agents-shipgate baseline save`, lacks an audit log row, has a malformed audit log row, or references a run_id not present in the audit log. A release gate that accepts silent baseline edits cannot claim to govern technical debt. fires_when: |- - The baseline file's SHA-256 differs from the latest audit log entry's hash_after, the audit log is missing, empty, or malformed, an entry's provenance.run_id is not in the audit log, or an entry pre-dates v0.5 and lacks provenance entirely. + The baseline file's SHA-256 differs from the hash_after of the latest audit log entry recorded for that baseline file, the audit log is missing, empty, or malformed, the audit log has no entry for that baseline file, an entry's provenance.run_id is not in the audit log, or an entry pre-dates v0.5 and lacks provenance entirely. evidence_fields: - fingerprint - check_id @@ -64,6 +64,8 @@ checks: - computed_hash - audit_log_path - latest_audit_run_id + - latest_audit_baseline_path + - recorded_baselines - error recommendation: |- Re-run `agents-shipgate baseline save` to refresh the baseline and audit log, or `agents-shipgate baseline verify` for the full report. Investigate the diff before accepting. diff --git a/llms-full.txt b/llms-full.txt index 713ec8751..d573cb512 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -4470,8 +4470,9 @@ unused write/admin scopes are `high`; other unused scopes are `medium`. Baseline file integrity check failed. Emitted when the baseline JSON has been edited outside `agents-shipgate baseline save` (hash mismatch against the -audit log), when the audit log is missing or empty for a non-empty baseline, -when the audit log is malformed, when an entry's `provenance.run_id` is not +latest audit log entry recorded for that baseline file), when the audit log is +missing or empty for a non-empty baseline or has no entry for that baseline +file, when the audit log is malformed, when an entry's `provenance.run_id` is not present in the audit log, or when an entry pre-dates the v0.5 provenance contract. In `baseline.integrity_mode: strict` the finding carries `blocks_release=true` diff --git a/src/agents_shipgate/checks/baseline_integrity.py b/src/agents_shipgate/checks/baseline_integrity.py index 390b358c9..f27e4ed0e 100644 --- a/src/agents_shipgate/checks/baseline_integrity.py +++ b/src/agents_shipgate/checks/baseline_integrity.py @@ -68,8 +68,9 @@ "reviewed." ), "missing_audit_log": ( - "Re-run `agents-shipgate baseline save` to regenerate the " - "audit log alongside the baseline file." + "Re-run `agents-shipgate baseline save` for this baseline file to " + "record an audit entry for it in the log beside it. A renamed or " + "copied baseline has no audit history under its new name." ), "invalid_audit_log": ( "Inspect the malformed audit log, then re-run " diff --git a/src/agents_shipgate/core/baseline.py b/src/agents_shipgate/core/baseline.py index 7246c8309..67c7f7801 100644 --- a/src/agents_shipgate/core/baseline.py +++ b/src/agents_shipgate/core/baseline.py @@ -15,8 +15,9 @@ append_audit_entry, compute_baseline_hash, compute_baseline_hash_from_file, - latest_audit_entry, + latest_entry_for_baseline, read_audit_log, + recorded_baseline_name, utc_now_isoformat, ) from agents_shipgate.core.check_ids import LEGACY_CHECK_ID_ALIASES @@ -191,7 +192,9 @@ def write_baseline( if path.exists(): try: hash_before = compute_baseline_hash_from_file(path) - except OSError: + except (OSError, ValueError): + # Unreadable, not UTF-8, or not a regular file: a corrupt prior + # baseline must not block the save that repairs it. hash_before = None path.parent.mkdir(parents=True, exist_ok=True) canonical = baseline.model_dump_json(indent=2, exclude_none=False) + "\n" @@ -651,12 +654,18 @@ def _verify_audit_log_alignment( log_path: Path, baseline: BaselineFile, ) -> list[BaselineIntegrityIssue]: - """Hash baseline vs. latest audit entry's hash_after; report mismatch. + """Hash baseline vs. its latest audit entry's hash_after; report mismatch. - A baseline with zero findings and no audit log is a fresh / unsaved + "Its" is load-bearing (#1030): several baselines can share one + directory and so one log, and only a row recorded for this baseline + (``audit_entry_records_baseline``) vouches for it. Comparing against + the log's last row let saving ``baseline-prod.json`` fail an untouched + ``baseline.json``, and let a copy of the sibling saved last pass. + + A baseline with zero findings and no audit row is a fresh / unsaved state — not an integrity violation. Only flag when the baseline contains entries (i.e., someone took the trouble to populate it) - but no audit row exists, or the most recent audit row's + but no audit row for it exists, or its most recent audit row's ``hash_after`` disagrees with the file on disk. """ if not log_path.exists(): @@ -676,7 +685,7 @@ def _verify_audit_log_alignment( ) ] try: - latest = latest_audit_entry(log_path) + entries = read_audit_log(log_path) except InputParseError as exc: return [ BaselineIntegrityIssue( @@ -692,18 +701,46 @@ def _verify_audit_log_alignment( }, ) ] + latest = latest_entry_for_baseline(entries, baseline_path) if latest is None: if not baseline.findings: return [] + if not entries: + return [ + BaselineIntegrityIssue( + kind="missing_audit_log", + default_severity="critical", + title="Audit log exists but is empty", + evidence={ + "baseline_path": str(baseline_path), + "audit_log_path": str(log_path), + "entry_count": len(baseline.findings), + "kind": "missing_audit_log", + }, + ) + ] + # The log has rows, but every one was written by a save of another + # baseline: a sibling in the same directory, or this file under an + # earlier name (a rename or copy carries no audit history). Name + # what the log does record so the fix is evident; never fall back + # to another baseline's row, which is what let one save fail or + # pass a different file (#1030). + recorded = sorted({recorded_baseline_name(entry.baseline_path) for entry in entries}) return [ BaselineIntegrityIssue( kind="missing_audit_log", default_severity="critical", - title="Audit log exists but is empty", + title=( + f"Audit log has no entry for {baseline_path.name}; its " + f"entries record {', '.join(recorded)}. Re-run " + f"`agents-shipgate baseline save` for {baseline_path.name} " + "to record it" + ), evidence={ "baseline_path": str(baseline_path), "audit_log_path": str(log_path), "entry_count": len(baseline.findings), + "recorded_baselines": recorded, "kind": "missing_audit_log", }, ) @@ -722,6 +759,7 @@ def _verify_audit_log_alignment( "computed_hash": on_disk_hash, "latest_audit_run_id": latest.run_id, "latest_audit_timestamp": latest.timestamp, + "latest_audit_baseline_path": latest.baseline_path, "kind": "hash_mismatch", }, ) diff --git a/src/agents_shipgate/core/baseline_audit.py b/src/agents_shipgate/core/baseline_audit.py index b4ced506b..9eb646cec 100644 --- a/src/agents_shipgate/core/baseline_audit.py +++ b/src/agents_shipgate/core/baseline_audit.py @@ -6,7 +6,9 @@ - ``git log`` the log to see when fingerprints joined the baseline. - Use ``agents-shipgate baseline verify`` to confirm the on-disk baseline - matches the most recent audit entry's ``hash_after``. + matches the ``hash_after`` of the most recent audit entry recorded for + *that* baseline (#1030). Several baselines can share one directory, and so + one log; a row only ever vouches for the baseline file it names. The audit log is **tamper-evident but not tamper-proof**. A well-resourced adversary who edits both the baseline JSON and the audit log atomically @@ -21,12 +23,14 @@ from __future__ import annotations import hashlib +import re from datetime import UTC, datetime from pathlib import Path from pydantic import BaseModel, ConfigDict, Field, ValidationError from agents_shipgate.core.errors import InputParseError +from agents_shipgate.core.static_inputs import read_static_input_bytes AUDIT_LOG_SCHEMA_VERSION = "0.1" @@ -42,8 +46,14 @@ class BaselineAuditEntry(BaseModel): ``hash_before`` is the SHA-256 of the prior baseline file's canonical content (see :func:`compute_baseline_hash`) or ``None`` if this was the first save. ``hash_after`` is the new file's hash. Verification - walks the log to find the most recent entry and compares its - ``hash_after`` against the current baseline hash. + walks the log to find the most recent entry recorded for the baseline + being verified (see :func:`audit_entry_records_baseline`) and compares + its ``hash_after`` against the current baseline hash. + + ``baseline_path`` is the path exactly as the saving command spelled it + — relative to that command's working directory, or absolute — so only + its final component identifies the baseline across machines and + working directories. ``added_fingerprints`` / ``removed_fingerprints`` describe the delta so reviewers can scan the log without diffing files. They are @@ -64,24 +74,48 @@ class BaselineAuditEntry(BaseModel): def compute_baseline_hash(canonical_json: str) -> str: - """SHA-256 of canonical baseline JSON. + """SHA-256 of canonical baseline JSON text. - ``canonical_json`` should be the exact bytes that ``write_baseline`` - writes to disk. We hash the literal file content rather than a model - dump so that ``verify_baseline`` can re-read the file and confirm - byte-equivalence without re-serializing. + ``canonical_json`` is the text ``write_baseline`` writes, whose line + endings are LF. We hash the file's text rather than a model dump so + that ``verify_baseline`` can re-read the file and confirm equivalence + without re-serializing; :func:`compute_baseline_hash_from_file` is the + reading half and defines exactly what "equivalent" means. """ return "sha256:" + hashlib.sha256(canonical_json.encode("utf-8")).hexdigest() -def compute_baseline_hash_from_file(path: Path) -> str: - """SHA-256 of a baseline file's exact on-disk bytes. +def normalize_baseline_line_endings(text: str) -> str: + """Rewrite CRLF and lone CR line endings as LF.""" + return text.replace("\r\n", "\n").replace("\r", "\n") + - Used by ``verify_baseline`` to detect any hand-edits. Reads the file - bytes directly so trailing whitespace / line endings / re-ordering - are all detected. +def compute_baseline_hash_from_file(path: Path) -> str: + """SHA-256 of a baseline file's UTF-8 text with line endings read as LF. + + Used by ``verify_baseline`` to detect hand-edits. Every edit to the + file's content changes the hash: a value, a key order, added or removed + whitespace (spaces, tabs, blank lines), a byte-order mark. A line-ending + conversion alone does not: CRLF and a lone CR hash as LF. + + That is deliberate (#1030), and it has been the behaviour of every + release since the audit log shipped: + + - a line-ending conversion cannot change what the baseline accepts. + JSON treats CR and LF alike as insignificant whitespace and cannot + carry a raw one inside a string, so the parsed baseline is identical; + - Git rewrites line endings on checkout (Git for Windows defaults to + ``core.autocrlf=true``), so hashing raw bytes would turn every such + checkout of an untouched baseline into a critical, unsuppressible + ``SHIP-BASELINE-INTEGRITY-MISMATCH``; + - ``hash_after`` values in logs written by released versions were + computed this way, and must keep verifying. + + The bytes come through the active static-input snapshot when there is + one, so the hash covers the same bytes ``load_baseline`` parsed. """ - return compute_baseline_hash(path.read_text(encoding="utf-8")) + raw = read_static_input_bytes(path) + return compute_baseline_hash(normalize_baseline_line_endings(raw.decode("utf-8"))) def append_audit_entry(log_path: Path, entry: BaselineAuditEntry) -> None: @@ -120,14 +154,60 @@ def read_audit_log(log_path: Path) -> list[BaselineAuditEntry]: return entries -def latest_audit_entry(log_path: Path) -> BaselineAuditEntry | None: +_PATH_SEPARATORS = re.compile(r"[\\/]") + + +def recorded_baseline_name(recorded_path: str) -> str: + """The file name an audit row's ``baseline_path`` records. + + ``baseline_path`` is spelled as the saving command spelled it: relative + to whatever working directory that command ran in, or absolute, with + POSIX or Windows separators. Its final component is the one part that + names the same file however and wherever it was saved, so it is the + part verification compares. + """ + return _PATH_SEPARATORS.split(recorded_path.rstrip("/\\"))[-1] + + +def audit_entry_records_baseline(entry: BaselineAuditEntry, baseline_path: Path) -> bool: + """Whether ``entry`` was written by a save of ``baseline_path`` (#1030). + + A row belongs to the baseline whose file name it records. Baselines in + one directory have distinct names, so each verifies against its own + rows: saving ``baseline-prod.json`` neither vouches for nor invalidates + ``baseline.json``. Two same-named baselines in different directories + can only meet in one log through ``baseline.audit_log``; they are not + told apart, and give them distinct names or distinct logs. + """ + return recorded_baseline_name(entry.baseline_path) == baseline_path.name + + +def latest_audit_entry( + log_path: Path, + *, + baseline_path: Path | None = None, +) -> BaselineAuditEntry | None: """Return the most recently appended audit entry, or ``None``. - The most recent entry's ``hash_after`` is the value - ``verify_baseline`` compares against the current baseline file's hash. + With ``baseline_path``, only rows recorded for that baseline count (see + :func:`audit_entry_records_baseline`); that entry's ``hash_after`` is + the value ``verify_baseline`` compares against the current file's hash. + Without it, the last row of the log, whichever baseline wrote it. """ - entries = read_audit_log(log_path) - return entries[-1] if entries else None + return latest_entry_for_baseline(read_audit_log(log_path), baseline_path) + + +def latest_entry_for_baseline( + entries: list[BaselineAuditEntry], + baseline_path: Path | None, +) -> BaselineAuditEntry | None: + """The last of ``entries`` recorded for ``baseline_path``, or ``None``.""" + if baseline_path is None: + return entries[-1] if entries else None + for entry in reversed(entries): + if audit_entry_records_baseline(entry, baseline_path): + return entry + return None def audit_entry_for_run( diff --git a/tests/test_baseline_integrity.py b/tests/test_baseline_integrity.py index 10be70844..710256e98 100644 --- a/tests/test_baseline_integrity.py +++ b/tests/test_baseline_integrity.py @@ -18,6 +18,7 @@ from datetime import date, timedelta from pathlib import Path +import pytest from typer.testing import CliRunner from agents_shipgate.checks.baseline_integrity import ( @@ -46,6 +47,7 @@ ) from agents_shipgate.core.context import ScanContext from agents_shipgate.core.domain import Agent +from agents_shipgate.core.errors import InputParseError from agents_shipgate.core.evaluation_clock import use_evaluation_date from agents_shipgate.schemas.baseline import ( BaselineFile, @@ -891,3 +893,227 @@ def test_cli_baseline_save_honors_manifest_audit_log_override(tmp_path): assert [ f for f in report.findings if f.check_id.startswith("SHIP-BASELINE-") ] == [] + + +# --- #1030: each baseline verifies against its own audit rows ------------- + + +def _invoke_verify(baseline: Path, *extra: str): + return CliRunner().invoke( + app, ["baseline", "verify", "--baseline", str(baseline), *extra] + ) + + +def test_two_baselines_in_one_directory_verify_independently(tmp_path, monkeypatch): + """#1030 acceptance: saving a sibling baseline must not fail an untouched one. + + Both baselines share ``.agents-shipgate/`` and so one + ``baseline-audit.log``. Before #1030, verification compared every + baseline with the log's last row, so the second save turned an untouched + ``baseline.json`` into a critical ``hash_mismatch`` (exit 6 under + ``--strict``). Tampering with either baseline must still be detected. + """ + sample_dir = tmp_path / "sample" + shutil.copytree(SAMPLE.parent, sample_dir, ignore=shutil.ignore_patterns("expected")) + monkeypatch.chdir(sample_dir) + runner = CliRunner() + for out, extra in ( + (".agents-shipgate/baseline.json", []), + (".agents-shipgate/baseline-prod.json", ["--owner", "other-team"]), + ): + result = runner.invoke( + app, ["baseline", "save", "--config", "shipgate.yaml", "--out", out, *extra] + ) + assert result.exit_code == 0, result.output + state = sample_dir / ".agents-shipgate" + rows = read_audit_log(state / "baseline-audit.log") + assert [Path(row.baseline_path).name for row in rows] == [ + "baseline.json", + "baseline-prod.json", + ] + assert rows[0].hash_after != rows[1].hash_after + for name in ("baseline.json", "baseline-prod.json"): + result = _invoke_verify(state / name, "--strict") + assert result.exit_code == 0, result.output + assert "Baseline OK" in result.output + + # `scan --baseline` on the untouched baseline raises no integrity finding. + report, _ = run_scan( + config_path=sample_dir / "shipgate.yaml", + output_dir=tmp_path / "out", + formats=["json"], + ci_mode="advisory", + baseline_path=state / "baseline.json", + ) + assert [ + f for f in report.findings if f.check_id == "SHIP-BASELINE-INTEGRITY-MISMATCH" + ] == [] + + # Tampering with either baseline is still detected. + for name in ("baseline.json", "baseline-prod.json"): + path = state / name + original = path.read_bytes() + path.write_bytes(original.replace(b"\n", b" \n", 1)) + result = _invoke_verify(path, "--strict", "--json") + assert result.exit_code == 6, result.output + kinds = [issue["kind"] for issue in json.loads(result.output)["issues"]] + assert "hash_mismatch" in kinds + path.write_bytes(original) + assert _invoke_verify(path, "--strict").exit_code == 0 + + +def test_a_sibling_baselines_content_is_not_vouched_for_by_its_row(tmp_path): + """Copying the sibling saved last over a baseline is tampering. + + Before #1030 the copy matched the log's last row and verified clean, so + one baseline's accepted debt could be swapped into another unnoticed. + """ + write_baseline(_stub_report(("SHIP-X", "tool_a")), tmp_path / "baseline.json") + write_baseline( + _stub_report(("SHIP-X", "tool_a"), ("SHIP-Y", "tool_b")), + tmp_path / "baseline-prod.json", + ) + (tmp_path / "baseline.json").write_bytes( + (tmp_path / "baseline-prod.json").read_bytes() + ) + issues = verify_baseline(tmp_path / "baseline.json", tmp_path / "baseline-audit.log") + assert "hash_mismatch" in [issue.kind for issue in issues] + assert has_hash_mismatch(issues) + + +def test_latest_audit_entry_selects_rows_for_the_named_baseline(tmp_path): + log_path = tmp_path / "baseline-audit.log" + for run_id, path in ( + ("run_1", ".agents-shipgate/baseline.json"), + ("run_2", ".agents-shipgate/baseline-prod.json"), + ("run_3", "/ci/work/repo/.agents-shipgate/baseline.json"), + ("run_4", r"C:\repo\.agents-shipgate\baseline-prod.json"), + ): + append_audit_entry( + log_path, + BaselineAuditEntry( + timestamp=utc_now_isoformat(), + run_id=run_id, + scanner_version="1.2.0", + baseline_path=path, + hash_before=None, + hash_after=f"sha256:{run_id}", + ), + ) + assert latest_audit_entry(log_path).run_id == "run_4" + assert latest_audit_entry(log_path, baseline_path=Path("baseline.json")).run_id == "run_3" + assert ( + latest_audit_entry(log_path, baseline_path=tmp_path / "baseline-prod.json").run_id + == "run_4" + ) + assert latest_audit_entry(log_path, baseline_path=Path("baseline-dev.json")) is None + + +def test_audit_log_from_a_released_version_still_verifies(tmp_path): + """Backward compatibility: rows a released scanner wrote keep verifying. + + Released versions recorded ``baseline_path`` as the saving command spelled + it — relative to its working directory, absolute, or with Windows + separators — and hashed the file's text with line endings read as LF. + The per-baseline lookup keys on the recorded file name, so every such + row still identifies its baseline and no migration is needed. + """ + base = tmp_path / ".agents-shipgate" + baseline = base / "baseline.json" + prod = base / "baseline-prod.json" + write_baseline(_stub_report(("SHIP-X", "tool_a")), baseline) + write_baseline(_stub_report(("SHIP-Y", "tool_b")), prod) + log_path = base / "baseline-audit.log" + spellings = { + "baseline.json": ".agents-shipgate/baseline.json", + "baseline-prod.json": r"D:\a\repo\repo\.agents-shipgate\baseline-prod.json", + } + # Rewrite the log as a released 1.2.0 scanner on two machines wrote it. + log_path.write_text( + "".join( + json.dumps( + { + **row.model_dump(mode="json"), + "scanner_version": "1.2.0", + "baseline_path": spellings[Path(row.baseline_path).name], + }, + separators=(",", ":"), + ) + + "\n" + for row in read_audit_log(log_path) + ), + encoding="utf-8", + ) + assert verify_baseline(baseline, log_path) == [] + assert verify_baseline(prod, log_path) == [] + + +def test_baseline_with_no_row_of_its_own_fails_with_an_actionable_message(tmp_path): + """A renamed or copied baseline has no audit history under its new name. + + The log has rows, all for another file. Verification must not borrow + them (that was #1030) and must not silently pass: it reports + ``missing_audit_log`` naming the file, what the log does record, and + the command that records it — a strict failure until it is re-saved. + """ + write_baseline(_stub_report(("SHIP-X", "tool_a")), tmp_path / "baseline.json") + renamed = tmp_path / "baseline-main.json" + (tmp_path / "baseline.json").rename(renamed) + issues = verify_baseline(renamed, tmp_path / "baseline-audit.log") + [issue] = [issue for issue in issues if issue.kind == "missing_audit_log"] + assert issue.default_severity == "critical" + assert "baseline-main.json" in issue.title + assert "record baseline.json" in issue.title + assert "baseline save" in issue.title + assert issue.evidence["recorded_baselines"] == ["baseline.json"] + assert has_hash_mismatch(issues) + result = _invoke_verify(renamed, "--strict") + assert result.exit_code == 6 + assert "no entry for baseline-main.json" in result.output + + # Re-saving under the new name records it, and the baseline verifies. + write_baseline(_stub_report(("SHIP-X", "tool_a")), renamed) + assert verify_baseline(renamed, tmp_path / "baseline-audit.log") == [] + + +def test_empty_baseline_with_no_row_of_its_own_is_not_an_issue(tmp_path): + """Same carve-out as a missing log: an empty baseline accepts nothing.""" + write_baseline(_stub_report(("SHIP-X", "tool_a")), tmp_path / "baseline.json") + empty = tmp_path / "baseline-empty.json" + empty.write_text( + BaselineFile( + created_at="2026-01-01T00:00:00Z", + source_report_run_id="run_empty", + ).model_dump_json(indent=2) + + "\n", + encoding="utf-8", + ) + assert verify_baseline(empty, tmp_path / "baseline-audit.log") == [] + + +def test_baseline_hash_reads_line_endings_as_lf_as_its_docstring_states(tmp_path): + """#1030 (#109): the CRLF behaviour and the docstring agree. + + ``compute_baseline_hash_from_file`` documents that a line-ending + conversion alone does not change the hash (a Git for Windows checkout + performs one on an untouched file) while every content edit does. The + docstring previously claimed line endings were detected. + """ + doc = " ".join((compute_baseline_hash_from_file.__doc__ or "").split()) + assert "A line-ending conversion alone does not" in doc + assert "CRLF and a lone CR hash as LF" in doc + + baseline_path, audit_path = _bootstrap_baseline(tmp_path) + lf = baseline_path.read_bytes() + assert b"\r" not in lf + for converted in (lf.replace(b"\n", b"\r\n"), lf.replace(b"\n", b"\r")): + baseline_path.write_bytes(converted) + assert verify_baseline(baseline_path, audit_path) == [] + result = _invoke_verify(baseline_path, "--audit-log", str(audit_path), "--strict") + assert result.exit_code == 0, result.output + # A content edit is detected whatever the line endings, and so is a BOM. + baseline_path.write_bytes(lf.replace(b"\n", b"\r\n").replace(b"\r\n", b" \r\n", 1)) + assert has_hash_mismatch(verify_baseline(baseline_path, audit_path)) + baseline_path.write_bytes(b"\xef\xbb\xbf" + lf) + with pytest.raises(InputParseError): + verify_baseline(baseline_path, audit_path) From 4f34eb580fb39bdba597c01fe0181c756cd8c3e0 Mon Sep 17 00:00:00 2001 From: Pengfei Hu Date: Mon, 19 Oct 2026 02:18:28 +0800 Subject: [PATCH 2/4] fix(overrides): judge the tier gate by the severity a finding was emitted at (#1030) The severity-override tier gate compared a downgrade with the catalog default. SHIP-SIDEFX-IDEMPOTENCY-MISSING is critical when a tool's retry policy is known (default high), so an override to high lowered it with no acknowledgement; a deprecated alias whose own default is lower than its expansions did the same. resolve_severity_overrides now takes the scan's pre-override findings. When an override would lower a finding it governs that was emitted above both defaults, that severity is the tier-crossing base. Governance is one rule, governing_configured_check_id, shared with apply_severity_overrides. The base never drops below the defaults. --- src/agents_shipgate/cli/scan/decision.py | 4 + src/agents_shipgate/core/check_ids.py | 19 ++ .../core/findings/mutations.py | 9 +- .../core/severity_overrides.py | 70 ++++- tests/test_severity_override_floor.py | 262 ++++++++++++++++++ 5 files changed, 356 insertions(+), 8 deletions(-) diff --git a/src/agents_shipgate/cli/scan/decision.py b/src/agents_shipgate/cli/scan/decision.py index 93249b249..4e87dc616 100644 --- a/src/agents_shipgate/cli/scan/decision.py +++ b/src/agents_shipgate/cli/scan/decision.py @@ -229,6 +229,10 @@ def _run_checks_and_decide( acknowledgements=manifest.acknowledge_overrides(), catalog=catalog, extra_known_check_defaults=effective_dynamic_defaults, + # Pre-override severities: the tier gate judges each override by + # the strongest finding it would lower, not only by the catalog + # default (#1030). + findings=findings, ) apply_severity_overrides(findings, override_resolution.override_by_check_id) apply_suppressions(findings, manifest.checks.ignore, tools_and_agent.tools) diff --git a/src/agents_shipgate/core/check_ids.py b/src/agents_shipgate/core/check_ids.py index 054d52eab..579147f1e 100644 --- a/src/agents_shipgate/core/check_ids.py +++ b/src/agents_shipgate/core/check_ids.py @@ -63,5 +63,24 @@ def expands_to_check_id(configured_check_id: str, emitted_check_id: str) -> bool ) +def governing_configured_check_id( + emitted_check_id: str, configured_check_ids: Iterable[str] +) -> str | None: + """The configured id whose setting applies to an emitted finding. + + An exact id wins; otherwise the first configured legacy alias or split + umbrella, in configuration order, that expands to it. This is the one + rule both applying a severity override and gating it use, so the gate + always judges the findings the override will actually change (#1030). + """ + configured = list(configured_check_ids) + if emitted_check_id in configured: + return emitted_check_id + for configured_check_id in configured: + if expands_to_check_id(configured_check_id, emitted_check_id): + return configured_check_id + return None + + def known_check_ids_with_legacy(check_ids: Iterable[str]) -> set[str]: return {*check_ids, *LEGACY_CHECK_ID_ALIASES} diff --git a/src/agents_shipgate/core/findings/mutations.py b/src/agents_shipgate/core/findings/mutations.py index cf8b887b3..f10de91c3 100644 --- a/src/agents_shipgate/core/findings/mutations.py +++ b/src/agents_shipgate/core/findings/mutations.py @@ -5,6 +5,7 @@ from agents_shipgate.core.check_ids import ( UNSUPPRESSIBLE_FINDING_CATEGORIES, expands_to_check_id, + governing_configured_check_id, ) from agents_shipgate.core.domain import Tool from agents_shipgate.core.tool_identity import tool_identity_aliases @@ -121,9 +122,5 @@ def _matching_suppression( def _severity_override_for_check( check_id: str, overrides: dict[str, Severity] ) -> Severity | None: - if override := overrides.get(check_id): - return override - for configured_check_id, override in overrides.items(): - if expands_to_check_id(configured_check_id, check_id): - return override - return None + configured_check_id = governing_configured_check_id(check_id, overrides) + return overrides[configured_check_id] if configured_check_id is not None else None diff --git a/src/agents_shipgate/core/severity_overrides.py b/src/agents_shipgate/core/severity_overrides.py index bec4fe4f3..2b8a30f46 100644 --- a/src/agents_shipgate/core/severity_overrides.py +++ b/src/agents_shipgate/core/severity_overrides.py @@ -33,16 +33,21 @@ from __future__ import annotations +from collections.abc import Iterable from dataclasses import dataclass, field from datetime import date from typing import TYPE_CHECKING -from agents_shipgate.core.check_ids import expands_to_check_id +from agents_shipgate.core.check_ids import ( + expands_to_check_id, + governing_configured_check_id, +) from agents_shipgate.core.errors import ConfigError from agents_shipgate.core.evaluation_clock import trust_expiry_date from agents_shipgate.schemas.checks import CheckMetadata from agents_shipgate.schemas.common import Severity from agents_shipgate.schemas.report import ( + Finding, PolicyAudit, SeverityOverrideAuditEntry, ) @@ -125,6 +130,7 @@ def resolve_severity_overrides( acknowledgements: list[OverrideAcknowledgement], catalog: list[CheckMetadata], extra_known_check_defaults: dict[str, Severity] | None = None, + findings: Iterable[Finding] = (), manifest_path_prefix: str = "shipgate.yaml#/checks/severity_overrides", today: date | None = None, ) -> SeverityOverrideResolution: @@ -169,12 +175,25 @@ def resolve_severity_overrides( default, the resolver uses the supplied value for tier-crossing and audit purposes. Floor enforcement still uses the catalog floor (the static gate floor for the check class). + + ``findings`` are this scan's findings *before* any override applies + (#1030). A check may emit above its catalog default from evidence no + manifest declares — ``SHIP-SIDEFX-IDEMPOTENCY-MISSING`` is ``critical`` + when the tool's retry policy is known, against a ``high`` default — so + neither of the two defaults above sees it. When the override would + lower a finding it governs (the same exact-id-then-alias rule + ``apply_severity_overrides`` uses) that was emitted above both + defaults, that finding's severity is the tier-crossing base, so + lowering it across a tier needs its acknowledgement like any other + downgrade. The base never drops below the defaults, so no override + that needed an acknowledgement stops needing one. """ today = today or trust_expiry_date() catalog_by_id = _catalog_index(catalog) known_ids = _known_check_ids(catalog) ack_by_id = _ack_by_check_id(acknowledgements) extra_defaults = extra_known_check_defaults or {} + emitted_by_override = _strongest_emitted_severity(findings, overrides) # 1. Expired acknowledgements are a config error regardless of # whether the matching override is tier-crossing — the user @@ -247,6 +266,21 @@ def resolve_severity_overrides( default_severity = dynamic_default else: default_severity = catalog_default_severity + # #1030: and never weaker than a finding this override would lower. + # The catalog and dynamic defaults describe the check; this + # describes what it actually emitted in this scan. Only a finding + # the override lowers raises the base, so an override that leaves + # every emitted severity as strong as it was keeps the audit row + # it had (an override to ``critical`` still reads as an upgrade). + emitted_severity = emitted_by_override.get(check_id) + emitted_above_default = False + if ( + emitted_severity is not None + and _is_weaker(default_severity, emitted_severity) + and _is_weaker(applied_severity, emitted_severity) + ): + default_severity = emitted_severity + emitted_above_default = True # 2. Floor enforcement. Hard. No ack bypass. Policy-pack rules # fall through with floor=None (extra_defaults path above). @@ -269,11 +303,19 @@ def resolve_severity_overrides( # Surface the alias path too — a user who configured # SHIP-API-OPERATIONAL-READINESS gets the *alias* name in # the diagnostic, not the expanded one. + emitted_note = ( + f" This scan emitted a finding it governs at " + f"{emitted_severity!r}, above the check's " + f"{catalog_default_severity!r} catalog default." + if emitted_above_default + else "" + ) raise ConfigError( f"checks.severity_overrides[{check_id!r}] downgrades " f"{default_severity!r} → {applied_severity!r}, " f"crossing the {severity_tier(default_severity)} → " - f"{severity_tier(applied_severity)} tier boundary. " + f"{severity_tier(applied_severity)} tier boundary." + f"{emitted_note} " f"Add an acknowledge_overrides entry with a reason." ) # Ack reason wins when both are set — ack is the explicit @@ -322,6 +364,30 @@ def resolve_severity_overrides( ) +def _strongest_emitted_severity( + findings: Iterable[Finding], + overrides: dict[str, SeverityOverrideEntry], +) -> dict[str, Severity]: + """Strongest pre-override severity among the findings each override governs. + + Keyed by the configured override id. Governance is + :func:`governing_configured_check_id`, the rule + ``apply_severity_overrides`` applies, over the same configuration order, + so an override is judged by exactly the findings it will change. + """ + strongest: dict[str, Severity] = {} + if not overrides: + return strongest + for finding in findings: + configured_check_id = governing_configured_check_id(finding.check_id, overrides) + if configured_check_id is None: + continue + current = strongest.get(configured_check_id) + if current is None or _is_weaker(current, finding.severity): + strongest[configured_check_id] = finding.severity + return strongest + + def _enforce_ack_expiry( acknowledgements: list[OverrideAcknowledgement], *, diff --git a/tests/test_severity_override_floor.py b/tests/test_severity_override_floor.py index c6e8613f8..879ea7bae 100644 --- a/tests/test_severity_override_floor.py +++ b/tests/test_severity_override_floor.py @@ -1088,3 +1088,265 @@ def test_aggregator_handles_override_without_manifest_section() -> None: assert audit_row.applied_severity == "medium" assert audit_row.direction == "downgrade" assert audit_row.tier_crossed is True + + +# --- #1030: the tier gate judges the severity each finding was emitted at --- + +_SIDEFX = "SHIP-SIDEFX-IDEMPOTENCY-MISSING" + + +def _emitted(check_id: str, severity: str, tool: str = "stripe.create_refund") -> Finding: + return Finding( + check_id=check_id, + title=f"{check_id} on {tool}", + severity=severity, + category="side_effects", + tool_name=tool, + recommendation="fix", + ) + + +def _builtin_catalog() -> list[CheckMetadata]: + from agents_shipgate.checks.registry import CHECK_METADATA + + return list(CHECK_METADATA) + + +def _builtin_dynamic_defaults(catalog: list[CheckMetadata]) -> dict[str, str]: + return {entry.id: entry.default_severity for entry in catalog if entry.dynamic_default} + + +def test_sidefx_catalog_default_is_below_its_dynamic_critical() -> None: + """Precondition for the tests below: the check emits above its default. + + ``SHIP-SIDEFX-IDEMPOTENCY-MISSING`` is ``critical`` when the tool's + retry policy is known, from tool evidence no manifest declares, so it + is not (and cannot be) a ``dynamic_default`` check. + """ + entry = next(e for e in _builtin_catalog() if e.id == _SIDEFX) + assert entry.default_severity == "high" + assert entry.dynamic_default is False + + +def test_override_lowering_a_finding_emitted_above_its_default_needs_ack() -> None: + """#1030: lowering a dynamically critical finding to high crosses a tier. + + The resolver compared the override with the catalog default (``high``), + saw ``high → high`` and accepted it with no acknowledgement, although + the finding it lowered was emitted ``critical``. + """ + catalog = _builtin_catalog() + with pytest.raises(ConfigError) as excinfo: + resolve_severity_overrides( + overrides={_SIDEFX: SeverityOverrideEntry(severity="high")}, + acknowledgements=[], + catalog=catalog, + extra_known_check_defaults=_builtin_dynamic_defaults(catalog), + findings=[ + _emitted(_SIDEFX, "high", tool="shopify.cancel_order"), + _emitted(_SIDEFX, "critical"), + ], + ) + message = str(excinfo.value) + assert "'critical' → 'high'" in message + assert "critical → high tier boundary" in message + assert "emitted a finding it governs at 'critical'" in message + assert "acknowledge_overrides" in message + + +def test_override_lowering_an_emitted_critical_finding_with_ack_is_audited() -> None: + catalog = _builtin_catalog() + resolution = resolve_severity_overrides( + overrides={_SIDEFX: SeverityOverrideEntry(severity="high")}, + acknowledgements=[ + OverrideAcknowledgement(check_id=_SIDEFX, reason="retries are bounded upstream") + ], + catalog=catalog, + extra_known_check_defaults=_builtin_dynamic_defaults(catalog), + findings=[_emitted(_SIDEFX, "critical")], + ) + [row] = resolution.audit.severity_overrides_applied + assert row.default_severity == "critical" + assert row.applied_severity == "high" + assert row.direction == "downgrade" + assert row.tier_crossed is True + assert row.reason == "retries are bounded upstream" + + +def test_override_matching_the_emitted_default_still_needs_no_ack() -> None: + """No finding above the default: the pre-#1030 answer is unchanged.""" + catalog = _builtin_catalog() + resolution = resolve_severity_overrides( + overrides={_SIDEFX: SeverityOverrideEntry(severity="high")}, + acknowledgements=[], + catalog=catalog, + extra_known_check_defaults=_builtin_dynamic_defaults(catalog), + findings=[_emitted(_SIDEFX, "high"), _emitted("SHIP-OTHER", "critical")], + ) + [row] = resolution.audit.severity_overrides_applied + assert (row.default_severity, row.direction, row.tier_crossed) == ("high", "same", False) + + +def test_override_that_lowers_no_emitted_finding_keeps_its_audit_row() -> None: + """Only a finding the override lowers raises the base. + + Overriding to ``critical`` lowers nothing, so the row reads as the + upgrade from the catalog default it read before #1030. + """ + catalog = _builtin_catalog() + resolution = resolve_severity_overrides( + overrides={_SIDEFX: SeverityOverrideEntry(severity="critical")}, + acknowledgements=[], + catalog=catalog, + extra_known_check_defaults=_builtin_dynamic_defaults(catalog), + findings=[_emitted(_SIDEFX, "critical"), _emitted(_SIDEFX, "high", tool="t2")], + ) + [row] = resolution.audit.severity_overrides_applied + assert (row.default_severity, row.applied_severity, row.direction) == ( + "high", + "critical", + "upgrade", + ) + + +def test_alias_override_is_judged_by_the_findings_it_governs() -> None: + """Another spelling reaches the same gate. + + ``SHIP-API-OPERATIONAL-READINESS`` is a deprecated catalog entry + (default ``medium``) that expands to the v0.4 API checks, among them + ``SHIP-API-RETRY-WITHOUT-IDEMPOTENCY`` (default ``high``). Judged by the + alias's own default, an override through it at ``medium`` was + ``medium → medium`` and lowered a ``high`` finding with no + acknowledgement. Judged by what it governs, it crosses a tier — and so + does ``high`` against a governed finding emitted ``critical``. A finding + an exact-id override governs instead is not the alias's to answer for: + the same exact-id-first rule ``apply_severity_overrides`` uses. + """ + catalog = _builtin_catalog() + alias = "SHIP-API-OPERATIONAL-READINESS" + alias_entry = next(e for e in catalog if e.id == alias) + assert alias_entry.default_severity == "medium" + with pytest.raises(ConfigError, match=r"'high' → 'medium'.*high → normal tier boundary"): + resolve_severity_overrides( + overrides={alias: SeverityOverrideEntry(severity="medium")}, + acknowledgements=[], + catalog=catalog, + extra_known_check_defaults=_builtin_dynamic_defaults(catalog), + findings=[_emitted("SHIP-API-RETRY-WITHOUT-IDEMPOTENCY", "high")], + ) + governed = _emitted("SHIP-API-RETRY-WITHOUT-IDEMPOTENCY", "critical") + with pytest.raises(ConfigError, match=r"SHIP-API-OPERATIONAL-READINESS.*critical → high"): + resolve_severity_overrides( + overrides={alias: SeverityOverrideEntry(severity="high")}, + acknowledgements=[], + catalog=catalog, + extra_known_check_defaults=_builtin_dynamic_defaults(catalog), + findings=[governed], + ) + resolution = resolve_severity_overrides( + overrides={ + "SHIP-API-OPERATIONAL-READINESS": SeverityOverrideEntry(severity="high"), + "SHIP-API-RETRY-WITHOUT-IDEMPOTENCY": SeverityOverrideEntry(severity="critical"), + }, + acknowledgements=[], + catalog=catalog, + extra_known_check_defaults=_builtin_dynamic_defaults(catalog), + findings=[governed], + ) + applied = {row.check_id: row for row in resolution.audit.severity_overrides_applied} + # The alias governs no finding here, so its row is judged by its own + # default: medium → high is an upgrade. The exact override lowers + # nothing (critical stays critical), so it too keeps its default base. + assert (applied[alias].default_severity, applied[alias].direction) == ("medium", "upgrade") + exact = applied["SHIP-API-RETRY-WITHOUT-IDEMPOTENCY"] + assert (exact.default_severity, exact.direction) == ("high", "upgrade") + apply_severity_overrides([governed], resolution.override_by_check_id) + assert governed.severity == "critical" + + +def test_no_builtin_check_emitted_above_its_default_lowers_without_ack() -> None: + """Catalog-wide: the gate does not depend on which emitters exceed defaults. + + For every built-in check whose default is below ``critical``, a finding + emitted ``critical`` and an override back to the catalog default (which + every floor permits) is a tier-crossing downgrade and is refused without + an acknowledgement. Before #1030 this held only for the + ``dynamic_default`` checks the aggregator knows about. + """ + catalog = _builtin_catalog() + dynamic = _builtin_dynamic_defaults(catalog) + accepted: list[str] = [] + checked = 0 + for entry in catalog: + if entry.default_severity == "critical": + continue + checked += 1 + try: + resolve_severity_overrides( + overrides={entry.id: SeverityOverrideEntry(severity=entry.default_severity)}, + acknowledgements=[], + catalog=catalog, + extra_known_check_defaults=dynamic, + findings=[_emitted(entry.id, "critical")], + ) + except ConfigError as exc: + assert "tier boundary" in str(exc), (entry.id, str(exc)) + else: + accepted.append(entry.id) + assert checked > 50 + assert accepted == [] + + +def test_scan_rejects_unacknowledged_override_of_dynamically_critical_sidefx(tmp_path) -> None: + """#1030 acceptance, end to end on the support-refund sample. + + ``stripe.create_refund`` declares a retry policy, so its + ``SHIP-SIDEFX-IDEMPOTENCY-MISSING`` finding is ``critical``. Lowering + the check to ``high`` without an acknowledgement is a config error + (exit 2); with one, the override applies and the audit row records + the real ``critical → high`` downgrade. + """ + import shutil + from pathlib import Path + + from agents_shipgate.cli.scan import run_scan + + sample = Path("samples/support_refund_agent") + work = tmp_path / "sample" + shutil.copytree(sample, work, ignore=shutil.ignore_patterns("expected")) + config = work / "shipgate.yaml" + original = config.read_text(encoding="utf-8") + assert original.count("checks:\n ignore:") == 1 + + baseline_report, _ = run_scan( + config_path=config, output_dir=tmp_path / "r0", formats=["json"], ci_mode="advisory" + ) + assert "critical" in { + f.severity for f in baseline_report.findings if f.check_id == _SIDEFX + }, "precondition: the sample emits the check at critical" + + override = f"checks:\n severity_overrides:\n {_SIDEFX}: high\n ignore:" + config.write_text(original.replace("checks:\n ignore:", override), encoding="utf-8") + with pytest.raises(ConfigError, match=r"'critical' → 'high'"): + run_scan( + config_path=config, output_dir=tmp_path / "r1", formats=["json"], ci_mode="advisory" + ) + + acknowledged = override.replace( + "\n ignore:", + f"\n acknowledge_overrides:\n - check_id: {_SIDEFX}\n" + " reason: retries are deduplicated by the payment gateway\n ignore:", + ) + config.write_text(original.replace("checks:\n ignore:", acknowledged), encoding="utf-8") + report, _ = run_scan( + config_path=config, output_dir=tmp_path / "r2", formats=["json"], ci_mode="advisory" + ) + assert {f.severity for f in report.findings if f.check_id == _SIDEFX} == {"high"} + [row] = [r for r in report.policy_audit.severity_overrides_applied if r.check_id == _SIDEFX] + assert (row.default_severity, row.applied_severity, row.direction, row.tier_crossed) == ( + "critical", + "high", + "downgrade", + True, + ) + assert row.reason == "retries are deduplicated by the payment gateway" From 9e015ebb9d81266a40f1290ea19f4475ac85c805 Mon Sep 17 00:00:00 2001 From: Pengfei Hu Date: Mon, 19 Oct 2026 02:18:28 +0800 Subject: [PATCH 3/4] fix(policy-packs): an optional pack tolerates only absence (#1030) load_policy_packs downgraded every load error of an optional pack to a warning, so a sha256 pin mismatch, or a pinned pack replaced by a malformed file, a directory or a link, dropped the pack's rules with only a warning. optional now tolerates exactly one failure: nothing exists at the declared path. Anything else is a config error, as for a required pack. Every pack-loading command goes through this loader. --- docs/manifest-v0.1.md | 4 +- docs/organization.md | 3 +- docs/policy-packs.md | 7 ++ src/agents_shipgate/inputs/policy_packs.py | 27 ++++- .../schemas/manifest/policy_packs.py | 9 +- tests/test_policy_packs.py | 112 ++++++++++++++++++ 6 files changed, 153 insertions(+), 9 deletions(-) diff --git a/docs/manifest-v0.1.md b/docs/manifest-v0.1.md index bd07e505a..61cd53596 100644 --- a/docs/manifest-v0.1.md +++ b/docs/manifest-v0.1.md @@ -1317,7 +1317,9 @@ checks: ``` Policy pack paths use the same manifest-relative containment policy as other -local inputs. External rule IDs must not start with `SHIP-`; use an +local inputs. `optional: true` tolerates only a missing pack (a warning); a pack +that is present must load and match its `sha256` pin, optional or not. External +rule IDs must not start with `SHIP-`; use an organization namespace such as `ORG-*`. Findings emitted by policy packs support the same suppressions, severity overrides, baselines, Markdown, JSON, and SARIF output as built-in findings. diff --git a/docs/organization.md b/docs/organization.md index aed848a91..520101d71 100644 --- a/docs/organization.md +++ b/docs/organization.md @@ -50,7 +50,8 @@ checks: `org status` reports sourced packs without a `sha256` as `policy_pack_unpinned`, and reports changed content as -`policy_pack_pin_mismatch`. Normal `scan` still fails closed on a pin mismatch. +`policy_pack_pin_mismatch`. Normal `scan` still fails closed on a pin mismatch, +for an `optional: true` pack too. For a read-only policy-pack inventory, run: ```bash diff --git a/docs/policy-packs.md b/docs/policy-packs.md index 140105556..67e4d1b9a 100644 --- a/docs/policy-packs.md +++ b/docs/policy-packs.md @@ -29,6 +29,13 @@ checks: agents-shipgate scan --config shipgate.yaml --policy-pack policies/org-release.yaml ``` +`optional: true` tolerates one thing only: no file at the declared path. That +is a source warning and the scan continues without the pack's rules. A pack +that is there must load and match its pin whether or not it is optional: a +`sha256` mismatch, a malformed or unreadable pack, or a link or directory in +its place fails the scan with a config error (exit 2), so editing a pinned +pack cannot quietly drop its rules (#1030). + External rule IDs must not start with `SHIP-`; that namespace is reserved for built-in checks. Use an organization namespace such as `ORG-*`. diff --git a/src/agents_shipgate/inputs/policy_packs.py b/src/agents_shipgate/inputs/policy_packs.py index cb2156d60..31af0cb16 100644 --- a/src/agents_shipgate/inputs/policy_packs.py +++ b/src/agents_shipgate/inputs/policy_packs.py @@ -1,6 +1,7 @@ from __future__ import annotations import hashlib +import os from dataclasses import dataclass from pathlib import Path @@ -58,6 +59,18 @@ def load_policy_packs( resolved_rules: list[ResolvedPolicyPackRule] = [] warnings: list[str] = [] for config in configs: + # ``optional`` tolerates exactly one failure: nothing exists at the + # declared path (#1030). A pack that is there must load and match its + # pin. Downgrading every load error of an optional pack to a warning + # let a pinned pack whose content changed — or that was swapped for a + # link, a directory or an oversized file — drop its rules from the + # scan with only a warning, which is what the pin exists to prevent. + if config.optional and _policy_pack_absent(base_dir, config.path): + warnings.append( + f"Optional policy pack {config.path!r} failed to load: no file " + "exists at that path; continuing without its rules." + ) + continue try: path = resolve_input_path(base_dir, config.path) sha256, sha256_status = _verify_pack_pin(path, config) @@ -84,9 +97,6 @@ def load_policy_packs( ResolvedPolicyPackRule(pack=pack, rule=rule) for rule in pack_file.rules ) except (ConfigError, InputParseError, ValidationError) as exc: - if config.optional: - warnings.append(f"Optional policy pack {config.path!r} failed to load: {exc}") - continue if isinstance(exc, ConfigError): raise raise ConfigError(f"Invalid policy pack {config.path!r}: {exc}") from exc @@ -94,6 +104,17 @@ def load_policy_packs( return LoadedPolicyPacks(rules=resolved_rules, loaded=loaded, warnings=warnings) +def _policy_pack_absent(base_dir: Path, value: str) -> bool: + """Whether nothing at all exists at a declared pack path. + + Lexical: a link is something, even one whose target is missing, so + replacing a pack with a dangling link is an error, not an absence. + """ + raw = Path(value) + candidate = raw if raw.is_absolute() else base_dir / raw + return not os.path.lexists(candidate) + + def run_policy_pack_rules( context: ScanContext, policy_packs: LoadedPolicyPacks, diff --git a/src/agents_shipgate/schemas/manifest/policy_packs.py b/src/agents_shipgate/schemas/manifest/policy_packs.py index 9b3c052e7..855c2e4c8 100644 --- a/src/agents_shipgate/schemas/manifest/policy_packs.py +++ b/src/agents_shipgate/schemas/manifest/policy_packs.py @@ -10,10 +10,11 @@ class PolicyPackConfig(ArtifactPathConfig): id: str | None = None source: str | None = None # v0.2 (org distribution): optional content pin. When set, the pack - # file's SHA-256 must match or the scan fails with a config error - # (optional packs degrade to a warning). Pin shared/org packs that are - # vendored or synced from another repo so a tampered pack cannot - # silently change the release policy. + # file's SHA-256 must match or the scan fails with a config error — + # for an ``optional: true`` pack too (#1030): ``optional`` tolerates a + # pack that does not exist, never one that fails its pin or does not + # load. Pin shared/org packs that are vendored or synced from another + # repo so a tampered pack cannot silently change the release policy. sha256: str | None = None diff --git a/tests/test_policy_packs.py b/tests/test_policy_packs.py index 1c23434da..ae7f21943 100644 --- a/tests/test_policy_packs.py +++ b/tests/test_policy_packs.py @@ -950,6 +950,118 @@ def test_optional_missing_policy_pack_warns(tmp_path): assert any("Optional policy pack 'missing-pack.yaml' failed to load" in item for item in report.source_warnings) +_PINNED_PACK_TEXT = """ +name: Org Pack +rules: + - id: ORG-PINNED-RULE + title: Pinned rule + severity: high + block: true + recommendation: n/a + match: + risk_tags: [financial_action] +""" + + +def _optional_pack_manifest(tmp_path: Path, *, sha256: str | None) -> Path: + pin = f"\n sha256: \"{sha256}\"" if sha256 is not None else "" + config = tmp_path / "shipgate.yaml" + config.write_text( + _manifest_without_policy_pack() + + f""" +checks: + policy_packs: + - path: org-pack.yaml + optional: true{pin} +""", + encoding="utf-8", + ) + return config + + +def test_optional_policy_pack_with_wrong_sha256_is_an_error(tmp_path): + """#1030 acceptance: ``optional`` never downgrades a pin mismatch. + + A PR that edits a pinned optional pack (removing a blocking rule, say) + used to load zero packs with a warning and scan on without the org + rules. A wrong pin is the tampering the pin exists to catch. + """ + _write_openapi(tmp_path) + (tmp_path / "org-pack.yaml").write_text(_PINNED_PACK_TEXT, encoding="utf-8") + pinned = hashlib.sha256(_PINNED_PACK_TEXT.encode("utf-8")).hexdigest() + config = _optional_pack_manifest(tmp_path, sha256=pinned) + report, _ = run_scan( + config_path=config, + output_dir=tmp_path / "reports", + formats=["json"], + ci_mode="advisory", + ) + assert [pack.sha256_status for pack in report.loaded_policy_packs] == ["verified"] + + # The pack is edited after it was pinned: the blocking rule is gone. + (tmp_path / "org-pack.yaml").write_text( + _PINNED_PACK_TEXT.replace("block: true", "block: false"), encoding="utf-8" + ) + with pytest.raises(ConfigError, match="does not match its pinned sha256"): + run_scan( + config_path=config, + output_dir=tmp_path / "reports2", + formats=["json"], + ci_mode="advisory", + ) + result = runner.invoke(app, ["doctor", "--config", str(config), "--json"]) + assert result.exit_code == 2, result.output + + +def test_optional_policy_pack_missing_with_a_pin_still_warns(tmp_path): + """#1030 acceptance: absence is the one failure ``optional`` tolerates.""" + _write_openapi(tmp_path) + config = _optional_pack_manifest(tmp_path, sha256="0" * 64) + report, _ = run_scan( + config_path=config, + output_dir=tmp_path / "reports", + formats=["json"], + ci_mode="advisory", + ) + assert report.loaded_policy_packs == [] + assert [ + item for item in report.source_warnings if "Optional policy pack 'org-pack.yaml'" in item + ] == [ + "Optional policy pack 'org-pack.yaml' failed to load: no file exists at " + "that path; continuing without its rules." + ] + + +@pytest.mark.parametrize("replacement", ["malformed", "directory", "dangling_link"]) +def test_optional_policy_pack_that_exists_but_does_not_load_is_an_error( + tmp_path, replacement +): + """Only a missing pack is tolerated; anything at the path must load. + + Each replacement used to degrade to a warning, so swapping a pinned + optional pack for a broken file, a directory or a dangling link dropped + its rules exactly as a failed pin did. + """ + from agents_shipgate.config.loader import load_manifest + from agents_shipgate.inputs.policy_packs import load_policy_packs + + _write_openapi(tmp_path) + pack = tmp_path / "org-pack.yaml" + if replacement == "malformed": + pack.write_text("name: [unclosed\n", encoding="utf-8") + elif replacement == "directory": + pack.mkdir() + else: + try: + pack.symlink_to(tmp_path / "gone.yaml") + except (OSError, NotImplementedError): + pytest.skip("symlinks unavailable") + for sha256 in (None, "0" * 64): + config = _optional_pack_manifest(tmp_path, sha256=sha256) + with pytest.raises(ConfigError, match="org-pack.yaml"): + load_policy_packs(load_manifest(config), tmp_path) + + def _write_openapi(tmp_path: Path) -> None: (tmp_path / "openapi.yaml").write_text( """ From 882bb84589abbdbd78f66333ed6f6082622a508c Mon Sep 17 00:00:00 2001 From: Pengfei Hu Date: Mon, 19 Oct 2026 02:18:28 +0800 Subject: [PATCH 4/4] docs: STABILITY migration note and CHANGELOG for #1030 Migration note #trust-config-1030; Baseline Integrity (which row vouches, what the hash covers), Severity-override floor (per-finding tier gate) and the loaded_policy_packs pin line updated. --- CHANGELOG.md | 1 + STABILITY.md | 150 ++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 145 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ed3249a3..9b668acc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ ### Changes +- **Each baseline verifies against its own audit rows, the override tier gate reads the severity a finding was emitted at, and an optional policy pack's pin is fatal.** Every baseline in a directory appends to one `baseline-audit.log`, and verification compared a baseline with the log's last row whoever wrote it: saving `baseline-prod.json` turned an untouched `baseline.json` into a critical `SHIP-BASELINE-INTEGRITY-MISMATCH` (`baseline verify --strict` exit 6, `scan` `blocked`), while a copy of the baseline saved last verified clean. Verification now uses the latest row recorded for that baseline's file name; a log with no row for it is `missing_audit_log`, naming the files the log does record and the `baseline save` that records it. Audit logs written by released versions verify unchanged. The baseline hash reads CRLF as LF, as it always has (a Windows checkout converts line endings on an untouched file); its docstring, which said line endings were detected, now says so. `checks.severity_overrides: {SHIP-SIDEFX-IDEMPOTENCY-MISSING: high}` lowered that check's `critical` finding (emitted when a retry policy is known) with no acknowledgement, because the tier gate compared with the `high` catalog default; it now compares with the strongest severity among the findings the override would lower, so the override needs an `acknowledge_overrides` entry (exit 2 otherwise), and the same holds through a legacy alias such as `SHIP-API-OPERATIONAL-READINESS`. An `optional: true` policy pack whose `sha256` pin did not match, or that had been replaced by a malformed file, a directory or a link, was a warning and its rules were dropped; `optional` now tolerates only a missing pack, and anything else is a config error. Migration: [STABILITY.md](STABILITY.md#trust-config-1030). (#1030) - **`checks.ignore` no longer turns a configured advisory gate green while the decision says `blocked`.** A suppression cannot waive a mandatory current-surface control (the four dedicated control families, such as `SHIP-ACTION-FINANCIAL-WRITE-CONTROL-MISSING`, and engine-raised control-pack obligations), so the release decision kept such a finding as a blocker; the exit code dropped it with every other suppressed finding before matching `fail_on`. `samples/support_refund_agent` in advisory mode with `fail_on: [critical]` and three `checks.ignore` entries printed `Decision: blocked`, `would_fail_ci=false` and exited `0`; without the entries it exited `20`. The exit code now also reads the decision's `blockers[]`: a blocker whose severity is in `fail_on` (including `verify`'s base `ci.fail_on` floor) exits `20` in either mode, for `scan`, `verify` and the Action, and `fail_policy.exit_code` / `would_fail_ci` in `report.json` and `verifier.json` say the same. Strict-mode exit codes are unchanged, because a blocker already meant a `blocked` decision, which strict mode fails; advisory mode without `fail_on` still exits `0`; and a suppression still waives every other finding. A report whose blocker in `fail_on` would not fail CI is refused when it is written. Separately, `verify-run.json`'s outcome, the verification receipt and the safety qualification script now read the decision-to-merge-verdict projection from `schemas/verifier.py` instead of carrying their own copies of its table, which would have refused every verify run the day that table changed; a test refuses a new copy. Migration: [STABILITY.md](STABILITY.md#release-decision-truth-table). (#1029) - **A Claude Code setting that runs a command, or an `env` entry that redirects requests or loads code, is a host-grant row, and a settings change no row shows is no longer "no change".** Adding `"statusLine": {"command": "curl … | sh"}`, `"apiKeyHelper": "curl … | sh"`, `env.ANTHROPIC_BASE_URL` or `env.NODE_OPTIONS` to `.claude/settings.json` gave no row: `diff` (text and `--json`), `verify`'s host comparison and the PR comment headlined `No static host-grant changes detected. No verdict is implied.`, and `audit --host` reported Claude Code coverage `complete`. Host-grants `0.9` (extended in place) now publishes `command_setting` grants for `statusLine`, `subagentStatusLine`, `fileSuggestion`, `apiKeyHelper`, `otelHeadersHelper`, `awsAuthRefresh` and `awsCredentialExport` (`high`, the command shown as a hook's is: name, digest and shape, never text) and `env` grants for endpoint, proxy, certificate, search-path and loader names (`*_BASE_URL`, OpenTelemetry exporter endpoints, `HTTPS_PROXY` and the other proxies, `NODE_EXTRA_CA_CERTS`, `NODE_TLS_REJECT_UNAUTHORIZED`, `SSL_CERT_FILE`, `NODE_OPTIONS`, `BASH_ENV`, `PATH`, `PYTHONPATH`, `LD_PRELOAD`, `GIT_SSH_COMMAND` and the other names `docs/host-boundary-support.md` lists, read without regard to case; `high`, the value digested and never published), so adding one is a `⚠ high added` row and editing one a `changed` row naming the two digests. A settings file's other keys are published as display-only `unmodelled_keys` with digests; when one changes with no row, the coverage item names it in `detail` and the headline reads `Changed, not compared: .claude/settings.json (keys this entry does not compare: …)`, as it does, without keys, for any settings, MCP or hook file that changed with no row. A key no reader models that is shaped as a command (an object with a `command`, or a `…Helper` string) is a blocking limit, so `audit --host` reports coverage `partial`, as for `policyHelper`. The cold-start benchmark's oracle moves those seven settings and those `env` names from unsupported to supported. Migration: [STABILITY.md](STABILITY.md#host-commands-env-1027). (#1027) - **An n8n MCP Client Tool that exposes every tool of its server is read as doing so.** n8n's node selects its tools with `include` (`all`, `selected` or `except`) and leaves it out of an exported workflow at its default, `all`; the reader recognised only `toolSelection`, `toolsToInclude` and `toolSelectionMode`, which n8n does not write. So the default export, `include: all` and `include: except` each entered the catalog as one concretely named tool, with no wildcard, no dynamic surface and no `SHIP-N8N-MCP-CLIENT-TOOLSET-UNFILTERED`. Each is now one `.*` action, which raises `SHIP-INVENTORY-WILDCARD-TOOLS` and, with no local inventory declared, `SHIP-N8N-DYNAMIC-TOOL-SURFACE-NOT-ENUMERABLE` and the unfiltered finding (`high` in `production_like` and `production`, `medium` otherwise); an `except` node's `excludeTools` are listed in the finding's `excluded_tools`; `selected` names its `includeTools`, and a `selected` node naming none is read as `all`, as n8n runs it. The earlier spellings remain aliases, and `include` decides when both are present. A workflow whose MCP Client Tool keeps n8n's default now reports these findings where it passed; select explicit tools in the node, or declare a local MCP inventory. Migration: [STABILITY.md](STABILITY.md#mcp-reader-conformance-1025). (#1025) diff --git a/STABILITY.md b/STABILITY.md index 1aad92dfe..0713dcd7c 100644 --- a/STABILITY.md +++ b/STABILITY.md @@ -356,6 +356,108 @@ the Action tag) for reproducible CI. --- + + +## Migration Note: Unreleased — each baseline verifies against its own audit rows, the override tier gate reads emitted severity, and an optional pack's pin is fatal (#1030) + +**What was wrong.** Three trust-configuration checks compared against the +wrong reference. + +- Every baseline in a directory appends to the one `baseline-audit.log` + there, and verification compared a baseline with the log's last row, + whichever baseline wrote it. Saving `baseline-prod.json` turned an untouched + `baseline.json` into a critical, unsuppressible + `SHIP-BASELINE-INTEGRITY-MISMATCH` (`baseline verify --strict` exited `6`, + and `scan --baseline` moved from `insufficient_evidence` to `blocked`), while + a copy of the baseline saved last, or a renamed baseline, verified clean. The + hash function's docstring also claimed line-ending edits were detected; they + never were. +- The severity-override tier gate measured a downgrade against the catalog + default, not the severity a finding was emitted at. + `SHIP-SIDEFX-IDEMPOTENCY-MISSING` is `critical` when the tool's retry policy + is known (catalog default `high`), so + `checks.severity_overrides: {SHIP-SIDEFX-IDEMPOTENCY-MISSING: high}` lowered it + with no acknowledgement and `policy_audit` recorded `high → high`, + `direction: same`. A deprecated alias whose own default is below what it + expands to did the same: `SHIP-API-OPERATIONAL-READINESS: medium` (its + default) lowered a `high` `SHIP-API-RETRY-WITHOUT-IDEMPOTENCY`. +- An `optional: true` policy pack whose `sha256` pin no longer matched — or + that had been replaced by a malformed file, a directory or a link — became a + source warning, and the scan ran without the pack's rules. + +**What changes.** + +- **Baseline audit lookup.** `baseline verify` and `scan --baseline` compare a + baseline with the latest audit row recorded **for that baseline**: the row + whose `baseline_path` names the same file. `baseline_path` is spelled as the + saving command spelled it — relative to that command's working directory, or + absolute, with either separator — so its final component (the file name) is + what is compared. A log that has rows but none for the baseline is + `missing_audit_log` (critical, exit `6` under `--strict`) whose title names + the baseline, the files the log does record and the `baseline save` that + records it, with `evidence.recorded_baselines`; an empty baseline is still + not an issue. + `hash_mismatch` evidence gains `latest_audit_baseline_path`, the row it was + compared with. +- **Line endings.** The baseline hash is, as in every release, the SHA-256 of + the file's UTF-8 text with CRLF and lone CR read as LF; the docstring now + says so. Any content edit changes it — a value, a key order, a space, a + blank line; a line-ending conversion alone does not, because it cannot + change the parsed baseline, and because Git for Windows' default + `core.autocrlf=true` performs one on checkout of an untouched file. +- **Override tier gate.** When an override would lower a finding it governs + that was emitted above both the catalog default and the manifest-effective + dynamic default, that finding's pre-override severity is the tier-crossing + base; otherwise the base is what it was. "Governs" is the rule that applies + the override: the exact check id, otherwise the first configured legacy + alias or split umbrella that expands to it. + `policy_audit.severity_overrides_applied[].default_severity` reports that + base, and the `ConfigError` names the emitted severity when it raised it. An + override that lowers no emitted finding (for example one to `critical`) + keeps the audit row it had. +- **Optional policy packs.** `optional: true` tolerates exactly one failure: + nothing exists at the declared path (a dangling link is something). That is + a warning, `Optional policy pack '' failed to load: no file exists at + that path; continuing without its rules.` Anything else — a pin mismatch, an + unreadable or non-regular file, a path outside the manifest directory, a + malformed pack, a schema or team-reference error — is a config error (exit + `2`), as for a required pack. Every command that loads packs goes through + the one loader (`scan`, `verify`, `doctor`, `baseline save`, and the Action + through `verify`/`scan`). + +**Released audit logs.** No migration is needed for a log a released version +wrote. Every row already records `baseline_path`, and in a directory holding +one baseline — the default — the latest row recorded for it is the log's last +row, so verification gives the answer it gave before; `hash_after` values +verify because the hash is unchanged. A baseline that verified only by +borrowing another file's row now fails with the message above: a baseline +renamed or copied without a save has no audit history under its new name, so +re-run `agents-shipgate baseline save --out `. The audit row shape and +`audit_schema_version` (`0.1`) are unchanged, so released readers keep reading +logs this version writes. + +**Who must act.** + +- A manifest whose override lowers a finding across a tier from the severity + it was emitted at now fails with exit `2` naming the override. Add a + `checks.acknowledge_overrides` entry with a reason, or raise the override. + Until the base branch carries the fix, `verify`'s base scan reports + `scan_failed` and leaves the head gate unchanged. +- Same-named baselines in different directories that share one log through + `baseline.audit_log` are not told apart: give them distinct file names or + distinct logs. +- An `optional: true` pack that is present but broken must be fixed, or its + declaration removed. + +**What does not change.** No CLI flag, schema, check ID, control state or exit +code is added or moved. The baseline and audit-log formats, `integrity_mode`, +floors, acknowledgement and expiry rules, the `dynamic_default` contract, a +required pack's behaviour and a missing optional pack's warning are unchanged. +An override that needed an acknowledgement still needs one: the tier base +never drops below the defaults it used before. + +--- + ## Migration Note: Unreleased — secret and injection observations are published, and redaction rules `0.2` (#1026) @@ -5011,7 +5113,7 @@ In `agents-shipgate-reports/report.json`, the following are guaranteed: - `release_decision.contribution_rules[].{finding_id, fingerprint, check_id, category, rule, rationale}` (v0.17+) — deterministic per-finding audit of how each finding contributed to the release decision. Required + always present. Exactly one row per `report.findings` entry, including suppressed findings. v0.33 adds `rule="unsupported_evidence"`, always with `category="excluded"`, for typed support that is not policy-eligible. - `baseline.{matched_count, new_count, resolved_count, path}` (when `--baseline` is used) - `tool_inventory[].{name, source_type, source_ref, risk_tags, auth_scopes, owner, confidence}` -- `loaded_policy_packs[].{id, name, version, path, source, sha256, sha256_status, owner, rule_count}` (v0.27+) — deterministic policy-pack distribution and ownership metadata for organization audit. `sha256_status` is `"verified"` when a manifest pin matched and `"unpinned"` otherwise. Hash mismatch still fails closed during pack loading; this metadata never introduces a second release verdict. +- `loaded_policy_packs[].{id, name, version, path, source, sha256, sha256_status, owner, rule_count}` (v0.27+) — deterministic policy-pack distribution and ownership metadata for organization audit. `sha256_status` is `"verified"` when a manifest pin matched and `"unpinned"` otherwise. Hash mismatch still fails closed during pack loading, for an `optional: true` pack too (#1030): `optional` tolerates a pack that does not exist, never one that fails its pin or does not load. This metadata never introduces a second release verdict. - `loaded_plugins[].{name, value, distribution, version, check_id}` - `loaded_plugins[].{validation_status, validation_errors, runtime_errors}` (v0.17+ / M5; `dynamic_default_not_supported` added v0.18) — plugin validation provenance, required + present on every entry. `validation_status` is one of `valid | load_failed | bad_signature | bad_metadata | dynamic_default_not_supported | id_collision | bad_floor`; the two error lists are always present and empty for clean plugins. Invalid plugins still appear in this array (with `check_id: null` for entries that failed before metadata parsing), so reviewers can see what was skipped without reading scanner logs. Plugin findings whose `check_id` does not match the declared metadata are dropped at runtime and recorded under `runtime_errors`. `dynamic_default_not_supported` (v0.18+) rejects plugins declaring `AGENTS_SHIPGATE_METADATA.dynamic_default=True` — plugins have no path to wire into `core/dynamic_defaults.py`'s aggregator, so a swing check would never receive a manifest-effective default and would be silently bypassable. - `policy_audit.severity_overrides_applied[].{check_id, default_severity, applied_severity, manifest_path, reason, tier_crossed, direction, expires}` (v0.17+ / M1) — top-of-report audit envelope for severity overrides applied during scan. Always present on emitted scans (empty when no overrides applied); required + non-nullable on the wire. `direction` is one of `downgrade | upgrade | same`. `tier_crossed=true` indicates the override crossed a severity tier boundary (critical / high / medium-low); tier-crossing downgrades require a matching `checks.acknowledge_overrides` entry, which is reflected in `reason`. `expires` is an ISO-8601 date carried from the matching acknowledgement (or the rich-form override entry); on/past this date the manifest fails to load with exit 2. Verify computes the hard-expiry date as the later of its wall-clock date and the content-bound `evaluation_date`; commit timestamps can never extend trust. @@ -5109,14 +5211,31 @@ the only such built-in is `SHIP-ACTION-POLICY-VIOLATION` (emits at `action_surface.policies[].severity`). Policy-pack rule IDs flow through the same `extra_known_check_defaults` mechanism but live outside the catalog. The severity-override resolver uses -`max(catalog default, manifest-effective default)` as the -tier-crossing comparison base, so a `severity: critical` action +`max(catalog default, manifest-effective default)` as the tier-crossing +comparison base — raised to the emitted severity when the override lowers +a finding emitted above it (below) — so a `severity: critical` action policy with override `high` cannot appear same-tier against the catalog's `high` default. The `policy_audit.severity_overrides_applied[].default_severity` row reports the effective (dynamic-aware) default so reviewers see the real before/after. +**Per-finding tier gate** (#1030). The emitted severity is the strongest +pre-override severity among this scan's findings that the override +governs — the exact check id, otherwise the first configured legacy alias +or split umbrella that expands to it, the same rule that applies the +override. When the override would lower it, and it is above both +defaults, it is the comparison base. A check can emit above its catalog +default from evidence no manifest declares +(`SHIP-SIDEFX-IDEMPOTENCY-MISSING` is `critical` when a tool's retry +policy is known, against a `high` default), so the gate does not rely on +`dynamic_default` to see it: an override that would lower any finding +across a tier boundary from the severity it was emitted at needs an +`acknowledge_overrides` entry, whatever the catalog says. The base never +drops below the defaults, so the gate only ever asks for more +acknowledgements than the defaults alone would, and an override that +lowers no emitted finding keeps the audit row it had. + Two contract rules pin the design (v0.18): - Built-in checks marked `dynamic_default=True` MUST also declare @@ -5625,7 +5744,8 @@ Each `agents-shipgate baseline save` appends one JSON line to - `run_id` — scan's run_id (matches `BaselineProvenance.run_id` for any fingerprints added in this save) - `scanner_version` — Agents Shipgate version that wrote the row -- `baseline_path` — string path saved at the time of the row +- `baseline_path` — string path saved at the time of the row, spelled as the + saving command spelled it (relative to its working directory, or absolute) - `hash_before` — `"sha256:…"` of the prior baseline file content, or `null` when this was the first save - `hash_after` — `"sha256:…"` of the new baseline file content @@ -5636,6 +5756,23 @@ a single `.agents-shipgate/` directory carries both. Commit both files together; reviewers can `git log .agents-shipgate/baseline-audit.log` to see when fingerprints joined the baseline. +**Which row vouches for a baseline** (#1030). Every baseline in a directory +appends to that directory's one log. Verification compares a baseline with +the latest row recorded for **that** baseline — the row whose `baseline_path` +has the same final component (file name), with either path separator — +never with a row another baseline wrote. A log with rows but none for the +baseline is `missing_audit_log`; re-run `baseline save --out ` to record +it (a renamed or copied baseline has no history under its new name). Two +same-named baselines in different directories can share a log only through +`baseline.audit_log`; they are not told apart, so give them distinct names or +distinct logs. + +**What the hash covers.** `hash_before`, `hash_after` and the verified hash +are the SHA-256 of the baseline's UTF-8 text with CRLF and lone CR read as +LF. Every content edit changes it; a line-ending conversion alone does not, +because it cannot change the parsed baseline and Git for Windows' default +`core.autocrlf=true` performs one on checkout of an untouched file. + `manifest.baseline.integrity_mode` controls behavior when `scan --baseline X` detects an integrity issue. Stable values: @@ -5650,8 +5787,9 @@ detects an integrity issue. Stable values: New stable check IDs (v0.11+): - `SHIP-BASELINE-INTEGRITY-MISMATCH` (critical) — file hash mismatch, missing - audit log, audit log empty or malformed, entry references unknown `run_id`, - or entry loaded from a legacy schema without provenance. + audit log, audit log empty or malformed or without a row for that baseline, + entry references unknown `run_id`, or entry loaded from a legacy schema + without provenance. - `SHIP-BASELINE-ENTRY-EXPIRED` (high) — `provenance.expires` < today. - `SHIP-BASELINE-ENTRY-STALE` (low) — deprecated check ID in the entry, or the entry matched no active finding (scan-aware; resolved-not-pruned).