diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index a9730b4..fb6bb28 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -7,4 +7,4 @@ Use only relevant modules. Preserve host/organization policy and existing user w This repository contains the framework, optional offline tooling, templates and historical v1 references. Do not load legacy packs for a v2 task unless maintaining an explicitly selected v1 adoption. Current work and evidence are in -[the current evolution record](../docs/exec_ctrl/V2_SESSION_EVOLUTION.md). +[the current publication record](../docs/exec_ctrl/V2_PUBLICATION_READINESS.md). diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index dc5b734..f0dcd4b 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -10,6 +10,7 @@ permissions: jobs: validate: strategy: + fail-fast: false matrix: os: [ubuntu-latest, windows-latest] python: ['3.10', '3.13'] diff --git a/AGENTS.md b/AGENTS.md index 85bf5f9..f2f5c68 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,7 +7,7 @@ Read only the modules relevant to the task. Do not recursively load the v1 libra When editing this repository: - Treat Markdown, adapters, catalog, examples and tools as one product. Keep their - behavior coherent. Current work: [project/session evolution](docs/exec_ctrl/V2_SESSION_EVOLUTION.md). + behavior coherent. Current work: [publication readiness](docs/exec_ctrl/V2_PUBLICATION_READINESS.md). - Preserve existing work and historical evidence. Do not rewrite v1 records as v2 proof. Label preserved v1 instructions as reference. - Use Python 3.10+ standard library only for the optional helper. It must remain diff --git a/CHANGELOG.md b/CHANGELOG.md index 86dc71d..e7f7ef1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## 2.0.0-preview.4 — 2026-10-05 + +- Reject observed file edits, truncation, replacement, deletion or redirection + during snapshot capture; preserve the portable manifest format and bounded scope. +- Handle Windows path/descriptor timestamp differences without rejecting stable files. +- Add a CI-reproducible project fix workflow: failing tenant-access regression, + scoped repair, new content identity, current passing evidence and changed-policy denial. +- Exercise all CLI commands under a test-only guard that denies writes, network and + process creation. Add routing-matrix and version/policy lifecycle acceptance. +- Document [test execution](tests/README.md) and + [publication evidence](docs/exec_ctrl/V2_PUBLICATION_READINESS.md). Real-project and + independent IDE adoption remain distinct acceptance gates. + ## 2.0.0-preview.3 — 2026-10-05 - Add proportionate session working agreements, checkpoints/resumption, daily task diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 759abfc..1392afe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,6 +20,9 @@ Verify generated pending records and example policies as well as changed command Test denial, stale identity, malformed input and read-only behavior for new helper capabilities. Explicit-file snapshots must disclose their bounded scope; never auto-read secrets or execute commands to infer project facts. +The [test guide](tests/README.md) describes the reproducible synthetic project +workflow, CLI side-effect guard, platform skips and publication acceptance limits. +All these checks run through ordinary unittest discovery in the existing CI matrix. For provider-specific behavior, cite dated official documentation and distinguish documentation support from observed field results. Update the field-trial record diff --git a/README.md b/README.md index bb3d452..e3a0b6f 100644 --- a/README.md +++ b/README.md @@ -72,12 +72,15 @@ flowchart LR ## Status -**2.0.0-preview.3** adds session/checkpoint guidance, concrete task playbooks, -pending-record generation, bounded content snapshots, typed policy/evidence gates -and performance/accessibility/compatibility routing. See the -[current review and evidence](docs/exec_ctrl/V2_SESSION_EVOLUTION.md). -Validation: all 97 framework tests passed on the four hosted Windows/Linux jobs; -the controlled lab passed 63 checks, 94 routing cases and 68 real HTTP requests. +**2.0.0-preview.4** adds reproducible project-lifecycle tests, audited CLI execution +and rejection of observed file changes during snapshot capture. It retains the +session/checkpoint guidance, task playbooks, pending records and typed policy gates. +See [publication readiness and evidence](docs/exec_ctrl/V2_PUBLICATION_READINESS.md) +and [test coverage](tests/README.md) for current results and how to reproduce them. + +The [preview 3 evolution record](docs/exec_ctrl/V2_SESSION_EVOLUTION.md) preserves +its 97-test hosted validation and controlled lab results. Those historical results +are not proof of this revision. Previous [controlled lab results](docs/exec_ctrl/V2_SECOND_PASS_TESTING.md) remain versioned evidence. Independent task-plus-link trials across IDEs and the stable diff --git a/START_HERE.md b/START_HERE.md index 1acf9cc..856ad40 100644 --- a/START_HERE.md +++ b/START_HERE.md @@ -1,6 +1,6 @@ # exec-ctrl: agent activation contract -Version: 2.0.0-preview.3. This is the v2 entry point. Use plain Markdown; no +Version: 2.0.0-preview.4. This is the v2 entry point. Use plain Markdown; no installation, command, account or plugin is required for the workflow. ## 1. Establish target, intent and access diff --git a/VERSION b/VERSION index 4df14bf..43b82e4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.0-preview.3 +2.0.0-preview.4 diff --git a/docs/exec_ctrl/V2_PUBLICATION_READINESS.md b/docs/exec_ctrl/V2_PUBLICATION_READINESS.md new file mode 100644 index 0000000..8beb2ae --- /dev/null +++ b/docs/exec_ctrl/V2_PUBLICATION_READINESS.md @@ -0,0 +1,79 @@ +# v2 publication readiness + +Started: 2026-10-05. Baseline: `2b9cfbd`, clean `main`, preview 3. +Method and target: this repository. Branch: `codex/publication-readiness`. +User scope: put tests in place, execute and improve the framework, iterate on +failures, and prepare a main-merge candidate before use in a real project. +Implementation/local acceptance: `complete`. Hosted verification/publication: +`complete` for the tested code below. Main merge: `not_performed`. + +## Scope and acceptance + +- Keep the link-first entry small and the optional helper Python 3.10+, stdlib, + offline and read-only; add no adoption writes or production integrations. +- Put a complete synthetic project/evidence lifecycle in the repository's normal + unittest/CI path, making it reproducible outside the author's separate local lab. +- Cover failure/repair, stale inputs, policy changes, user-work preservation and + forbidden helper side effects. Prove discovered defects fail before the repair. +- Keep versions, catalog, examples and documentation coherent; preserve historical + records and explicit limitations. Review the final diff and published revision. +- Run repository validation, all tests and diff checks locally and on the hosted + Windows/Linux Python 3.10/3.13 matrix. Prepare a PR; main merge remains pending. + +## Findings and repair evidence + +| Finding | Reproduction and correction | +| --- | --- | +| Snapshot capture accepted files edited during hashing | Same-size edit and truncation tests both failed against preview 3 because no invalid result was raised. Compare opened-file identity/size/modification metadata before and after reading. | +| Capture accepted a replaced path after the read | Replacement at the close boundary also failed against preview 3. Re-resolve and compare path identity after close; deletion likewise returns invalid input. | +| Windows `stat`/`fstat` disagree on `ctime` for unchanged rewrites | The first repair caused false rejections in stable-file tests. A local repeated-rewrite probe isolated the timestamp difference. On Windows compare file ID, size and modification time; include `ctime` on other platforms. | +| Whole-project method acceptance depended on a separate lab | Add a checked-in test that creates an actual defective service, sees its regression fail, repairs it and closes only with fresh passing evidence. This is helper acceptance, not a blind agent-adoption trial. | +| Hosted Windows mutation fixtures used a temporary-directory alias | First hosted runs failed four mutation assertions because the fixture's short TEMP path differed from the helper's resolved path, so the mutation hook never ran. Resolve the fixture root and assert each mutation occurred; retain every matrix job with `fail-fast: false`. | + +All three initial mutation regressions failed before the helper repair. The first +replacement fixture hit Windows file locking while trying to rename an open file; +moving that mutation to the close boundary made the regression portable. Preserve +that distinction: a fixture error was not counted as a product failure or pass. + +The [test guide](../../tests/README.md) describes commands, scenario scope and audit +guard limits. Existing schema/CLI/snapshot formats remain unchanged. VERSION, +catalog, activation version and current synthetic examples advance to preview 4. + +## Current evidence + +- Baseline validation: 10 modules, 270 supported local links; 97 discovered tests, + 95 passed and two symlink skips on local Windows/Python 3.13. +- Repaired local suite: 108 discovered tests, 106 passed and the same two symlink + skips. The project scenario runs two application tests before and after repair; + these are nested acceptance assertions, not additional framework test counts. +- Routing acceptance: 72 kind/risk/project cases plus all 22 single signals, checking + baseline gates, sensitive escalation, all four modes and all ten modules. +- All five CLI commands and invalid JSON run under a side-effect-denying audit hook; + four negative probes check the guard itself. No network or production access. +- Local final validation: 10 modules, 284 supported local links; activation entry + remains 1,264 words within its 1,700-word ceiling. `git diff --check` passed. +- Initial published candidate `b2be76e` failed its + [PR matrix](https://github.com/LeiterConsulting/exec-ctrl/actions/runs/37351620006) + and [push matrix](https://github.com/LeiterConsulting/exec-ctrl/actions/runs/37351611046) + on the Windows fixture alias described above; cancelled jobs were not counted as + passes. The fixture repair and complete-matrix setting require a new hosted run. +- Repaired code revision: `02e753a5983177126a837bff7bd298b022951805`. + [PR validation](https://github.com/LeiterConsulting/exec-ctrl/actions/runs/37351837594) + and [push validation](https://github.com/LeiterConsulting/exec-ctrl/actions/runs/37351830995) + succeeded on all four Windows/Linux Python 3.10/3.13 jobs. Each job passed all + 108 tests with no skips, including both symlink checks and all mutation hooks. +- Publication candidate: [PR #3](https://github.com/LeiterConsulting/exec-ctrl/pull/3), + `codex/publication-readiness` into `main`. This record's documentation update + receives its own CI validation; the code identity above remains the repair evidence. + +## Remaining acceptance + +This increment prepares a preview for main merge and real-project trial. It does +not certify every agent/IDE, semantic module behavior, real organization rules, +scanner findings, external connector writes or production readiness. Independent +task-plus-link adoption remains a [field gate](../v2/FIELD_TRIALS.md). Stable-release +reuse/distribution terms remain an owner decision recorded in the +[roadmap](../v2/ROADMAP.md); no license terms are invented here. + +The prior [session evolution](V2_SESSION_EVOLUTION.md) and lab records retain their +original revisions/results. Main merge and a stable v2 release are separate actions. diff --git a/docs/v2/MIGRATION.md b/docs/v2/MIGRATION.md index 6ee2c9f..f0adf6e 100644 --- a/docs/v2/MIGRATION.md +++ b/docs/v2/MIGRATION.md @@ -37,3 +37,8 @@ generation require no target adoption files. Existing helpers cannot interpret t new gate field, so keep historical records/helpers pinned. For ongoing work, review required evidence kinds and current input coverage before collecting fresh evidence. The new nonfunctional routing signals add controls without changing existing facts. + +Preview.4 retains schema 1, the snapshot format and CLI commands. Snapshot capture +now rejects observed changes during a file read. A previously accepted unstable +capture may return exit 2; isolate the inputs and capture again. Preserve earlier +records with their original helper/version rather than relabeling old evidence. diff --git a/docs/v2/ROADMAP.md b/docs/v2/ROADMAP.md index a032ffe..e51c24c 100644 --- a/docs/v2/ROADMAP.md +++ b/docs/v2/ROADMAP.md @@ -39,6 +39,11 @@ team systems. Current validation belongs to that review, not prior preview resul ## Next: field acceptance +[Preview 4 publication preparation](../exec_ctrl/V2_PUBLICATION_READINESS.md) puts +a synthetic project lifecycle into ordinary CI and checks offline/read-only helper +execution. It also rejects observed mid-capture file changes. These are repeatable +helper acceptance checks, leaving actual agent adoption to the trials below. + [Preview 2 testing](../exec_ctrl/V2_SECOND_PASS_TESTING.md) adds adversarial helper regressions and concurrent/failed-operation lab coverage. It fixes policy argument ordering, expands supported local Markdown links and bounds JSON inputs. These diff --git a/docs/v2/TOOLING.md b/docs/v2/TOOLING.md index fc4901d..c03fc97 100644 --- a/docs/v2/TOOLING.md +++ b/docs/v2/TOOLING.md @@ -99,8 +99,16 @@ missing or renamed selected files, tampered metadata or a different record subje produce exit 2. Keep old manifests/reports historical and collect new evidence. Snapshots exclude unlisted files, permissions, runtime/tool versions and environment -state. Reads are per file, not an atomic repository capture. During concurrent edits, -use an isolated immutable input or repeat comparison at the appropriate checkpoint. +state. The helper compares file identity, size and modification metadata before, +during and after each read; observed edits, truncation, replacement, deletion or +redirection during capture produce exit 2 rather than a usable manifest. Windows +uses file identity/size/modification time because path and descriptor `ctime` can +differ for an unchanged file; other platforms also compare `ctime`. +Python documents the [platform-dependent timestamp semantics](https://docs.python.org/3.13/library/os.html#os.stat_result). + +These checks detect observable changes, not every possible concurrent write. Reads +remain per file, not an atomic repository capture or a filesystem security boundary. +Use isolated immutable inputs during concurrent work and compare again at closure. ## Records @@ -149,3 +157,7 @@ collapsed or shortcut references. Fenced code, inline code and HTML comments are ignored. First reference definitions win; unused definitions are not checked. This is not a full CommonMark renderer: HTML links, indented code, block-container syntax, external URLs and anchors are outside its validation scope. + +For repeatable publication checks and the synthetic project lifecycle, see +[test coverage and execution](../../tests/README.md). Those tests exercise the +helper's behavior; they do not prove an IDE retrieved or followed these instructions. diff --git a/examples/v2/docs-record.json b/examples/v2/docs-record.json index 8224d5b..2a6a0f8 100644 --- a/examples/v2/docs-record.json +++ b/examples/v2/docs-record.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "framework_version": "2.0.0-preview.3", + "framework_version": "2.0.0-preview.4", "id": "synthetic-docs-example", "objective": "Illustrate a reviewed spelling correction; no real task evidence", "subject": "synthetic-example-snapshot-1", diff --git a/examples/v2/pending-release-record.json b/examples/v2/pending-release-record.json index 59faf93..ce7e04c 100644 --- a/examples/v2/pending-release-record.json +++ b/examples/v2/pending-release-record.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "framework_version": "2.0.0-preview.3", + "framework_version": "2.0.0-preview.4", "id": "synthetic-pending-release", "objective": "Illustrate pending release obligations; no real approvals or evidence", "subject": "synthetic-unverified-inputs", diff --git a/framework/catalog.json b/framework/catalog.json index 0d8370a..18baf13 100644 --- a/framework/catalog.json +++ b/framework/catalog.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "framework_version": "2.0.0-preview.3", + "framework_version": "2.0.0-preview.4", "entry": "START_HERE.md", "kinds": ["docs", "fix", "feature", "refactor", "audit", "incident", "migration", "release", "planning"], "risks": ["low", "medium", "high", "unknown"], diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..55cad7a --- /dev/null +++ b/tests/README.md @@ -0,0 +1,67 @@ +# Test coverage and publication checks + +Use Python 3.10+ and the standard library. From the repository root: + +```sh +python tools/exec_ctrl.py validate +python -m unittest discover -s tests -v +git diff --check +``` + +The existing [CI workflow](../.github/workflows/validate.yml) runs these checks on +Windows/Linux with Python 3.10 and 3.13, for pushes and pull requests. No package +installation, account, network service or production target is needed by the suite. +The test harness creates disposable local inputs and runs local Python processes; +the optional helper itself remains offline and read-only. + +| Coverage | Behavior exercised | +| --- | --- | +| `test_exec_ctrl.py` | Routing, authority/risk facts, additive policy identity, evidence gates, closure denials, CLI exit codes | +| `test_adversarial.py` | Malformed JSON/type mutations, size limits, policy order and supported local Markdown links | +| `test_session_tools.py` | Pending records, typed evidence requirements, current-file snapshots, containment and resource bounds | +| `test_publication_readiness.py` | Complete disposable project workflow, all modes/modules through 94 routing cases, audited CLI behavior, history preservation and unstable capture | + +## Reproducible project acceptance + +The publication test creates a small Python service with a deliberate tenant-access +bug and two actual application tests. It records a pending auth fix with a fictional +policy requiring test evidence, observes the denial test fail, and checks that failed +evidence cannot pass the gate. The harness repairs only its disposable service, runs +the same application tests successfully, captures a new identity and supplies current +evidence. Old snapshots/subjects and changed policy content are rejected. + +The helper is called as a real CLI from outside its own checkout, including a target +path with spaces. Existing project instructions and unfinished user notes retain their +bytes; no helper-generated files appear in the target. Passing fixture evidence is +explicitly synthetic acceptance, with no production approval or independent reviewer. +The deliberate application failure is an expected assertion inside a passing framework +test; it is not hidden by skipping a failed test. + +## Offline/read-only execution and capture consistency + +`audit_cli.py` is a test-only subprocess wrapper. It installs a Python audit hook +before running the helper and denies file mutations, process creation and socket +operations. All five helper commands and an invalid-input path run under it. +Separate probes verify that the guard rejects actual writes, directory creation, +process attempts and sockets. Executable-looking and URL evidence references remain +data. This is a regression guard for exercised paths, not a general host sandbox. + +Deterministic capture tests edit the same-size content, truncate a file, and replace +or delete the selected path at read/close boundaries. They require an invalid result +and separately accept stable files immediately after repeated rewrites. Metadata +checks cannot establish an atomic multi-file snapshot or defeat every concurrent +writer; immutable inputs and closure comparison remain necessary. + +Two symlink tests skip only when the host cannot create symlinks. Report skips +separately from passes and inspect hosted job logs before claiming those paths passed. +Capture mutation tests use controlled read boundaries, not timing-dependent sleeps. + +## What remains a field gate + +These tests validate the helper, catalog and supported links. The behavioral modules +also require an agent to inspect the real target and carry out their guidance. Offline +results cannot prove instruction retrieval, semantic defect detection, actual enterprise +policy compliance, connector authorization, independent approval or customer acceptance. +Use the [field-trial protocol](../docs/v2/FIELD_TRIALS.md) for task-plus-link adoption +and the [publication record](../docs/exec_ctrl/V2_PUBLICATION_READINESS.md) for this +increment's evidence. Keep earlier runs/versioned records unchanged. diff --git a/tests/audit_cli.py b/tests/audit_cli.py new file mode 100644 index 0000000..ef49e67 --- /dev/null +++ b/tests/audit_cli.py @@ -0,0 +1,38 @@ +"""Test-only subprocess guard. Never installed into a user's project.""" + +# Preload helper/probe dependencies before installing the hook so it observes +# application behavior without incidental interpreter import activity. +import argparse +import hashlib +import html +import json +import math +import os +import re +import runpy +import socket +import stat +import subprocess +import sys +from pathlib import Path +from urllib.parse import unquote, urlsplit + + +def deny_side_effects(event, args): + mutations = {"os.remove", "os.rmdir", "os.rename", "os.mkdir", "os.chmod", + "os.truncate", "os.symlink", "os.link", "os.utime", "os.chdir", + "os.putenv", "os.unsetenv", "os.system", "os.startfile"} + denied = event in mutations or event.startswith(("socket.", "subprocess.", "os.spawn", "os.exec")) + if event == "open": + mode, flags = args[1:3] + denied = bool(isinstance(mode, str) and any(char in mode for char in "wax+")) + denied |= bool(isinstance(flags, int) and flags & (os.O_WRONLY | os.O_RDWR | os.O_APPEND | os.O_CREAT | os.O_TRUNC)) + if denied: + raise RuntimeError("read-only audit denied: " + event) + + +if __name__ == "__main__": + sys.dont_write_bytecode = True + sys.argv = sys.argv[1:] + sys.addaudithook(deny_side_effects) + runpy.run_path(sys.argv[0], run_name="__main__") diff --git a/tests/test_publication_readiness.py b/tests/test_publication_readiness.py new file mode 100644 index 0000000..5129f31 --- /dev/null +++ b/tests/test_publication_readiness.py @@ -0,0 +1,294 @@ +"""Reproducible acceptance workflows; synthetic targets, no provider claims.""" + +import copy +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools")) +import exec_ctrl as ec + + +class PublicationReadiness(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.catalog = ec.load_catalog() + + def test_routing_matrix_keeps_baselines_and_escalates_sensitive_work(self): + observed_modes = set() + observed_modules = set() + for kind in self.catalog["kinds"]: + for risk in self.catalog["risks"]: + for whole in (False, True): + facts = {"kind": kind, "risk": risk, "signals": [], "whole_project": whole} + with self.subTest(facts=facts): + plan = ec.route(facts, self.catalog) + self.assertTrue({"scope", "policy", "verification", "review", "git-review"} <= set(plan["required_gates"])) + self.assertTrue({"policy", "git"} <= {m["id"] for m in plan["modules"]}) + self.assertEqual(plan["mode"] == "project", whole) + if kind in {"incident", "migration"}: + self.assertEqual(plan["effective_risk"], "high") + observed_modes.add(plan["mode"]) + observed_modules.update(m["id"] for m in plan["modules"]) + for signal in self.catalog["signals"]: + facts = {"kind": "docs", "risk": "low", "signals": [signal], "whole_project": False} + with self.subTest(signal=signal): + plan = ec.route(facts, self.catalog) + self.assertNotEqual(plan["mode"], "inline") + if signal in self.catalog["high_risk_signals"]: + self.assertEqual(plan["effective_risk"], "high") + self.assertIn("security-review", plan["required_gates"]) + observed_modules.update(m["id"] for m in plan["modules"]) + self.assertEqual(observed_modes, {"inline", "task", "initiative", "project"}) + self.assertEqual(observed_modules, {m["id"] for m in self.catalog["modules"]}) + + def cli(self, cwd, *args, expected=0, audited=False): + command = [sys.executable, "-B"] + if audited: + command.append(str(ec.ROOT / "tests/audit_cli.py")) + command.extend([str(ec.ROOT / "tools/exec_ctrl.py"), *args]) + result = subprocess.run(command, cwd=cwd, capture_output=True, text=True, timeout=20) + self.assertEqual(result.returncode, expected, result.stderr) + self.assertNotIn("Traceback", result.stderr) + self.assertEqual(result.stdout == "", expected == 2) + self.assertEqual(result.stderr == "", expected != 2) + return json.loads(result.stderr if expected == 2 else result.stdout) + + def save(self, path, value): + path.write_text(json.dumps(value, indent=2), encoding="utf-8") + + def test_project_fix_failure_repair_and_current_evidence_closure(self): + with tempfile.TemporaryDirectory(prefix="exec-ctrl-project-") as temp: + root = Path(temp) / "project with spaces" + root.mkdir() + (root / "tests").mkdir() + instructions = b"Synthetic project. Preserve user work; Python standard library only.\n" + user_work = b"Unfinished user notes; do not overwrite.\n" + (root / "AGENTS.md").write_bytes(instructions) + (root / "USER_NOTES.md").write_bytes(user_work) + (root / "service.py").write_text("def can_read(actor, owner):\n return True\n", encoding="utf-8") + (root / "tests/test_access.py").write_text( + "import unittest\nfrom service import can_read\n" + "class Access(unittest.TestCase):\n" + " def test_owner(self):\n self.assertTrue(can_read('blue', 'blue'))\n" + " def test_other_tenant(self):\n self.assertFalse(can_read('blue', 'green'))\n", encoding="utf-8") + rules = copy.deepcopy(ec.read_json(ec.ROOT / "examples/v2/team-rules.json")) + rules["id"] = "synthetic-regression-policy" + rules["gates"] = [{"id": "tenant-regression", "description": "Run synthetic tenant denial tests", + "source": "fixture://policy", "owner": "Fixture", "requires": ["test"]}] + policy = Path(temp) / "policy.json" + manifest_path = Path(temp) / "snapshot.json" + record_path = Path(temp) / "record.json" + self.save(policy, rules) + + def capture(): + return self.cli(temp, "snapshot", "--target", str(root), "--path", "service.py", + "--path", "tests/test_access.py", "--path", "AGENTS.md", audited=True) + + def generate(subject): + return self.cli(temp, "record-template", "--kind", "fix", "--risk", "low", "--signal", "auth", + "--rules", str(policy), "--id", "tenant-fix", "--objective", "Deny cross-tenant access", + "--subject", subject, audited=True) + + def check(expected): + return self.cli(temp, "check-record", str(record_path), "--rules", str(policy), + "--snapshot", str(manifest_path), "--target", str(root), expected=expected, audited=True) + + def run_project_tests(): + return subprocess.run([sys.executable, "-B", "-m", "unittest", "discover", "-s", "tests", "-v"], + cwd=root, capture_output=True, text=True, timeout=20) + + original = capture() + self.save(manifest_path, original) + pending = generate(original["subject"]) + self.save(record_path, pending) + self.assertIn("tenant-regression", check(1)["unresolved_gates"]) + failed = run_project_tests() + self.assertEqual(failed.returncode, 1, failed.stderr) + self.assertIn("test_other_tenant", failed.stderr) + failing = copy.deepcopy(pending) + failing["state"] = "in_progress" + failing["evidence"] = [{"id": "denial-test", "kind": "test", "subject": original["subject"], + "result": "fail", "reference": "fixture://original-test-run", "summary": failed.stderr}] + gate = next(g for g in failing["gates"] if g["id"] == "tenant-regression") + gate.update(result="fail", evidence=["denial-test"], reason="Cross-tenant denial failed") + self.save(record_path, failing) + self.assertFalse(check(1)["ready_to_close"]) + gate["result"] = "pass" + self.save(record_path, failing) + self.assertIn("failed evidence", check(2)["error"]) + + # The test harness repairs its disposable app; the helper never edits it. + (root / "service.py").write_text("def can_read(actor, owner):\n return actor == owner\n", encoding="utf-8") + self.assertIn("changed", check(2)["error"]) + passed = run_project_tests() + self.assertEqual(passed.returncode, 0, passed.stderr) + current = capture() + self.assertNotEqual(current["subject"], original["subject"]) + self.save(manifest_path, current) + self.assertIn("current snapshot", check(2)["error"]) + ready = generate(current["subject"]) + ready["state"] = "complete" + ready["evidence"] = [ + {"id": "current-tests", "kind": "test", "subject": current["subject"], "result": "pass", + "reference": "fixture://repaired-test-run", "summary": passed.stderr}, + {"id": "fixture-inspection", "kind": "inspection", "subject": current["subject"], "result": "pass", + "reference": "fixture://scope-diff", "summary": "Synthetic harness inspected one equality fix; no production approval"}] + for gate in ready["gates"]: + gate.update(result="pass", evidence=["current-tests", "fixture-inspection"], reason="Synthetic acceptance only") + self.save(record_path, ready) + self.assertTrue(check(0)["ready_to_close"]) + changed_policy = copy.deepcopy(rules) + changed_policy["gates"][0]["requires"].append("review") + self.save(policy, changed_policy) + self.assertIn("ruleset identities", check(2)["error"]) + self.assertEqual((root / "AGENTS.md").read_bytes(), instructions) + self.assertEqual((root / "USER_NOTES.md").read_bytes(), user_work) + self.assertEqual({p.relative_to(root).as_posix() for p in root.rglob("*") if p.is_file()}, + {"AGENTS.md", "USER_NOTES.md", "service.py", "tests/test_access.py"}) + + def test_every_cli_command_is_offline_and_read_only_under_audit(self): + with tempfile.TemporaryDirectory(prefix="exec-ctrl-audit-") as temp: + root = Path(temp) + sentinel = root / "must-not-exist.txt" + # Both a network URL and executable-looking text remain inert metadata. + record = ec.read_json(ec.ROOT / "examples/v2/docs-record.json") + for evidence in record["evidence"]: + evidence["reference"] = "__import__('pathlib').Path(" + repr(str(sentinel)) + ").write_text('executed')" + evidence["summary"] = "URL metadata https://example.invalid/ must not be fetched" + path = root / "record.json" + self.save(path, record) + before = path.read_bytes() + self.cli(temp, "validate", audited=True) + self.cli(temp, "route", "--kind", "release", "--signal", "production", audited=True) + self.cli(temp, "record-template", "--kind", "docs", "--risk", "low", "--id", "audit", + "--objective", "Audit helper", "--subject", "fixture", audited=True) + self.cli(temp, "snapshot", "--target", temp, "--path", "record.json", audited=True) + self.assertTrue(self.cli(temp, "check-record", str(path), audited=True)["ready_to_close"]) + path.write_text('{"duplicate": 1, "duplicate": 2}', encoding="utf-8") + self.cli(temp, "check-record", str(path), expected=2, audited=True) + self.assertFalse(sentinel.exists()) + self.assertEqual({p.name for p in root.iterdir()}, {"record.json"}) + self.assertIn(b"example.invalid", before) + + def test_audit_guard_itself_denies_file_writes_and_network_and_processes(self): + for operation in ["open('forbidden', 'w')", "__import__('os').mkdir('forbidden')", + "__import__('socket').socket()", "__import__('subprocess').run(['unused'])"]: + with self.subTest(operation=operation), tempfile.TemporaryDirectory() as temp: + script = Path(temp) / "probe.py" + script.write_text(operation, encoding="utf-8") + result = subprocess.run([sys.executable, "-B", str(ec.ROOT / "tests/audit_cli.py"), str(script)], + cwd=temp, capture_output=True, text=True, timeout=20) + self.assertNotEqual(result.returncode, 0) + self.assertIn("read-only audit denied", result.stderr) + self.assertFalse((Path(temp) / "forbidden").exists()) + + def test_unknown_policy_is_pending_until_resolved_without_overwriting_history(self): + facts = {"kind": "docs", "risk": "low", "signals": ["policy-unknown"], "whole_project": False} + pending = ec.record_template(facts, self.catalog, "policy", "Resolve policy", "current") + historical = copy.deepcopy(pending) + gate = next(g for g in pending["gates"] if g["id"] == "policy") + gate.update(result="blocked", reason="Expected policy unavailable") + self.assertIn("policy", ec.check_record(pending, self.catalog)["unresolved_gates"]) + pending["state"] = "complete" + with self.assertRaisesRegex(ec.Invalid, "unresolved gates"): + ec.check_record(pending, self.catalog) + revised_facts = {**facts, "signals": []} + revised = ec.record_template(revised_facts, self.catalog, "policy", "Resolve policy", "current") + self.assertEqual(ec.route(revised_facts, self.catalog)["mode"], "inline") + self.assertFalse(ec.check_record(revised, self.catalog)["ready_to_close"]) + self.assertEqual(historical["state"], "not_started") + self.assertEqual(historical["facts"]["signals"], ["policy-unknown"]) + + def test_historical_version_is_rejected_without_rewriting_record(self): + record = ec.read_json(ec.ROOT / "examples/v2/docs-record.json") + record["framework_version"] = "2.0.0-preview.2" + original = copy.deepcopy(record) + with self.assertRaisesRegex(ec.Invalid, "version mismatch"): + ec.check_record(record, self.catalog) + self.assertEqual(record, original) + + def unstable_snapshot(self, mutate, after_close=False): + with tempfile.TemporaryDirectory(prefix="exec-ctrl-capture-") as temp: + # Windows runners may use an 8.3 alias in TEMP. Match the helper's + # canonical path so every injected mutation actually reaches it. + root = Path(temp).resolve() + path = root / "source.txt" + path.write_bytes(b"original") + original_open = Path.open + mutations = [] + + class MutatingReader: + def __init__(self, stream): + self.stream = stream + self.changed = False + + def __enter__(self): + self.stream.__enter__() + return self + + def __exit__(self, *args): + result = self.stream.__exit__(*args) + if after_close: + mutate(path, root) + mutations.append("closed") + return result + + def fileno(self): + return self.stream.fileno() + + def read(self, size): + value = self.stream.read(size) + if value and not self.changed and not after_close: + self.changed = True + mutate(path, root) + mutations.append("read") + return value + + def hooked_open(candidate, *args, **kwargs): + stream = original_open(candidate, *args, **kwargs) + if candidate == path and args == ("rb",): + return MutatingReader(stream) + return stream + + with patch.object(Path, "open", hooked_open), self.assertRaisesRegex(ec.Invalid, "changed during capture"): + ec.snapshot(root, ["source.txt"]) + self.assertEqual(mutations, ["closed" if after_close else "read"]) + + def test_snapshot_rejects_same_size_edit_during_capture(self): + def change(path, root): + before = path.stat() + path.write_bytes(b"modified") + os.utime(path, ns=(before.st_atime_ns, before.st_mtime_ns + 2_000_000_000)) + self.unstable_snapshot(change) + + def test_snapshot_rejects_file_replacement_during_capture(self): + def replace(path, root): + path.rename(root / "historical.txt") + path.write_bytes(b"replaced") + self.unstable_snapshot(replace, after_close=True) + + def test_snapshot_rejects_truncation_during_capture(self): + self.unstable_snapshot(lambda path, root: path.write_bytes(b"")) + + def test_snapshot_rejects_deletion_during_capture(self): + self.unstable_snapshot(lambda path, root: path.unlink(), after_close=True) + + def test_snapshot_accepts_stable_files_immediately_after_rewrites(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "source.txt" + for count in range(20): + value = str(count).encode("ascii") + path.write_bytes(value) + manifest = ec.snapshot(temp, ["source.txt"]) + self.assertEqual(manifest["files"][0]["bytes"], len(value)) + self.assertEqual(ec.check_snapshot(manifest, temp), manifest["subject"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/exec_ctrl.py b/tools/exec_ctrl.py index 1ca36d1..963b375 100644 --- a/tools/exec_ctrl.py +++ b/tools/exec_ctrl.py @@ -6,7 +6,9 @@ import html import json import math +import os import re +import stat import sys from pathlib import Path from urllib.parse import unquote, urlsplit @@ -100,6 +102,12 @@ def local_file(root, name): def snapshot(target, names): """Hash only explicitly selected local files, never discover or execute inputs.""" + def metadata_key(metadata): + result = (metadata.st_dev, metadata.st_ino, metadata.st_size, metadata.st_mtime_ns) + # Windows path stat and descriptor stat can report different ctime values + # for the same unchanged file. Compare modification time and file ID there. + return result if os.name == "nt" else result + (metadata.st_ctime_ns,) + root = Path(target).resolve() require(root.is_dir(), "snapshot target must be a directory") strings(names, "snapshot paths") @@ -114,9 +122,15 @@ def snapshot(target, names): "snapshot paths cannot contain parent traversal, symbolic links or redirected directories") require(path not in seen, "snapshot contains aliases of the same file") seen.add(path) + before = path.stat() + require(before.st_size <= MAX_SNAPSHOT_FILE_BYTES, "snapshot file exceeds 16 MiB limit") + require(total + before.st_size <= MAX_SNAPSHOT_TOTAL_BYTES, "snapshot exceeds 64 MiB total limit") + changed = "snapshot input changed during capture: " + name digest = hashlib.sha256() size = 0 with path.open("rb") as stream: + opened = os.fstat(stream.fileno()) + require(stat.S_ISREG(opened.st_mode) and metadata_key(before) == metadata_key(opened), changed) while True: chunk = stream.read(1024 * 1024) if not chunk: @@ -126,6 +140,11 @@ def snapshot(target, names): require(size <= MAX_SNAPSHOT_FILE_BYTES, "snapshot file exceeds 16 MiB limit") require(total <= MAX_SNAPSHOT_TOTAL_BYTES, "snapshot exceeds 64 MiB total limit") digest.update(chunk) + require(size == before.st_size and metadata_key(before) == metadata_key(os.fstat(stream.fileno())), changed) + try: + require(candidate.resolve(strict=True) == path and metadata_key(before) == metadata_key(path.stat()), changed) + except OSError as exc: + raise Invalid(changed) from exc files.append({"path": path.relative_to(root).as_posix(), "sha256": digest.hexdigest(), "bytes": size}) files.sort(key=lambda item: item["path"]) identity = json.dumps(files, sort_keys=True, separators=(",", ":"), ensure_ascii=True)