From 844510d465ebf5a3cc118918681fc90924572014 Mon Sep 17 00:00:00 2001 From: Trecek Date: Wed, 7 Oct 2026 22:27:06 -0700 Subject: [PATCH 01/20] feat: add seeded sandbox implementation E2E validation --- .github/workflows/e2e.yml | 3 +- scripts/e2e/README.md | 101 +++- scripts/e2e/autoskillit-config.yaml | 3 + scripts/e2e/catalog.json | 29 + scripts/e2e/e2e_catalog.py | 182 +++++- scripts/e2e/e2e_harness.py | 697 +++++++++++++++++++++- scripts/e2e/e2e_select.py | 48 +- scripts/e2e/e2e_sessions.py | 879 ++++++++++++++++++++++++++++ tests/infra/test_e2e_harness.py | 717 ++++++++++++++++++++++- tests/infra/test_e2e_select.py | 216 +++++++ tests/infra/test_e2e_select_cli.py | 29 + tests/infra/test_e2e_sessions.py | 580 ++++++++++++++++++ tests/infra/test_e2e_workflow.py | 18 +- 13 files changed, 3424 insertions(+), 78 deletions(-) create mode 100644 scripts/e2e/e2e_sessions.py create mode 100644 tests/infra/test_e2e_sessions.py diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index ce9c788876..6ad180bb68 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -104,12 +104,13 @@ jobs: env: MINIMAX_API_KEY: ${{ matrix.kind != 'clean-install' && secrets.MINIMAX_API_KEY || '' }} E2E_SANDBOX_TOKEN: ${{ matrix.kind != 'clean-install' && secrets.E2E_SANDBOX_TOKEN || '' }} + E2E_IMPLEMENTATION_ISSUE_URL: ${{ vars.E2E_IMPLEMENTATION_ISSUE_URL }} TEST: ${{ matrix.test }} run: | set -euo pipefail docker_args=() if [[ '${{ matrix.kind }}' != 'clean-install' ]]; then - docker_args+=(--env MINIMAX_API_KEY --env E2E_SANDBOX_TOKEN) + docker_args+=(--env MINIMAX_API_KEY --env E2E_SANDBOX_TOKEN --env E2E_IMPLEMENTATION_ISSUE_URL) docker_args+=(--volume "$RUNNER_TEMP/e2e/data:/home/autoskillit/.local/share/autoskillit") fi if [[ "$TEST" == 'headless-smoke' ]]; then diff --git a/scripts/e2e/README.md b/scripts/e2e/README.md index bb732de838..dc041abb0a 100644 --- a/scripts/e2e/README.md +++ b/scripts/e2e/README.md @@ -11,6 +11,7 @@ This directory holds everything the workflow runs: | `e2e_catalog.py` | Stdlib loader and validator for the catalog | | `e2e_select.py` | Picks the tests an event runs (`select`) and judges the required check (`gate`) | | `e2e_harness.py` | Runs one test (`run`), performs exact sandbox teardown (`cleanup`) and redacts artifacts (`redact`) | +| `e2e_sessions.py` | Captures synchronous lifecycle events, measures concurrency and pairs ownership evidence | | `autoskillit-config.yaml` | AutoSkillit user-layer config the harness installs | | `post-e2e-failure.sh` | Appends a failure to the test's tracking issue | @@ -40,6 +41,11 @@ model tests: because MiniMax serves the Anthropic Messages API. 3. Create the `e2e` label in this repository. 4. In branch protection, mark only `e2e-gate` as required — never `select` or `run (…)`. +5. Before publishing a branch with the selectable `implementation` entry, obtain a passing + #5232 sandbox smoke CI run. A maintainer must seed a fixed OPEN sandbox issue describing + a small feature and its test, then set repository variable `E2E_IMPLEMENTATION_ISSUE_URL` + to its canonical `https://github.com/OWNER/REPO/issues/NUMBER` URL. Reuse that issue on + every run. Missing configuration fails preflight; it is not an expected product defect. ## Sandbox repository requirements @@ -167,7 +173,11 @@ matrix is always parseable (`{"include":[]}` when nothing is selected). bypasses sampling because a labelled pull request whose sample fell out would otherwise never run for that commit. - **Picking:** tests whose `trigger_paths` match a changed path, up to `MAX_TESTS` (2); when - none match, exactly one test. Ties are ordered by the SHA-256 hex digest of + none match, one eligible ordinary test. A pipeline requires actual size at or above a + threshold and a matching trigger path, even with the `e2e` label. This filter also applies + to fallback; a pipeline-only ineligible pool selects nothing. At most one pipeline is + selected within the total cap. Named workflow dispatch bypasses PR eligibility. + Ties are ordered by the SHA-256 hex digest of `"{seed}:{name}"`, so the decision is a pure function of the payload and the changed paths. Patterns use `fnmatch.fnmatchcase`, where `*` also matches `/` (`scripts/e2e/*` matches every file below `scripts/e2e/`). @@ -183,11 +193,25 @@ succeeded or nothing was selected and `run` was skipped. Every other combination The `run` job uses one repository-wide concurrency group (`group: e2e`, `queue: max`, `cancel-in-progress: false`) and `max-parallel: 1`, so every selected test of every run waits its turn and never cancels another. `select` and `e2e-gate` are not queued. A -catalog entry's `peak_sessions` is capped at `MAX_PEAK_SESSIONS` (8): one step's fan-out of -up to six subagents plus the orchestrator and step session. With one test running at a -time, CI stays at or below eight of the roughly twelve concurrent MiniMax sessions. +catalog entry's `peak_sessions` is capped at `MAX_PEAK_SESSIONS` (8). For an uncommissioned +pipeline this declares capacity, not a measured fan-out. The child limit stays six. `clean-install` requires exactly zero `peak_sessions`. +The implementation pipeline installs synchronous command hooks for `SessionStart`, +`SessionEnd`, `SubagentStart` and `SubagentStop`, preserving existing hooks and MiniMax +settings. These hooks follow the [Claude lifecycle schema](https://code.claude.com/docs/en/hooks). +Each identity spans its first start through its final end, so compaction and resume yield +a conservative upper bound. Half-open intervals end before simultaneous starts. The +summary reconciles the dispatched root, managed lineage and enumerable native transcripts; +missing endpoints, identities or owned transcripts fail coverage. Transcript activity is +never used to infer a lifetime. A peak exceeding capacity fails validation. + +Commission with a successful MiniMax CI run and repeat against the same seed. Inspect +planning, dry-walkthrough, implementation, test, integration, audit GO and open-PR evidence, +plus cleanup and the sandbox CI association. Replace provisional `peak_sessions` with +the observed conservative peak and set `peak_sessions_evidence` to the CI run URL, tested +commit and first-start/final-end method. Expected failures do not satisfy commissioning. + Every harness subprocess has a timeout that fires before the job's: `fleet run` gets the test's `timeout_sec` plus `HARNESS_GRACE_SEC`, and the job gets that budget in minutes plus `JOB_OVERHEAD_MINUTES` for building the image, redaction and upload. So `result.json`, PR @@ -202,6 +226,12 @@ and are often `0600`, so a second container run copies `out/` and `data/logs/` i fails if any secret survives. Only `upload/` is uploaded, and only when redaction succeeded. +Pipeline evidence includes `session-events.jsonl`, `session-concurrency.json`, compact +`ownership.json`, the seed, effective configuration, fleet stdout/stderr/envelope, captured +PR metadata/files, detached-head test commands and associated CI runs/jobs. Collection +occurs inside the container before transcript paths disappear. Incomplete evidence is +retained and fails the scenario; redaction and upload still run after failure. + `clean-install` mounts no home directories and retains evidence under `out/`. Its redactor receives the same required `--secret-env` names with variables unset and skips the absent home-log tree. Redaction and upload still run after test failures. @@ -258,6 +288,34 @@ row and close the bug. Fleet translates the L3 result's `reason` into the failed command envelope's `error` and `user_visible_message` fields; the harness matches that envelope rather than the recipe payload's `failed_step`/`reason`. +For the implementation pipeline, the project-owned +[`DispatchCompleted.to_envelope`](../../src/autoskillit/fleet/campaign_state/state_outcomes.py) +must report success, `dispatch_status="success"` and root/dispatch identities. The nested +[`implementation` terminal](../../src/autoskillit/recipes/implementation.yaml) must have +`reason="implementation_complete"`. Successful no-change and already-done terminals remain +valid recipe behavior but fail this scenario. No PR URL is required in the envelope. + +The harness resolves the configured seed once and dispatches its exact URL and title/body +task once. Effective project-aware config comes from `autoskillit config show`, whose +implementation calls `load_config(Path.cwd())`; both configured and sandbox default branches +must be `main`. No caller `base_branch` ingredient is sent. Sandbox setup and baseline tests +must succeed before launch. + +Paired root transcript tool calls/results prove the created branch; paired root/managed +claim results prove any added claim label. Discovery paginates through the PR baseline +and leaves unrelated PRs untouched. Require exactly one owned open, unmerged sandbox PR +targeting `main`, without auto-merge, with the canonical seed URL and non-removed changes +under both `sandbox/` and `tests/`. Fetch and detach at the captured head before setup and +tests. Require a successful associated sandbox `CI` run and `test` job whose nested PR +association has the exact branch revision; an outer merge SHA is not the branch SHA. +Finally re-read the head and state to reject movement during validation. + +Teardown runs after validation, failure or exception. It independently collects metrics, +closes owned open PRs, deletes evidenced branches (including branches without a PR), and +removes only the effective claim label proven added by this run. The seed remains OPEN +and other labels are preserved. Missing ownership proof preserves uncertain resources and +fails cleanup. Cleanup errors remain fatal alongside the original failure. + ## Catalog schema ```json @@ -286,15 +344,38 @@ the harness matches that envelope rather than the recipe payload's `failed_step` ``` `recipe`, `ingredients` and `expected_pull_request_state` are required for `recipe` tests -and forbidden for `canary` and `clean-install` tests. `expected_failures` is optional for -`clean-install` and recipe tests. Clean-install rows use the exact doctor `severity`, `check` -and `message`; recipe rows require `severity: error`, `check` and `message` matching the failed -envelope. Every row includes an open bug issue URL -(`https://github.com/TalonT-Org/AutoSkillit/issues/`). Duplicate diagnostic identities -and unknown keys are rejected. `peak_sessions` must be +and forbidden for `canary` and `clean-install` tests. Optional recipe-only fields are +`pipeline` (boolean, default false), `issue_url_env` (seed variable name), +`required_changed_paths` (nonempty relative prefixes), `test_command` (nonempty argv), and +`peak_sessions_evidence` (nonempty provenance string). Environment-owned seeds cannot also +specify literal `task` or `issue_url` ingredients. Path prefixes cannot be absolute or +traverse parents. For example, the implementation entry uses: + +```json +{"pipeline": true, "issue_url_env": "E2E_IMPLEMENTATION_ISSUE_URL", + "required_changed_paths": ["sandbox/", "tests/"], "test_command": ["task", "test-check"]} +``` + +The recipe_fixture field names a staged YAML fixture for the headless smoke scenario. + +The expected_failures field is optional for recipes and clean-install. Pipeline rows +contain exactly check, message, issue; ordinary recipe and clean-install rows also +contain severity. Ordinary recipes require severity: error and match the failed +envelope's error and user_visible_message; clean-install allows error or warning +and matches doctor diagnostics. Pipeline rows match structured implementation findings. +Every row links to an open AutoSkillit bug issue. Duplicate diagnostic identities and +unknown keys are rejected. The peak_sessions value must be 1–8 for model tests and exactly zero for clean-install. `timeout_sec` must keep the derived job timeout within 360 minutes. +Recipe expectations match exact stable `(check, message)` product findings linked to real +bug issues. Setup, discovery, malformed evidence and cleanup failures cannot be admitted. +Unknown findings and absent expected findings fail. Every recipe records `outcome`: +`passed`, `expected_failure` (with `passed=false`), or `failed`. A linked known defect may +use the accepted CLI exit convention, but only `passed` with no expected findings counts +as successful commissioning. File or deduplicate observed product bugs separately; do not +predeclare a provider defect or replace the MiniMax settings-file route. + ### Adding a test 1. Add the entry to `catalog.json` with a realistic `peak_sessions` and `timeout_sec`. diff --git a/scripts/e2e/autoskillit-config.yaml b/scripts/e2e/autoskillit-config.yaml index 577e8df598..e4a8c3de29 100644 --- a/scripts/e2e/autoskillit-config.yaml +++ b/scripts/e2e/autoskillit-config.yaml @@ -3,6 +3,9 @@ quota_guard: enabled: false +branching: + default_base_branch: main + features: fleet: true fleet_headless_run: true diff --git a/scripts/e2e/catalog.json b/scripts/e2e/catalog.json index 00bf957ce1..ba100c3b0b 100644 --- a/scripts/e2e/catalog.json +++ b/scripts/e2e/catalog.json @@ -70,6 +70,35 @@ "recipe_fixture": "sandbox-smoke.yaml", "ingredients": {}, "expected_pull_request_state": "closed" + }, + { + "name": "implementation", + "kind": "recipe", + "pipeline": true, + "peak_sessions": 8, + "timeout_sec": 7200, + "trigger_paths": [ + "src/autoskillit/recipes/*", + "src/autoskillit/skills/*", + "src/autoskillit/skills_extended/*", + "src/autoskillit/server/*", + "src/autoskillit/execution/*" + ], + "recipe": "implementation", + "ingredients": { + "source_dir": "/workspace/sandbox", + "open_pr": "true", + "auto_merge": "false", + "arch_lenses": "false", + "audit_impl": "true" + }, + "expected_pull_request_state": "open", + "issue_url_env": "E2E_IMPLEMENTATION_ISSUE_URL", + "required_changed_paths": [ + "sandbox/", + "tests/" + ], + "test_command": ["task", "test-check"] } ] } diff --git a/scripts/e2e/e2e_catalog.py b/scripts/e2e/e2e_catalog.py index 46354d43f4..2940e8e472 100644 --- a/scripts/e2e/e2e_catalog.py +++ b/scripts/e2e/e2e_catalog.py @@ -7,7 +7,7 @@ import re from collections.abc import Mapping from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath from types import MappingProxyType CATALOG_PATH = Path(__file__).with_name("catalog.json") @@ -28,7 +28,21 @@ _CATALOG_KEYS = frozenset({"sandbox_repository", "tests"}) _COMMON_KEYS = frozenset({"name", "kind", "peak_sessions", "timeout_sec", "trigger_paths"}) _RECIPE_KEYS = frozenset({"recipe", "ingredients", "expected_pull_request_state"}) +_PIPELINE_METADATA_KEYS = frozenset( + { + "issue_url_env", + "required_changed_paths", + "test_command", + "peak_sessions_evidence", + } +) +_RECIPE_OPTIONAL_KEYS = _PIPELINE_METADATA_KEYS | { + "pipeline", + "expected_failures", + "recipe_fixture", +} _EXPECTED_FAILURE_KEYS = frozenset({"severity", "check", "message", "issue"}) +_PIPELINE_EXPECTED_FAILURE_KEYS = frozenset({"check", "message", "issue"}) _BUG_URL_PATTERN = re.compile(r"https://github\.com/TalonT-Org/AutoSkillit/issues/[1-9][0-9]*") @@ -50,6 +64,11 @@ class CatalogTest: expected_pull_request_state: str | None expected_failures: tuple[Mapping[str, str], ...] = () recipe_fixture: str | None = None + pipeline: bool = False + issue_url_env: str | None = None + required_changed_paths: tuple[str, ...] = () + test_command: tuple[str, ...] = () + peak_sessions_evidence: str | None = None @dataclass(frozen=True, slots=True) @@ -83,6 +102,13 @@ def _require_str(value: object, where: str) -> str: return value +def _require_nonblank_str(value: object, where: str) -> str: + result = _require_str(value, where) + if not result.strip(): + raise CatalogError(f"{where} must be non-empty") + return result + + def _require_int(value: object, where: str, *, low: int, high: int | None = None) -> int: if isinstance(value, bool) or not isinstance(value, int): raise CatalogError(f"{where} must be an integer") @@ -111,7 +137,7 @@ def _parse_ingredients(value: object, where: str) -> tuple[tuple[str, str], ...] def _parse_recipe_fields( raw: Mapping[str, object], where: str -) -> tuple[str, tuple[tuple[str, str], ...], str]: +) -> tuple[str, tuple[tuple[str, str], ...], str, str | None]: recipe = _require_str(raw["recipe"], f"{where}.recipe") ingredients = _parse_ingredients(raw["ingredients"], f"{where}.ingredients") state = _require_str( @@ -121,28 +147,40 @@ def _parse_recipe_fields( raise CatalogError( f"{where}.expected_pull_request_state must be one of {PULL_REQUEST_STATES}" ) - return recipe, ingredients, state + fixture = None + if "recipe_fixture" in raw: + fixture = _require_str(raw["recipe_fixture"], f"{where}.recipe_fixture") + if not _RECIPE_FIXTURE_PATTERN.fullmatch(fixture): + raise CatalogError(f"{where}.recipe_fixture must be a .yaml basename") + return recipe, ingredients, state, fixture -def _parse_expected_failures(value: object, where: str) -> tuple[Mapping[str, str], ...]: +def _parse_expected_failures( + value: object, where: str, *, pipeline: bool = False +) -> tuple[Mapping[str, str], ...]: where = f"{where}.expected_failures" if not isinstance(value, list): raise CatalogError(f"{where} must be a list") + keys = _PIPELINE_EXPECTED_FAILURE_KEYS if pipeline else _EXPECTED_FAILURE_KEYS rows: list[Mapping[str, str]] = [] - seen: set[tuple[str, str, str]] = set() + seen: set[tuple[str, ...]] = set() for index, raw in enumerate(value): location = f"{where}[{index}]" if not isinstance(raw, dict): raise CatalogError(f"{location} must be an object") - _require_keys(raw, _EXPECTED_FAILURE_KEYS, location) - row = {key: _require_str(raw[key], f"{location}.{key}") for key in _EXPECTED_FAILURE_KEYS} + _require_keys(raw, keys, location) + row = {key: _require_str(raw[key], f"{location}.{key}") for key in keys} if any(not item.strip() for item in row.values()): raise CatalogError(f"{location} fields must be non-empty") - if row["severity"] not in ("error", "warning"): + if not pipeline and row["severity"] not in ("error", "warning"): raise CatalogError(f"{location}.severity must be error or warning") if not _BUG_URL_PATTERN.fullmatch(row["issue"]): raise CatalogError(f"{location}.issue must link to an AutoSkillit bug issue") - identity = (row["severity"], row["check"], row["message"]) + identity = ( + (row["check"], row["message"]) + if pipeline + else (row["severity"], row["check"], row["message"]) + ) if identity in seen: raise CatalogError(f"{location}: duplicate diagnostic {identity}") seen.add(identity) @@ -150,6 +188,85 @@ def _parse_expected_failures(value: object, where: str) -> tuple[Mapping[str, st return tuple(rows) +def _parse_recipe_options( + raw: Mapping[str, object], where: str, ingredients: tuple[tuple[str, str], ...] +) -> tuple[str | None, tuple[str, ...], tuple[str, ...], str | None]: + issue_url_env = ( + _require_nonblank_str(raw["issue_url_env"], f"{where}.issue_url_env") + if "issue_url_env" in raw + else None + ) + if issue_url_env and {"issue_url", "task"}.intersection(key for key, _ in ingredients): + raise CatalogError( + f"{where}.ingredients cannot contain issue_url or task when issue_url_env is set" + ) + changed_paths = ( + _parse_required_changed_paths( + raw["required_changed_paths"], f"{where}.required_changed_paths" + ) + if "required_changed_paths" in raw + else () + ) + test_command = ( + _parse_test_command(raw["test_command"], f"{where}.test_command") + if "test_command" in raw + else () + ) + evidence = ( + _require_nonblank_str(raw["peak_sessions_evidence"], f"{where}.peak_sessions_evidence") + if "peak_sessions_evidence" in raw + else None + ) + return issue_url_env, changed_paths, test_command, evidence + + +def _parse_required_changed_paths(value: object, where: str) -> tuple[str, ...]: + if not isinstance(value, list) or not value: + raise CatalogError(f"{where} must be a non-empty list") + changed_paths: list[str] = [] + for item in value: + prefix = _require_nonblank_str(item, f"{where}[]") + posix_path = PurePosixPath(prefix) + windows_path = PureWindowsPath(prefix) + invalid_path = any( + ( + posix_path.is_absolute(), + windows_path.is_absolute(), + bool(windows_path.drive), + ".." in posix_path.parts, + ".." in windows_path.parts, + ) + ) + if invalid_path: + raise CatalogError(f"{where}[] must be a relative path prefix without '..'") + changed_paths.append(prefix) + return tuple(changed_paths) + + +def _parse_test_command(command: object, where: str) -> tuple[str, ...]: + if not isinstance(command, list) or not command: + raise CatalogError(f"{where} must be a non-empty argv list") + return tuple(_require_nonblank_str(item, f"{where}[]") for item in command) + + +def _validate_test_keys(raw: Mapping[str, object], kind: str, where: str) -> bool: + keys = _COMMON_KEYS | _RECIPE_KEYS if kind == "recipe" else _COMMON_KEYS + if kind == "clean-install" and "expected_failures" in raw: + keys |= {"expected_failures"} + if kind != "recipe": + _require_keys(raw, keys, where) + return False + + missing = sorted(keys - raw.keys()) + unknown = sorted(raw.keys() - keys - _RECIPE_OPTIONAL_KEYS) + if missing or unknown: + raise CatalogError(f"{where}: missing keys {missing}, unknown keys {unknown}") + pipeline = raw.get("pipeline", False) + if not isinstance(pipeline, bool): + raise CatalogError(f"{where}.pipeline must be a boolean") + return pipeline + + def _parse_test(raw: object, index: int) -> CatalogTest: where = f"tests[{index}]" if not isinstance(raw, dict): @@ -157,26 +274,38 @@ def _parse_test(raw: object, index: int) -> CatalogTest: kind = raw.get("kind") if kind not in KINDS: raise CatalogError(f"{where}.kind must be one of {KINDS}") - keys = _COMMON_KEYS | _RECIPE_KEYS if kind == "recipe" else _COMMON_KEYS - if kind == "recipe" and "recipe_fixture" in raw: - keys |= {"recipe_fixture"} - if kind in ("recipe", "clean-install") and "expected_failures" in raw: - keys |= {"expected_failures"} - _require_keys(raw, keys, where) + pipeline = _validate_test_keys(raw, kind, where) name = _require_str(raw["name"], f"{where}.name") if not _NAME_PATTERN.fullmatch(name): raise CatalogError(f"{where}.name must match {_NAME_PATTERN.pattern}") - recipe, ingredients, state = ( - _parse_recipe_fields(raw, where) if kind == "recipe" else (None, (), None) + recipe, ingredients, state, recipe_fixture = ( + _parse_recipe_fields(raw, where) if kind == "recipe" else (None, (), None, None) ) - recipe_fixture = None - if kind == "recipe" and "recipe_fixture" in raw: - recipe_fixture = _require_str(raw["recipe_fixture"], f"{where}.recipe_fixture") - if not _RECIPE_FIXTURE_PATTERN.fullmatch(recipe_fixture): - raise CatalogError(f"{where}.recipe_fixture must be a .yaml basename") - expected_failures = _parse_expected_failures(raw.get("expected_failures", []), where) - if kind == "recipe" and any(row["severity"] != "error" for row in expected_failures): + issue_url_env: str | None = None + required_changed_paths: tuple[str, ...] = () + test_command: tuple[str, ...] = () + peak_sessions_evidence: str | None = None + if kind == "recipe": + expected_failures = _parse_expected_failures( + raw.get("expected_failures", []), where, pipeline=pipeline + ) + elif kind == "clean-install": + expected_failures = _parse_expected_failures(raw.get("expected_failures", []), where) + else: + expected_failures = () + if ( + kind == "recipe" + and not pipeline + and any(row["severity"] != "error" for row in expected_failures) + ): raise CatalogError(f"{where}.expected_failures severity must be error for recipe tests") + if kind == "recipe": + ( + issue_url_env, + required_changed_paths, + test_command, + peak_sessions_evidence, + ) = _parse_recipe_options(raw, where, ingredients) test = CatalogTest( name=name, kind=kind, @@ -193,6 +322,11 @@ def _parse_test(raw: object, index: int) -> CatalogTest: expected_pull_request_state=state, expected_failures=expected_failures, recipe_fixture=recipe_fixture, + pipeline=pipeline, + issue_url_env=issue_url_env, + required_changed_paths=required_changed_paths, + test_command=test_command, + peak_sessions_evidence=peak_sessions_evidence, ) if job_timeout_minutes(test) > MAX_JOB_MINUTES: raise CatalogError(f"{where}.timeout_sec exceeds the {MAX_JOB_MINUTES}-minute job limit") diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index 3a89b895de..281d46a3c1 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -8,6 +8,7 @@ import json import os import re +import shlex import shutil import subprocess import sys @@ -18,8 +19,10 @@ from collections.abc import Callable, Mapping, Sequence from pathlib import Path from typing import Any +from urllib.parse import quote import e2e_catalog +import e2e_sessions MINIMAX_BASE_URL = "https://api.minimax.io/anthropic" MINIMAX_MODEL = "MiniMax-M3[1m]" @@ -257,28 +260,40 @@ def latest_pull_request_number( def new_pull_requests( - repository: str, baseline: int, env: Mapping[str, str], runner: Runner + repository: str, + baseline: int, + env: Mapping[str, str], + runner: Runner, + *, + pipeline: bool = False, ) -> tuple[list[dict[str, Any]] | None, str | None]: - rows, failure = _list_pull_requests( - repository, 20, "number,state,additions,deletions", env, runner - ) - if rows is None: - return None, failure - return [row for row in rows if int(row["number"]) > baseline], None + limit = 20 + fields = "number,state,additions,deletions" + if pipeline: + fields += ( + ",body,headRefOid,headRefName,headRepository,baseRefName,autoMergeRequest,mergedAt" + ) + while True: + rows, failure = _list_pull_requests(repository, limit, fields, env, runner) + if rows is None: + return None, failure + if not pipeline or len(rows) < limit or any(row["number"] <= baseline for row in rows): + return [row for row in rows if row["number"] > baseline], None + limit *= 2 def fleet_run_argv( - test: e2e_catalog.CatalogTest, runtime_ingredients: Mapping[str, str] | None = None + test: e2e_catalog.CatalogTest, *, ingredients: Mapping[str, str] | None = None ) -> list[str]: - effective = dict(test.ingredients) - effective.update(runtime_ingredients or {}) - ingredients = [arg for key, value in effective.items() for arg in ("-i", f"{key}={value}")] + values = dict(test.ingredients) + values.update(ingredients or {}) + arguments = [arg for key, value in values.items() for arg in ("-i", f"{key}={value}")] return [ "autoskillit", "fleet", "run", str(test.recipe), - *ingredients, + *arguments, "--disable-quota-guard", "--timeout-sec", str(test.timeout_sec), @@ -305,28 +320,77 @@ def check_envelope(returncode: int, stdout: str) -> tuple[list[str], dict[str, A return failures, envelope +def check_implementation_envelope( + envelope: Mapping[str, Any], +) -> tuple[list[str], list[dict[str, str]]]: + failures: list[str] = [] + findings: list[dict[str, str]] = [] + payload = envelope.get("l3_payload") + reason = payload.get("reason") if isinstance(payload, dict) else None + status = envelope.get("dispatch_status") + if envelope.get("success") is not True or status != "success": + stable_reason = envelope.get("reason") or reason + if isinstance(stable_reason, str) and stable_reason: + findings.append({"check": "fleet_envelope", "message": f"{status}: {stable_reason}"}) + else: + failures.append("fleet run: unsuccessful envelope has no specific failure reason") + elif reason != "implementation_complete": + if isinstance(reason, str) and reason: + findings.append({"check": "implementation_terminal", "message": reason}) + else: + failures.append("fleet run: implementation terminal reason is missing") + if not all( + isinstance(envelope.get(key), str) and envelope[key] + for key in ("dispatch_id", "dispatched_session_id") + ): + failures.append("fleet run: dispatch/session identity is missing") + return failures, findings + + def run_fleet( test: e2e_catalog.CatalogTest, out: Path, env: Mapping[str, str], runner: Runner, - runtime_ingredients: Mapping[str, str] | None = None, -) -> tuple[list[str], dict[str, Any] | None]: + *, + ingredients: Mapping[str, str] | None = None, +) -> tuple[dict[str, Any] | None, list[str], list[dict[str, str]]]: result, failure = _call( runner, - fleet_run_argv(test, runtime_ingredients), + fleet_run_argv(test, ingredients=ingredients), what="fleet run", + evidence=out / "fleet-run", cwd=SANDBOX_CLONE, env=env, timeout=test.timeout_sec + e2e_catalog.HARNESS_GRACE_SEC, - evidence=out / "fleet-run", ) + stdout = (out / "fleet-run.stdout.txt").read_text(encoding="utf-8", errors="replace") + shutil.copyfile(out / "fleet-run.stdout.txt", out / "fleet-run.stdout.log") + shutil.copyfile(out / "fleet-run.stderr.txt", out / "fleet-run.stderr.log") + line = last_line(stdout) + (out / "envelope.json").write_text(line + "\n", encoding="utf-8") + if not test.pipeline: + if result is None: + return None, [str(failure)], [] + envelope_failures, envelope = check_envelope(result.returncode, stdout) + return envelope, envelope_failures, [] + failures: list[str] = [] + findings: list[dict[str, str]] = [] + try: + envelope = json.loads(line) + except json.JSONDecodeError: + envelope = None + if not isinstance(envelope, dict): + envelope = None + failures.append("fleet run: the last stdout line is not a JSON object envelope") if result is None: - return [str(failure)], None - (out / "fleet-run.stdout.log").write_text(result.stdout, encoding="utf-8") - (out / "fleet-run.stderr.log").write_text(result.stderr, encoding="utf-8") - (out / "envelope.json").write_text(last_line(result.stdout) + "\n", encoding="utf-8") - return check_envelope(result.returncode, result.stdout) + failures.append(str(failure)) + elif envelope is not None and test.pipeline: + envelope_failures, findings = check_implementation_envelope(envelope) + failures += envelope_failures + if failure and not findings: + failures.append(failure) + return envelope, failures, findings def match_recipe_failure( @@ -456,7 +520,7 @@ def run_smoke_recipe( if runtime is None: return failures, [] (out / "smoke-launched").write_text(test.name + "\n", encoding="utf-8") - fleet_failures, envelope = run_fleet(test, out, env, runner, runtime) + envelope, fleet_failures, _ = run_fleet(test, out, env, runner, ingredients=runtime) fleet_failures, matched = match_recipe_failure(test, fleet_failures, envelope) failures += fleet_failures if ( @@ -866,6 +930,456 @@ def close_open_pull_requests( return failures +def install_session_hooks(home: Path, trace: Path) -> None: + path = home / ".claude" / "settings.json" + settings = json.loads(path.read_text(encoding="utf-8")) + command = f"python3 /opt/e2e/e2e_sessions.py {shlex.quote(str(trace))}" + hooks = settings.setdefault("hooks", {}) + for event in ("SessionStart", "SessionEnd", "SubagentStart", "SubagentStop"): + hooks.setdefault(event, []).append( + {"hooks": [{"type": "command", "command": command, "timeout": 10}]} + ) + path.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8") + + +def _json_call( + argv: Sequence[str], + env: Mapping[str, str], + runner: Runner, + *, + evidence: Path | None = None, + cwd: Path | None = None, +) -> Any: + result, failure = _setup_call(runner, argv, env, evidence=evidence, cwd=cwd) + if failure or result is None: + raise ValueError(failure or "missing command result") + try: + return json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise ValueError(f"{' '.join(argv[:3])}: invalid JSON") from exc + + +def _api_pages(endpoint: str, env: Mapping[str, str], runner: Runner, evidence: Path) -> list[Any]: + pages = _json_call( + ["gh", "api", endpoint, "--paginate", "--slurp"], env, runner, evidence=evidence + ) + if not isinstance(pages, list): + raise ValueError("GitHub API: expected paginated JSON array") + return pages + + +def prepare_seed( + test: e2e_catalog.CatalogTest, + repository: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> tuple[dict[str, Any], dict[str, str], str]: + url = env.get(test.issue_url_env or "", "") + match = re.fullmatch( + r"https://github\.com/" + re.escape(repository) + r"/issues/([1-9][0-9]*)", url + ) + if not match: + raise ValueError("seed: missing or noncanonical sandbox issue URL") + config = _json_call( + ["autoskillit", "config", "show"], + env, + runner, + evidence=out / "effective-config", + cwd=SANDBOX_CLONE, + ) + label = config["github"]["in_progress_label"] + seed = _json_call( + ["gh", "issue", "view", url, "--json", "number,url,state,title,body,labels"], + env, + runner, + evidence=out / "seed", + ) + if ( + not isinstance(seed, dict) + or seed.get("url") != url + or type(seed.get("number")) is not int + or seed["number"] != int(match[1]) + or seed.get("state") != "OPEN" + or not isinstance(seed.get("title"), str) + or not seed["title"].strip() + or not isinstance(seed.get("body"), str) + or not isinstance(seed.get("labels"), list) + or any( + not isinstance(row, dict) or not isinstance(row.get("name"), str) + for row in seed["labels"] + ) + ): + raise ValueError("seed: expected the configured OPEN sandbox issue with valid metadata") + if label in {row["name"] for row in seed["labels"]}: + raise ValueError("seed: issue is already claimed") + repo = _json_call( + ["gh", "repo", "view", repository, "--json", "defaultBranchRef"], env, runner + ) + default_branch = repo.get("defaultBranchRef") if isinstance(repo, dict) else None + if ( + not isinstance(default_branch, dict) + or default_branch.get("name") != "main" + or config["branching"]["default_base_branch"] != "main" + ): + raise ValueError("seed: sandbox and effective configured base branch must both be main") + ingredients = dict(test.ingredients) + ingredients.update(issue_url=url, task=f"{seed['title']}\n\n{seed['body']}") + for stage, argv in ( + ("baseline-install", ["task", "install-worktree"]), + ("baseline-test", ["task", "test-check"]), + ): + _, failure = _call( + runner, + argv, + what=stage, + evidence=out / stage, + cwd=SANDBOX_CLONE, + env=env, + timeout=SETUP_COMMAND_TIMEOUT_SEC, + ) + if failure: + raise ValueError(failure) + return seed, ingredients, label + + +def _repo_matches(value: Any, repository: str) -> bool: + return isinstance(value, dict) and ( + value.get("full_name") == repository + or value.get("url") == f"https://api.github.com/repos/{repository}" + ) + + +def owned_pull_requests( + rows: Sequence[Mapping[str, Any]], + proof: Mapping[str, Any], + repository: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[dict[str, Any]]: + owned = [] + for row in rows: + pr = _json_call( + ["gh", "api", f"repos/{repository}/pulls/{row['number']}"], + env, + runner, + evidence=out / f"pr-{row['number']}", + ) + if not isinstance(pr, dict): + raise ValueError("PR discovery: expected an object") + head = pr.get("head") + if ( + isinstance(head, dict) + and head.get("ref") in proof["branches"] + and _repo_matches(head.get("repo"), repository) + ): + pr["_listed_head"] = row.get("headRefOid") + owned.append(pr) + return owned + + +def validate_sandbox_ci( + repository: str, + number: int, + head: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> None: + workflows = _api_pages( + f"repos/{repository}/actions/workflows?per_page=100", env, runner, out / "ci-workflows" + ) + candidates = [ + workflow + for page in workflows + for workflow in page.get("workflows", []) + if workflow.get("name") == "CI" and workflow.get("state") == "active" + ] + if len(candidates) != 1: + raise ValueError("sandbox CI: expected one active CI workflow") + for attempt in range(6): + pages = _api_pages( + f"repos/{repository}/actions/workflows/{candidates[0]['id']}/runs?event=pull_request&per_page=100", + env, + runner, + out / "ci-runs", + ) + runs = [ + run + for page in pages + for run in page.get("workflow_runs", []) + if any( + pr.get("number") == number + and pr.get("head", {}).get("sha") == head + and _repo_matches(pr.get("head", {}).get("repo"), repository) + and _repo_matches(pr.get("base", {}).get("repo"), repository) + for pr in run.get("pull_requests", []) + ) + ] + if runs: + run = max(runs, key=lambda value: (value["id"], value.get("run_attempt", 1))) + if run.get("status") == "completed": + jobs = _api_pages( + f"repos/{repository}/actions/runs/{run['id']}/jobs?per_page=100", + env, + runner, + out / "ci-jobs", + ) + tests = [ + job + for page in jobs + for job in page.get("jobs", []) + if job.get("name") == "test" + ] + if ( + run.get("conclusion") != "success" + or not tests + or any( + job.get("status") != "completed" or job.get("conclusion") != "success" + for job in tests + ) + ): + raise ValueError("sandbox CI: CI run and test job must succeed") + return + if attempt < 5: + time.sleep(10) + raise ValueError("sandbox CI: no completed CI run associated with the exact PR head") + + +def validate_implementation_pr( + test: e2e_catalog.CatalogTest, + prs: Sequence[Mapping[str, Any]], + seed: Mapping[str, Any], + repository: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + if len(prs) != 1: + return [f"expected exactly one new owned sandbox pull request, found {len(prs)}"] + pr = prs[0] + number = pr["number"] + head = pr.get("head", {}) + base = pr.get("base", {}) + oid = head.get("sha") + if ( + pr.get("state") != "open" + or pr.get("merged") is not False + or pr.get("merged_at") + or pr.get("auto_merge") is not None + or base.get("ref") != "main" + or head.get("ref") == "main" + or not isinstance(oid, str) + or not oid + or not _repo_matches(base.get("repo"), repository) + or not _repo_matches(head.get("repo"), repository) + ): + return [ + "implementation PR: expected an open, unmerged sandbox PR targeting main " + "without auto-merge" + ] + if oid != pr.get("_listed_head"): + return ["implementation PR: REST head differs from captured headRefOid"] + if not re.search(re.escape(seed["url"]) + r"(?![A-Za-z0-9_/#?=-])", str(pr.get("body", ""))): + return ["implementation PR: body does not reference the canonical seed URL"] + if ( + type(pr.get("additions")) is not int + or type(pr.get("deletions")) is not int + or (pr["additions"] + pr["deletions"] <= 0) + ): + return ["implementation PR: empty or malformed diff"] + pages = _api_pages( + f"repos/{repository}/pulls/{number}/files?per_page=100", env, runner, out / "pr-files" + ) + files = [row for page in pages for row in page] + for prefix in test.required_changed_paths: + if not any( + row.get("status") != "removed" + and isinstance(row.get("filename"), str) + and row["filename"].startswith(prefix) + and row.get("additions", 0) + row.get("deletions", 0) > 0 + for row in files + ): + return [f"implementation PR: missing non-removed changes under {prefix}"] + failures = test_pr_head(test, number, oid, out, env, runner) + if failures: + return failures + validate_sandbox_ci(repository, number, oid, out, env, runner) + final = _json_call( + ["gh", "api", f"repos/{repository}/pulls/{number}"], env, runner, evidence=out / "pr-final" + ) + if ( + final.get("head", {}).get("sha") != oid + or final.get("state") != "open" + or final.get("merged") is not False + or final.get("merged_at") + ): + return ["implementation PR: head or open/unmerged state changed during validation"] + return [] + + +def test_pr_head( + test: e2e_catalog.CatalogTest, + number: int, + oid: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + commands = ( + ("pr-fetch", ["git", "fetch", "origin", f"refs/pull/{number}/head"]), + ("pr-oid", ["git", "rev-parse", "FETCH_HEAD"]), + ("pr-checkout", ["git", "checkout", "--detach", oid]), + ("pr-install", ["task", "install-worktree"]), + ("pr-test", list(test.test_command)), + ) + for stage, argv in commands: + result, failure = _call( + runner, + argv, + what=stage, + evidence=out / stage, + cwd=SANDBOX_CLONE, + env=env, + timeout=SETUP_COMMAND_TIMEOUT_SEC, + ) + if failure: + return [failure] + if stage == "pr-oid" and (result is None or result.stdout.strip() != oid): + return ["implementation PR: fetched head differs from captured head"] + return [] + + +def discover_cleanup_prs( + repository: str, + baseline: int, + proof: Mapping[str, Any], + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> tuple[list[dict[str, Any]], list[str]]: + failures: list[str] = [] + owned: list[dict[str, Any]] = [] + for attempt in range(2): + try: + rows, failure = new_pull_requests(repository, baseline, env, runner, pipeline=True) + if rows is None: + raise ValueError(failure) + owned = owned_pull_requests(rows, proof, repository, out, env, runner) + if owned or attempt == 1: + if rows and not proof["branches"]: + failures.append("cleanup: PR ownership could not be established") + break + except Exception as exc: + if attempt == 1: + failures.append(f"cleanup discovery: {exc}") + if attempt == 0: + time.sleep(1) + return owned, failures + + +def delete_owned_branch( + repository: str, branch: str, env: Mapping[str, str], runner: Runner +) -> None: + if branch == "main" or not branch or branch.startswith("refs/"): + raise ValueError("refusing to delete an unsafe/default branch") + endpoint = f"repos/{repository}/git/refs/heads/{quote(branch, safe='')}" + result, failure = _setup_call(runner, ["gh", "api", endpoint], env) + if failure and result is not None and "404" in result.stderr: + return + if failure: + raise ValueError(failure) + time.sleep(1) + _, failure = _setup_call(runner, ["gh", "api", "--method", "DELETE", endpoint], env) + if failure: + raise ValueError(failure) + + +def release_owned_claim( + proof: Mapping[str, Any], + seed: Mapping[str, Any], + label: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + current = _json_call( + ["gh", "issue", "view", seed["url"], "--json", "state,labels"], + env, + runner, + evidence=out / "seed-final", + ) + failures = [] if current.get("state") == "OPEN" else ["cleanup: seed issue is no longer OPEN"] + if label not in {row["name"] for row in current["labels"]}: + return failures + if not proof["claimed"]: + return [*failures, "cleanup: claim label is present without this run's claim proof"] + time.sleep(1) + _, failure = _setup_call( + runner, ["gh", "issue", "edit", seed["url"], "--remove-label", label], env + ) + if failure: + failures.append(f"cleanup: {failure}") + return failures + + +def cleanup_implementation( + repository: str, + baseline: int, + proof: Mapping[str, Any], + seed: Mapping[str, Any], + label: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + owned, failures = discover_cleanup_prs(repository, baseline, proof, out, env, runner) + for pr in owned: + try: + if pr.get("state") == "open": + time.sleep(1) + _, failure = _setup_call( + runner, ["gh", "pr", "close", str(pr["number"]), "--repo", repository], env + ) + if failure: + failures.append(f"cleanup: {failure}") + except Exception as exc: + failures.append(f"cleanup PR: {exc}") + for branch in proof["branches"]: + try: + delete_owned_branch(repository, branch, env, runner) + except Exception as exc: + failures.append(f"cleanup branch: {exc}") + try: + failures += release_owned_claim(proof, seed, label, out, env, runner) + except Exception as exc: + failures.append(f"cleanup claim: {exc}") + return failures + + +def match_recipe_findings( + failures: list[str], + findings: Sequence[Mapping[str, str]], + expected: Sequence[Mapping[str, str]], +) -> tuple[list[str], list[dict[str, str]]]: + annotations = {(row["check"], row["message"]): row for row in expected} + matched: dict[tuple[str, str], dict[str, str]] = {} + for row in findings: + identity = (row["check"], row["message"]) + if identity in annotations: + matched[identity] = dict(annotations[identity]) + else: + failures.append(f"{row['check']}: {row['message']}") + for identity, annotation in annotations.items(): + if identity not in matched: + failures.append( + f"expected recipe finding absent {identity}; remove stale expected failure " + f"for {annotation['issue']}" + ) + return failures, list(matched.values()) + + def run_recipe( test: e2e_catalog.CatalogTest, repository: str, @@ -873,6 +1387,8 @@ def run_recipe( out: Path, env: Mapping[str, str], runner: Runner, + *, + home: Path | None = None, ) -> tuple[list[str], list[dict[str, str]]]: failure = ( configure_git(env, runner) @@ -883,15 +1399,44 @@ def run_recipe( return [failure], [] if test.recipe_fixture is not None: return run_smoke_recipe(test, repository, out, env, runner) + seed: dict[str, Any] = {} + ingredients = dict(test.ingredients) + label = "" + home = home or Path.home() + trace = out / "session-events.jsonl" + if test.pipeline or test.issue_url_env: + try: + seed, ingredients, label = prepare_seed(test, repository, out, env, runner) + if test.pipeline: + trace.write_text("", encoding="utf-8") + install_session_hooks(home, trace) + except Exception as exc: + return [f"preflight: {exc}"], [] baseline, failure = latest_pull_request_number(repository, env, runner) if baseline is None: return [str(failure)], [] + if not test.pipeline: + return launch_regular_recipe(test, repository, baseline, ingredients, out, env, runner) + return launch_recipe( + test, repository, baseline, seed, ingredients, label, out, home, env, runner + ) + + +def launch_regular_recipe( + test: e2e_catalog.CatalogTest, + repository: str, + baseline: int, + ingredients: Mapping[str, str], + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> tuple[list[str], list[dict[str, str]]]: failures: list[str] = [] + matched: list[dict[str, str]] = [] listed: list[dict[str, Any]] | None = None try: - fleet_failures, envelope = run_fleet(test, out, env, runner) - fleet_failures, matched = match_recipe_failure(test, fleet_failures, envelope) - failures += fleet_failures + envelope, failures, _ = run_fleet(test, out, env, runner, ingredients=ingredients) + failures, matched = match_recipe_failure(test, failures, envelope) if not matched: listed, failure = new_pull_requests(repository, baseline, env, runner) if listed is None: @@ -903,6 +1448,106 @@ def run_recipe( return failures, matched +def save_ownership( + trace: Path, + envelope: dict[str, Any] | None, + log_root: Path, + seed: Mapping[str, Any], + repository: str, + out: Path, + label: str, +) -> dict[str, Any]: + proof = e2e_sessions.collect_ownership( + trace, envelope, log_root, seed["url"], repository, claim_label=label + ) + (out / "ownership.json").write_text(json.dumps(proof, indent=2) + "\n", encoding="utf-8") + return proof + + +def finish_pipeline( + test: e2e_catalog.CatalogTest, + repository: str, + baseline: int, + seed: Mapping[str, Any], + label: str, + out: Path, + home: Path, + env: Mapping[str, str], + runner: Runner, + envelope: dict[str, Any] | None, + proof: dict[str, Any] | None, +) -> list[str]: + failures: list[str] = [] + trace = out / "session-events.jsonl" + log_root = home / ".local" / "share" / "autoskillit" / "logs" + try: + summary = e2e_sessions.summarize(trace, envelope, log_root) + if not summary["coverage"]["complete"]: + failures.append("session measurement: incomplete lifecycle coverage") + if summary["peak_sessions"] > test.peak_sessions: + failures.append("session measurement: observed peak exceeds catalog capacity") + except Exception as exc: + failures.append(f"session measurement: {exc}") + if proof is None: + try: + proof = save_ownership(trace, envelope, log_root, seed, repository, out, label) + failures += [f"ownership: {value}" for value in proof["violations"]] + except Exception as exc: + failures.append(f"cleanup ownership: {exc}") + proof = proof or {"branches": [], "claimed": False} + failures += cleanup_implementation(repository, baseline, proof, seed, label, out, env, runner) + return failures + + +def launch_recipe( + test: e2e_catalog.CatalogTest, + repository: str, + baseline: int, + seed: Mapping[str, Any], + ingredients: Mapping[str, str], + label: str, + out: Path, + home: Path, + env: Mapping[str, str], + runner: Runner, +) -> tuple[list[str], list[dict[str, str]]]: + failures: list[str] = [] + findings: list[dict[str, str]] = [] + envelope: dict[str, Any] | None = None + proof: dict[str, Any] | None = None + try: + envelope, fleet_failures, findings = run_fleet( + test, out, env, runner, ingredients=ingredients + ) + failures += fleet_failures + proof = save_ownership( + out / "session-events.jsonl", + envelope, + home / ".local" / "share" / "autoskillit" / "logs", + seed, + repository, + out, + label, + ) + failures += [f"ownership: {value}" for value in proof["violations"]] + listed, failure = new_pull_requests(repository, baseline, env, runner, pipeline=True) + if listed is None: + failures.append(str(failure)) + else: + owned = owned_pull_requests(listed, proof, repository, out, env, runner) + if not fleet_failures and not findings: + failures += validate_implementation_pr( + test, owned, seed, repository, out, env, runner + ) + except Exception as exc: + failures.append(f"harness error: {exc}") + finally: + failures += finish_pipeline( + test, repository, baseline, seed, label, out, home, env, runner, envelope, proof + ) + return match_recipe_findings(failures, findings, test.expected_failures) + + def check_clean_install_doctor( stdout: str, expected_failures: Sequence[Mapping[str, str]] ) -> tuple[list[str], list[dict[str, str]]]: @@ -1014,7 +1659,7 @@ def _run_test( if test.kind == "canary": return run_canary(test, out, scrubbed, runner), [] token = env["E2E_SANDBOX_TOKEN"] - return run_recipe(test, catalog.sandbox_repository, token, out, scrubbed, runner) + return run_recipe(test, catalog.sandbox_repository, token, out, scrubbed, runner, home=home) def run_test( diff --git a/scripts/e2e/e2e_select.py b/scripts/e2e/e2e_select.py index 906e66a69b..56a8dbd246 100644 --- a/scripts/e2e/e2e_select.py +++ b/scripts/e2e/e2e_select.py @@ -21,6 +21,7 @@ MIN_CHANGED_LINES = 300 SAMPLE_PROBABILITY = 0.5 MAX_TESTS = 2 +MAX_PIPELINE_TESTS = 1 @dataclass(frozen=True, slots=True) @@ -68,10 +69,12 @@ def selected(self) -> bool: def is_eligible(pr: PullRequestFacts) -> bool: + return E2E_LABEL in pr.labels or _is_large_change(pr) + + +def _is_large_change(pr: PullRequestFacts) -> bool: return ( - E2E_LABEL in pr.labels - or pr.changed_files >= MIN_CHANGED_FILES - or pr.additions + pr.deletions >= MIN_CHANGED_LINES + pr.changed_files >= MIN_CHANGED_FILES or pr.additions + pr.deletions >= MIN_CHANGED_LINES ) @@ -97,13 +100,32 @@ def _touches(test: e2e_catalog.CatalogTest, changed_paths: Sequence[str]) -> boo def pick_tests( - catalog: e2e_catalog.Catalog, changed_paths: Sequence[str], seed: str + catalog: e2e_catalog.Catalog, + changed_paths: Sequence[str], + seed: str, + *, + large_change: bool = False, ) -> tuple[str, ...]: - ranked = sorted(catalog.tests, key=lambda test: _rank(seed, test.name)) - touched = [test.name for test in ranked if _touches(test, changed_paths)] + eligible = [ + test + for test in catalog.tests + if not test.pipeline or (large_change and _touches(test, changed_paths)) + ] + ranked = sorted(eligible, key=lambda test: _rank(seed, test.name)) + touched = [test for test in ranked if _touches(test, changed_paths)] if touched: - return tuple(touched[:MAX_TESTS]) - return (ranked[0].name,) + picked: list[str] = [] + pipeline_count = 0 + for test in touched: + if test.pipeline: + if pipeline_count >= MAX_PIPELINE_TESTS: + continue + pipeline_count += 1 + picked.append(test.name) + if len(picked) == MAX_TESTS: + break + return tuple(picked) + return (ranked[0].name,) if ranked else () def select_for_pull_request( @@ -122,10 +144,16 @@ def select_for_pull_request( ) seed = f"{pr.number}:{pr.head_sha}" if E2E_LABEL in pr.labels: - return Selection(pick_tests(catalog, changed_paths, seed), f"{E2E_LABEL} label") + return Selection( + pick_tests(catalog, changed_paths, seed, large_change=_is_large_change(pr)), + f"{E2E_LABEL} label", + ) if not is_sampled(pr, SAMPLE_PROBABILITY): return Selection((), "eligible but not sampled for this head commit") - return Selection(pick_tests(catalog, changed_paths, seed), "eligible and sampled") + return Selection( + pick_tests(catalog, changed_paths, seed, large_change=_is_large_change(pr)), + "eligible and sampled", + ) def select_for_dispatch(requested: str, catalog: e2e_catalog.Catalog) -> Selection: diff --git a/scripts/e2e/e2e_sessions.py b/scripts/e2e/e2e_sessions.py new file mode 100644 index 0000000000..bab3aec9be --- /dev/null +++ b/scripts/e2e/e2e_sessions.py @@ -0,0 +1,879 @@ +#!/usr/bin/env python3 +"""Capture and summarize Claude session lifecycle evidence for E2E runs.""" + +from __future__ import annotations + +import json +import math +import os +import re +import sys +import time +from collections.abc import Mapping +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit + +_SESSION_EVENTS = frozenset({"SessionStart", "SessionEnd"}) +_AGENT_EVENTS = frozenset({"SubagentStart", "SubagentStop"}) +_LIFECYCLE_EVENTS = _SESSION_EVENTS | _AGENT_EVENTS +_MAX_HOOK_INPUT_BYTES = 65_536 +_MAX_TRACE_RECORD_BYTES = 8_192 + + +def capture_hook_event(trace_path: Path, payload: Mapping[str, Any]) -> bool: + """Append one allow-listed lifecycle record; return False for other hooks.""" + record = _hook_record(payload) + if record is None: + return False + encoded = (json.dumps(record, separators=(",", ":")) + "\n").encode("utf-8") + if len(encoded) > _MAX_TRACE_RECORD_BYTES: + raise ValueError("lifecycle hook record exceeds the size limit") + trace_path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(trace_path, os.O_APPEND | os.O_CREAT | os.O_WRONLY, 0o600) + try: + written = os.write(fd, encoded) + if written != len(encoded): + raise OSError("short lifecycle trace append") + finally: + os.close(fd) + return True + + +def _hook_record(payload: Mapping[str, Any]) -> dict[str, Any] | None: + event = payload.get("hook_event_name", payload.get("event")) + if event is None: + return None + if not isinstance(event, str): + raise ValueError("lifecycle hook event must be a string") + if event not in _LIFECYCLE_EVENTS: + return None + + session_id = payload.get("session_id") + if not isinstance(session_id, str) or not session_id: + raise ValueError("lifecycle hook has no session_id") + + record: dict[str, Any] = { + "event": event, + "timestamp": time.time(), + "session_id": session_id, + } + for key in ( + "agent_id", + "agent_type", + "source", + "transcript_path", + "agent_transcript_path", + ): + value = payload.get(key) + if value is None: + continue + if not isinstance(value, str): + raise ValueError(f"lifecycle hook {key} must be a string") + record[key] = value + if event in _AGENT_EVENTS and not record.get("agent_id"): + raise ValueError("subagent lifecycle hook has no agent_id") + return record + + +def _read_jsonl( + path: Path, label: str, violations: list[str], *, collect: bool = True +) -> list[dict[str, Any]]: + try: + handle = path.open(encoding="utf-8", errors="replace") + except OSError: + violations.append(f"{label}:unavailable") + return [] + + records: list[dict[str, Any]] = [] + with handle: + for line_number, line in enumerate(handle, 1): + if not line.strip(): + violations.append(f"{label}:empty_line:{line_number}") + continue + try: + item = json.loads(line) + except (json.JSONDecodeError, UnicodeDecodeError): + violations.append(f"{label}:invalid_json:{line_number}") + continue + if not isinstance(item, dict): + violations.append(f"{label}:invalid_record:{line_number}") + continue + if collect: + records.append(item) + return records + + +def _load_trace(trace_path: Path, violations: list[str]) -> list[dict[str, Any]]: + records = _read_jsonl(trace_path, "trace", violations) + valid: list[dict[str, Any]] = [] + for index, record in enumerate(records, 1): + event = record.get("event") + timestamp = record.get("timestamp") + session_id = record.get("session_id") + if not isinstance(event, str) or event not in _LIFECYCLE_EVENTS: + violations.append(f"trace:unknown_event:{index}") + continue + finite_timestamp = False + if isinstance(timestamp, (int, float)) and not isinstance(timestamp, bool): + try: + finite_timestamp = math.isfinite(timestamp) + except OverflowError: + pass + if not finite_timestamp: + violations.append(f"trace:invalid_timestamp:{index}") + continue + if not isinstance(session_id, str) or not session_id: + violations.append(f"trace:invalid_session_id:{index}") + continue + optional_invalid = any( + key in record and record[key] is not None and not isinstance(record[key], str) + for key in ( + "agent_id", + "agent_type", + "source", + "transcript_path", + "agent_transcript_path", + ) + ) + if optional_invalid: + violations.append(f"trace:invalid_optional_field:{index}") + continue + if event in _AGENT_EVENTS and not record.get("agent_id"): + violations.append(f"trace:missing_agent_id:{index}") + continue + valid.append(record) + return valid + + +def _read_envelope(envelope: dict[str, Any] | None, violations: list[str]) -> tuple[str, str]: + if not isinstance(envelope, dict): + violations.append("envelope:missing") + return "", "" + session_id = envelope.get("dispatched_session_id") + dispatch_id = envelope.get("dispatch_id") + if not isinstance(session_id, str) or not session_id: + violations.append("envelope:missing_dispatched_session_id") + session_id = "" + if not isinstance(dispatch_id, str) or not dispatch_id: + violations.append("envelope:missing_dispatch_id") + dispatch_id = "" + return session_id, dispatch_id + + +def _load_index(log_root: Path, violations: list[str]) -> list[dict[str, Any]]: + index_path = log_root / "sessions.jsonl" + records = _read_jsonl(index_path, "sessions_index", violations) + for line_number, row in enumerate(records, 1): + if not isinstance(row.get("session_id"), str) or not row.get("session_id"): + violations.append(f"sessions_index:missing_session_id:{line_number}") + return records + + +def _safe_session_dir(log_root: Path, row: Mapping[str, Any]) -> Path | None: + name = row.get("dir_name") + if not isinstance(name, str) or not name or Path(name).name != name: + return None + return log_root / "sessions" / name + + +def _read_summary(session_dir: Path, expected_session_id: str) -> dict[str, Any] | None: + try: + summary = json.loads((session_dir / "summary.json").read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError, UnicodeDecodeError): + return None + if not isinstance(summary, dict) or summary.get("session_id") != expected_session_id: + return None + identity = summary.get("execution_identity") + if not isinstance(identity, dict) or not isinstance(identity.get("parent_session_id"), str): + return None + return summary + + +def _discover_owned( + trace_path: Path, + envelope: dict[str, Any] | None, + log_root: Path | None, +) -> dict[str, Any]: + """Resolve only the dispatched session and fresh, linked managed descendants.""" + root = log_root or Path.home() / ".local" / "share" / "autoskillit" / "logs" + violations: list[str] = [] + events = _load_trace(trace_path, violations) + root_id, dispatch_id = _read_envelope(envelope, violations) + trace_session_ids = { + record["session_id"] for record in events if record["event"] in _SESSION_EVENTS + } + rows = _load_index(root, violations) + rows_by_id: dict[str, list[dict[str, Any]]] = {} + for row in rows: + session_id = row.get("session_id") + if isinstance(session_id, str) and session_id: + rows_by_id.setdefault(session_id, []).append(row) + owned: dict[str, dict[str, Any]] = {} + root_rows = rows_by_id.get(root_id, []) + if not root_id or len(root_rows) != 1: + violations.append("sessions_index:root_missing_or_ambiguous") + else: + if root_id not in trace_session_ids: + violations.append("trace:root_session_missing") + root_row = root_rows[0] + if not dispatch_id or root_row.get("dispatch_id") != dispatch_id: + violations.append("sessions_index:root_dispatch_mismatch") + _add_owned_session(root, root_row, root_id, violations, owned) + candidates = _fresh_candidates(root, root_id, dispatch_id, trace_session_ids, rows_by_id) + _add_linked_descendants(root, dispatch_id, candidates, owned, violations) + + return { + "root_id": root_id, + "dispatch_id": dispatch_id, + "log_root": root, + "events": events, + "owned": owned, + "violations": violations, + "trace_session_ids": trace_session_ids, + } + + +def _fresh_candidates( + log_root: Path, + root_id: str, + dispatch_id: str, + trace_session_ids: set[str], + rows_by_id: Mapping[str, list[dict[str, Any]]], +) -> dict[str, tuple[dict[str, Any], dict[str, Any] | None]]: + candidates: dict[str, tuple[dict[str, Any], dict[str, Any] | None]] = {} + candidate_ids = trace_session_ids | { + session_id + for session_id, rows in rows_by_id.items() + if dispatch_id and any(row.get("dispatch_id") == dispatch_id for row in rows) + } + for session_id in candidate_ids: + if session_id == root_id: + continue + matching_rows = rows_by_id.get(session_id, []) + if len(matching_rows) != 1: + continue + row = matching_rows[0] + session_dir = _safe_session_dir(log_root, row) + summary = _read_summary(session_dir, session_id) if session_dir else None + candidates[session_id] = row, summary + return candidates + + +def _add_linked_descendants( + log_root: Path, + dispatch_id: str, + candidates: dict[str, tuple[dict[str, Any], dict[str, Any] | None]], + owned: dict[str, dict[str, Any]], + violations: list[str], +) -> None: + remaining = dict(candidates) + while remaining: + changed = False + owned_ids = set(owned) + for session_id, (row, summary) in list(remaining.items()): + parents = { + value + for key in ("parent_session_id", "caller_session_id") + if isinstance((value := row.get(key)), str) and value + } + summary_parent = summary["execution_identity"]["parent_session_id"] if summary else "" + if not parents.intersection(owned_ids) and summary_parent not in owned_ids: + continue + if not dispatch_id or row.get("dispatch_id") != dispatch_id: + violations.append(f"sessions_index:{session_id}:dispatch_mismatch") + del remaining[session_id] + continue + if summary is None: + violations.append(f"summary:{session_id}:missing_or_malformed") + elif summary_parent not in owned_ids: + violations.append(f"summary:{session_id}:parent_mismatch") + if parents and not parents.intersection(owned_ids): + violations.append(f"sessions_index:{session_id}:parent_mismatch") + _add_owned_session(log_root, row, session_id, violations, owned, summary=summary) + del remaining[session_id] + changed = True + if not changed: + break + + +def _add_owned_session( + log_root: Path, + row: Mapping[str, Any], + session_id: str, + violations: list[str], + owned: dict[str, dict[str, Any]], + *, + summary: dict[str, Any] | None = None, +) -> None: + session_dir = _safe_session_dir(log_root, row) + if session_dir is None: + violations.append(f"sessions_index:{session_id}:invalid_dir_name") + return + required_index_fields = ( + "session_id", + "dir_name", + "dispatch_id", + "parent_session_id", + "caller_session_id", + "claude_code_log", + ) + if any(key not in row for key in required_index_fields): + violations.append(f"sessions_index:{session_id}:missing_fields") + if summary is None: + summary = _read_summary(session_dir, session_id) + if summary is None: + violations.append(f"summary:{session_id}:missing_or_malformed") + elif summary.get("dispatch_id", "") != row.get("dispatch_id", ""): + violations.append(f"summary:{session_id}:dispatch_mismatch") + transcript_value = row.get("claude_code_log") + transcript: Path | None = None + if isinstance(transcript_value, str) and transcript_value: + transcript = Path(transcript_value) + if not transcript.is_absolute(): + transcript = log_root / transcript + else: + violations.append(f"sessions_index:{session_id}:missing_transcript") + if transcript is None or not transcript.is_file(): + violations.append(f"transcript:{session_id}:missing") + else: + _read_jsonl(transcript, f"transcript:{session_id}", violations, collect=False) + owned[session_id] = { + "row": dict(row), + "summary": summary, + "session_dir": session_dir, + "transcript": transcript, + } + + +def _lifecycle_intervals( + events: list[dict[str, Any]], + session_ids: set[str], + violations: list[str], +) -> tuple[list[dict[str, Any]], set[str]]: + grouped = _group_lifecycle_events(events, session_ids) + intervals: list[dict[str, Any]] = [] + seen_sessions: set[str] = set() + for identity, endpoints in grouped.items(): + item = _make_interval(identity, endpoints, violations) + if item is None: + continue + if identity[0] == "session": + seen_sessions.add(identity[1]) + intervals.append(item) + return intervals, seen_sessions + + +def _group_lifecycle_events( + events: list[dict[str, Any]], session_ids: set[str] +) -> dict[tuple[str, ...], dict[str, list[dict[str, Any]]]]: + grouped: dict[tuple[str, ...], dict[str, list[dict[str, Any]]]] = {} + for event in events: + name = event["event"] + session_id = event["session_id"] + if session_id not in session_ids: + continue + identity: tuple[str, ...] + if name in _SESSION_EVENTS: + identity = ("session", session_id) + role = "start" if name == "SessionStart" else "end" + else: + identity = ("agent", session_id, event["agent_id"]) + role = "start" if name == "SubagentStart" else "end" + grouped.setdefault(identity, {"start": [], "end": []})[role].append(event) + return grouped + + +def _make_interval( + identity: tuple[str, ...], + endpoints: Mapping[str, list[dict[str, Any]]], + violations: list[str], +) -> dict[str, Any] | None: + starts, stops = endpoints["start"], endpoints["end"] + label = ":".join(identity) + if not starts: + violations.append(f"lifecycle:{label}:orphan_end") + return None + if not stops: + violations.append(f"lifecycle:{label}:missing_end") + return None + start = min(float(item["timestamp"]) for item in starts) + end = max(float(item["timestamp"]) for item in stops) + if any(float(item["timestamp"]) < start for item in stops): + violations.append(f"lifecycle:{label}:orphan_end_before_start") + if end < start or any(float(event["timestamp"]) > end for event in starts): + reason = "end_before_start" if end < start else "missing_final_end" + violations.append(f"lifecycle:{label}:{reason}") + return None + item = {"kind": identity[0], "session_id": identity[1], "start": start, "end": end} + if identity[0] == "agent": + item["agent_id"] = identity[2] + path_field = "agent_transcript_path" + else: + path_field = "transcript_path" + paths = { + str(Path(event[path_field]).resolve()) + for event in starts + stops + if isinstance(event.get(path_field), str) and event[path_field] + } + if len(paths) > 1: + violations.append(f"lifecycle:{label}:transcript_mismatch") + if paths: + item["transcript_path"] = sorted(paths)[0] + for key in ("agent_type", "source"): + values = {event[key] for event in starts + stops if isinstance(event.get(key), str)} + if len(values) == 1: + item[key] = next(iter(values)) + return item + + +def _check_transcript_coverage( + events: list[dict[str, Any]], + owned: Mapping[str, Mapping[str, Any]], + violations: list[str], +) -> list[dict[str, str]]: + session_ids = set(owned) + session_paths: dict[str, set[str]] = {session_id: set() for session_id in session_ids} + observed: dict[tuple[str, str], list[dict[str, Any]]] = {} + for event in events: + session_id = event["session_id"] + if session_id not in session_ids: + continue + if event["event"] in _SESSION_EVENTS and event.get("transcript_path"): + session_paths[session_id].add(str(Path(event["transcript_path"]).resolve())) + elif event["event"] in _AGENT_EVENTS: + key = (session_id, event["agent_id"]) + observed.setdefault(key, []).append(event) + for session_id, paths in session_paths.items(): + if len(paths) > 1: + violations.append(f"lifecycle:{session_id}:transcript_mismatch") + enumerated = _enumerate_native_transcripts(owned, session_paths, violations) + + native_agents: list[dict[str, str]] = [] + agent_keys = set(observed) + agent_keys.update( + (session_id, agent_id) for session_id, files in enumerated.items() for agent_id in files + ) + for session_id, agent_id in sorted(agent_keys): + transcript = enumerated.get(session_id, {}).get(agent_id) + proof = _verify_native_transcript( + session_id, agent_id, transcript, observed.get((session_id, agent_id), []), violations + ) + if proof is not None: + native_agents.append(proof) + return native_agents + + +def _enumerate_native_transcripts( + owned: Mapping[str, Mapping[str, Any]], + session_paths: Mapping[str, set[str]], + violations: list[str], +) -> dict[str, dict[str, Path]]: + enumerated: dict[str, dict[str, Path]] = {} + for session_id, item in owned.items(): + transcript = item.get("transcript") + if not isinstance(transcript, Path) or not transcript.is_file(): + continue + paths = session_paths.get(session_id, set()) + if not paths: + violations.append(f"lifecycle:{session_id}:transcript_path_missing") + elif paths != {str(transcript.resolve())}: + violations.append(f"lifecycle:{session_id}:transcript_path_mismatch") + subagents_dir = transcript.parent / session_id / "subagents" + files: dict[str, Path] = {} + for path in sorted(subagents_dir.glob("agent-*.jsonl")): + agent_id = path.name.removeprefix("agent-").removesuffix(".jsonl") + if not agent_id: + continue + files[agent_id] = path + if not path.is_file(): + violations.append(f"native:{session_id}:{agent_id}:transcript_missing") + enumerated[session_id] = files + return enumerated + + +def _verify_native_transcript( + session_id: str, + agent_id: str, + transcript: Path | None, + events: list[dict[str, Any]], + violations: list[str], +) -> dict[str, str] | None: + label = f"native:{session_id}:{agent_id}" + names = {event["event"] for event in events} + if not {"SubagentStart", "SubagentStop"} <= names: + violations.append(f"{label}:lifecycle_incomplete") + if transcript is None: + violations.append(f"{label}:transcript_unenumerated") + return None + expected_path = str(transcript.resolve()) + paths = { + str(Path(event["agent_transcript_path"]).resolve()) + for event in events + if isinstance(event.get("agent_transcript_path"), str) + } + if paths != {expected_path}: + violations.append(f"{label}:transcript_mismatch") + if not paths: + violations.append(f"{label}:transcript_path_missing") + if not transcript.is_file(): + violations.append(f"{label}:transcript_missing") + return None + _read_jsonl(transcript, f"transcript:{session_id}:{agent_id}", violations, collect=False) + return {"session_id": session_id, "agent_id": agent_id, "transcript_path": expected_path} + + +def _peak_sessions(intervals: list[dict[str, Any]]) -> int: + points: list[tuple[float, int]] = [] + for item in intervals: + if item["start"] == item["end"]: + continue + points.append((item["end"], -1)) + points.append((item["start"], 1)) + active = peak = 0 + for _timestamp, delta in sorted(points, key=lambda point: (point[0], point[1])): + active += delta + peak = max(peak, active) + return peak + + +def summarize( + trace_path: Path, + envelope: dict[str, Any] | None, + log_root: Path | None = None, +) -> dict[str, Any]: + """Write a bounded concurrency summary beside the raw lifecycle trace.""" + context = _discover_owned(trace_path, envelope, log_root) + violations = list(context["violations"]) + owned: dict[str, dict[str, Any]] = context["owned"] + owned_ids = set(owned) + intervals, seen_sessions = _lifecycle_intervals(context["events"], owned_ids, violations) + for session_id in sorted(owned_ids - seen_sessions): + violations.append(f"lifecycle:{session_id}:missing_endpoints") + native_agents = _check_transcript_coverage(context["events"], owned, violations) + root_id = context["root_id"] + root_seen = root_id in seen_sessions + summary = { + "method": "hook_lifecycle_intervals", + "trace_path": str(trace_path), + "peak_sessions": _peak_sessions(intervals), + "identities": sorted( + intervals, + key=lambda item: ( + item["start"], + item["kind"], + item["session_id"], + item.get("agent_id", ""), + ), + ), + "coverage": { + "complete": not violations, + "root_session_id": root_id or None, + "root_session_seen": root_seen, + "managed_session_ids": sorted(owned_ids - ({root_id} if root_id else set())), + "owned_session_ids": sorted(owned_ids), + "native_agents": native_agents, + }, + "violations": sorted(set(violations)), + } + output_path = trace_path.with_name("session-concurrency.json") + output_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return summary + + +def _issue_number(issue_url: str) -> str: + match = re.search(r"/issues/(\d+)/?$", urlsplit(issue_url).path) + if match: + return match.group(1) + match = re.search(r"#(\d+)$", issue_url) + return match.group(1) if match else "" + + +def _issue_matches_repository(issue_url: str, repository: str) -> bool: + parsed = urlsplit(issue_url) + return ( + parsed.scheme == "https" + and parsed.netloc.lower() == "github.com" + and (parsed.path.strip("/").split("/issues/", 1)[0].lower() == repository.lower()) + ) + + +def _remote_matches_repository(remote_url: Any, repository: str) -> bool: + if not isinstance(remote_url, str): + return False + https = re.fullmatch(r"https://github\.com/([^/]+/[^/]+?)(?:\.git)?/?", remote_url) + ssh = re.fullmatch(r"git@github\.com:([^/]+/[^/]+)\.git", remote_url) + match = https or ssh + return bool(match and match.group(1).lower() == repository.lower()) + + +def _result_object(value: Any) -> dict[str, Any] | None: + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError: + return None + if isinstance(value, dict): + if any( + key in value for key in ("success", "claimed", "merge_target", "error", "error_type") + ): + return value + nested = value.get("structuredContent", value.get("result", value.get("content"))) + if isinstance(nested, list): + return _result_text_blocks(nested) + elif isinstance(nested, str): + return _result_object(nested) + if isinstance(value, list): + return _result_text_blocks(value) + return None + + +def _result_text_blocks(blocks: list[Any]) -> dict[str, Any] | None: + for block in blocks: + if not isinstance(block, dict) or not isinstance(block.get("text"), str): + continue + try: + decoded = json.loads(block["text"]) + except json.JSONDecodeError: + continue + if isinstance(decoded, dict): + return decoded + return None + + +def _transcript_tool_pairs( + transcript: Path, + violation_prefix: str, + violations: list[str], +) -> list[tuple[str, dict[str, Any], dict[str, Any] | None, bool]]: + """Return paired target calls as name, input, result, and result-error flag.""" + try: + handle = transcript.open(encoding="utf-8", errors="replace") + except OSError: + violations.append(f"transcript:{violation_prefix}:unavailable") + return [] + pending: dict[str, tuple[str, dict[str, Any]]] = {} + paired: list[tuple[str, dict[str, Any], dict[str, Any] | None, bool]] = [] + with handle: + for line_number, line in enumerate(handle, 1): + try: + row = json.loads(line) + except (json.JSONDecodeError, UnicodeDecodeError): + violations.append(f"transcript:{violation_prefix}:invalid_json:{line_number}") + continue + if isinstance(row, dict): + _pair_record(row, line_number, violation_prefix, pending, paired, violations) + for tool_id, (name, tool_input) in pending.items(): + violations.append(f"transcript:{violation_prefix}:unpaired_tool_use:{tool_id}") + paired.append((name, tool_input, None, False)) + return paired + + +def _pair_record( + row: dict[str, Any], + line_number: int, + violation_prefix: str, + pending: dict[str, tuple[str, dict[str, Any]]], + paired: list[tuple[str, dict[str, Any], dict[str, Any] | None, bool]], + violations: list[str], +) -> None: + message = row.get("message") + if not isinstance(message, dict): + message = row + content = message.get("content", row.get("content", [])) + if not isinstance(content, list): + return + role = row.get("type", message.get("role")) + for block in content: + if not isinstance(block, dict): + continue + if role == "assistant" and block.get("type") == "tool_use": + _remember_tool_use(block, line_number, violation_prefix, pending, violations) + elif role == "user" and block.get("type") == "tool_result": + tool_id = block.get("tool_use_id") + if isinstance(tool_id, str) and tool_id in pending: + name, tool_input = pending.pop(tool_id) + result = _result_object(block.get("content")) + paired.append((name, tool_input, result, block.get("is_error") is True)) + + +def _remember_tool_use( + block: dict[str, Any], + line_number: int, + prefix: str, + pending: dict[str, tuple[str, dict[str, Any]]], + violations: list[str], +) -> None: + name = block.get("name") + name = name.removeprefix("mcp__autoskillit__") if isinstance(name, str) else name + if name not in {"claim_and_resolve_issue", "create_and_publish_branch"}: + return + tool_id, tool_input = block.get("id"), block.get("input") + if not isinstance(tool_id, str) or not tool_id or not isinstance(tool_input, dict): + violations.append(f"transcript:{prefix}:malformed_tool_use:{line_number}") + return + if tool_id in pending: + violations.append(f"transcript:{prefix}:duplicate_tool_use:{line_number}") + pending[tool_id] = (name, tool_input) + + +def collect_ownership( + trace_path: Path, + envelope: dict[str, Any] | None, + log_root: Path | None = None, + issue_url: str = "", + repository: str = "", + claim_label: str | None = None, +) -> dict[str, Any]: + """Collect compact branch and claim proofs from the owned transcript lineage.""" + context = _discover_owned(trace_path, envelope, log_root) + violations = list(context["violations"]) + root_id = context["root_id"] + expected_number = _issue_number(issue_url) + valid_issue_reference = bool( + expected_number and repository and _issue_matches_repository(issue_url, repository) + ) + if not valid_issue_reference: + violations.append("ownership:invalid_issue_reference") + + branches, claim_calls, transcript_evidence = _owned_tool_proofs( + context["owned"], + root_id, + valid_issue_reference, + expected_number, + issue_url, + repository, + violations, + ) + claimed = _claim_succeeded(claim_calls, expected_number, claim_label, violations) + return { + "branches": sorted(set(branches)), + "claimed": claimed, + "transcripts": sorted(transcript_evidence, key=lambda item: item["session_id"]), + "violations": sorted(set(violations)), + } + + +def _owned_tool_proofs( + owned: Mapping[str, Mapping[str, Any]], + root_id: str, + valid_issue_reference: bool, + expected_number: str, + issue_url: str, + repository: str, + violations: list[str], +) -> tuple[ + list[str], list[tuple[dict[str, Any], dict[str, Any] | None, bool]], list[dict[str, str]] +]: + branches: list[str] = [] + claims: list[tuple[dict[str, Any], dict[str, Any] | None, bool]] = [] + transcripts: list[dict[str, str]] = [] + for session_id, item in owned.items(): + transcript = item.get("transcript") + if not isinstance(transcript, Path) or not transcript.is_file(): + continue + transcripts.append({"session_id": session_id, "path": str(transcript)}) + for name, tool_input, result, is_error in _transcript_tool_pairs( + transcript, session_id, violations + ): + target = _branch_target( + session_id, + root_id, + name, + tool_input, + result, + is_error, + valid_issue_reference, + expected_number, + repository, + violations, + ) + if target is not None: + branches.append(target) + if name == "claim_and_resolve_issue" and valid_issue_reference: + if tool_input.get("issue_url") == issue_url: + claims.append((tool_input, result, is_error)) + return branches, claims, transcripts + + +def _branch_target( + session_id: str, + root_id: str, + name: str, + tool_input: dict[str, Any], + result: dict[str, Any] | None, + is_error: bool, + valid_issue_reference: bool, + expected_number: str, + repository: str, + violations: list[str], +) -> str | None: + if name != "create_and_publish_branch" or session_id != root_id or not valid_issue_reference: + return None + if tool_input.get("issue_number") != expected_number: + return None + if not _remote_matches_repository(tool_input.get("remote_url"), repository): + return None + if ( + is_error + or result is None + or "error" in result + or "error_type" in result + or not isinstance(result.get("merge_target"), str) + or not result["merge_target"].strip() + ): + violations.append("ownership:branch_result_failed") + return None + return result["merge_target"].strip() + + +def _claim_succeeded( + claims: list[tuple[dict[str, Any], dict[str, Any] | None, bool]], + expected_number: str, + claim_label: str | None, + violations: list[str], +) -> bool: + if len(claims) > 1: + violations.append("ownership:claim_reentry") + return False + if not claims: + return False + tool_input, result, is_error = claims[0] + requested_label = tool_input.get("label") + effective_label = claim_label is None or requested_label in (None, "", claim_label) + claimed = bool( + not is_error + and effective_label + and tool_input.get("allow_reentry") is not True + and result is not None + and result.get("success") is True + and result.get("claimed") is True + and str(result.get("issue_number", "")) == expected_number + and result.get("reentry") is not True + ) + if not claimed: + violations.append("ownership:claim_result_failed") + return claimed + + +def main(argv: list[str] | None = None) -> int: + args = sys.argv[1:] if argv is None else argv + if len(args) != 1: + print("usage: e2e_sessions.py ", file=sys.stderr) + return 2 + try: + raw = sys.stdin.buffer.read(_MAX_HOOK_INPUT_BYTES + 1) + if len(raw) > _MAX_HOOK_INPUT_BYTES: + raise ValueError("lifecycle hook input exceeds the size limit") + payload = json.loads(raw) + if not isinstance(payload, dict): + raise ValueError("lifecycle hook input must be an object") + capture_hook_event(Path(args[0]), payload) + except (OSError, ValueError, json.JSONDecodeError, UnicodeDecodeError) as exc: + print(f"e2e lifecycle capture failed: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index 121a2c9a86..cc2ea845d6 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -12,6 +12,7 @@ from dataclasses import dataclass, replace from pathlib import Path from types import SimpleNamespace +from typing import Any import pytest @@ -50,9 +51,13 @@ "message": "model reported failure", "issue": "https://github.com/TalonT-Org/AutoSkillit/issues/5232", } +_IMPLEMENTATION_ISSUE = "https://github.com/owner/sandbox/issues/5233" +_IMPLEMENTATION_EXPECTED_ISSUE = "https://github.com/TalonT-Org/AutoSkillit/issues/5233" +_PIPELINE_HEAD = "0123456789abcdef" +_PIPELINE_BRANCH = "codex/issue-5233" -@dataclass +@dataclass(eq=False) class RecordedCall: argv: list[str] cwd: Path | None @@ -86,6 +91,39 @@ def _completed(argv, stdout: str = "", returncode: int = 0, stderr: str = ""): return subprocess.CompletedProcess(list(argv), returncode, stdout, stderr) +def _recorded_calls( + calls: list[RecordedCall], + *, + prefix: list[str] | None = None, + exact: list[str] | None = None, + contains: str | None = None, +) -> list[RecordedCall]: + return [ + call + for call in calls + if (prefix is None or call.argv[: len(prefix)] == prefix) + and (exact is None or call.argv == exact) + and (contains is None or contains in " ".join(call.argv)) + ] + + +def _recorded_pr_list_calls(calls: list[RecordedCall], *, baseline: bool) -> list[RecordedCall]: + lists = _recorded_calls(calls, prefix=["gh", "pr", "list"]) + selected = [] + for call in lists: + is_baseline = call.argv[call.argv.index("--limit") + 1] == "1" + if (baseline and is_baseline) or (not baseline and not is_baseline): + selected.append(call) + return selected + + +def _recorded_pr_detail_calls(calls: list[RecordedCall], number: int) -> list[RecordedCall]: + suffix = f"/pulls/{number}" + return [ + call for call in calls if call.argv[:2] == ["gh", "api"] and call.argv[2].endswith(suffix) + ] + + def _pr(number: int, state: str, additions: int = 3, deletions: int = 1) -> dict: return {"number": number, "state": state, "additions": additions, "deletions": deletions} @@ -144,6 +182,273 @@ def _env(**overrides: str) -> dict[str, str]: return {name: value for name, value in env.items() if value} +def _pipeline_catalog(expected_failures: list[dict[str, str]] | None = None): + entry = { + "name": "implementation", + "kind": "recipe", + "pipeline": True, + "peak_sessions": 6, + "timeout_sec": 600, + "trigger_paths": ["src/*"], + "recipe": "implementation", + "ingredients": {"source_dir": "/workspace/sandbox", "open_pr": "true"}, + "expected_pull_request_state": "open", + "issue_url_env": "E2E_IMPLEMENTATION_ISSUE_URL", + "required_changed_paths": ["sandbox/", "tests/"], + "test_command": ["task", "test-check"], + "peak_sessions_evidence": "recorded session events", + } + if expected_failures is not None: + entry["expected_failures"] = expected_failures + return e2e_catalog.parse_catalog({"sandbox_repository": SANDBOX, "tests": [entry]}) + + +def _pipeline_pr(number: int = 8, *, branch: str = _PIPELINE_BRANCH) -> dict[str, Any]: + return { + "number": number, + "state": "open", + "merged": False, + "merged_at": None, + "auto_merge": None, + "body": f"Implements {_IMPLEMENTATION_ISSUE}", + "additions": 5, + "deletions": 2, + "head": { + "ref": branch, + "sha": _PIPELINE_HEAD, + "repo": {"full_name": SANDBOX}, + }, + "base": {"ref": "main", "repo": {"full_name": SANDBOX}}, + } + + +def _pipeline_setup_response(argv: list[str], state: dict[str, Any]): + if argv[:3] == ["autoskillit", "config", "show"]: + return _completed(argv, json.dumps(state["config"])) + if argv[:3] == ["gh", "issue", "view"]: + payload = ( + state["seed"] if "number,url,state,title,body,labels" in argv else state["final_issue"] + ) + return _completed(argv, json.dumps(payload)) + if argv[:3] == ["gh", "repo", "view"]: + payload = {"defaultBranchRef": {"name": state["default_branch"]}} + return _completed(argv, json.dumps(payload)) + if argv[:3] == ["gh", "pr", "list"]: + rows = [{"number": 7}] if argv[argv.index("--limit") + 1] == "1" else state["listed_prs"] + return _completed(argv, json.dumps(rows)) + return None + + +def _pipeline_execution_response(argv: list[str], state: dict[str, Any]): + if argv[:3] == ["autoskillit", "fleet", "run"]: + if state["fleet_exception"] is not None: + raise state["fleet_exception"] + return _completed(argv, json.dumps(state["fleet_envelope"]) + "\n") + if argv[:2] == ["task", "test-check"]: + state["test_check_count"] += 1 + if state["test_check_count"] > 1 and state["pr_test_returncode"]: + return _completed(argv, returncode=state["pr_test_returncode"], stderr="tests failed") + return None + if argv[:2] == ["git", "rev-parse"]: + return _completed(argv, state["fetch_oid"] + "\n") + return None + + +def _pipeline_ci_api_response(argv: list[str], endpoint: str, state: dict[str, Any]): + if endpoint.endswith("/actions/workflows?per_page=100"): + workflows = [{"id": 1, "name": "CI", "state": "active"}] + return _completed(argv, json.dumps([{"workflows": workflows}])) + if "/actions/workflows/1/runs?" in endpoint: + run = { + "id": 2, + "status": "completed", + "conclusion": state["workflow_conclusion"], + "pull_requests": [ + { + "number": 8, + "head": {"sha": _PIPELINE_HEAD, "repo": {"full_name": SANDBOX}}, + "base": {"repo": {"full_name": SANDBOX}}, + } + ], + } + return _completed(argv, json.dumps([{"workflow_runs": [run]}])) + if endpoint.endswith("/actions/runs/2/jobs?per_page=100"): + job = {"name": "test", "status": "completed", "conclusion": state["job_conclusion"]} + return _completed(argv, json.dumps([{"jobs": [job]}])) + return None + + +def _pipeline_pull_request_api_response(argv: list[str], endpoint: str, state: dict[str, Any]): + if "/pulls/" not in endpoint: + return None + if endpoint.endswith("/files?per_page=100"): + return _completed(argv, json.dumps([state["files"]])) + number = int(endpoint.split("/pulls/", 1)[1].split("/", 1)[0]) + return _completed(argv, json.dumps(state["pr_details"][number])) + + +def _pipeline_cleanup_api_response(argv: list[str], endpoint: str, state: dict[str, Any]): + if argv[2] == "--method": + error = state["cleanup_errors"].get("branch_delete") + if error is not None: + raise error + return None + if "/git/refs/heads/" in endpoint: + error = state["cleanup_errors"].get("branch_get") + if error is not None: + raise error + return _completed(argv, "{}") + return None + + +def _pipeline_api_response(argv: list[str], state: dict[str, Any]): + if argv[:2] != ["gh", "api"]: + return None + endpoint = argv[4] if argv[2] == "--method" else argv[2] + for route in ( + _pipeline_ci_api_response, + _pipeline_pull_request_api_response, + _pipeline_cleanup_api_response, + ): + response = route(argv, endpoint, state) + if response is not None: + return response + return None + + +def _pipeline_cleanup_response(argv: list[str], state: dict[str, Any]): + if argv[:3] == ["gh", "pr", "close"]: + error = state["cleanup_errors"].get("pr_close") + if error is not None: + raise error + return None + + +def _pipeline_response(argv: list[str], state: dict[str, Any]): + for route in ( + _pipeline_setup_response, + _pipeline_execution_response, + _pipeline_api_response, + _pipeline_cleanup_response, + ): + response = route(argv, state) + if response is not None: + return response + return None + + +def _pipeline_runner() -> tuple[FakeRunner, dict[str, Any]]: + seed = { + "number": 5233, + "url": _IMPLEMENTATION_ISSUE, + "state": "OPEN", + "title": "Add a greeting", + "body": "Implement the requested behavior.", + "labels": [], + } + listed = { + "number": 8, + "state": "OPEN", + "additions": 5, + "deletions": 2, + "body": f"Implements {_IMPLEMENTATION_ISSUE}", + "headRefOid": _PIPELINE_HEAD, + "headRefName": _PIPELINE_BRANCH, + "headRepository": {"nameWithOwner": SANDBOX}, + "baseRefName": "main", + "autoMergeRequest": None, + "mergedAt": None, + } + state: dict[str, Any] = { + "seed": seed, + "default_branch": "main", + "config": { + "github": {"in_progress_label": "in-progress"}, + "branching": {"default_base_branch": "main"}, + }, + "listed_prs": [listed], + "pr_details": {8: _pipeline_pr()}, + "files": [ + {"filename": "sandbox/worker.py", "status": "added", "additions": 3, "deletions": 0}, + { + "filename": "tests/test_worker.py", + "status": "modified", + "additions": 2, + "deletions": 1, + }, + ], + "final_issue": {"state": "OPEN", "labels": [{"name": "in-progress"}]}, + "fetch_oid": _PIPELINE_HEAD, + "test_check_count": 0, + "pr_test_returncode": 0, + "workflow_conclusion": "success", + "job_conclusion": "success", + "fleet_envelope": { + "success": True, + "dispatch_status": "success", + "dispatch_id": "dispatch-5233", + "dispatched_session_id": "session-5233", + "l3_payload": {"reason": "implementation_complete"}, + }, + "fleet_exception": None, + "cleanup_errors": {}, + "proof": {"branches": [_PIPELINE_BRANCH], "claimed": True, "violations": []}, + "summary": {"coverage": {"complete": True}, "peak_sessions": 4, "violations": []}, + } + + return FakeRunner(lambda argv: _pipeline_response(argv, state)), state + + +def _patch_pipeline_measurement(monkeypatch, state: dict[str, Any]): + observed: dict[str, list[tuple]] = {"ownership": [], "summaries": []} + + def collect(trace_path, envelope, log_root, issue_url, repository, *, claim_label=None): + observed["ownership"].append( + (trace_path, envelope, log_root, issue_url, repository, claim_label) + ) + return state["proof"] + + def summarize(trace_path, envelope, log_root): + observed["summaries"].append((trace_path, envelope, log_root)) + return state["summary"] + + monkeypatch.setattr(harness.e2e_sessions, "collect_ownership", collect) + monkeypatch.setattr(harness.e2e_sessions, "summarize", summarize) + monkeypatch.setattr(harness.time, "sleep", lambda _seconds: None) + return observed + + +def _run_pipeline( + tmp_path: Path, + runner: FakeRunner, + state: dict[str, Any], + monkeypatch, + *, + env=None, + expected_failures=None, +): + catalog = _pipeline_catalog(expected_failures) + out = tmp_path / "out" + home = tmp_path / "home" + observed = _patch_pipeline_measurement(monkeypatch, state) + failures = harness.run_test( + catalog.get("implementation"), + catalog, + out=out, + home=home, + env=env if env is not None else _env(E2E_IMPLEMENTATION_ISSUE_URL=_IMPLEMENTATION_ISSUE), + runner=runner, + ) + return ( + failures, + json.loads((out / "result.json").read_text(encoding="utf-8")), + out, + home, + catalog, + observed, + ) + + def _run_recipe(tmp_path: Path, runner: FakeRunner, *, state: str = "open", env=None): catalog = _recipe_catalog(state) out = tmp_path / "out" @@ -1470,11 +1775,15 @@ def reject_login(argv: list[str]): assert ("autoskillit", "fleet", "run") not in prefixes def test_fleet_timeout_is_a_failure_and_cleanup_still_runs(self, tmp_path: Path) -> None: - timeout = subprocess.TimeoutExpired(["autoskillit"], 720) + timeout = subprocess.TimeoutExpired( + ["autoskillit"], 720, output=b"partial stdout", stderr=b"partial stderr" + ) runner = FakeRunner(_recipe_handler(fleet=timeout)) failures, result = _run_recipe(tmp_path, runner) assert result["passed"] is False assert any("fleet run: timed out" in failure for failure in failures) + assert (tmp_path / "out" / "fleet-run.stdout.log").read_text() == "partial stdout" + assert (tmp_path / "out" / "fleet-run.stderr.log").read_text() == "partial stderr" assert runner.calls[-1].argv[:4] == ["gh", "pr", "close", "8"] def test_unexpected_fleet_error_records_failure_and_still_cleans_up( @@ -1550,6 +1859,410 @@ def test_missing_sandbox_token_fails_before_any_launch(self, tmp_path: Path) -> assert runner.calls == [] +class TestPipelineRecipeFlow: + def test_pipeline_uses_the_seed_issue_and_cleans_only_its_owned_pr( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + unrelated = {**state["listed_prs"][0], "number": 9, "headRefName": "other/branch"} + state["listed_prs"].append(unrelated) + state["pr_details"][9] = _pipeline_pr(9, branch="other/branch") + failures, result, out, home, catalog, observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert failures == [] + assert result == { + "test": "implementation", + "passed": True, + "failures": [], + "outcome": "passed", + "expected_findings": [], + } + test = catalog.get("implementation") + assert test.pipeline is True + assert test.issue_url_env == "E2E_IMPLEMENTATION_ISSUE_URL" + assert test.required_changed_paths == ("sandbox/", "tests/") + assert test.test_command == ("task", "test-check") + + calls = runner.calls + config_call = _recorded_calls(calls, prefix=["autoskillit", "config", "show"])[0] + assert config_call.cwd == harness.SANDBOX_CLONE + assert config_call.env == harness.child_env( + _env(E2E_IMPLEMENTATION_ISSUE_URL=_IMPLEMENTATION_ISSUE) + ) + seed_call = _recorded_calls(calls, prefix=["gh", "issue", "view"])[0] + repo_call = _recorded_calls(calls, prefix=["gh", "repo", "view"])[0] + baseline_install = _recorded_calls(calls, exact=["task", "install-worktree"])[0] + checks = _recorded_calls(calls, exact=["task", "test-check"]) + baseline_list = _recorded_pr_list_calls(calls, baseline=True)[0] + fleet_call = _recorded_calls(calls, prefix=["autoskillit", "fleet", "run"])[0] + assert ( + calls.index(config_call) + < calls.index(seed_call) + < calls.index(repo_call) + < calls.index(baseline_install) + < calls.index(checks[0]) + < calls.index(baseline_list) + < calls.index(fleet_call) + ) + assert baseline_install.cwd == checks[0].cwd == harness.SANDBOX_CLONE + assert len(checks) == 2 and checks[1].cwd == harness.SANDBOX_CLONE + + listed_calls = _recorded_pr_list_calls(calls, baseline=False) + file_call = _recorded_calls(calls, contains="/files?per_page=100")[0] + fetch_call = _recorded_calls(calls, prefix=["git", "fetch"])[0] + oid_call = _recorded_calls(calls, prefix=["git", "rev-parse"])[0] + checkout_call = _recorded_calls(calls, prefix=["git", "checkout", "--detach"])[0] + install_calls = _recorded_calls(calls, exact=["task", "install-worktree"]) + workflow_call = _recorded_calls( + calls, prefix=["gh", "api"], contains="/actions/workflows?per_page=100" + )[0] + runs_call = _recorded_calls(calls, contains="/actions/workflows/1/runs?")[0] + jobs_call = _recorded_calls(calls, contains="/actions/runs/2/jobs?")[0] + pr_reads = _recorded_pr_detail_calls(calls, 8) + close = _recorded_calls(calls, prefix=["gh", "pr", "close"])[0] + branch_delete = _recorded_calls(calls, prefix=["gh", "api", "--method", "DELETE"])[0] + final_issue = _recorded_calls( + calls, prefix=["gh", "issue", "view"], contains="state,labels" + )[-1] + issue_edit = _recorded_calls(calls, prefix=["gh", "issue", "edit"])[0] + assert len(listed_calls) == 2 and len(install_calls) == 2 and len(pr_reads) == 3 + assert ( + calls.index(fleet_call) + < calls.index(listed_calls[0]) + < calls.index(file_call) + < calls.index(fetch_call) + < calls.index(oid_call) + < calls.index(checkout_call) + < calls.index(install_calls[1]) + < calls.index(checks[1]) + < calls.index(workflow_call) + < calls.index(runs_call) + < calls.index(jobs_call) + < calls.index(pr_reads[1]) + < calls.index(listed_calls[1]) + < calls.index(close) + < calls.index(branch_delete) + < calls.index(final_issue) + < calls.index(issue_edit) + ) + + ingredient_args = [ + fleet_call.argv[index + 1] + for index, arg in enumerate(fleet_call.argv[:-1]) + if arg == "-i" + ] + assert ingredient_args.count(f"issue_url={_IMPLEMENTATION_ISSUE}") == 1 + assert ( + ingredient_args.count("task=Add a greeting\n\nImplement the requested behavior.") == 1 + ) + ingredient_names = {key for key, _value in test.ingredients} + assert "issue_url" not in ingredient_names and "task" not in ingredient_names + assert fleet_call.timeout == test.timeout_sec + e2e_catalog.HARNESS_GRACE_SEC + assert fleet_call.cwd == harness.SANDBOX_CLONE + assert [call.input for call in calls if call.input] == [SANDBOX_TOKEN] + assert all(SANDBOX_TOKEN not in " ".join(call.argv) for call in calls) + assert all( + call.env == harness.child_env(_env(E2E_IMPLEMENTATION_ISSUE_URL=_IMPLEMENTATION_ISSUE)) + for call in calls + ) + + trace = out / "session-events.jsonl" + log_root = home / ".local" / "share" / "autoskillit" / "logs" + envelope = json.loads((out / "envelope.json").read_text(encoding="utf-8")) + assert observed["ownership"] == [ + (trace, envelope, log_root, _IMPLEMENTATION_ISSUE, SANDBOX, "in-progress") + ] + assert observed["summaries"] == [(trace, envelope, log_root)] + settings = json.loads((home / ".claude" / "settings.json").read_text(encoding="utf-8")) + assert settings["env"] == _EXPECTED_SETTINGS_ENV + assert all( + len(settings["hooks"][event]) == 1 + for event in ("SessionStart", "SessionEnd", "SubagentStart", "SubagentStop") + ) + + close = next(call for call in calls if call.argv[:3] == ["gh", "pr", "close"]) + assert close.argv == ["gh", "pr", "close", "8", "--repo", SANDBOX] + assert not any(call.argv[:4] == ["gh", "pr", "close", "9"] for call in calls) + assert any( + call.argv[:4] == ["gh", "api", "--method", "DELETE"] + and call.argv[4].endswith("codex%2Fissue-5233") + for call in calls + ) + assert any( + call.argv[:3] == ["gh", "issue", "edit"] + and call.argv[-2:] == ["--remove-label", "in-progress"] + for call in calls + ) + + @pytest.mark.parametrize( + ("problem", "expected"), + [ + ("malformed", "seed: expected the configured OPEN sandbox issue"), + ("closed", "seed: expected the configured OPEN sandbox issue"), + ("claimed", "seed: issue is already claimed"), + ], + ) + def test_invalid_seed_stops_before_baseline_or_fleet( + self, tmp_path: Path, monkeypatch, problem: str, expected: str + ) -> None: + runner, state = _pipeline_runner() + if problem == "malformed": + state["seed"].pop("title") + elif problem == "closed": + state["seed"]["state"] = "CLOSED" + else: + state["seed"]["labels"] = [{"name": "in-progress"}] + + failures, result, _out, _home, _catalog, observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert any(expected in failure for failure in failures) + assert result["outcome"] == "failed" + assert not any(call.argv[:2] == ["task", "install-worktree"] for call in runner.calls) + assert not any(call.argv[:2] == ["autoskillit", "fleet"] for call in runner.calls) + assert not any(call.argv[:3] == ["gh", "pr", "list"] for call in runner.calls) + assert observed["ownership"] == observed["summaries"] == [] + + def test_missing_seed_url_stops_before_github_issue_lookup( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, + runner, + state, + monkeypatch, + env=_env(E2E_IMPLEMENTATION_ISSUE_URL=""), + ) + + assert failures == ["preflight: seed: missing or noncanonical sandbox issue URL"] + assert result["outcome"] == "failed" + assert not any(call.argv[:3] == ["gh", "issue", "view"] for call in runner.calls) + assert not any(call.argv[:3] == ["autoskillit", "fleet", "run"] for call in runner.calls) + + @pytest.mark.parametrize("wrong_base", ["sandbox", "effective-config"]) + def test_seed_requires_the_main_branch_in_repository_and_config( + self, tmp_path: Path, monkeypatch, wrong_base: str + ) -> None: + runner, state = _pipeline_runner() + if wrong_base == "sandbox": + state["default_branch"] = "develop" + else: + state["config"]["branching"]["default_base_branch"] = "trunk" + + failures, _result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert any( + "sandbox and effective configured base branch must both be main" in item + for item in failures + ) + assert not any(call.argv[:2] == ["task", "install-worktree"] for call in runner.calls) + assert not any(call.argv[:3] == ["autoskillit", "fleet", "run"] for call in runner.calls) + + def test_linked_stable_finding_is_recorded_and_skips_pr_assertions( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["fleet_envelope"] = { + "success": False, + "dispatch_status": "retry_exhausted", + "dispatch_id": "dispatch-5233", + "dispatched_session_id": "session-5233", + "reason": "known orchestration failure", + } + state["listed_prs"] = [] + state["pr_details"] = {} + state["final_issue"] = {"state": "OPEN", "labels": []} + state["proof"] = {"branches": [], "claimed": False, "violations": []} + expected = { + "check": "fleet_envelope", + "message": "retry_exhausted: known orchestration failure", + "issue": _IMPLEMENTATION_EXPECTED_ISSUE, + } + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch, expected_failures=[expected] + ) + + assert failures == [] + assert result["outcome"] == "expected_failure" + assert result["passed"] is False + assert result["expected_findings"] == [expected] + assert not any(call.argv[:2] == ["git", "fetch"] for call in runner.calls) + assert not any( + "/pulls/" in " ".join(call.argv) and call.argv[0:2] == ["gh", "api"] + for call in runner.calls + ) + assert not any("actions/workflows" in " ".join(call.argv) for call in runner.calls) + + @pytest.mark.parametrize("envelope_kind", ["stale", "fatal"]) + def test_stale_or_fatal_pipeline_failure_cannot_be_allowlisted( + self, tmp_path: Path, monkeypatch, envelope_kind: str + ) -> None: + runner, state = _pipeline_runner() + expected = { + "check": "fleet_envelope", + "message": "retry_exhausted: known orchestration failure", + "issue": _IMPLEMENTATION_EXPECTED_ISSUE, + } + state["listed_prs"] = [] + state["pr_details"] = {} + state["final_issue"] = {"state": "OPEN", "labels": []} + state["proof"] = {"branches": [], "claimed": False, "violations": []} + if envelope_kind == "fatal": + state["fleet_envelope"] = { + "success": False, + "dispatch_status": "failed", + "dispatch_id": "dispatch-5233", + "dispatched_session_id": "session-5233", + "l3_payload": {}, + } + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch, expected_failures=[expected] + ) + + assert result["outcome"] == "failed" + assert result["expected_findings"] == [] + if envelope_kind == "stale": + assert any("expected recipe finding absent" in failure for failure in failures) + else: + assert any( + "unsuccessful envelope has no specific failure reason" in f for f in failures + ) + assert any("expected recipe finding absent" in failure for failure in failures) + assert not any(call.argv[:2] == ["git", "fetch"] for call in runner.calls) + + @pytest.mark.parametrize( + ("problem", "expected"), + [ + ("pr", "implementation PR: expected an open, unmerged sandbox PR"), + ("body", "implementation PR: body does not reference the canonical seed URL"), + ("paths", "implementation PR: missing non-removed changes under tests/"), + ("ci", "sandbox CI: CI run and test job must succeed"), + ("head", "implementation PR: fetched head differs from captured head"), + ("ci-job", "sandbox CI: CI run and test job must succeed"), + ("test", "pr-test: exit 1: tests failed"), + ], + ) + def test_pipeline_rejects_unverified_pr_or_validation_failure( + self, tmp_path: Path, monkeypatch, problem: str, expected: str + ) -> None: + runner, state = _pipeline_runner() + pr = state["pr_details"][8] + if problem == "pr": + pr["base"]["ref"] = "develop" + elif problem == "body": + pr["body"] = "No linked issue" + elif problem == "paths": + state["files"] = state["files"][:1] + elif problem == "ci": + state["workflow_conclusion"] = "failure" + elif problem == "ci-job": + state["job_conclusion"] = "failure" + elif problem == "head": + state["fetch_oid"] = "fetched-a-different-head" + else: + state["pr_test_returncode"] = 1 + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert any(expected in failure for failure in failures) + + def test_listed_pr_head_must_match_rest_head_before_validation( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["listed_prs"][0]["headRefOid"] = "stale-listed-head" + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert any("head" in failure.lower() or "oid" in failure.lower() for failure in failures) + assert not any("/files?" in " ".join(call.argv) for call in runner.calls) + assert not any("actions/workflows" in " ".join(call.argv) for call in runner.calls) + + @pytest.mark.parametrize( + ("problem", "expected"), + [ + ("coverage", "session measurement: incomplete lifecycle coverage"), + ("capacity", "session measurement: observed peak exceeds catalog capacity"), + ], + ) + def test_session_evidence_must_be_complete_and_within_catalog_capacity( + self, tmp_path: Path, monkeypatch, problem: str, expected: str + ) -> None: + runner, state = _pipeline_runner() + if problem == "coverage": + state["summary"]["coverage"]["complete"] = False + else: + state["summary"]["peak_sessions"] = 7 + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert any(expected in failure for failure in failures) + + def test_ownership_violations_fail_the_pipeline_result( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["proof"]["violations"] = ["transcript evidence is incomplete"] + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert "ownership: transcript evidence is incomplete" in failures + + def test_cleanup_errors_do_not_prevent_remaining_owned_cleanup( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["cleanup_errors"] = { + "pr_close": RuntimeError("close failed"), + "branch_delete": RuntimeError("branch delete failed"), + } + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert any("cleanup PR: close failed" in failure for failure in failures) + assert any("cleanup branch: branch delete failed" in failure for failure in failures) + assert any(call.argv[:3] == ["gh", "issue", "edit"] for call in runner.calls) + + def test_fleet_exception_and_cleanup_exception_are_both_preserved( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["fleet_exception"] = RuntimeError("fleet launch crashed") + state["cleanup_errors"] = {"branch_delete": RuntimeError("branch delete failed")} + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert any("harness error: fleet launch crashed" in failure for failure in failures) + assert any("cleanup branch: branch delete failed" in failure for failure in failures) + assert any(call.argv[:3] == ["gh", "issue", "edit"] for call in runner.calls) + + class TestCanaryFlow: def test_canary_runs_one_claude_prompt(self, tmp_path: Path, monkeypatch) -> None: runner = FakeRunner(lambda argv: _completed(argv, _CANARY_REPLY)) diff --git a/tests/infra/test_e2e_select.py b/tests/infra/test_e2e_select.py index ecec41bf95..1c2610d36b 100644 --- a/tests/infra/test_e2e_select.py +++ b/tests/infra/test_e2e_select.py @@ -23,6 +23,13 @@ REPOSITORY = "TalonT-Org/AutoSkillit" SHA_ZERO = "0" * 40 SYNTHETIC_SHAS = [hashlib.sha1(str(index).encode()).hexdigest() for index in range(2000)] +_IMPLEMENTATION_PATHS = ( + "src/autoskillit/recipes/", + "src/autoskillit/skills/", + "src/autoskillit/skills_extended/", + "src/autoskillit/server/", + "src/autoskillit/execution/", +) def _facts( @@ -188,6 +195,44 @@ def test_deleted_head_repository_is_not_selected(self) -> None: assert pr.head_repository is None assert not selection.selected + @pytest.mark.parametrize( + ("changed_files", "additions", "deletions"), + [(10, 0, 0), (1, 300, 0), (1, 100, 200)], + ) + def test_large_change_with_matching_root_selects_pipeline( + self, changed_files: int, additions: int, deletions: int + ) -> None: + pr = _facts( + head_sha=_sha_with_sample(5230, sampled=True), + changed_files=changed_files, + additions=additions, + deletions=deletions, + ) + selection = select.select_for_pull_request( + pr, + ["src/autoskillit/recipes/changed.py"], + _catalog(_pipeline_entry()), + REPOSITORY, + ) + assert selection.tests == ("implementation",) + + def test_label_does_not_make_a_small_change_pipeline_eligible(self) -> None: + pr = _facts(changed_files=1, additions=10, labels=("e2e",)) + selection = select.select_for_pull_request( + pr, + ["src/autoskillit/recipes/changed.py"], + _catalog(_pipeline_entry()), + REPOSITORY, + ) + assert not selection.selected + + def test_pipeline_is_not_an_unrelated_fallback(self) -> None: + pr = _facts(head_sha=_sha_with_sample(5230, sampled=True), changed_files=10) + selection = select.select_for_pull_request( + pr, ["README.md"], _catalog(_pipeline_entry()), REPOSITORY + ) + assert not selection.selected + class TestPickTests: _SEED = f"5230:{SHA_ZERO}" @@ -228,6 +273,25 @@ def test_star_matches_across_directories(self) -> None: catalog = _catalog(_entry("deep", ("src/*",)), _entry("other", ("docs/*",))) assert select.pick_tests(catalog, ["src/a/b/c.py"], self._SEED) == ("deep",) + @pytest.mark.parametrize("root", _IMPLEMENTATION_PATHS) + def test_pipeline_trigger_roots_match(self, root: str) -> None: + assert select.pick_tests( + _catalog(_pipeline_entry()), + [f"{root}changed.py"], + self._SEED, + large_change=True, + ) == ("implementation",) + + def test_at_most_one_pipeline_is_selected(self) -> None: + catalog = _catalog( + _pipeline_entry("pipeline-a", trigger_paths=["src/*"]), + _pipeline_entry("pipeline-b", trigger_paths=["src/*"]), + _entry("ordinary", ("src/*",)), + ) + picked = select.pick_tests(catalog, ["src/changed.py"], self._SEED, large_change=True) + assert len(picked) == select.MAX_TESTS + assert sum(catalog.get(name).pipeline for name in picked) == 1 + class TestSelectForEvent: def test_merge_group_is_never_selected(self) -> None: @@ -301,6 +365,18 @@ def test_matrix_includes_kind_from_each_catalog_entry() -> None: "ingredients": {"task": "Add a greeting"}, "expected_pull_request_state": "merged", } +_RECIPE_EXPECTED_FAILURE = { + "check": "implementation-check", + "message": "expected implementation failure", + "issue": "https://github.com/TalonT-Org/AutoSkillit/issues/123", +} +_IMPLEMENTATION_INGREDIENTS = { + "source_dir": "/workspace/sandbox", + "open_pr": "true", + "auto_merge": "false", + "arch_lenses": "false", + "audit_impl": "true", +} _MAX_TIMEOUT_SEC = ( e2e_catalog.MAX_JOB_MINUTES - e2e_catalog.JOB_OVERHEAD_MINUTES ) * 60 - e2e_catalog.HARNESS_GRACE_SEC @@ -311,6 +387,28 @@ def _recipe_entry_without(field: str) -> dict: return _entry("impl", kind="recipe", **fields) +def _pipeline_entry(name: str = "implementation", **overrides) -> dict: + entry = _entry( + name, + tuple(f"{path}*" for path in _IMPLEMENTATION_PATHS), + kind="recipe", + peak_sessions=8, + timeout_sec=7200, + recipe="implementation", + ingredients=dict(_IMPLEMENTATION_INGREDIENTS), + expected_pull_request_state="open", + pipeline=True, + issue_url_env="E2E_IMPLEMENTATION_ISSUE_URL", + required_changed_paths=list(_IMPLEMENTATION_PATHS), + test_command=["task", "test-check"], + peak_sessions_evidence=( + "configured capacity limit of eight concurrent sessions; no measured peak supplied" + ), + ) + entry.update(overrides) + return entry + + _INVALID_CATALOGS = { "duplicate-names": _raw_catalog(_entry("a"), _entry("a")), "bad-name": _raw_catalog(_entry("Bad_Name")), @@ -352,6 +450,9 @@ def _recipe_entry_without(field: str) -> dict: **_RECIPE_FIELDS, ) ), + "expected-failures-on-recipe": _raw_catalog( + _pipeline_entry(expected_failures=[_EXPECTED_FAILURE]) + ), "expected-failures-missing-field": _raw_catalog( _clean_install_entry( expected_failures=[ @@ -465,6 +566,7 @@ def test_recipe_entry_parses(self) -> None: "impl", kind="recipe", recipe_fixture="sandbox-smoke.yaml", + expected_failures=[{"severity": "error", **_RECIPE_EXPECTED_FAILURE}], **_RECIPE_FIELDS, ) ) @@ -473,6 +575,12 @@ def test_recipe_entry_parses(self) -> None: assert test.ingredients == (("task", "Add a greeting"),) assert test.expected_pull_request_state == "merged" assert test.recipe_fixture == "sandbox-smoke.yaml" + assert test.pipeline is False + assert test.issue_url_env is None + assert test.required_changed_paths == () + assert test.test_command == () + assert test.peak_sessions_evidence is None + assert test.expected_failures == ({"severity": "error", **_RECIPE_EXPECTED_FAILURE},) def test_recipe_without_fixture_defaults_to_none(self) -> None: test = _catalog(_entry("impl", kind="recipe", **_RECIPE_FIELDS)).get("impl") @@ -524,3 +632,111 @@ def test_unknown_test_name_raises_catalog_error(self) -> None: def test_invalid_catalog_raises(self, raw: dict) -> None: with pytest.raises(e2e_catalog.CatalogError): e2e_catalog.parse_catalog(raw) + + +class TestPipelineCatalog: + def test_shipped_implementation_targets_sandbox_source_and_tests_without_a_measurement( + self, + ) -> None: + test = e2e_catalog.load_catalog().get("implementation") + assert test.required_changed_paths == ("sandbox/", "tests/") + assert test.peak_sessions == 8 + assert test.peak_sessions_evidence is None + assert test.timeout_sec == 7200 + assert dict(test.ingredients) == _IMPLEMENTATION_INGREDIENTS + + def test_pipeline_recipe_parses_its_contract_and_expected_failures(self) -> None: + rows = [ + _RECIPE_EXPECTED_FAILURE, + {**_RECIPE_EXPECTED_FAILURE, "check": "another-check"}, + ] + test = _catalog(_pipeline_entry(expected_failures=rows)).get("implementation") + + assert test.pipeline is True + assert test.issue_url_env == "E2E_IMPLEMENTATION_ISSUE_URL" + assert test.required_changed_paths == _IMPLEMENTATION_PATHS + assert test.test_command == ("task", "test-check") + assert test.peak_sessions_evidence == ( + "configured capacity limit of eight concurrent sessions; no measured peak supplied" + ) + assert test.expected_failures == tuple(rows) + with pytest.raises(TypeError): + cast(MutableMapping[str, str], test.expected_failures[0])["check"] = "changed" + + def test_recipe_options_are_independently_optional(self) -> None: + entry = _pipeline_entry() + for key in ( + "issue_url_env", + "required_changed_paths", + "test_command", + "peak_sessions_evidence", + ): + entry.pop(key) + test = _catalog(entry).get("implementation") + assert test.pipeline is True + assert test.issue_url_env is None + assert test.required_changed_paths == () + assert test.test_command == () + assert test.peak_sessions_evidence is None + + def test_pipeline_rejects_unknown_keys(self) -> None: + with pytest.raises(e2e_catalog.CatalogError, match="unknown keys"): + _catalog(_pipeline_entry(unrecognized=True)) + + @pytest.mark.parametrize("ingredient", ["issue_url", "task"]) + def test_issue_url_env_conflicts_with_recipe_ingredient(self, ingredient: str) -> None: + ingredients = {**_IMPLEMENTATION_INGREDIENTS, ingredient: "value"} + with pytest.raises(e2e_catalog.CatalogError, match="cannot contain issue_url or task"): + _catalog(_pipeline_entry(ingredients=ingredients)) + + def test_recipe_options_do_not_require_pipeline_true(self) -> None: + entry = _pipeline_entry(pipeline=False) + assert _catalog(entry).get("implementation").pipeline is False + + @pytest.mark.parametrize("value", [None, "true", 1]) + def test_pipeline_flag_must_be_boolean(self, value: object) -> None: + with pytest.raises(e2e_catalog.CatalogError, match="pipeline must be a boolean"): + _catalog(_pipeline_entry(pipeline=value)) + + @pytest.mark.parametrize("value", ["", " ", 123]) + def test_issue_url_environment_name_must_be_nonempty(self, value: object) -> None: + with pytest.raises(e2e_catalog.CatalogError, match="issue_url_env"): + _catalog(_pipeline_entry(issue_url_env=value)) + + @pytest.mark.parametrize( + "prefix", + ["", " ", 123, "/absolute/path", "../parent", "src/../other", "C:\\absolute"], + ) + def test_required_changed_paths_must_be_relative_prefixes(self, prefix: object) -> None: + with pytest.raises(e2e_catalog.CatalogError): + _catalog(_pipeline_entry(required_changed_paths=[prefix])) + + @pytest.mark.parametrize("command", [[], [""], [1]]) + def test_pipeline_test_command_must_be_nonempty_argv(self, command: object) -> None: + with pytest.raises(e2e_catalog.CatalogError, match="test_command"): + _catalog(_pipeline_entry(test_command=command)) + + @pytest.mark.parametrize("evidence", ["", " ", 123]) + def test_pipeline_peak_sessions_evidence_must_be_nonempty(self, evidence: object) -> None: + with pytest.raises(e2e_catalog.CatalogError, match="peak_sessions_evidence"): + _catalog(_pipeline_entry(peak_sessions_evidence=evidence)) + + @pytest.mark.parametrize( + "row", + [ + {key: value for key, value in _RECIPE_EXPECTED_FAILURE.items() if key != "check"}, + {**_RECIPE_EXPECTED_FAILURE, "severity": "warning"}, + {**_RECIPE_EXPECTED_FAILURE, "issue": "https://github.com/other/repo/issues/123"}, + ], + ) + def test_recipe_expected_failures_have_exact_canonical_rows(self, row: dict) -> None: + with pytest.raises(e2e_catalog.CatalogError): + _catalog(_pipeline_entry(expected_failures=[row])) + + def test_recipe_expected_failures_reject_duplicate_check_and_message(self) -> None: + duplicate = { + **_RECIPE_EXPECTED_FAILURE, + "issue": "https://github.com/TalonT-Org/AutoSkillit/issues/124", + } + with pytest.raises(e2e_catalog.CatalogError, match="duplicate diagnostic"): + _catalog(_pipeline_entry(expected_failures=[_RECIPE_EXPECTED_FAILURE, duplicate])) diff --git a/tests/infra/test_e2e_select_cli.py b/tests/infra/test_e2e_select_cli.py index 25a7f2c1e2..0f0fb0f317 100644 --- a/tests/infra/test_e2e_select_cli.py +++ b/tests/infra/test_e2e_select_cli.py @@ -24,6 +24,9 @@ _HEADLESS_SMOKE_MATRIX = ( '{"include":[{"test":"headless-smoke","kind":"recipe","timeout_minutes":47}]}' ) +_IMPLEMENTATION_MATRIX = ( + '{"include":[{"test":"implementation","kind":"recipe","timeout_minutes":152}]}' +) def _run( @@ -112,6 +115,19 @@ def test_small_unlabelled_pull_request_selects_nothing(tmp_path: Path) -> None: assert outputs["matrix"] == '{"include":[]}' +def test_large_pull_request_selects_implementation_pipeline(tmp_path: Path) -> None: + changed = tmp_path / "changed-paths.txt" + changed.write_text("src/autoskillit/recipes/changed.py\n", encoding="utf-8") + payload = _pull_request(changed_files=10, additions=0, labels=("e2e",)) + + result = _select(tmp_path, changed, "pull_request", payload) + + assert result.returncode == 0, result.stderr + outputs = _outputs(result.stdout) + assert outputs["selected"] == "true" + assert outputs["matrix"] == _IMPLEMENTATION_MATRIX + + def test_merge_group_never_reads_the_changed_paths_file(tmp_path: Path) -> None: result = _select(tmp_path, tmp_path / "missing.txt", "merge_group", {"merge_group": {}}) assert result.returncode == 0, result.stderr @@ -151,6 +167,19 @@ def test_dispatch_selects_both_named_tests_in_request_order(tmp_path: Path) -> N ) +def test_dispatch_selects_named_pipeline_without_pull_request_eligibility(tmp_path: Path) -> None: + result = _select( + tmp_path, + tmp_path / "missing.txt", + "workflow_dispatch", + {"inputs": {"tests": "implementation"}}, + ) + assert result.returncode == 0, result.stderr + outputs = _outputs(result.stdout) + assert outputs["selected"] == "true" + assert outputs["matrix"] == _IMPLEMENTATION_MATRIX + + def test_dispatch_of_an_unknown_test_fails(tmp_path: Path) -> None: payload = {"inputs": {"tests": "nope"}} result = _select(tmp_path, tmp_path / "missing.txt", "workflow_dispatch", payload) diff --git a/tests/infra/test_e2e_sessions.py b/tests/infra/test_e2e_sessions.py new file mode 100644 index 0000000000..89ad47d9a9 --- /dev/null +++ b/tests/infra/test_e2e_sessions.py @@ -0,0 +1,580 @@ +"""Tests for the E2E session lifecycle and ownership evidence helper.""" + +from __future__ import annotations + +import json +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any + +import pytest + +from tests.infra._complexity_helpers import load_check_script + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] + +REPO_ROOT = Path(__file__).resolve().parents[2] +_SCRIPT = REPO_ROOT / "scripts" / "e2e" / "e2e_sessions.py" +sessions = load_check_script("_autoskillit_e2e_sessions", _SCRIPT) + +DISPATCH_ID = "dispatch-5233" +ROOT_ID = "session-root" +ISSUE_URL = "https://github.com/owner/repo/issues/5233" +REPOSITORY = "owner/repo" + + +def _jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + "".join(json.dumps(row, separators=(",", ":")) + "\n" for row in rows), + encoding="utf-8", + ) + + +def _session( + log_root: Path, + session_id: str, + *, + parent_session_id: str = "", + caller_session_id: str = "", + records: list[dict[str, Any]] | None = None, +) -> Path: + transcript = log_root / "transcripts" / f"{session_id}.jsonl" + _jsonl(transcript, records or []) + session_dir = log_root / "sessions" / session_id + session_dir.mkdir(parents=True, exist_ok=True) + (session_dir / "summary.json").write_text( + json.dumps( + { + "session_id": session_id, + "dispatch_id": DISPATCH_ID, + "execution_identity": {"parent_session_id": parent_session_id}, + } + ), + encoding="utf-8", + ) + return transcript + + +def _index(log_root: Path, entries: list[dict[str, Any]]) -> None: + _jsonl(log_root / "sessions.jsonl", entries) + + +def _entry( + session_id: str, + transcript: Path, + *, + parent_session_id: str = "", + caller_session_id: str = "", +) -> dict[str, Any]: + return { + "session_id": session_id, + "dir_name": session_id, + "dispatch_id": DISPATCH_ID, + "parent_session_id": parent_session_id, + "caller_session_id": caller_session_id, + "claude_code_log": str(transcript), + } + + +def _event( + name: str, + session_id: str, + timestamp: float, + *, + transcript: Path | None = None, + agent_id: str | None = None, + agent_transcript: Path | None = None, +) -> dict[str, Any]: + row: dict[str, Any] = { + "event": name, + "timestamp": timestamp, + "session_id": session_id, + } + if transcript is not None: + row["transcript_path"] = str(transcript) + if agent_id is not None: + row["agent_id"] = agent_id + if agent_transcript is not None: + row["agent_transcript_path"] = str(agent_transcript) + return row + + +def _trace(path: Path, events: list[dict[str, Any]]) -> None: + _jsonl(path, events) + + +def _envelope(session_id: str = ROOT_ID) -> dict[str, str]: + return {"dispatched_session_id": session_id, "dispatch_id": DISPATCH_ID} + + +def _base_sessions( + tmp_path: Path, + *, + child_id: str | None = None, + child_records: list[dict[str, Any]] | None = None, +) -> tuple[Path, Path, Path | None]: + log_root = tmp_path / "logs" + root_transcript = _session(log_root, ROOT_ID) + entries = [_entry(ROOT_ID, root_transcript)] + child_transcript = None + if child_id is not None: + child_transcript = _session( + log_root, + child_id, + parent_session_id=ROOT_ID, + records=child_records, + ) + entries.append( + _entry( + child_id, + child_transcript, + parent_session_id=ROOT_ID, + caller_session_id=ROOT_ID, + ) + ) + _index(log_root, entries) + trace = tmp_path / "run" / "session-lifecycle.jsonl" + return log_root, trace, child_transcript + + +def _tool_use(tool_id: str, name: str, tool_input: dict[str, Any]) -> dict[str, Any]: + return { + "type": "assistant", + "message": { + "content": [ + { + "type": "tool_use", + "id": tool_id, + "name": f"mcp__autoskillit__{name}", + "input": tool_input, + } + ] + }, + } + + +def _tool_result(tool_id: str, result: dict[str, Any]) -> dict[str, Any]: + return { + "type": "user", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": tool_id, + "content": json.dumps(result), + } + ] + }, + } + + +def test_hook_capture_appends_concurrent_allow_listed_records(tmp_path: Path) -> None: + trace = tmp_path / "events.jsonl" + with ThreadPoolExecutor(max_workers=8) as pool: + results = list( + pool.map( + lambda index: sessions.capture_hook_event( + trace, + { + "hook_event_name": "SessionStart", + "session_id": f"session-{index}", + "source": "startup", + "transcript_path": f"/tmp/{index}.jsonl", + "prompt": "must not be persisted", + "environment": {"TOKEN": "must not be persisted"}, + }, + ), + range(32), + ) + ) + + records = [json.loads(line) for line in trace.read_text(encoding="utf-8").splitlines()] + assert all(results) + assert len(records) == 32 + assert {record["session_id"] for record in records} == { + f"session-{index}" for index in range(32) + } + assert all(isinstance(record["timestamp"], (int, float)) for record in records) + assert all( + set(record) <= {"event", "timestamp", "session_id", "source", "transcript_path"} + for record in records + ) + assert ( + sessions.capture_hook_event(trace, {"event": "OtherHook", "session_id": "ignored"}) + is False + ) + assert len(trace.read_text(encoding="utf-8").splitlines()) == 32 + + +def test_summary_collapses_repeated_events_excludes_unrelated_and_sweeps_half_open( + tmp_path: Path, +) -> None: + log_root, trace, child_transcript = _base_sessions(tmp_path, child_id="session-child") + assert child_transcript is not None + unrelated = _session(log_root, "session-unrelated", parent_session_id="other-parent") + rows = json.loads((log_root / "sessions.jsonl").read_text(encoding="utf-8").splitlines()[0]) + entries = [ + rows, + _entry( + "session-child", child_transcript, parent_session_id=ROOT_ID, caller_session_id=ROOT_ID + ), + ] + entries.append( + _entry( + "session-unrelated", + unrelated, + parent_session_id="other-parent", + caller_session_id="other-parent", + ) + ) + _index(log_root, entries) + _trace( + trace, + [ + _event( + "SessionStart", + ROOT_ID, + 1, + transcript=log_root / "transcripts" / f"{ROOT_ID}.jsonl", + ), + _event( + "SessionStart", + ROOT_ID, + 1.5, + transcript=log_root / "transcripts" / f"{ROOT_ID}.jsonl", + ), + _event("SessionStart", "session-child", 3, transcript=child_transcript), + _event("SessionStart", "session-unrelated", 1, transcript=unrelated), + _event( + "SessionEnd", ROOT_ID, 2, transcript=log_root / "transcripts" / f"{ROOT_ID}.jsonl" + ), + _event( + "SessionEnd", ROOT_ID, 3, transcript=log_root / "transcripts" / f"{ROOT_ID}.jsonl" + ), + _event("SessionEnd", "session-child", 4, transcript=child_transcript), + _event("SessionEnd", "session-unrelated", 5, transcript=unrelated), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["peak_sessions"] == 1 + assert summary["coverage"]["complete"] is True + assert summary["coverage"]["managed_session_ids"] == ["session-child"] + assert [identity["session_id"] for identity in summary["identities"]] == [ + ROOT_ID, + "session-child", + ] + root_interval = summary["identities"][0] + assert (root_interval["start"], root_interval["end"]) == (1.0, 3.0) + assert (trace.parent / "session-concurrency.json").is_file() + + +@pytest.mark.parametrize( + ("child_event", "expected_violation"), + [("SessionStart", "missing_end"), ("SessionEnd", "orphan_end")], +) +def test_summary_does_not_infer_child_end( + tmp_path: Path, child_event: str, expected_violation: str +) -> None: + log_root, trace, child_transcript = _base_sessions(tmp_path, child_id="session-child") + assert child_transcript is not None + _trace( + trace, + [ + _event( + "SessionStart", + ROOT_ID, + 1, + transcript=log_root / "transcripts" / f"{ROOT_ID}.jsonl", + ), + _event(child_event, "session-child", 2, transcript=child_transcript), + _event( + "SessionEnd", ROOT_ID, 5, transcript=log_root / "transcripts" / f"{ROOT_ID}.jsonl" + ), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["peak_sessions"] == 1 + assert summary["coverage"]["complete"] is False + assert f"lifecycle:session:session-child:{expected_violation}" in summary["violations"] + + +def test_summary_enumerates_native_agent_transcripts_and_requires_endpoints( + tmp_path: Path, +) -> None: + log_root, trace, _child = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + agent_transcript = log_root / "transcripts" / ROOT_ID / "subagents" / "agent-agent-a.jsonl" + agent_transcript.parent.mkdir(parents=True) + agent_transcript.write_text("{}\n", encoding="utf-8") + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SubagentStart", ROOT_ID, 2, agent_id="agent-a"), + _event( + "SubagentStop", ROOT_ID, 4, agent_id="agent-a", agent_transcript=agent_transcript + ), + _event("SessionEnd", ROOT_ID, 5, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["complete"] is True + assert summary["peak_sessions"] == 2 + assert summary["coverage"]["native_agents"] == [ + { + "session_id": ROOT_ID, + "agent_id": "agent-a", + "transcript_path": str(agent_transcript.resolve()), + } + ] + + +def test_indexed_owned_child_without_hooks_is_incomplete(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path, child_id="missing-hooks") + transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=transcript), + _event("SessionEnd", ROOT_ID, 5, transcript=transcript), + ], + ) + summary = sessions.summarize(trace, _envelope(), log_root) + assert not summary["coverage"]["complete"] + assert "lifecycle:missing-hooks:missing_endpoints" in summary["violations"] + + +def test_resumed_identity_requires_a_final_end(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=transcript), + _event("SessionStart", ROOT_ID, 3, transcript=transcript), + ], + ) + summary = sessions.summarize(trace, _envelope(), log_root) + assert not summary["coverage"]["complete"] + assert f"lifecycle:session:{ROOT_ID}:missing_final_end" in summary["violations"] + + +def test_empty_half_open_interval_does_not_reduce_concurrent_peak() -> None: + assert ( + sessions._peak_sessions( + [{"start": 1, "end": 3}, {"start": 2, "end": 2}, {"start": 2, "end": 4}] + ) + == 2 + ) + + +def test_no_attempted_resource_calls_leave_cleanup_proof_empty(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=transcript), + ], + ) + proof = sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) + assert proof["branches"] == [] + assert proof["claimed"] is False + assert proof["violations"] == [] + + +def test_missing_root_hooks_do_not_discard_available_branch_proof(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + transcript, + [ + _tool_use( + "branch", + "create_and_publish_branch", + {"issue_number": "5233", "remote_url": "https://github.com/owner/repo.git"}, + ), + _tool_result("branch", {"merge_target": "impl/5233-fix", "was_unique": True}), + ], + ) + _trace(trace, []) + proof = sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) + assert proof["branches"] == ["impl/5233-fix"] + assert "trace:root_session_missing" in proof["violations"] + + +def test_missing_root_and_malformed_trace_are_incomplete(tmp_path: Path) -> None: + log_root = tmp_path / "logs" + trace = tmp_path / "run" / "session-lifecycle.jsonl" + trace.parent.mkdir(parents=True) + trace.write_text("{malformed\n", encoding="utf-8") + + summary = sessions.summarize(trace, None, log_root) + + assert summary["coverage"]["complete"] is False + assert "envelope:missing" in summary["violations"] + assert "sessions_index:unavailable" in summary["violations"] + assert "trace:invalid_json:1" in summary["violations"] + + +def test_summary_rejects_malformed_owned_index_summary_and_transcript( + tmp_path: Path, +) -> None: + log_root, trace, _child = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + index_path = log_root / "sessions.jsonl" + index_path.write_text( + index_path.read_text(encoding="utf-8") + "{malformed\n", encoding="utf-8" + ) + (log_root / "sessions" / ROOT_ID / "summary.json").write_text("{malformed\n", encoding="utf-8") + root_transcript.write_text("{malformed\n", encoding="utf-8") + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["complete"] is False + assert "sessions_index:invalid_json:2" in summary["violations"] + assert f"summary:{ROOT_ID}:missing_or_malformed" in summary["violations"] + assert f"transcript:{ROOT_ID}:invalid_json:1" in summary["violations"] + + +@pytest.mark.parametrize( + ("claim_input_label", "expected_claimed"), + [(None, True), ("in-progress", True), ("different-label", False)], +) +def test_ownership_pairs_root_branch_with_managed_claim_and_keeps_compact_proof( + tmp_path: Path, + claim_input_label: str | None, + expected_claimed: bool, +) -> None: + claim_input: dict[str, Any] = {"issue_url": ISSUE_URL, "allow_reentry": False} + if claim_input_label is not None: + claim_input["label"] = claim_input_label + claim_records = [ + _tool_use("claim-1", "claim_and_resolve_issue", claim_input), + _tool_result( + "claim-1", + { + "success": True, + "claimed": True, + "issue_number": 5233, + "issue_title": "private title", + }, + ), + ] + log_root, trace, child_transcript = _base_sessions( + tmp_path, + child_id="session-child", + child_records=claim_records, + ) + assert child_transcript is not None + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + root_transcript, + [ + _tool_use( + "branch-1", + "create_and_publish_branch", + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + ), + _tool_result("branch-1", {"merge_target": "impl/5233-fix", "was_unique": True}), + ], + ) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionStart", "session-child", 2, transcript=child_transcript), + _event("SessionEnd", "session-child", 4, transcript=child_transcript), + _event("SessionEnd", ROOT_ID, 5, transcript=root_transcript), + ], + ) + + proof = sessions.collect_ownership( + trace, _envelope(), log_root, ISSUE_URL, REPOSITORY, claim_label="in-progress" + ) + + assert proof["branches"] == ["impl/5233-fix"] + assert proof["claimed"] is expected_claimed + if expected_claimed: + assert proof["violations"] == [] + else: + assert "ownership:claim_result_failed" in proof["violations"] + assert {item["session_id"] for item in proof["transcripts"]} == {ROOT_ID, "session-child"} + + +def test_ownership_ignores_child_branch_and_rejects_reentry_and_unpaired_results( + tmp_path: Path, +) -> None: + child_records = [ + _tool_use( + "child-branch", + "create_and_publish_branch", + {"issue_number": "5233", "remote_url": "git@github.com:owner/repo.git"}, + ), + _tool_result("child-branch", {"merge_target": "wrong-owner/branch"}), + ] + log_root, trace, child_transcript = _base_sessions( + tmp_path, + child_id="session-child", + child_records=child_records, + ) + assert child_transcript is not None + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + root_transcript, + [ + _tool_use( + "root-branch", + "create_and_publish_branch", + {"issue_number": "5233", "remote_url": "git@github.com:owner/repo.git"}, + ), + _tool_use( + "claim-1", + "claim_and_resolve_issue", + {"issue_url": ISSUE_URL, "allow_reentry": False}, + ), + _tool_result("claim-1", {"success": True, "claimed": True, "issue_number": 5233}), + _tool_use( + "claim-2", + "claim_and_resolve_issue", + {"issue_url": ISSUE_URL, "allow_reentry": False}, + ), + _tool_result("claim-2", {"success": True, "claimed": True, "issue_number": 5233}), + _tool_result("wrong-branch-id", {"merge_target": "must-not-be-paired"}), + ], + ) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionStart", "session-child", 2, transcript=child_transcript), + _event("SessionEnd", "session-child", 4, transcript=child_transcript), + _event("SessionEnd", ROOT_ID, 5, transcript=root_transcript), + ], + ) + + proof = sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) + + assert proof["branches"] == [] + assert proof["claimed"] is False + assert "ownership:claim_reentry" in proof["violations"] + assert any("unpaired_tool_use:root-branch" in item for item in proof["violations"]) diff --git a/tests/infra/test_e2e_workflow.py b/tests/infra/test_e2e_workflow.py index c1fa814d2e..16c7f1a384 100644 --- a/tests/infra/test_e2e_workflow.py +++ b/tests/infra/test_e2e_workflow.py @@ -370,9 +370,17 @@ def test_docker_run_forwards_secrets_by_name_only(self, workflow: dict, step_id: forwarded = re.findall(r"--env ([A-Z0-9_]+)", run) expected = ["MINIMAX_API_KEY", "E2E_SANDBOX_TOKEN"] if step_id == "e2e": - expected.append("E2E_SANDBOX_TOKEN") + expected.extend(["E2E_SANDBOX_TOKEN", "E2E_IMPLEMENTATION_ISSUE_URL"]) assert Counter(forwarded) == Counter(expected) + def test_seed_repository_variable_reaches_test_container(self, workflow: dict) -> None: + step = _step(workflow, "run", step_id="e2e") + assert ( + step["env"]["E2E_IMPLEMENTATION_ISSUE_URL"] + == "${{ vars.E2E_IMPLEMENTATION_ISSUE_URL }}" + ) + assert "--env E2E_IMPLEMENTATION_ISSUE_URL" in step["run"] + def test_redaction_keeps_required_secret_names(self, workflow: dict) -> None: run = _step(workflow, "run", step_id="redact")["run"] branch, outside = _model_branch(run) @@ -421,10 +429,10 @@ def test_clean_install_uses_the_common_user_image_and_artifacts( ] assert len(credential_lines) == 1 assert credential_lines[0].startswith("docker_args+=(") - assert re.findall(r"--env ([A-Z0-9_]+)", credential_lines[0]) == [ - "MINIMAX_API_KEY", - "E2E_SANDBOX_TOKEN", - ] + forwarded = ["MINIMAX_API_KEY", "E2E_SANDBOX_TOKEN"] + if step_id == "e2e": + forwarded.append("E2E_IMPLEMENTATION_ISSUE_URL") + assert re.findall(r"--env ([A-Z0-9_]+)", credential_lines[0]) == forwarded if step_id == "e2e": assert not re.search(r"--env MINIMAX_API_KEY\b", outside) assert re.findall(r"--env ([A-Z0-9_]+)", outside) == ["E2E_SANDBOX_TOKEN"] From 9efed3c90651668817acc1325adb8c572e457e1e Mon Sep 17 00:00:00 2001 From: Trecek Date: Thu, 8 Oct 2026 07:35:47 -0700 Subject: [PATCH 02/20] fix(review): centralize E2E session log root construction --- scripts/e2e/e2e_harness.py | 4 ++-- scripts/e2e/e2e_sessions.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index 281d46a3c1..d6e76057e9 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -1479,7 +1479,7 @@ def finish_pipeline( ) -> list[str]: failures: list[str] = [] trace = out / "session-events.jsonl" - log_root = home / ".local" / "share" / "autoskillit" / "logs" + log_root = e2e_sessions.default_log_root(home) try: summary = e2e_sessions.summarize(trace, envelope, log_root) if not summary["coverage"]["complete"]: @@ -1523,7 +1523,7 @@ def launch_recipe( proof = save_ownership( out / "session-events.jsonl", envelope, - home / ".local" / "share" / "autoskillit" / "logs", + e2e_sessions.default_log_root(home), seed, repository, out, diff --git a/scripts/e2e/e2e_sessions.py b/scripts/e2e/e2e_sessions.py index bab3aec9be..86cfd89d35 100644 --- a/scripts/e2e/e2e_sessions.py +++ b/scripts/e2e/e2e_sessions.py @@ -190,13 +190,17 @@ def _read_summary(session_dir: Path, expected_session_id: str) -> dict[str, Any] return summary +def default_log_root(home: Path) -> Path: + return home / ".local" / "share" / "autoskillit" / "logs" + + def _discover_owned( trace_path: Path, envelope: dict[str, Any] | None, log_root: Path | None, ) -> dict[str, Any]: """Resolve only the dispatched session and fresh, linked managed descendants.""" - root = log_root or Path.home() / ".local" / "share" / "autoskillit" / "logs" + root = log_root or default_log_root(Path.home()) violations: list[str] = [] events = _load_trace(trace_path, violations) root_id, dispatch_id = _read_envelope(envelope, violations) From 4c41bd9f566799f1c8bd55fab213f2a9024add77 Mon Sep 17 00:00:00 2001 From: Trecek Date: Thu, 8 Oct 2026 07:36:13 -0700 Subject: [PATCH 03/20] fix(review): name the E2E session trace artifact --- scripts/e2e/e2e_harness.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index d6e76057e9..9e5c95937f 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -41,6 +41,7 @@ _SCRUBBED_ENV = frozenset({"CLAUDE_CODE_OAUTH_TOKEN", *SECRET_ENV}) _STDERR_TAIL_CHARS = 2000 +_SESSION_TRACE_FILENAME = "session-events.jsonl" CLEAN_INSTALL_ALLOWED_WARNINGS: dict[tuple[str, str], str] = { ( "pytest_temp_capacity", @@ -1403,7 +1404,7 @@ def run_recipe( ingredients = dict(test.ingredients) label = "" home = home or Path.home() - trace = out / "session-events.jsonl" + trace = out / _SESSION_TRACE_FILENAME if test.pipeline or test.issue_url_env: try: seed, ingredients, label = prepare_seed(test, repository, out, env, runner) @@ -1478,7 +1479,7 @@ def finish_pipeline( proof: dict[str, Any] | None, ) -> list[str]: failures: list[str] = [] - trace = out / "session-events.jsonl" + trace = out / _SESSION_TRACE_FILENAME log_root = e2e_sessions.default_log_root(home) try: summary = e2e_sessions.summarize(trace, envelope, log_root) @@ -1521,7 +1522,7 @@ def launch_recipe( ) failures += fleet_failures proof = save_ownership( - out / "session-events.jsonl", + out / _SESSION_TRACE_FILENAME, envelope, e2e_sessions.default_log_root(home), seed, From 84db95fe820def8f7a4805aca0bf415078d0b9f1 Mon Sep 17 00:00:00 2001 From: Trecek Date: Thu, 8 Oct 2026 07:36:40 -0700 Subject: [PATCH 04/20] fix(review): name sandbox CI polling settings --- scripts/e2e/e2e_harness.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index 9e5c95937f..a8df99d646 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -42,6 +42,8 @@ _SCRUBBED_ENV = frozenset({"CLAUDE_CODE_OAUTH_TOKEN", *SECRET_ENV}) _STDERR_TAIL_CHARS = 2000 _SESSION_TRACE_FILENAME = "session-events.jsonl" +_CI_POLL_ATTEMPTS = 6 +_CI_POLL_INTERVAL_SEC = 10 CLEAN_INSTALL_ALLOWED_WARNINGS: dict[tuple[str, str], str] = { ( "pytest_temp_capacity", @@ -1099,7 +1101,7 @@ def validate_sandbox_ci( ] if len(candidates) != 1: raise ValueError("sandbox CI: expected one active CI workflow") - for attempt in range(6): + for attempt in range(_CI_POLL_ATTEMPTS): pages = _api_pages( f"repos/{repository}/actions/workflows/{candidates[0]['id']}/runs?event=pull_request&per_page=100", env, @@ -1143,8 +1145,8 @@ def validate_sandbox_ci( ): raise ValueError("sandbox CI: CI run and test job must succeed") return - if attempt < 5: - time.sleep(10) + if attempt + 1 < _CI_POLL_ATTEMPTS: + time.sleep(_CI_POLL_INTERVAL_SEC) raise ValueError("sandbox CI: no completed CI run associated with the exact PR head") From 54d56bf1c0016aabb9968bd3c6fa42215d4030c2 Mon Sep 17 00:00:00 2001 From: Trecek Date: Thu, 8 Oct 2026 07:37:06 -0700 Subject: [PATCH 05/20] fix(review): clarify claim label predicate naming --- scripts/e2e/e2e_sessions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/e2e/e2e_sessions.py b/scripts/e2e/e2e_sessions.py index 86cfd89d35..3785066543 100644 --- a/scripts/e2e/e2e_sessions.py +++ b/scripts/e2e/e2e_sessions.py @@ -844,10 +844,10 @@ def _claim_succeeded( return False tool_input, result, is_error = claims[0] requested_label = tool_input.get("label") - effective_label = claim_label is None or requested_label in (None, "", claim_label) + label_matches_request = claim_label is None or requested_label in (None, "", claim_label) claimed = bool( not is_error - and effective_label + and label_matches_request and tool_input.get("allow_reentry") is not True and result is not None and result.get("success") is True From a8acd0d22959ff1fe8dc3214be145a5b611eae20 Mon Sep 17 00:00:00 2001 From: Trecek Date: Thu, 8 Oct 2026 07:37:41 -0700 Subject: [PATCH 06/20] fix(review): ignore non-string transcript tool names --- scripts/e2e/e2e_sessions.py | 4 +++- tests/infra/test_e2e_sessions.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/scripts/e2e/e2e_sessions.py b/scripts/e2e/e2e_sessions.py index 3785066543..d483903417 100644 --- a/scripts/e2e/e2e_sessions.py +++ b/scripts/e2e/e2e_sessions.py @@ -709,7 +709,9 @@ def _remember_tool_use( violations: list[str], ) -> None: name = block.get("name") - name = name.removeprefix("mcp__autoskillit__") if isinstance(name, str) else name + if not isinstance(name, str): + return + name = name.removeprefix("mcp__autoskillit__") if name not in {"claim_and_resolve_issue", "create_and_publish_branch"}: return tool_id, tool_input = block.get("id"), block.get("input") diff --git a/tests/infra/test_e2e_sessions.py b/tests/infra/test_e2e_sessions.py index 89ad47d9a9..f2f2e72f20 100644 --- a/tests/infra/test_e2e_sessions.py +++ b/tests/infra/test_e2e_sessions.py @@ -521,6 +521,21 @@ def test_ownership_pairs_root_branch_with_managed_claim_and_keeps_compact_proof( assert {item["session_id"] for item in proof["transcripts"]} == {ROOT_ID, "session-child"} +def test_ownership_ignores_unhashable_tool_names(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + record = _tool_use("unrelated", "ignored", {}) + record["message"]["content"][0]["name"] = [] + _jsonl(root_transcript, [record]) + _trace(trace, [_event("SessionStart", ROOT_ID, 1, transcript=root_transcript)]) + + proof = sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) + + assert proof["branches"] == [] + assert proof["claimed"] is False + assert proof["violations"] == [] + + def test_ownership_ignores_child_branch_and_rejects_reentry_and_unpaired_results( tmp_path: Path, ) -> None: From ea59abd813398d2e772afa5aac16574712dbaaf2 Mon Sep 17 00:00:00 2001 From: Trecek Date: Thu, 8 Oct 2026 07:38:14 -0700 Subject: [PATCH 07/20] fix(review): reject dot session directory components --- scripts/e2e/e2e_sessions.py | 2 +- tests/infra/test_e2e_sessions.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/e2e/e2e_sessions.py b/scripts/e2e/e2e_sessions.py index d483903417..7e7d79f5ac 100644 --- a/scripts/e2e/e2e_sessions.py +++ b/scripts/e2e/e2e_sessions.py @@ -172,7 +172,7 @@ def _load_index(log_root: Path, violations: list[str]) -> list[dict[str, Any]]: def _safe_session_dir(log_root: Path, row: Mapping[str, Any]) -> Path | None: name = row.get("dir_name") - if not isinstance(name, str) or not name or Path(name).name != name: + if not isinstance(name, str) or not name or name in {".", ".."} or Path(name).name != name: return None return log_root / "sessions" / name diff --git a/tests/infra/test_e2e_sessions.py b/tests/infra/test_e2e_sessions.py index f2f2e72f20..cb469cc107 100644 --- a/tests/infra/test_e2e_sessions.py +++ b/tests/infra/test_e2e_sessions.py @@ -426,6 +426,21 @@ def test_missing_root_and_malformed_trace_are_incomplete(tmp_path: Path) -> None assert "trace:invalid_json:1" in summary["violations"] +@pytest.mark.parametrize("dir_name", [".", ".."]) +def test_summary_rejects_dot_session_directories(tmp_path: Path, dir_name: str) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + entry = _entry(ROOT_ID, root_transcript) + entry["dir_name"] = dir_name + _index(log_root, [entry]) + _trace(trace, [_event("SessionStart", ROOT_ID, 1, transcript=root_transcript)]) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["complete"] is False + assert f"sessions_index:{ROOT_ID}:invalid_dir_name" in summary["violations"] + + def test_summary_rejects_malformed_owned_index_summary_and_transcript( tmp_path: Path, ) -> None: From 3ba1d69da263d353cd94fe5a2a29b0a3b01ffa66 Mon Sep 17 00:00:00 2001 From: Trecek Date: Thu, 8 Oct 2026 07:39:33 -0700 Subject: [PATCH 08/20] test(review): isolate ownership proof scenarios --- tests/infra/test_e2e_sessions.py | 57 +++++++++++++++++--------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/tests/infra/test_e2e_sessions.py b/tests/infra/test_e2e_sessions.py index cb469cc107..c23e160532 100644 --- a/tests/infra/test_e2e_sessions.py +++ b/tests/infra/test_e2e_sessions.py @@ -551,8 +551,10 @@ def test_ownership_ignores_unhashable_tool_names(tmp_path: Path) -> None: assert proof["violations"] == [] -def test_ownership_ignores_child_branch_and_rejects_reentry_and_unpaired_results( +@pytest.mark.parametrize("scenario", ["child_branch", "unpaired_root_branch", "claim_reentry"]) +def test_ownership_isolates_branch_and_claim_scenarios( tmp_path: Path, + scenario: str, ) -> None: child_records = [ _tool_use( @@ -564,19 +566,12 @@ def test_ownership_ignores_child_branch_and_rejects_reentry_and_unpaired_results ] log_root, trace, child_transcript = _base_sessions( tmp_path, - child_id="session-child", - child_records=child_records, + child_id="session-child" if scenario == "child_branch" else None, + child_records=child_records if scenario == "child_branch" else None, ) - assert child_transcript is not None root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" - _jsonl( - root_transcript, - [ - _tool_use( - "root-branch", - "create_and_publish_branch", - {"issue_number": "5233", "remote_url": "git@github.com:owner/repo.git"}, - ), + if scenario == "claim_reentry": + root_records = [ _tool_use( "claim-1", "claim_and_resolve_issue", @@ -589,22 +584,30 @@ def test_ownership_ignores_child_branch_and_rejects_reentry_and_unpaired_results {"issue_url": ISSUE_URL, "allow_reentry": False}, ), _tool_result("claim-2", {"success": True, "claimed": True, "issue_number": 5233}), - _tool_result("wrong-branch-id", {"merge_target": "must-not-be-paired"}), - ], - ) - _trace( - trace, - [ - _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), - _event("SessionStart", "session-child", 2, transcript=child_transcript), - _event("SessionEnd", "session-child", 4, transcript=child_transcript), - _event("SessionEnd", ROOT_ID, 5, transcript=root_transcript), - ], - ) + ] + else: + root_records = [ + _tool_use( + "root-branch", + "create_and_publish_branch", + {"issue_number": "5233", "remote_url": "git@github.com:owner/repo.git"}, + ), + _tool_result( + "root-branch" if scenario == "child_branch" else "wrong-branch-id", + {"merge_target": "impl/5233-root"}, + ), + ] + _jsonl(root_transcript, root_records) + _trace(trace, [_event("SessionStart", ROOT_ID, 1, transcript=root_transcript)]) proof = sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) - assert proof["branches"] == [] + assert proof["branches"] == (["impl/5233-root"] if scenario == "child_branch" else []) assert proof["claimed"] is False - assert "ownership:claim_reentry" in proof["violations"] - assert any("unpaired_tool_use:root-branch" in item for item in proof["violations"]) + if scenario == "claim_reentry": + assert proof["violations"] == ["ownership:claim_reentry"] + elif scenario == "unpaired_root_branch": + assert len(proof["violations"]) == 1 + assert "unpaired_tool_use:root-branch" in proof["violations"][0] + else: + assert proof["violations"] == [] From 021beadb687a96882fd717dabf36171732a97598 Mon Sep 17 00:00:00 2001 From: Trecek Date: Thu, 8 Oct 2026 07:44:45 -0700 Subject: [PATCH 09/20] test(review): assert both unpaired branch diagnostics --- tests/infra/test_e2e_sessions.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/infra/test_e2e_sessions.py b/tests/infra/test_e2e_sessions.py index c23e160532..b2bce94d4e 100644 --- a/tests/infra/test_e2e_sessions.py +++ b/tests/infra/test_e2e_sessions.py @@ -607,7 +607,9 @@ def test_ownership_isolates_branch_and_claim_scenarios( if scenario == "claim_reentry": assert proof["violations"] == ["ownership:claim_reentry"] elif scenario == "unpaired_root_branch": - assert len(proof["violations"]) == 1 - assert "unpaired_tool_use:root-branch" in proof["violations"][0] + assert proof["violations"] == [ + "ownership:branch_result_failed", + f"transcript:{ROOT_ID}:unpaired_tool_use:root-branch", + ] else: assert proof["violations"] == [] From 2e3a5ed67b8a6161aee335951cee75f974282f8d Mon Sep 17 00:00:00 2001 From: Trecek Date: Wed, 7 Oct 2026 22:27:06 -0700 Subject: [PATCH 10/20] feat: add seeded sandbox implementation E2E validation --- .github/workflows/e2e.yml | 3 +- scripts/e2e/README.md | 101 +++- scripts/e2e/autoskillit-config.yaml | 3 + scripts/e2e/catalog.json | 29 + scripts/e2e/e2e_catalog.py | 182 +++++- scripts/e2e/e2e_harness.py | 697 +++++++++++++++++++++- scripts/e2e/e2e_select.py | 48 +- scripts/e2e/e2e_sessions.py | 879 ++++++++++++++++++++++++++++ tests/infra/test_e2e_harness.py | 717 ++++++++++++++++++++++- tests/infra/test_e2e_select.py | 216 +++++++ tests/infra/test_e2e_select_cli.py | 29 + tests/infra/test_e2e_sessions.py | 580 ++++++++++++++++++ tests/infra/test_e2e_workflow.py | 18 +- 13 files changed, 3424 insertions(+), 78 deletions(-) create mode 100644 scripts/e2e/e2e_sessions.py create mode 100644 tests/infra/test_e2e_sessions.py diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a35b998ce8..aedb28d507 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -104,12 +104,13 @@ jobs: env: MINIMAX_API_KEY: ${{ matrix.kind != 'clean-install' && secrets.MINIMAX_API_KEY || '' }} E2E_SANDBOX_TOKEN: ${{ matrix.kind != 'clean-install' && secrets.E2E_SANDBOX_TOKEN || '' }} + E2E_IMPLEMENTATION_ISSUE_URL: ${{ vars.E2E_IMPLEMENTATION_ISSUE_URL }} TEST: ${{ matrix.test }} run: | set -euo pipefail docker_args=(--shm-size=3g) if [[ '${{ matrix.kind }}' != 'clean-install' ]]; then - docker_args+=(--env MINIMAX_API_KEY --env E2E_SANDBOX_TOKEN) + docker_args+=(--env MINIMAX_API_KEY --env E2E_SANDBOX_TOKEN --env E2E_IMPLEMENTATION_ISSUE_URL) docker_args+=(--volume "$RUNNER_TEMP/e2e/data:/home/autoskillit/.local/share/autoskillit") fi if [[ "$TEST" == 'headless-smoke' ]]; then diff --git a/scripts/e2e/README.md b/scripts/e2e/README.md index 8883f7d720..0c296402a2 100644 --- a/scripts/e2e/README.md +++ b/scripts/e2e/README.md @@ -11,6 +11,7 @@ This directory holds everything the workflow runs: | `e2e_catalog.py` | Stdlib loader and validator for the catalog | | `e2e_select.py` | Picks the tests an event runs (`select`) and judges the required check (`gate`) | | `e2e_harness.py` | Runs one test (`run`), performs exact sandbox teardown (`cleanup`) and redacts artifacts (`redact`) | +| `e2e_sessions.py` | Captures synchronous lifecycle events, measures concurrency and pairs ownership evidence | | `autoskillit-config.yaml` | AutoSkillit user-layer config the harness installs | | `post-e2e-failure.sh` | Appends a failure to the test's tracking issue | @@ -40,6 +41,11 @@ model tests: because MiniMax serves the Anthropic Messages API. 3. Create the `e2e` label in this repository. 4. In branch protection, mark only `e2e-gate` as required — never `select` or `run (…)`. +5. Before publishing a branch with the selectable `implementation` entry, obtain a passing + #5232 sandbox smoke CI run. A maintainer must seed a fixed OPEN sandbox issue describing + a small feature and its test, then set repository variable `E2E_IMPLEMENTATION_ISSUE_URL` + to its canonical `https://github.com/OWNER/REPO/issues/NUMBER` URL. Reuse that issue on + every run. Missing configuration fails preflight; it is not an expected product defect. ## Sandbox repository requirements @@ -167,7 +173,11 @@ matrix is always parseable (`{"include":[]}` when nothing is selected). bypasses sampling because a labelled pull request whose sample fell out would otherwise never run for that commit. - **Picking:** tests whose `trigger_paths` match a changed path, up to `MAX_TESTS` (2); when - none match, exactly one test. Ties are ordered by the SHA-256 hex digest of + none match, one eligible ordinary test. A pipeline requires actual size at or above a + threshold and a matching trigger path, even with the `e2e` label. This filter also applies + to fallback; a pipeline-only ineligible pool selects nothing. At most one pipeline is + selected within the total cap. Named workflow dispatch bypasses PR eligibility. + Ties are ordered by the SHA-256 hex digest of `"{seed}:{name}"`, so the decision is a pure function of the payload and the changed paths. Patterns use `fnmatch.fnmatchcase`, where `*` also matches `/` (`scripts/e2e/*` matches every file below `scripts/e2e/`). @@ -183,11 +193,25 @@ succeeded or nothing was selected and `run` was skipped. Every other combination The `run` job uses one repository-wide concurrency group (`group: e2e`, `queue: max`, `cancel-in-progress: false`) and `max-parallel: 1`, so every selected test of every run waits its turn and never cancels another. `select` and `e2e-gate` are not queued. A -catalog entry's `peak_sessions` is capped at `MAX_PEAK_SESSIONS` (8): one step's fan-out of -up to six subagents plus the orchestrator and step session. With one test running at a -time, CI stays at or below eight of the roughly twelve concurrent MiniMax sessions. +catalog entry's `peak_sessions` is capped at `MAX_PEAK_SESSIONS` (8). For an uncommissioned +pipeline this declares capacity, not a measured fan-out. The child limit stays six. `clean-install` requires exactly zero `peak_sessions`. +The implementation pipeline installs synchronous command hooks for `SessionStart`, +`SessionEnd`, `SubagentStart` and `SubagentStop`, preserving existing hooks and MiniMax +settings. These hooks follow the [Claude lifecycle schema](https://code.claude.com/docs/en/hooks). +Each identity spans its first start through its final end, so compaction and resume yield +a conservative upper bound. Half-open intervals end before simultaneous starts. The +summary reconciles the dispatched root, managed lineage and enumerable native transcripts; +missing endpoints, identities or owned transcripts fail coverage. Transcript activity is +never used to infer a lifetime. A peak exceeding capacity fails validation. + +Commission with a successful MiniMax CI run and repeat against the same seed. Inspect +planning, dry-walkthrough, implementation, test, integration, audit GO and open-PR evidence, +plus cleanup and the sandbox CI association. Replace provisional `peak_sessions` with +the observed conservative peak and set `peak_sessions_evidence` to the CI run URL, tested +commit and first-start/final-end method. Expected failures do not satisfy commissioning. + Every harness subprocess has a timeout that fires before the job's: `fleet run` gets the test's `timeout_sec` plus `HARNESS_GRACE_SEC`, and the job gets that budget in minutes plus `JOB_OVERHEAD_MINUTES` for building the image, redaction and upload. So `result.json`, PR @@ -211,6 +235,12 @@ redaction and upload steps as the other output artifacts. Collection also runs f expected failure or a harness exception. A container killed before or during collection may leave no evidence or only partial evidence. +Pipeline evidence includes `session-events.jsonl`, `session-concurrency.json`, compact +`ownership.json`, the seed, effective configuration, fleet stdout/stderr/envelope, captured +PR metadata/files, detached-head test commands and associated CI runs/jobs. Collection +occurs inside the container before transcript paths disappear. Incomplete evidence is +retained and fails the scenario; redaction and upload still run after failure. + `clean-install` mounts no home directories and retains evidence under `out/`. Its redactor receives the same required `--secret-env` names with variables unset and skips the absent home-log tree. Redaction and upload still run after test failures. @@ -267,6 +297,34 @@ row and close the bug. Fleet translates the L3 result's `reason` into the failed command envelope's `error` and `user_visible_message` fields; the harness matches that envelope rather than the recipe payload's `failed_step`/`reason`. +For the implementation pipeline, the project-owned +[`DispatchCompleted.to_envelope`](../../src/autoskillit/fleet/campaign_state/state_outcomes.py) +must report success, `dispatch_status="success"` and root/dispatch identities. The nested +[`implementation` terminal](../../src/autoskillit/recipes/implementation.yaml) must have +`reason="implementation_complete"`. Successful no-change and already-done terminals remain +valid recipe behavior but fail this scenario. No PR URL is required in the envelope. + +The harness resolves the configured seed once and dispatches its exact URL and title/body +task once. Effective project-aware config comes from `autoskillit config show`, whose +implementation calls `load_config(Path.cwd())`; both configured and sandbox default branches +must be `main`. No caller `base_branch` ingredient is sent. Sandbox setup and baseline tests +must succeed before launch. + +Paired root transcript tool calls/results prove the created branch; paired root/managed +claim results prove any added claim label. Discovery paginates through the PR baseline +and leaves unrelated PRs untouched. Require exactly one owned open, unmerged sandbox PR +targeting `main`, without auto-merge, with the canonical seed URL and non-removed changes +under both `sandbox/` and `tests/`. Fetch and detach at the captured head before setup and +tests. Require a successful associated sandbox `CI` run and `test` job whose nested PR +association has the exact branch revision; an outer merge SHA is not the branch SHA. +Finally re-read the head and state to reject movement during validation. + +Teardown runs after validation, failure or exception. It independently collects metrics, +closes owned open PRs, deletes evidenced branches (including branches without a PR), and +removes only the effective claim label proven added by this run. The seed remains OPEN +and other labels are preserved. Missing ownership proof preserves uncertain resources and +fails cleanup. Cleanup errors remain fatal alongside the original failure. + ## Catalog schema ```json @@ -295,15 +353,38 @@ the harness matches that envelope rather than the recipe payload's `failed_step` ``` `recipe`, `ingredients` and `expected_pull_request_state` are required for `recipe` tests -and forbidden for `canary` and `clean-install` tests. `expected_failures` is optional for -`clean-install` and recipe tests. Clean-install rows use the exact doctor `severity`, `check` -and `message`; recipe rows require `severity: error`, `check` and `message` matching the failed -envelope. Every row includes an open bug issue URL -(`https://github.com/TalonT-Org/AutoSkillit/issues/`). Duplicate diagnostic identities -and unknown keys are rejected. `peak_sessions` must be +and forbidden for `canary` and `clean-install` tests. Optional recipe-only fields are +`pipeline` (boolean, default false), `issue_url_env` (seed variable name), +`required_changed_paths` (nonempty relative prefixes), `test_command` (nonempty argv), and +`peak_sessions_evidence` (nonempty provenance string). Environment-owned seeds cannot also +specify literal `task` or `issue_url` ingredients. Path prefixes cannot be absolute or +traverse parents. For example, the implementation entry uses: + +```json +{"pipeline": true, "issue_url_env": "E2E_IMPLEMENTATION_ISSUE_URL", + "required_changed_paths": ["sandbox/", "tests/"], "test_command": ["task", "test-check"]} +``` + +The recipe_fixture field names a staged YAML fixture for the headless smoke scenario. + +The expected_failures field is optional for recipes and clean-install. Pipeline rows +contain exactly check, message, issue; ordinary recipe and clean-install rows also +contain severity. Ordinary recipes require severity: error and match the failed +envelope's error and user_visible_message; clean-install allows error or warning +and matches doctor diagnostics. Pipeline rows match structured implementation findings. +Every row links to an open AutoSkillit bug issue. Duplicate diagnostic identities and +unknown keys are rejected. The peak_sessions value must be 1–8 for model tests and exactly zero for clean-install. `timeout_sec` must keep the derived job timeout within 360 minutes. +Recipe expectations match exact stable `(check, message)` product findings linked to real +bug issues. Setup, discovery, malformed evidence and cleanup failures cannot be admitted. +Unknown findings and absent expected findings fail. Every recipe records `outcome`: +`passed`, `expected_failure` (with `passed=false`), or `failed`. A linked known defect may +use the accepted CLI exit convention, but only `passed` with no expected findings counts +as successful commissioning. File or deduplicate observed product bugs separately; do not +predeclare a provider defect or replace the MiniMax settings-file route. + ### Adding a test 1. Add the entry to `catalog.json` with a realistic `peak_sessions` and `timeout_sec`. diff --git a/scripts/e2e/autoskillit-config.yaml b/scripts/e2e/autoskillit-config.yaml index 577e8df598..e4a8c3de29 100644 --- a/scripts/e2e/autoskillit-config.yaml +++ b/scripts/e2e/autoskillit-config.yaml @@ -3,6 +3,9 @@ quota_guard: enabled: false +branching: + default_base_branch: main + features: fleet: true fleet_headless_run: true diff --git a/scripts/e2e/catalog.json b/scripts/e2e/catalog.json index 00bf957ce1..ba100c3b0b 100644 --- a/scripts/e2e/catalog.json +++ b/scripts/e2e/catalog.json @@ -70,6 +70,35 @@ "recipe_fixture": "sandbox-smoke.yaml", "ingredients": {}, "expected_pull_request_state": "closed" + }, + { + "name": "implementation", + "kind": "recipe", + "pipeline": true, + "peak_sessions": 8, + "timeout_sec": 7200, + "trigger_paths": [ + "src/autoskillit/recipes/*", + "src/autoskillit/skills/*", + "src/autoskillit/skills_extended/*", + "src/autoskillit/server/*", + "src/autoskillit/execution/*" + ], + "recipe": "implementation", + "ingredients": { + "source_dir": "/workspace/sandbox", + "open_pr": "true", + "auto_merge": "false", + "arch_lenses": "false", + "audit_impl": "true" + }, + "expected_pull_request_state": "open", + "issue_url_env": "E2E_IMPLEMENTATION_ISSUE_URL", + "required_changed_paths": [ + "sandbox/", + "tests/" + ], + "test_command": ["task", "test-check"] } ] } diff --git a/scripts/e2e/e2e_catalog.py b/scripts/e2e/e2e_catalog.py index 46354d43f4..2940e8e472 100644 --- a/scripts/e2e/e2e_catalog.py +++ b/scripts/e2e/e2e_catalog.py @@ -7,7 +7,7 @@ import re from collections.abc import Mapping from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath from types import MappingProxyType CATALOG_PATH = Path(__file__).with_name("catalog.json") @@ -28,7 +28,21 @@ _CATALOG_KEYS = frozenset({"sandbox_repository", "tests"}) _COMMON_KEYS = frozenset({"name", "kind", "peak_sessions", "timeout_sec", "trigger_paths"}) _RECIPE_KEYS = frozenset({"recipe", "ingredients", "expected_pull_request_state"}) +_PIPELINE_METADATA_KEYS = frozenset( + { + "issue_url_env", + "required_changed_paths", + "test_command", + "peak_sessions_evidence", + } +) +_RECIPE_OPTIONAL_KEYS = _PIPELINE_METADATA_KEYS | { + "pipeline", + "expected_failures", + "recipe_fixture", +} _EXPECTED_FAILURE_KEYS = frozenset({"severity", "check", "message", "issue"}) +_PIPELINE_EXPECTED_FAILURE_KEYS = frozenset({"check", "message", "issue"}) _BUG_URL_PATTERN = re.compile(r"https://github\.com/TalonT-Org/AutoSkillit/issues/[1-9][0-9]*") @@ -50,6 +64,11 @@ class CatalogTest: expected_pull_request_state: str | None expected_failures: tuple[Mapping[str, str], ...] = () recipe_fixture: str | None = None + pipeline: bool = False + issue_url_env: str | None = None + required_changed_paths: tuple[str, ...] = () + test_command: tuple[str, ...] = () + peak_sessions_evidence: str | None = None @dataclass(frozen=True, slots=True) @@ -83,6 +102,13 @@ def _require_str(value: object, where: str) -> str: return value +def _require_nonblank_str(value: object, where: str) -> str: + result = _require_str(value, where) + if not result.strip(): + raise CatalogError(f"{where} must be non-empty") + return result + + def _require_int(value: object, where: str, *, low: int, high: int | None = None) -> int: if isinstance(value, bool) or not isinstance(value, int): raise CatalogError(f"{where} must be an integer") @@ -111,7 +137,7 @@ def _parse_ingredients(value: object, where: str) -> tuple[tuple[str, str], ...] def _parse_recipe_fields( raw: Mapping[str, object], where: str -) -> tuple[str, tuple[tuple[str, str], ...], str]: +) -> tuple[str, tuple[tuple[str, str], ...], str, str | None]: recipe = _require_str(raw["recipe"], f"{where}.recipe") ingredients = _parse_ingredients(raw["ingredients"], f"{where}.ingredients") state = _require_str( @@ -121,28 +147,40 @@ def _parse_recipe_fields( raise CatalogError( f"{where}.expected_pull_request_state must be one of {PULL_REQUEST_STATES}" ) - return recipe, ingredients, state + fixture = None + if "recipe_fixture" in raw: + fixture = _require_str(raw["recipe_fixture"], f"{where}.recipe_fixture") + if not _RECIPE_FIXTURE_PATTERN.fullmatch(fixture): + raise CatalogError(f"{where}.recipe_fixture must be a .yaml basename") + return recipe, ingredients, state, fixture -def _parse_expected_failures(value: object, where: str) -> tuple[Mapping[str, str], ...]: +def _parse_expected_failures( + value: object, where: str, *, pipeline: bool = False +) -> tuple[Mapping[str, str], ...]: where = f"{where}.expected_failures" if not isinstance(value, list): raise CatalogError(f"{where} must be a list") + keys = _PIPELINE_EXPECTED_FAILURE_KEYS if pipeline else _EXPECTED_FAILURE_KEYS rows: list[Mapping[str, str]] = [] - seen: set[tuple[str, str, str]] = set() + seen: set[tuple[str, ...]] = set() for index, raw in enumerate(value): location = f"{where}[{index}]" if not isinstance(raw, dict): raise CatalogError(f"{location} must be an object") - _require_keys(raw, _EXPECTED_FAILURE_KEYS, location) - row = {key: _require_str(raw[key], f"{location}.{key}") for key in _EXPECTED_FAILURE_KEYS} + _require_keys(raw, keys, location) + row = {key: _require_str(raw[key], f"{location}.{key}") for key in keys} if any(not item.strip() for item in row.values()): raise CatalogError(f"{location} fields must be non-empty") - if row["severity"] not in ("error", "warning"): + if not pipeline and row["severity"] not in ("error", "warning"): raise CatalogError(f"{location}.severity must be error or warning") if not _BUG_URL_PATTERN.fullmatch(row["issue"]): raise CatalogError(f"{location}.issue must link to an AutoSkillit bug issue") - identity = (row["severity"], row["check"], row["message"]) + identity = ( + (row["check"], row["message"]) + if pipeline + else (row["severity"], row["check"], row["message"]) + ) if identity in seen: raise CatalogError(f"{location}: duplicate diagnostic {identity}") seen.add(identity) @@ -150,6 +188,85 @@ def _parse_expected_failures(value: object, where: str) -> tuple[Mapping[str, st return tuple(rows) +def _parse_recipe_options( + raw: Mapping[str, object], where: str, ingredients: tuple[tuple[str, str], ...] +) -> tuple[str | None, tuple[str, ...], tuple[str, ...], str | None]: + issue_url_env = ( + _require_nonblank_str(raw["issue_url_env"], f"{where}.issue_url_env") + if "issue_url_env" in raw + else None + ) + if issue_url_env and {"issue_url", "task"}.intersection(key for key, _ in ingredients): + raise CatalogError( + f"{where}.ingredients cannot contain issue_url or task when issue_url_env is set" + ) + changed_paths = ( + _parse_required_changed_paths( + raw["required_changed_paths"], f"{where}.required_changed_paths" + ) + if "required_changed_paths" in raw + else () + ) + test_command = ( + _parse_test_command(raw["test_command"], f"{where}.test_command") + if "test_command" in raw + else () + ) + evidence = ( + _require_nonblank_str(raw["peak_sessions_evidence"], f"{where}.peak_sessions_evidence") + if "peak_sessions_evidence" in raw + else None + ) + return issue_url_env, changed_paths, test_command, evidence + + +def _parse_required_changed_paths(value: object, where: str) -> tuple[str, ...]: + if not isinstance(value, list) or not value: + raise CatalogError(f"{where} must be a non-empty list") + changed_paths: list[str] = [] + for item in value: + prefix = _require_nonblank_str(item, f"{where}[]") + posix_path = PurePosixPath(prefix) + windows_path = PureWindowsPath(prefix) + invalid_path = any( + ( + posix_path.is_absolute(), + windows_path.is_absolute(), + bool(windows_path.drive), + ".." in posix_path.parts, + ".." in windows_path.parts, + ) + ) + if invalid_path: + raise CatalogError(f"{where}[] must be a relative path prefix without '..'") + changed_paths.append(prefix) + return tuple(changed_paths) + + +def _parse_test_command(command: object, where: str) -> tuple[str, ...]: + if not isinstance(command, list) or not command: + raise CatalogError(f"{where} must be a non-empty argv list") + return tuple(_require_nonblank_str(item, f"{where}[]") for item in command) + + +def _validate_test_keys(raw: Mapping[str, object], kind: str, where: str) -> bool: + keys = _COMMON_KEYS | _RECIPE_KEYS if kind == "recipe" else _COMMON_KEYS + if kind == "clean-install" and "expected_failures" in raw: + keys |= {"expected_failures"} + if kind != "recipe": + _require_keys(raw, keys, where) + return False + + missing = sorted(keys - raw.keys()) + unknown = sorted(raw.keys() - keys - _RECIPE_OPTIONAL_KEYS) + if missing or unknown: + raise CatalogError(f"{where}: missing keys {missing}, unknown keys {unknown}") + pipeline = raw.get("pipeline", False) + if not isinstance(pipeline, bool): + raise CatalogError(f"{where}.pipeline must be a boolean") + return pipeline + + def _parse_test(raw: object, index: int) -> CatalogTest: where = f"tests[{index}]" if not isinstance(raw, dict): @@ -157,26 +274,38 @@ def _parse_test(raw: object, index: int) -> CatalogTest: kind = raw.get("kind") if kind not in KINDS: raise CatalogError(f"{where}.kind must be one of {KINDS}") - keys = _COMMON_KEYS | _RECIPE_KEYS if kind == "recipe" else _COMMON_KEYS - if kind == "recipe" and "recipe_fixture" in raw: - keys |= {"recipe_fixture"} - if kind in ("recipe", "clean-install") and "expected_failures" in raw: - keys |= {"expected_failures"} - _require_keys(raw, keys, where) + pipeline = _validate_test_keys(raw, kind, where) name = _require_str(raw["name"], f"{where}.name") if not _NAME_PATTERN.fullmatch(name): raise CatalogError(f"{where}.name must match {_NAME_PATTERN.pattern}") - recipe, ingredients, state = ( - _parse_recipe_fields(raw, where) if kind == "recipe" else (None, (), None) + recipe, ingredients, state, recipe_fixture = ( + _parse_recipe_fields(raw, where) if kind == "recipe" else (None, (), None, None) ) - recipe_fixture = None - if kind == "recipe" and "recipe_fixture" in raw: - recipe_fixture = _require_str(raw["recipe_fixture"], f"{where}.recipe_fixture") - if not _RECIPE_FIXTURE_PATTERN.fullmatch(recipe_fixture): - raise CatalogError(f"{where}.recipe_fixture must be a .yaml basename") - expected_failures = _parse_expected_failures(raw.get("expected_failures", []), where) - if kind == "recipe" and any(row["severity"] != "error" for row in expected_failures): + issue_url_env: str | None = None + required_changed_paths: tuple[str, ...] = () + test_command: tuple[str, ...] = () + peak_sessions_evidence: str | None = None + if kind == "recipe": + expected_failures = _parse_expected_failures( + raw.get("expected_failures", []), where, pipeline=pipeline + ) + elif kind == "clean-install": + expected_failures = _parse_expected_failures(raw.get("expected_failures", []), where) + else: + expected_failures = () + if ( + kind == "recipe" + and not pipeline + and any(row["severity"] != "error" for row in expected_failures) + ): raise CatalogError(f"{where}.expected_failures severity must be error for recipe tests") + if kind == "recipe": + ( + issue_url_env, + required_changed_paths, + test_command, + peak_sessions_evidence, + ) = _parse_recipe_options(raw, where, ingredients) test = CatalogTest( name=name, kind=kind, @@ -193,6 +322,11 @@ def _parse_test(raw: object, index: int) -> CatalogTest: expected_pull_request_state=state, expected_failures=expected_failures, recipe_fixture=recipe_fixture, + pipeline=pipeline, + issue_url_env=issue_url_env, + required_changed_paths=required_changed_paths, + test_command=test_command, + peak_sessions_evidence=peak_sessions_evidence, ) if job_timeout_minutes(test) > MAX_JOB_MINUTES: raise CatalogError(f"{where}.timeout_sec exceeds the {MAX_JOB_MINUTES}-minute job limit") diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index 157f948203..2e811479da 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -9,6 +9,7 @@ import json import os import re +import shlex import shutil import subprocess import sys @@ -19,8 +20,10 @@ from collections.abc import Callable, Mapping, Sequence from pathlib import Path from typing import Any +from urllib.parse import quote import e2e_catalog +import e2e_sessions MINIMAX_BASE_URL = "https://api.minimax.io/anthropic" MINIMAX_MODEL = "MiniMax-M3[1m]" @@ -259,28 +262,40 @@ def latest_pull_request_number( def new_pull_requests( - repository: str, baseline: int, env: Mapping[str, str], runner: Runner + repository: str, + baseline: int, + env: Mapping[str, str], + runner: Runner, + *, + pipeline: bool = False, ) -> tuple[list[dict[str, Any]] | None, str | None]: - rows, failure = _list_pull_requests( - repository, 20, "number,state,additions,deletions", env, runner - ) - if rows is None: - return None, failure - return [row for row in rows if int(row["number"]) > baseline], None + limit = 20 + fields = "number,state,additions,deletions" + if pipeline: + fields += ( + ",body,headRefOid,headRefName,headRepository,baseRefName,autoMergeRequest,mergedAt" + ) + while True: + rows, failure = _list_pull_requests(repository, limit, fields, env, runner) + if rows is None: + return None, failure + if not pipeline or len(rows) < limit or any(row["number"] <= baseline for row in rows): + return [row for row in rows if row["number"] > baseline], None + limit *= 2 def fleet_run_argv( - test: e2e_catalog.CatalogTest, runtime_ingredients: Mapping[str, str] | None = None + test: e2e_catalog.CatalogTest, *, ingredients: Mapping[str, str] | None = None ) -> list[str]: - effective = dict(test.ingredients) - effective.update(runtime_ingredients or {}) - ingredients = [arg for key, value in effective.items() for arg in ("-i", f"{key}={value}")] + values = dict(test.ingredients) + values.update(ingredients or {}) + arguments = [arg for key, value in values.items() for arg in ("-i", f"{key}={value}")] return [ "autoskillit", "fleet", "run", str(test.recipe), - *ingredients, + *arguments, "--disable-quota-guard", "--timeout-sec", str(test.timeout_sec), @@ -307,28 +322,77 @@ def check_envelope(returncode: int, stdout: str) -> tuple[list[str], dict[str, A return failures, envelope +def check_implementation_envelope( + envelope: Mapping[str, Any], +) -> tuple[list[str], list[dict[str, str]]]: + failures: list[str] = [] + findings: list[dict[str, str]] = [] + payload = envelope.get("l3_payload") + reason = payload.get("reason") if isinstance(payload, dict) else None + status = envelope.get("dispatch_status") + if envelope.get("success") is not True or status != "success": + stable_reason = envelope.get("reason") or reason + if isinstance(stable_reason, str) and stable_reason: + findings.append({"check": "fleet_envelope", "message": f"{status}: {stable_reason}"}) + else: + failures.append("fleet run: unsuccessful envelope has no specific failure reason") + elif reason != "implementation_complete": + if isinstance(reason, str) and reason: + findings.append({"check": "implementation_terminal", "message": reason}) + else: + failures.append("fleet run: implementation terminal reason is missing") + if not all( + isinstance(envelope.get(key), str) and envelope[key] + for key in ("dispatch_id", "dispatched_session_id") + ): + failures.append("fleet run: dispatch/session identity is missing") + return failures, findings + + def run_fleet( test: e2e_catalog.CatalogTest, out: Path, env: Mapping[str, str], runner: Runner, - runtime_ingredients: Mapping[str, str] | None = None, -) -> tuple[list[str], dict[str, Any] | None]: + *, + ingredients: Mapping[str, str] | None = None, +) -> tuple[dict[str, Any] | None, list[str], list[dict[str, str]]]: result, failure = _call( runner, - fleet_run_argv(test, runtime_ingredients), + fleet_run_argv(test, ingredients=ingredients), what="fleet run", + evidence=out / "fleet-run", cwd=SANDBOX_CLONE, env=env, timeout=test.timeout_sec + e2e_catalog.HARNESS_GRACE_SEC, - evidence=out / "fleet-run", ) + stdout = (out / "fleet-run.stdout.txt").read_text(encoding="utf-8", errors="replace") + shutil.copyfile(out / "fleet-run.stdout.txt", out / "fleet-run.stdout.log") + shutil.copyfile(out / "fleet-run.stderr.txt", out / "fleet-run.stderr.log") + line = last_line(stdout) + (out / "envelope.json").write_text(line + "\n", encoding="utf-8") + if not test.pipeline: + if result is None: + return None, [str(failure)], [] + envelope_failures, envelope = check_envelope(result.returncode, stdout) + return envelope, envelope_failures, [] + failures: list[str] = [] + findings: list[dict[str, str]] = [] + try: + envelope = json.loads(line) + except json.JSONDecodeError: + envelope = None + if not isinstance(envelope, dict): + envelope = None + failures.append("fleet run: the last stdout line is not a JSON object envelope") if result is None: - return [str(failure)], None - (out / "fleet-run.stdout.log").write_text(result.stdout, encoding="utf-8") - (out / "fleet-run.stderr.log").write_text(result.stderr, encoding="utf-8") - (out / "envelope.json").write_text(last_line(result.stdout) + "\n", encoding="utf-8") - return check_envelope(result.returncode, result.stdout) + failures.append(str(failure)) + elif envelope is not None and test.pipeline: + envelope_failures, findings = check_implementation_envelope(envelope) + failures += envelope_failures + if failure and not findings: + failures.append(failure) + return envelope, failures, findings def match_recipe_failure( @@ -458,7 +522,7 @@ def run_smoke_recipe( if runtime is None: return failures, [] (out / "smoke-launched").write_text(test.name + "\n", encoding="utf-8") - fleet_failures, envelope = run_fleet(test, out, env, runner, runtime) + envelope, fleet_failures, _ = run_fleet(test, out, env, runner, ingredients=runtime) fleet_failures, matched = match_recipe_failure(test, fleet_failures, envelope) failures += fleet_failures if ( @@ -868,6 +932,456 @@ def close_open_pull_requests( return failures +def install_session_hooks(home: Path, trace: Path) -> None: + path = home / ".claude" / "settings.json" + settings = json.loads(path.read_text(encoding="utf-8")) + command = f"python3 /opt/e2e/e2e_sessions.py {shlex.quote(str(trace))}" + hooks = settings.setdefault("hooks", {}) + for event in ("SessionStart", "SessionEnd", "SubagentStart", "SubagentStop"): + hooks.setdefault(event, []).append( + {"hooks": [{"type": "command", "command": command, "timeout": 10}]} + ) + path.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8") + + +def _json_call( + argv: Sequence[str], + env: Mapping[str, str], + runner: Runner, + *, + evidence: Path | None = None, + cwd: Path | None = None, +) -> Any: + result, failure = _setup_call(runner, argv, env, evidence=evidence, cwd=cwd) + if failure or result is None: + raise ValueError(failure or "missing command result") + try: + return json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise ValueError(f"{' '.join(argv[:3])}: invalid JSON") from exc + + +def _api_pages(endpoint: str, env: Mapping[str, str], runner: Runner, evidence: Path) -> list[Any]: + pages = _json_call( + ["gh", "api", endpoint, "--paginate", "--slurp"], env, runner, evidence=evidence + ) + if not isinstance(pages, list): + raise ValueError("GitHub API: expected paginated JSON array") + return pages + + +def prepare_seed( + test: e2e_catalog.CatalogTest, + repository: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> tuple[dict[str, Any], dict[str, str], str]: + url = env.get(test.issue_url_env or "", "") + match = re.fullmatch( + r"https://github\.com/" + re.escape(repository) + r"/issues/([1-9][0-9]*)", url + ) + if not match: + raise ValueError("seed: missing or noncanonical sandbox issue URL") + config = _json_call( + ["autoskillit", "config", "show"], + env, + runner, + evidence=out / "effective-config", + cwd=SANDBOX_CLONE, + ) + label = config["github"]["in_progress_label"] + seed = _json_call( + ["gh", "issue", "view", url, "--json", "number,url,state,title,body,labels"], + env, + runner, + evidence=out / "seed", + ) + if ( + not isinstance(seed, dict) + or seed.get("url") != url + or type(seed.get("number")) is not int + or seed["number"] != int(match[1]) + or seed.get("state") != "OPEN" + or not isinstance(seed.get("title"), str) + or not seed["title"].strip() + or not isinstance(seed.get("body"), str) + or not isinstance(seed.get("labels"), list) + or any( + not isinstance(row, dict) or not isinstance(row.get("name"), str) + for row in seed["labels"] + ) + ): + raise ValueError("seed: expected the configured OPEN sandbox issue with valid metadata") + if label in {row["name"] for row in seed["labels"]}: + raise ValueError("seed: issue is already claimed") + repo = _json_call( + ["gh", "repo", "view", repository, "--json", "defaultBranchRef"], env, runner + ) + default_branch = repo.get("defaultBranchRef") if isinstance(repo, dict) else None + if ( + not isinstance(default_branch, dict) + or default_branch.get("name") != "main" + or config["branching"]["default_base_branch"] != "main" + ): + raise ValueError("seed: sandbox and effective configured base branch must both be main") + ingredients = dict(test.ingredients) + ingredients.update(issue_url=url, task=f"{seed['title']}\n\n{seed['body']}") + for stage, argv in ( + ("baseline-install", ["task", "install-worktree"]), + ("baseline-test", ["task", "test-check"]), + ): + _, failure = _call( + runner, + argv, + what=stage, + evidence=out / stage, + cwd=SANDBOX_CLONE, + env=env, + timeout=SETUP_COMMAND_TIMEOUT_SEC, + ) + if failure: + raise ValueError(failure) + return seed, ingredients, label + + +def _repo_matches(value: Any, repository: str) -> bool: + return isinstance(value, dict) and ( + value.get("full_name") == repository + or value.get("url") == f"https://api.github.com/repos/{repository}" + ) + + +def owned_pull_requests( + rows: Sequence[Mapping[str, Any]], + proof: Mapping[str, Any], + repository: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[dict[str, Any]]: + owned = [] + for row in rows: + pr = _json_call( + ["gh", "api", f"repos/{repository}/pulls/{row['number']}"], + env, + runner, + evidence=out / f"pr-{row['number']}", + ) + if not isinstance(pr, dict): + raise ValueError("PR discovery: expected an object") + head = pr.get("head") + if ( + isinstance(head, dict) + and head.get("ref") in proof["branches"] + and _repo_matches(head.get("repo"), repository) + ): + pr["_listed_head"] = row.get("headRefOid") + owned.append(pr) + return owned + + +def validate_sandbox_ci( + repository: str, + number: int, + head: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> None: + workflows = _api_pages( + f"repos/{repository}/actions/workflows?per_page=100", env, runner, out / "ci-workflows" + ) + candidates = [ + workflow + for page in workflows + for workflow in page.get("workflows", []) + if workflow.get("name") == "CI" and workflow.get("state") == "active" + ] + if len(candidates) != 1: + raise ValueError("sandbox CI: expected one active CI workflow") + for attempt in range(6): + pages = _api_pages( + f"repos/{repository}/actions/workflows/{candidates[0]['id']}/runs?event=pull_request&per_page=100", + env, + runner, + out / "ci-runs", + ) + runs = [ + run + for page in pages + for run in page.get("workflow_runs", []) + if any( + pr.get("number") == number + and pr.get("head", {}).get("sha") == head + and _repo_matches(pr.get("head", {}).get("repo"), repository) + and _repo_matches(pr.get("base", {}).get("repo"), repository) + for pr in run.get("pull_requests", []) + ) + ] + if runs: + run = max(runs, key=lambda value: (value["id"], value.get("run_attempt", 1))) + if run.get("status") == "completed": + jobs = _api_pages( + f"repos/{repository}/actions/runs/{run['id']}/jobs?per_page=100", + env, + runner, + out / "ci-jobs", + ) + tests = [ + job + for page in jobs + for job in page.get("jobs", []) + if job.get("name") == "test" + ] + if ( + run.get("conclusion") != "success" + or not tests + or any( + job.get("status") != "completed" or job.get("conclusion") != "success" + for job in tests + ) + ): + raise ValueError("sandbox CI: CI run and test job must succeed") + return + if attempt < 5: + time.sleep(10) + raise ValueError("sandbox CI: no completed CI run associated with the exact PR head") + + +def validate_implementation_pr( + test: e2e_catalog.CatalogTest, + prs: Sequence[Mapping[str, Any]], + seed: Mapping[str, Any], + repository: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + if len(prs) != 1: + return [f"expected exactly one new owned sandbox pull request, found {len(prs)}"] + pr = prs[0] + number = pr["number"] + head = pr.get("head", {}) + base = pr.get("base", {}) + oid = head.get("sha") + if ( + pr.get("state") != "open" + or pr.get("merged") is not False + or pr.get("merged_at") + or pr.get("auto_merge") is not None + or base.get("ref") != "main" + or head.get("ref") == "main" + or not isinstance(oid, str) + or not oid + or not _repo_matches(base.get("repo"), repository) + or not _repo_matches(head.get("repo"), repository) + ): + return [ + "implementation PR: expected an open, unmerged sandbox PR targeting main " + "without auto-merge" + ] + if oid != pr.get("_listed_head"): + return ["implementation PR: REST head differs from captured headRefOid"] + if not re.search(re.escape(seed["url"]) + r"(?![A-Za-z0-9_/#?=-])", str(pr.get("body", ""))): + return ["implementation PR: body does not reference the canonical seed URL"] + if ( + type(pr.get("additions")) is not int + or type(pr.get("deletions")) is not int + or (pr["additions"] + pr["deletions"] <= 0) + ): + return ["implementation PR: empty or malformed diff"] + pages = _api_pages( + f"repos/{repository}/pulls/{number}/files?per_page=100", env, runner, out / "pr-files" + ) + files = [row for page in pages for row in page] + for prefix in test.required_changed_paths: + if not any( + row.get("status") != "removed" + and isinstance(row.get("filename"), str) + and row["filename"].startswith(prefix) + and row.get("additions", 0) + row.get("deletions", 0) > 0 + for row in files + ): + return [f"implementation PR: missing non-removed changes under {prefix}"] + failures = test_pr_head(test, number, oid, out, env, runner) + if failures: + return failures + validate_sandbox_ci(repository, number, oid, out, env, runner) + final = _json_call( + ["gh", "api", f"repos/{repository}/pulls/{number}"], env, runner, evidence=out / "pr-final" + ) + if ( + final.get("head", {}).get("sha") != oid + or final.get("state") != "open" + or final.get("merged") is not False + or final.get("merged_at") + ): + return ["implementation PR: head or open/unmerged state changed during validation"] + return [] + + +def test_pr_head( + test: e2e_catalog.CatalogTest, + number: int, + oid: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + commands = ( + ("pr-fetch", ["git", "fetch", "origin", f"refs/pull/{number}/head"]), + ("pr-oid", ["git", "rev-parse", "FETCH_HEAD"]), + ("pr-checkout", ["git", "checkout", "--detach", oid]), + ("pr-install", ["task", "install-worktree"]), + ("pr-test", list(test.test_command)), + ) + for stage, argv in commands: + result, failure = _call( + runner, + argv, + what=stage, + evidence=out / stage, + cwd=SANDBOX_CLONE, + env=env, + timeout=SETUP_COMMAND_TIMEOUT_SEC, + ) + if failure: + return [failure] + if stage == "pr-oid" and (result is None or result.stdout.strip() != oid): + return ["implementation PR: fetched head differs from captured head"] + return [] + + +def discover_cleanup_prs( + repository: str, + baseline: int, + proof: Mapping[str, Any], + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> tuple[list[dict[str, Any]], list[str]]: + failures: list[str] = [] + owned: list[dict[str, Any]] = [] + for attempt in range(2): + try: + rows, failure = new_pull_requests(repository, baseline, env, runner, pipeline=True) + if rows is None: + raise ValueError(failure) + owned = owned_pull_requests(rows, proof, repository, out, env, runner) + if owned or attempt == 1: + if rows and not proof["branches"]: + failures.append("cleanup: PR ownership could not be established") + break + except Exception as exc: + if attempt == 1: + failures.append(f"cleanup discovery: {exc}") + if attempt == 0: + time.sleep(1) + return owned, failures + + +def delete_owned_branch( + repository: str, branch: str, env: Mapping[str, str], runner: Runner +) -> None: + if branch == "main" or not branch or branch.startswith("refs/"): + raise ValueError("refusing to delete an unsafe/default branch") + endpoint = f"repos/{repository}/git/refs/heads/{quote(branch, safe='')}" + result, failure = _setup_call(runner, ["gh", "api", endpoint], env) + if failure and result is not None and "404" in result.stderr: + return + if failure: + raise ValueError(failure) + time.sleep(1) + _, failure = _setup_call(runner, ["gh", "api", "--method", "DELETE", endpoint], env) + if failure: + raise ValueError(failure) + + +def release_owned_claim( + proof: Mapping[str, Any], + seed: Mapping[str, Any], + label: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + current = _json_call( + ["gh", "issue", "view", seed["url"], "--json", "state,labels"], + env, + runner, + evidence=out / "seed-final", + ) + failures = [] if current.get("state") == "OPEN" else ["cleanup: seed issue is no longer OPEN"] + if label not in {row["name"] for row in current["labels"]}: + return failures + if not proof["claimed"]: + return [*failures, "cleanup: claim label is present without this run's claim proof"] + time.sleep(1) + _, failure = _setup_call( + runner, ["gh", "issue", "edit", seed["url"], "--remove-label", label], env + ) + if failure: + failures.append(f"cleanup: {failure}") + return failures + + +def cleanup_implementation( + repository: str, + baseline: int, + proof: Mapping[str, Any], + seed: Mapping[str, Any], + label: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + owned, failures = discover_cleanup_prs(repository, baseline, proof, out, env, runner) + for pr in owned: + try: + if pr.get("state") == "open": + time.sleep(1) + _, failure = _setup_call( + runner, ["gh", "pr", "close", str(pr["number"]), "--repo", repository], env + ) + if failure: + failures.append(f"cleanup: {failure}") + except Exception as exc: + failures.append(f"cleanup PR: {exc}") + for branch in proof["branches"]: + try: + delete_owned_branch(repository, branch, env, runner) + except Exception as exc: + failures.append(f"cleanup branch: {exc}") + try: + failures += release_owned_claim(proof, seed, label, out, env, runner) + except Exception as exc: + failures.append(f"cleanup claim: {exc}") + return failures + + +def match_recipe_findings( + failures: list[str], + findings: Sequence[Mapping[str, str]], + expected: Sequence[Mapping[str, str]], +) -> tuple[list[str], list[dict[str, str]]]: + annotations = {(row["check"], row["message"]): row for row in expected} + matched: dict[tuple[str, str], dict[str, str]] = {} + for row in findings: + identity = (row["check"], row["message"]) + if identity in annotations: + matched[identity] = dict(annotations[identity]) + else: + failures.append(f"{row['check']}: {row['message']}") + for identity, annotation in annotations.items(): + if identity not in matched: + failures.append( + f"expected recipe finding absent {identity}; remove stale expected failure " + f"for {annotation['issue']}" + ) + return failures, list(matched.values()) + + def run_recipe( test: e2e_catalog.CatalogTest, repository: str, @@ -875,6 +1389,8 @@ def run_recipe( out: Path, env: Mapping[str, str], runner: Runner, + *, + home: Path | None = None, ) -> tuple[list[str], list[dict[str, str]]]: failure = ( configure_git(env, runner) @@ -885,15 +1401,44 @@ def run_recipe( return [failure], [] if test.recipe_fixture is not None: return run_smoke_recipe(test, repository, out, env, runner) + seed: dict[str, Any] = {} + ingredients = dict(test.ingredients) + label = "" + home = home or Path.home() + trace = out / "session-events.jsonl" + if test.pipeline or test.issue_url_env: + try: + seed, ingredients, label = prepare_seed(test, repository, out, env, runner) + if test.pipeline: + trace.write_text("", encoding="utf-8") + install_session_hooks(home, trace) + except Exception as exc: + return [f"preflight: {exc}"], [] baseline, failure = latest_pull_request_number(repository, env, runner) if baseline is None: return [str(failure)], [] + if not test.pipeline: + return launch_regular_recipe(test, repository, baseline, ingredients, out, env, runner) + return launch_recipe( + test, repository, baseline, seed, ingredients, label, out, home, env, runner + ) + + +def launch_regular_recipe( + test: e2e_catalog.CatalogTest, + repository: str, + baseline: int, + ingredients: Mapping[str, str], + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> tuple[list[str], list[dict[str, str]]]: failures: list[str] = [] + matched: list[dict[str, str]] = [] listed: list[dict[str, Any]] | None = None try: - fleet_failures, envelope = run_fleet(test, out, env, runner) - fleet_failures, matched = match_recipe_failure(test, fleet_failures, envelope) - failures += fleet_failures + envelope, failures, _ = run_fleet(test, out, env, runner, ingredients=ingredients) + failures, matched = match_recipe_failure(test, failures, envelope) if not matched: listed, failure = new_pull_requests(repository, baseline, env, runner) if listed is None: @@ -905,6 +1450,106 @@ def run_recipe( return failures, matched +def save_ownership( + trace: Path, + envelope: dict[str, Any] | None, + log_root: Path, + seed: Mapping[str, Any], + repository: str, + out: Path, + label: str, +) -> dict[str, Any]: + proof = e2e_sessions.collect_ownership( + trace, envelope, log_root, seed["url"], repository, claim_label=label + ) + (out / "ownership.json").write_text(json.dumps(proof, indent=2) + "\n", encoding="utf-8") + return proof + + +def finish_pipeline( + test: e2e_catalog.CatalogTest, + repository: str, + baseline: int, + seed: Mapping[str, Any], + label: str, + out: Path, + home: Path, + env: Mapping[str, str], + runner: Runner, + envelope: dict[str, Any] | None, + proof: dict[str, Any] | None, +) -> list[str]: + failures: list[str] = [] + trace = out / "session-events.jsonl" + log_root = home / ".local" / "share" / "autoskillit" / "logs" + try: + summary = e2e_sessions.summarize(trace, envelope, log_root) + if not summary["coverage"]["complete"]: + failures.append("session measurement: incomplete lifecycle coverage") + if summary["peak_sessions"] > test.peak_sessions: + failures.append("session measurement: observed peak exceeds catalog capacity") + except Exception as exc: + failures.append(f"session measurement: {exc}") + if proof is None: + try: + proof = save_ownership(trace, envelope, log_root, seed, repository, out, label) + failures += [f"ownership: {value}" for value in proof["violations"]] + except Exception as exc: + failures.append(f"cleanup ownership: {exc}") + proof = proof or {"branches": [], "claimed": False} + failures += cleanup_implementation(repository, baseline, proof, seed, label, out, env, runner) + return failures + + +def launch_recipe( + test: e2e_catalog.CatalogTest, + repository: str, + baseline: int, + seed: Mapping[str, Any], + ingredients: Mapping[str, str], + label: str, + out: Path, + home: Path, + env: Mapping[str, str], + runner: Runner, +) -> tuple[list[str], list[dict[str, str]]]: + failures: list[str] = [] + findings: list[dict[str, str]] = [] + envelope: dict[str, Any] | None = None + proof: dict[str, Any] | None = None + try: + envelope, fleet_failures, findings = run_fleet( + test, out, env, runner, ingredients=ingredients + ) + failures += fleet_failures + proof = save_ownership( + out / "session-events.jsonl", + envelope, + home / ".local" / "share" / "autoskillit" / "logs", + seed, + repository, + out, + label, + ) + failures += [f"ownership: {value}" for value in proof["violations"]] + listed, failure = new_pull_requests(repository, baseline, env, runner, pipeline=True) + if listed is None: + failures.append(str(failure)) + else: + owned = owned_pull_requests(listed, proof, repository, out, env, runner) + if not fleet_failures and not findings: + failures += validate_implementation_pr( + test, owned, seed, repository, out, env, runner + ) + except Exception as exc: + failures.append(f"harness error: {exc}") + finally: + failures += finish_pipeline( + test, repository, baseline, seed, label, out, home, env, runner, envelope, proof + ) + return match_recipe_findings(failures, findings, test.expected_failures) + + def check_clean_install_doctor( stdout: str, expected_failures: Sequence[Mapping[str, str]] ) -> tuple[list[str], list[dict[str, str]]]: @@ -1016,7 +1661,7 @@ def _run_test( if test.kind == "canary": return run_canary(test, out, scrubbed, runner), [] token = env["E2E_SANDBOX_TOKEN"] - return run_recipe(test, catalog.sandbox_repository, token, out, scrubbed, runner) + return run_recipe(test, catalog.sandbox_repository, token, out, scrubbed, runner, home=home) def _copy_failure_files( diff --git a/scripts/e2e/e2e_select.py b/scripts/e2e/e2e_select.py index 906e66a69b..56a8dbd246 100644 --- a/scripts/e2e/e2e_select.py +++ b/scripts/e2e/e2e_select.py @@ -21,6 +21,7 @@ MIN_CHANGED_LINES = 300 SAMPLE_PROBABILITY = 0.5 MAX_TESTS = 2 +MAX_PIPELINE_TESTS = 1 @dataclass(frozen=True, slots=True) @@ -68,10 +69,12 @@ def selected(self) -> bool: def is_eligible(pr: PullRequestFacts) -> bool: + return E2E_LABEL in pr.labels or _is_large_change(pr) + + +def _is_large_change(pr: PullRequestFacts) -> bool: return ( - E2E_LABEL in pr.labels - or pr.changed_files >= MIN_CHANGED_FILES - or pr.additions + pr.deletions >= MIN_CHANGED_LINES + pr.changed_files >= MIN_CHANGED_FILES or pr.additions + pr.deletions >= MIN_CHANGED_LINES ) @@ -97,13 +100,32 @@ def _touches(test: e2e_catalog.CatalogTest, changed_paths: Sequence[str]) -> boo def pick_tests( - catalog: e2e_catalog.Catalog, changed_paths: Sequence[str], seed: str + catalog: e2e_catalog.Catalog, + changed_paths: Sequence[str], + seed: str, + *, + large_change: bool = False, ) -> tuple[str, ...]: - ranked = sorted(catalog.tests, key=lambda test: _rank(seed, test.name)) - touched = [test.name for test in ranked if _touches(test, changed_paths)] + eligible = [ + test + for test in catalog.tests + if not test.pipeline or (large_change and _touches(test, changed_paths)) + ] + ranked = sorted(eligible, key=lambda test: _rank(seed, test.name)) + touched = [test for test in ranked if _touches(test, changed_paths)] if touched: - return tuple(touched[:MAX_TESTS]) - return (ranked[0].name,) + picked: list[str] = [] + pipeline_count = 0 + for test in touched: + if test.pipeline: + if pipeline_count >= MAX_PIPELINE_TESTS: + continue + pipeline_count += 1 + picked.append(test.name) + if len(picked) == MAX_TESTS: + break + return tuple(picked) + return (ranked[0].name,) if ranked else () def select_for_pull_request( @@ -122,10 +144,16 @@ def select_for_pull_request( ) seed = f"{pr.number}:{pr.head_sha}" if E2E_LABEL in pr.labels: - return Selection(pick_tests(catalog, changed_paths, seed), f"{E2E_LABEL} label") + return Selection( + pick_tests(catalog, changed_paths, seed, large_change=_is_large_change(pr)), + f"{E2E_LABEL} label", + ) if not is_sampled(pr, SAMPLE_PROBABILITY): return Selection((), "eligible but not sampled for this head commit") - return Selection(pick_tests(catalog, changed_paths, seed), "eligible and sampled") + return Selection( + pick_tests(catalog, changed_paths, seed, large_change=_is_large_change(pr)), + "eligible and sampled", + ) def select_for_dispatch(requested: str, catalog: e2e_catalog.Catalog) -> Selection: diff --git a/scripts/e2e/e2e_sessions.py b/scripts/e2e/e2e_sessions.py new file mode 100644 index 0000000000..bab3aec9be --- /dev/null +++ b/scripts/e2e/e2e_sessions.py @@ -0,0 +1,879 @@ +#!/usr/bin/env python3 +"""Capture and summarize Claude session lifecycle evidence for E2E runs.""" + +from __future__ import annotations + +import json +import math +import os +import re +import sys +import time +from collections.abc import Mapping +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit + +_SESSION_EVENTS = frozenset({"SessionStart", "SessionEnd"}) +_AGENT_EVENTS = frozenset({"SubagentStart", "SubagentStop"}) +_LIFECYCLE_EVENTS = _SESSION_EVENTS | _AGENT_EVENTS +_MAX_HOOK_INPUT_BYTES = 65_536 +_MAX_TRACE_RECORD_BYTES = 8_192 + + +def capture_hook_event(trace_path: Path, payload: Mapping[str, Any]) -> bool: + """Append one allow-listed lifecycle record; return False for other hooks.""" + record = _hook_record(payload) + if record is None: + return False + encoded = (json.dumps(record, separators=(",", ":")) + "\n").encode("utf-8") + if len(encoded) > _MAX_TRACE_RECORD_BYTES: + raise ValueError("lifecycle hook record exceeds the size limit") + trace_path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(trace_path, os.O_APPEND | os.O_CREAT | os.O_WRONLY, 0o600) + try: + written = os.write(fd, encoded) + if written != len(encoded): + raise OSError("short lifecycle trace append") + finally: + os.close(fd) + return True + + +def _hook_record(payload: Mapping[str, Any]) -> dict[str, Any] | None: + event = payload.get("hook_event_name", payload.get("event")) + if event is None: + return None + if not isinstance(event, str): + raise ValueError("lifecycle hook event must be a string") + if event not in _LIFECYCLE_EVENTS: + return None + + session_id = payload.get("session_id") + if not isinstance(session_id, str) or not session_id: + raise ValueError("lifecycle hook has no session_id") + + record: dict[str, Any] = { + "event": event, + "timestamp": time.time(), + "session_id": session_id, + } + for key in ( + "agent_id", + "agent_type", + "source", + "transcript_path", + "agent_transcript_path", + ): + value = payload.get(key) + if value is None: + continue + if not isinstance(value, str): + raise ValueError(f"lifecycle hook {key} must be a string") + record[key] = value + if event in _AGENT_EVENTS and not record.get("agent_id"): + raise ValueError("subagent lifecycle hook has no agent_id") + return record + + +def _read_jsonl( + path: Path, label: str, violations: list[str], *, collect: bool = True +) -> list[dict[str, Any]]: + try: + handle = path.open(encoding="utf-8", errors="replace") + except OSError: + violations.append(f"{label}:unavailable") + return [] + + records: list[dict[str, Any]] = [] + with handle: + for line_number, line in enumerate(handle, 1): + if not line.strip(): + violations.append(f"{label}:empty_line:{line_number}") + continue + try: + item = json.loads(line) + except (json.JSONDecodeError, UnicodeDecodeError): + violations.append(f"{label}:invalid_json:{line_number}") + continue + if not isinstance(item, dict): + violations.append(f"{label}:invalid_record:{line_number}") + continue + if collect: + records.append(item) + return records + + +def _load_trace(trace_path: Path, violations: list[str]) -> list[dict[str, Any]]: + records = _read_jsonl(trace_path, "trace", violations) + valid: list[dict[str, Any]] = [] + for index, record in enumerate(records, 1): + event = record.get("event") + timestamp = record.get("timestamp") + session_id = record.get("session_id") + if not isinstance(event, str) or event not in _LIFECYCLE_EVENTS: + violations.append(f"trace:unknown_event:{index}") + continue + finite_timestamp = False + if isinstance(timestamp, (int, float)) and not isinstance(timestamp, bool): + try: + finite_timestamp = math.isfinite(timestamp) + except OverflowError: + pass + if not finite_timestamp: + violations.append(f"trace:invalid_timestamp:{index}") + continue + if not isinstance(session_id, str) or not session_id: + violations.append(f"trace:invalid_session_id:{index}") + continue + optional_invalid = any( + key in record and record[key] is not None and not isinstance(record[key], str) + for key in ( + "agent_id", + "agent_type", + "source", + "transcript_path", + "agent_transcript_path", + ) + ) + if optional_invalid: + violations.append(f"trace:invalid_optional_field:{index}") + continue + if event in _AGENT_EVENTS and not record.get("agent_id"): + violations.append(f"trace:missing_agent_id:{index}") + continue + valid.append(record) + return valid + + +def _read_envelope(envelope: dict[str, Any] | None, violations: list[str]) -> tuple[str, str]: + if not isinstance(envelope, dict): + violations.append("envelope:missing") + return "", "" + session_id = envelope.get("dispatched_session_id") + dispatch_id = envelope.get("dispatch_id") + if not isinstance(session_id, str) or not session_id: + violations.append("envelope:missing_dispatched_session_id") + session_id = "" + if not isinstance(dispatch_id, str) or not dispatch_id: + violations.append("envelope:missing_dispatch_id") + dispatch_id = "" + return session_id, dispatch_id + + +def _load_index(log_root: Path, violations: list[str]) -> list[dict[str, Any]]: + index_path = log_root / "sessions.jsonl" + records = _read_jsonl(index_path, "sessions_index", violations) + for line_number, row in enumerate(records, 1): + if not isinstance(row.get("session_id"), str) or not row.get("session_id"): + violations.append(f"sessions_index:missing_session_id:{line_number}") + return records + + +def _safe_session_dir(log_root: Path, row: Mapping[str, Any]) -> Path | None: + name = row.get("dir_name") + if not isinstance(name, str) or not name or Path(name).name != name: + return None + return log_root / "sessions" / name + + +def _read_summary(session_dir: Path, expected_session_id: str) -> dict[str, Any] | None: + try: + summary = json.loads((session_dir / "summary.json").read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError, UnicodeDecodeError): + return None + if not isinstance(summary, dict) or summary.get("session_id") != expected_session_id: + return None + identity = summary.get("execution_identity") + if not isinstance(identity, dict) or not isinstance(identity.get("parent_session_id"), str): + return None + return summary + + +def _discover_owned( + trace_path: Path, + envelope: dict[str, Any] | None, + log_root: Path | None, +) -> dict[str, Any]: + """Resolve only the dispatched session and fresh, linked managed descendants.""" + root = log_root or Path.home() / ".local" / "share" / "autoskillit" / "logs" + violations: list[str] = [] + events = _load_trace(trace_path, violations) + root_id, dispatch_id = _read_envelope(envelope, violations) + trace_session_ids = { + record["session_id"] for record in events if record["event"] in _SESSION_EVENTS + } + rows = _load_index(root, violations) + rows_by_id: dict[str, list[dict[str, Any]]] = {} + for row in rows: + session_id = row.get("session_id") + if isinstance(session_id, str) and session_id: + rows_by_id.setdefault(session_id, []).append(row) + owned: dict[str, dict[str, Any]] = {} + root_rows = rows_by_id.get(root_id, []) + if not root_id or len(root_rows) != 1: + violations.append("sessions_index:root_missing_or_ambiguous") + else: + if root_id not in trace_session_ids: + violations.append("trace:root_session_missing") + root_row = root_rows[0] + if not dispatch_id or root_row.get("dispatch_id") != dispatch_id: + violations.append("sessions_index:root_dispatch_mismatch") + _add_owned_session(root, root_row, root_id, violations, owned) + candidates = _fresh_candidates(root, root_id, dispatch_id, trace_session_ids, rows_by_id) + _add_linked_descendants(root, dispatch_id, candidates, owned, violations) + + return { + "root_id": root_id, + "dispatch_id": dispatch_id, + "log_root": root, + "events": events, + "owned": owned, + "violations": violations, + "trace_session_ids": trace_session_ids, + } + + +def _fresh_candidates( + log_root: Path, + root_id: str, + dispatch_id: str, + trace_session_ids: set[str], + rows_by_id: Mapping[str, list[dict[str, Any]]], +) -> dict[str, tuple[dict[str, Any], dict[str, Any] | None]]: + candidates: dict[str, tuple[dict[str, Any], dict[str, Any] | None]] = {} + candidate_ids = trace_session_ids | { + session_id + for session_id, rows in rows_by_id.items() + if dispatch_id and any(row.get("dispatch_id") == dispatch_id for row in rows) + } + for session_id in candidate_ids: + if session_id == root_id: + continue + matching_rows = rows_by_id.get(session_id, []) + if len(matching_rows) != 1: + continue + row = matching_rows[0] + session_dir = _safe_session_dir(log_root, row) + summary = _read_summary(session_dir, session_id) if session_dir else None + candidates[session_id] = row, summary + return candidates + + +def _add_linked_descendants( + log_root: Path, + dispatch_id: str, + candidates: dict[str, tuple[dict[str, Any], dict[str, Any] | None]], + owned: dict[str, dict[str, Any]], + violations: list[str], +) -> None: + remaining = dict(candidates) + while remaining: + changed = False + owned_ids = set(owned) + for session_id, (row, summary) in list(remaining.items()): + parents = { + value + for key in ("parent_session_id", "caller_session_id") + if isinstance((value := row.get(key)), str) and value + } + summary_parent = summary["execution_identity"]["parent_session_id"] if summary else "" + if not parents.intersection(owned_ids) and summary_parent not in owned_ids: + continue + if not dispatch_id or row.get("dispatch_id") != dispatch_id: + violations.append(f"sessions_index:{session_id}:dispatch_mismatch") + del remaining[session_id] + continue + if summary is None: + violations.append(f"summary:{session_id}:missing_or_malformed") + elif summary_parent not in owned_ids: + violations.append(f"summary:{session_id}:parent_mismatch") + if parents and not parents.intersection(owned_ids): + violations.append(f"sessions_index:{session_id}:parent_mismatch") + _add_owned_session(log_root, row, session_id, violations, owned, summary=summary) + del remaining[session_id] + changed = True + if not changed: + break + + +def _add_owned_session( + log_root: Path, + row: Mapping[str, Any], + session_id: str, + violations: list[str], + owned: dict[str, dict[str, Any]], + *, + summary: dict[str, Any] | None = None, +) -> None: + session_dir = _safe_session_dir(log_root, row) + if session_dir is None: + violations.append(f"sessions_index:{session_id}:invalid_dir_name") + return + required_index_fields = ( + "session_id", + "dir_name", + "dispatch_id", + "parent_session_id", + "caller_session_id", + "claude_code_log", + ) + if any(key not in row for key in required_index_fields): + violations.append(f"sessions_index:{session_id}:missing_fields") + if summary is None: + summary = _read_summary(session_dir, session_id) + if summary is None: + violations.append(f"summary:{session_id}:missing_or_malformed") + elif summary.get("dispatch_id", "") != row.get("dispatch_id", ""): + violations.append(f"summary:{session_id}:dispatch_mismatch") + transcript_value = row.get("claude_code_log") + transcript: Path | None = None + if isinstance(transcript_value, str) and transcript_value: + transcript = Path(transcript_value) + if not transcript.is_absolute(): + transcript = log_root / transcript + else: + violations.append(f"sessions_index:{session_id}:missing_transcript") + if transcript is None or not transcript.is_file(): + violations.append(f"transcript:{session_id}:missing") + else: + _read_jsonl(transcript, f"transcript:{session_id}", violations, collect=False) + owned[session_id] = { + "row": dict(row), + "summary": summary, + "session_dir": session_dir, + "transcript": transcript, + } + + +def _lifecycle_intervals( + events: list[dict[str, Any]], + session_ids: set[str], + violations: list[str], +) -> tuple[list[dict[str, Any]], set[str]]: + grouped = _group_lifecycle_events(events, session_ids) + intervals: list[dict[str, Any]] = [] + seen_sessions: set[str] = set() + for identity, endpoints in grouped.items(): + item = _make_interval(identity, endpoints, violations) + if item is None: + continue + if identity[0] == "session": + seen_sessions.add(identity[1]) + intervals.append(item) + return intervals, seen_sessions + + +def _group_lifecycle_events( + events: list[dict[str, Any]], session_ids: set[str] +) -> dict[tuple[str, ...], dict[str, list[dict[str, Any]]]]: + grouped: dict[tuple[str, ...], dict[str, list[dict[str, Any]]]] = {} + for event in events: + name = event["event"] + session_id = event["session_id"] + if session_id not in session_ids: + continue + identity: tuple[str, ...] + if name in _SESSION_EVENTS: + identity = ("session", session_id) + role = "start" if name == "SessionStart" else "end" + else: + identity = ("agent", session_id, event["agent_id"]) + role = "start" if name == "SubagentStart" else "end" + grouped.setdefault(identity, {"start": [], "end": []})[role].append(event) + return grouped + + +def _make_interval( + identity: tuple[str, ...], + endpoints: Mapping[str, list[dict[str, Any]]], + violations: list[str], +) -> dict[str, Any] | None: + starts, stops = endpoints["start"], endpoints["end"] + label = ":".join(identity) + if not starts: + violations.append(f"lifecycle:{label}:orphan_end") + return None + if not stops: + violations.append(f"lifecycle:{label}:missing_end") + return None + start = min(float(item["timestamp"]) for item in starts) + end = max(float(item["timestamp"]) for item in stops) + if any(float(item["timestamp"]) < start for item in stops): + violations.append(f"lifecycle:{label}:orphan_end_before_start") + if end < start or any(float(event["timestamp"]) > end for event in starts): + reason = "end_before_start" if end < start else "missing_final_end" + violations.append(f"lifecycle:{label}:{reason}") + return None + item = {"kind": identity[0], "session_id": identity[1], "start": start, "end": end} + if identity[0] == "agent": + item["agent_id"] = identity[2] + path_field = "agent_transcript_path" + else: + path_field = "transcript_path" + paths = { + str(Path(event[path_field]).resolve()) + for event in starts + stops + if isinstance(event.get(path_field), str) and event[path_field] + } + if len(paths) > 1: + violations.append(f"lifecycle:{label}:transcript_mismatch") + if paths: + item["transcript_path"] = sorted(paths)[0] + for key in ("agent_type", "source"): + values = {event[key] for event in starts + stops if isinstance(event.get(key), str)} + if len(values) == 1: + item[key] = next(iter(values)) + return item + + +def _check_transcript_coverage( + events: list[dict[str, Any]], + owned: Mapping[str, Mapping[str, Any]], + violations: list[str], +) -> list[dict[str, str]]: + session_ids = set(owned) + session_paths: dict[str, set[str]] = {session_id: set() for session_id in session_ids} + observed: dict[tuple[str, str], list[dict[str, Any]]] = {} + for event in events: + session_id = event["session_id"] + if session_id not in session_ids: + continue + if event["event"] in _SESSION_EVENTS and event.get("transcript_path"): + session_paths[session_id].add(str(Path(event["transcript_path"]).resolve())) + elif event["event"] in _AGENT_EVENTS: + key = (session_id, event["agent_id"]) + observed.setdefault(key, []).append(event) + for session_id, paths in session_paths.items(): + if len(paths) > 1: + violations.append(f"lifecycle:{session_id}:transcript_mismatch") + enumerated = _enumerate_native_transcripts(owned, session_paths, violations) + + native_agents: list[dict[str, str]] = [] + agent_keys = set(observed) + agent_keys.update( + (session_id, agent_id) for session_id, files in enumerated.items() for agent_id in files + ) + for session_id, agent_id in sorted(agent_keys): + transcript = enumerated.get(session_id, {}).get(agent_id) + proof = _verify_native_transcript( + session_id, agent_id, transcript, observed.get((session_id, agent_id), []), violations + ) + if proof is not None: + native_agents.append(proof) + return native_agents + + +def _enumerate_native_transcripts( + owned: Mapping[str, Mapping[str, Any]], + session_paths: Mapping[str, set[str]], + violations: list[str], +) -> dict[str, dict[str, Path]]: + enumerated: dict[str, dict[str, Path]] = {} + for session_id, item in owned.items(): + transcript = item.get("transcript") + if not isinstance(transcript, Path) or not transcript.is_file(): + continue + paths = session_paths.get(session_id, set()) + if not paths: + violations.append(f"lifecycle:{session_id}:transcript_path_missing") + elif paths != {str(transcript.resolve())}: + violations.append(f"lifecycle:{session_id}:transcript_path_mismatch") + subagents_dir = transcript.parent / session_id / "subagents" + files: dict[str, Path] = {} + for path in sorted(subagents_dir.glob("agent-*.jsonl")): + agent_id = path.name.removeprefix("agent-").removesuffix(".jsonl") + if not agent_id: + continue + files[agent_id] = path + if not path.is_file(): + violations.append(f"native:{session_id}:{agent_id}:transcript_missing") + enumerated[session_id] = files + return enumerated + + +def _verify_native_transcript( + session_id: str, + agent_id: str, + transcript: Path | None, + events: list[dict[str, Any]], + violations: list[str], +) -> dict[str, str] | None: + label = f"native:{session_id}:{agent_id}" + names = {event["event"] for event in events} + if not {"SubagentStart", "SubagentStop"} <= names: + violations.append(f"{label}:lifecycle_incomplete") + if transcript is None: + violations.append(f"{label}:transcript_unenumerated") + return None + expected_path = str(transcript.resolve()) + paths = { + str(Path(event["agent_transcript_path"]).resolve()) + for event in events + if isinstance(event.get("agent_transcript_path"), str) + } + if paths != {expected_path}: + violations.append(f"{label}:transcript_mismatch") + if not paths: + violations.append(f"{label}:transcript_path_missing") + if not transcript.is_file(): + violations.append(f"{label}:transcript_missing") + return None + _read_jsonl(transcript, f"transcript:{session_id}:{agent_id}", violations, collect=False) + return {"session_id": session_id, "agent_id": agent_id, "transcript_path": expected_path} + + +def _peak_sessions(intervals: list[dict[str, Any]]) -> int: + points: list[tuple[float, int]] = [] + for item in intervals: + if item["start"] == item["end"]: + continue + points.append((item["end"], -1)) + points.append((item["start"], 1)) + active = peak = 0 + for _timestamp, delta in sorted(points, key=lambda point: (point[0], point[1])): + active += delta + peak = max(peak, active) + return peak + + +def summarize( + trace_path: Path, + envelope: dict[str, Any] | None, + log_root: Path | None = None, +) -> dict[str, Any]: + """Write a bounded concurrency summary beside the raw lifecycle trace.""" + context = _discover_owned(trace_path, envelope, log_root) + violations = list(context["violations"]) + owned: dict[str, dict[str, Any]] = context["owned"] + owned_ids = set(owned) + intervals, seen_sessions = _lifecycle_intervals(context["events"], owned_ids, violations) + for session_id in sorted(owned_ids - seen_sessions): + violations.append(f"lifecycle:{session_id}:missing_endpoints") + native_agents = _check_transcript_coverage(context["events"], owned, violations) + root_id = context["root_id"] + root_seen = root_id in seen_sessions + summary = { + "method": "hook_lifecycle_intervals", + "trace_path": str(trace_path), + "peak_sessions": _peak_sessions(intervals), + "identities": sorted( + intervals, + key=lambda item: ( + item["start"], + item["kind"], + item["session_id"], + item.get("agent_id", ""), + ), + ), + "coverage": { + "complete": not violations, + "root_session_id": root_id or None, + "root_session_seen": root_seen, + "managed_session_ids": sorted(owned_ids - ({root_id} if root_id else set())), + "owned_session_ids": sorted(owned_ids), + "native_agents": native_agents, + }, + "violations": sorted(set(violations)), + } + output_path = trace_path.with_name("session-concurrency.json") + output_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return summary + + +def _issue_number(issue_url: str) -> str: + match = re.search(r"/issues/(\d+)/?$", urlsplit(issue_url).path) + if match: + return match.group(1) + match = re.search(r"#(\d+)$", issue_url) + return match.group(1) if match else "" + + +def _issue_matches_repository(issue_url: str, repository: str) -> bool: + parsed = urlsplit(issue_url) + return ( + parsed.scheme == "https" + and parsed.netloc.lower() == "github.com" + and (parsed.path.strip("/").split("/issues/", 1)[0].lower() == repository.lower()) + ) + + +def _remote_matches_repository(remote_url: Any, repository: str) -> bool: + if not isinstance(remote_url, str): + return False + https = re.fullmatch(r"https://github\.com/([^/]+/[^/]+?)(?:\.git)?/?", remote_url) + ssh = re.fullmatch(r"git@github\.com:([^/]+/[^/]+)\.git", remote_url) + match = https or ssh + return bool(match and match.group(1).lower() == repository.lower()) + + +def _result_object(value: Any) -> dict[str, Any] | None: + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError: + return None + if isinstance(value, dict): + if any( + key in value for key in ("success", "claimed", "merge_target", "error", "error_type") + ): + return value + nested = value.get("structuredContent", value.get("result", value.get("content"))) + if isinstance(nested, list): + return _result_text_blocks(nested) + elif isinstance(nested, str): + return _result_object(nested) + if isinstance(value, list): + return _result_text_blocks(value) + return None + + +def _result_text_blocks(blocks: list[Any]) -> dict[str, Any] | None: + for block in blocks: + if not isinstance(block, dict) or not isinstance(block.get("text"), str): + continue + try: + decoded = json.loads(block["text"]) + except json.JSONDecodeError: + continue + if isinstance(decoded, dict): + return decoded + return None + + +def _transcript_tool_pairs( + transcript: Path, + violation_prefix: str, + violations: list[str], +) -> list[tuple[str, dict[str, Any], dict[str, Any] | None, bool]]: + """Return paired target calls as name, input, result, and result-error flag.""" + try: + handle = transcript.open(encoding="utf-8", errors="replace") + except OSError: + violations.append(f"transcript:{violation_prefix}:unavailable") + return [] + pending: dict[str, tuple[str, dict[str, Any]]] = {} + paired: list[tuple[str, dict[str, Any], dict[str, Any] | None, bool]] = [] + with handle: + for line_number, line in enumerate(handle, 1): + try: + row = json.loads(line) + except (json.JSONDecodeError, UnicodeDecodeError): + violations.append(f"transcript:{violation_prefix}:invalid_json:{line_number}") + continue + if isinstance(row, dict): + _pair_record(row, line_number, violation_prefix, pending, paired, violations) + for tool_id, (name, tool_input) in pending.items(): + violations.append(f"transcript:{violation_prefix}:unpaired_tool_use:{tool_id}") + paired.append((name, tool_input, None, False)) + return paired + + +def _pair_record( + row: dict[str, Any], + line_number: int, + violation_prefix: str, + pending: dict[str, tuple[str, dict[str, Any]]], + paired: list[tuple[str, dict[str, Any], dict[str, Any] | None, bool]], + violations: list[str], +) -> None: + message = row.get("message") + if not isinstance(message, dict): + message = row + content = message.get("content", row.get("content", [])) + if not isinstance(content, list): + return + role = row.get("type", message.get("role")) + for block in content: + if not isinstance(block, dict): + continue + if role == "assistant" and block.get("type") == "tool_use": + _remember_tool_use(block, line_number, violation_prefix, pending, violations) + elif role == "user" and block.get("type") == "tool_result": + tool_id = block.get("tool_use_id") + if isinstance(tool_id, str) and tool_id in pending: + name, tool_input = pending.pop(tool_id) + result = _result_object(block.get("content")) + paired.append((name, tool_input, result, block.get("is_error") is True)) + + +def _remember_tool_use( + block: dict[str, Any], + line_number: int, + prefix: str, + pending: dict[str, tuple[str, dict[str, Any]]], + violations: list[str], +) -> None: + name = block.get("name") + name = name.removeprefix("mcp__autoskillit__") if isinstance(name, str) else name + if name not in {"claim_and_resolve_issue", "create_and_publish_branch"}: + return + tool_id, tool_input = block.get("id"), block.get("input") + if not isinstance(tool_id, str) or not tool_id or not isinstance(tool_input, dict): + violations.append(f"transcript:{prefix}:malformed_tool_use:{line_number}") + return + if tool_id in pending: + violations.append(f"transcript:{prefix}:duplicate_tool_use:{line_number}") + pending[tool_id] = (name, tool_input) + + +def collect_ownership( + trace_path: Path, + envelope: dict[str, Any] | None, + log_root: Path | None = None, + issue_url: str = "", + repository: str = "", + claim_label: str | None = None, +) -> dict[str, Any]: + """Collect compact branch and claim proofs from the owned transcript lineage.""" + context = _discover_owned(trace_path, envelope, log_root) + violations = list(context["violations"]) + root_id = context["root_id"] + expected_number = _issue_number(issue_url) + valid_issue_reference = bool( + expected_number and repository and _issue_matches_repository(issue_url, repository) + ) + if not valid_issue_reference: + violations.append("ownership:invalid_issue_reference") + + branches, claim_calls, transcript_evidence = _owned_tool_proofs( + context["owned"], + root_id, + valid_issue_reference, + expected_number, + issue_url, + repository, + violations, + ) + claimed = _claim_succeeded(claim_calls, expected_number, claim_label, violations) + return { + "branches": sorted(set(branches)), + "claimed": claimed, + "transcripts": sorted(transcript_evidence, key=lambda item: item["session_id"]), + "violations": sorted(set(violations)), + } + + +def _owned_tool_proofs( + owned: Mapping[str, Mapping[str, Any]], + root_id: str, + valid_issue_reference: bool, + expected_number: str, + issue_url: str, + repository: str, + violations: list[str], +) -> tuple[ + list[str], list[tuple[dict[str, Any], dict[str, Any] | None, bool]], list[dict[str, str]] +]: + branches: list[str] = [] + claims: list[tuple[dict[str, Any], dict[str, Any] | None, bool]] = [] + transcripts: list[dict[str, str]] = [] + for session_id, item in owned.items(): + transcript = item.get("transcript") + if not isinstance(transcript, Path) or not transcript.is_file(): + continue + transcripts.append({"session_id": session_id, "path": str(transcript)}) + for name, tool_input, result, is_error in _transcript_tool_pairs( + transcript, session_id, violations + ): + target = _branch_target( + session_id, + root_id, + name, + tool_input, + result, + is_error, + valid_issue_reference, + expected_number, + repository, + violations, + ) + if target is not None: + branches.append(target) + if name == "claim_and_resolve_issue" and valid_issue_reference: + if tool_input.get("issue_url") == issue_url: + claims.append((tool_input, result, is_error)) + return branches, claims, transcripts + + +def _branch_target( + session_id: str, + root_id: str, + name: str, + tool_input: dict[str, Any], + result: dict[str, Any] | None, + is_error: bool, + valid_issue_reference: bool, + expected_number: str, + repository: str, + violations: list[str], +) -> str | None: + if name != "create_and_publish_branch" or session_id != root_id or not valid_issue_reference: + return None + if tool_input.get("issue_number") != expected_number: + return None + if not _remote_matches_repository(tool_input.get("remote_url"), repository): + return None + if ( + is_error + or result is None + or "error" in result + or "error_type" in result + or not isinstance(result.get("merge_target"), str) + or not result["merge_target"].strip() + ): + violations.append("ownership:branch_result_failed") + return None + return result["merge_target"].strip() + + +def _claim_succeeded( + claims: list[tuple[dict[str, Any], dict[str, Any] | None, bool]], + expected_number: str, + claim_label: str | None, + violations: list[str], +) -> bool: + if len(claims) > 1: + violations.append("ownership:claim_reentry") + return False + if not claims: + return False + tool_input, result, is_error = claims[0] + requested_label = tool_input.get("label") + effective_label = claim_label is None or requested_label in (None, "", claim_label) + claimed = bool( + not is_error + and effective_label + and tool_input.get("allow_reentry") is not True + and result is not None + and result.get("success") is True + and result.get("claimed") is True + and str(result.get("issue_number", "")) == expected_number + and result.get("reentry") is not True + ) + if not claimed: + violations.append("ownership:claim_result_failed") + return claimed + + +def main(argv: list[str] | None = None) -> int: + args = sys.argv[1:] if argv is None else argv + if len(args) != 1: + print("usage: e2e_sessions.py ", file=sys.stderr) + return 2 + try: + raw = sys.stdin.buffer.read(_MAX_HOOK_INPUT_BYTES + 1) + if len(raw) > _MAX_HOOK_INPUT_BYTES: + raise ValueError("lifecycle hook input exceeds the size limit") + payload = json.loads(raw) + if not isinstance(payload, dict): + raise ValueError("lifecycle hook input must be an object") + capture_hook_event(Path(args[0]), payload) + except (OSError, ValueError, json.JSONDecodeError, UnicodeDecodeError) as exc: + print(f"e2e lifecycle capture failed: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index 7c9c671567..fe8826b7a6 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -12,6 +12,7 @@ from dataclasses import dataclass, replace from pathlib import Path from types import SimpleNamespace +from typing import Any import pytest @@ -250,9 +251,13 @@ def test_pipeline_runs_directory_matches_clone_source_of_truth() -> None: "message": "model reported failure", "issue": "https://github.com/TalonT-Org/AutoSkillit/issues/5232", } +_IMPLEMENTATION_ISSUE = "https://github.com/owner/sandbox/issues/5233" +_IMPLEMENTATION_EXPECTED_ISSUE = "https://github.com/TalonT-Org/AutoSkillit/issues/5233" +_PIPELINE_HEAD = "0123456789abcdef" +_PIPELINE_BRANCH = "codex/issue-5233" -@dataclass +@dataclass(eq=False) class RecordedCall: argv: list[str] cwd: Path | None @@ -286,6 +291,39 @@ def _completed(argv, stdout: str = "", returncode: int = 0, stderr: str = ""): return subprocess.CompletedProcess(list(argv), returncode, stdout, stderr) +def _recorded_calls( + calls: list[RecordedCall], + *, + prefix: list[str] | None = None, + exact: list[str] | None = None, + contains: str | None = None, +) -> list[RecordedCall]: + return [ + call + for call in calls + if (prefix is None or call.argv[: len(prefix)] == prefix) + and (exact is None or call.argv == exact) + and (contains is None or contains in " ".join(call.argv)) + ] + + +def _recorded_pr_list_calls(calls: list[RecordedCall], *, baseline: bool) -> list[RecordedCall]: + lists = _recorded_calls(calls, prefix=["gh", "pr", "list"]) + selected = [] + for call in lists: + is_baseline = call.argv[call.argv.index("--limit") + 1] == "1" + if (baseline and is_baseline) or (not baseline and not is_baseline): + selected.append(call) + return selected + + +def _recorded_pr_detail_calls(calls: list[RecordedCall], number: int) -> list[RecordedCall]: + suffix = f"/pulls/{number}" + return [ + call for call in calls if call.argv[:2] == ["gh", "api"] and call.argv[2].endswith(suffix) + ] + + def _pr(number: int, state: str, additions: int = 3, deletions: int = 1) -> dict: return {"number": number, "state": state, "additions": additions, "deletions": deletions} @@ -344,6 +382,273 @@ def _env(**overrides: str) -> dict[str, str]: return {name: value for name, value in env.items() if value} +def _pipeline_catalog(expected_failures: list[dict[str, str]] | None = None): + entry = { + "name": "implementation", + "kind": "recipe", + "pipeline": True, + "peak_sessions": 6, + "timeout_sec": 600, + "trigger_paths": ["src/*"], + "recipe": "implementation", + "ingredients": {"source_dir": "/workspace/sandbox", "open_pr": "true"}, + "expected_pull_request_state": "open", + "issue_url_env": "E2E_IMPLEMENTATION_ISSUE_URL", + "required_changed_paths": ["sandbox/", "tests/"], + "test_command": ["task", "test-check"], + "peak_sessions_evidence": "recorded session events", + } + if expected_failures is not None: + entry["expected_failures"] = expected_failures + return e2e_catalog.parse_catalog({"sandbox_repository": SANDBOX, "tests": [entry]}) + + +def _pipeline_pr(number: int = 8, *, branch: str = _PIPELINE_BRANCH) -> dict[str, Any]: + return { + "number": number, + "state": "open", + "merged": False, + "merged_at": None, + "auto_merge": None, + "body": f"Implements {_IMPLEMENTATION_ISSUE}", + "additions": 5, + "deletions": 2, + "head": { + "ref": branch, + "sha": _PIPELINE_HEAD, + "repo": {"full_name": SANDBOX}, + }, + "base": {"ref": "main", "repo": {"full_name": SANDBOX}}, + } + + +def _pipeline_setup_response(argv: list[str], state: dict[str, Any]): + if argv[:3] == ["autoskillit", "config", "show"]: + return _completed(argv, json.dumps(state["config"])) + if argv[:3] == ["gh", "issue", "view"]: + payload = ( + state["seed"] if "number,url,state,title,body,labels" in argv else state["final_issue"] + ) + return _completed(argv, json.dumps(payload)) + if argv[:3] == ["gh", "repo", "view"]: + payload = {"defaultBranchRef": {"name": state["default_branch"]}} + return _completed(argv, json.dumps(payload)) + if argv[:3] == ["gh", "pr", "list"]: + rows = [{"number": 7}] if argv[argv.index("--limit") + 1] == "1" else state["listed_prs"] + return _completed(argv, json.dumps(rows)) + return None + + +def _pipeline_execution_response(argv: list[str], state: dict[str, Any]): + if argv[:3] == ["autoskillit", "fleet", "run"]: + if state["fleet_exception"] is not None: + raise state["fleet_exception"] + return _completed(argv, json.dumps(state["fleet_envelope"]) + "\n") + if argv[:2] == ["task", "test-check"]: + state["test_check_count"] += 1 + if state["test_check_count"] > 1 and state["pr_test_returncode"]: + return _completed(argv, returncode=state["pr_test_returncode"], stderr="tests failed") + return None + if argv[:2] == ["git", "rev-parse"]: + return _completed(argv, state["fetch_oid"] + "\n") + return None + + +def _pipeline_ci_api_response(argv: list[str], endpoint: str, state: dict[str, Any]): + if endpoint.endswith("/actions/workflows?per_page=100"): + workflows = [{"id": 1, "name": "CI", "state": "active"}] + return _completed(argv, json.dumps([{"workflows": workflows}])) + if "/actions/workflows/1/runs?" in endpoint: + run = { + "id": 2, + "status": "completed", + "conclusion": state["workflow_conclusion"], + "pull_requests": [ + { + "number": 8, + "head": {"sha": _PIPELINE_HEAD, "repo": {"full_name": SANDBOX}}, + "base": {"repo": {"full_name": SANDBOX}}, + } + ], + } + return _completed(argv, json.dumps([{"workflow_runs": [run]}])) + if endpoint.endswith("/actions/runs/2/jobs?per_page=100"): + job = {"name": "test", "status": "completed", "conclusion": state["job_conclusion"]} + return _completed(argv, json.dumps([{"jobs": [job]}])) + return None + + +def _pipeline_pull_request_api_response(argv: list[str], endpoint: str, state: dict[str, Any]): + if "/pulls/" not in endpoint: + return None + if endpoint.endswith("/files?per_page=100"): + return _completed(argv, json.dumps([state["files"]])) + number = int(endpoint.split("/pulls/", 1)[1].split("/", 1)[0]) + return _completed(argv, json.dumps(state["pr_details"][number])) + + +def _pipeline_cleanup_api_response(argv: list[str], endpoint: str, state: dict[str, Any]): + if argv[2] == "--method": + error = state["cleanup_errors"].get("branch_delete") + if error is not None: + raise error + return None + if "/git/refs/heads/" in endpoint: + error = state["cleanup_errors"].get("branch_get") + if error is not None: + raise error + return _completed(argv, "{}") + return None + + +def _pipeline_api_response(argv: list[str], state: dict[str, Any]): + if argv[:2] != ["gh", "api"]: + return None + endpoint = argv[4] if argv[2] == "--method" else argv[2] + for route in ( + _pipeline_ci_api_response, + _pipeline_pull_request_api_response, + _pipeline_cleanup_api_response, + ): + response = route(argv, endpoint, state) + if response is not None: + return response + return None + + +def _pipeline_cleanup_response(argv: list[str], state: dict[str, Any]): + if argv[:3] == ["gh", "pr", "close"]: + error = state["cleanup_errors"].get("pr_close") + if error is not None: + raise error + return None + + +def _pipeline_response(argv: list[str], state: dict[str, Any]): + for route in ( + _pipeline_setup_response, + _pipeline_execution_response, + _pipeline_api_response, + _pipeline_cleanup_response, + ): + response = route(argv, state) + if response is not None: + return response + return None + + +def _pipeline_runner() -> tuple[FakeRunner, dict[str, Any]]: + seed = { + "number": 5233, + "url": _IMPLEMENTATION_ISSUE, + "state": "OPEN", + "title": "Add a greeting", + "body": "Implement the requested behavior.", + "labels": [], + } + listed = { + "number": 8, + "state": "OPEN", + "additions": 5, + "deletions": 2, + "body": f"Implements {_IMPLEMENTATION_ISSUE}", + "headRefOid": _PIPELINE_HEAD, + "headRefName": _PIPELINE_BRANCH, + "headRepository": {"nameWithOwner": SANDBOX}, + "baseRefName": "main", + "autoMergeRequest": None, + "mergedAt": None, + } + state: dict[str, Any] = { + "seed": seed, + "default_branch": "main", + "config": { + "github": {"in_progress_label": "in-progress"}, + "branching": {"default_base_branch": "main"}, + }, + "listed_prs": [listed], + "pr_details": {8: _pipeline_pr()}, + "files": [ + {"filename": "sandbox/worker.py", "status": "added", "additions": 3, "deletions": 0}, + { + "filename": "tests/test_worker.py", + "status": "modified", + "additions": 2, + "deletions": 1, + }, + ], + "final_issue": {"state": "OPEN", "labels": [{"name": "in-progress"}]}, + "fetch_oid": _PIPELINE_HEAD, + "test_check_count": 0, + "pr_test_returncode": 0, + "workflow_conclusion": "success", + "job_conclusion": "success", + "fleet_envelope": { + "success": True, + "dispatch_status": "success", + "dispatch_id": "dispatch-5233", + "dispatched_session_id": "session-5233", + "l3_payload": {"reason": "implementation_complete"}, + }, + "fleet_exception": None, + "cleanup_errors": {}, + "proof": {"branches": [_PIPELINE_BRANCH], "claimed": True, "violations": []}, + "summary": {"coverage": {"complete": True}, "peak_sessions": 4, "violations": []}, + } + + return FakeRunner(lambda argv: _pipeline_response(argv, state)), state + + +def _patch_pipeline_measurement(monkeypatch, state: dict[str, Any]): + observed: dict[str, list[tuple]] = {"ownership": [], "summaries": []} + + def collect(trace_path, envelope, log_root, issue_url, repository, *, claim_label=None): + observed["ownership"].append( + (trace_path, envelope, log_root, issue_url, repository, claim_label) + ) + return state["proof"] + + def summarize(trace_path, envelope, log_root): + observed["summaries"].append((trace_path, envelope, log_root)) + return state["summary"] + + monkeypatch.setattr(harness.e2e_sessions, "collect_ownership", collect) + monkeypatch.setattr(harness.e2e_sessions, "summarize", summarize) + monkeypatch.setattr(harness.time, "sleep", lambda _seconds: None) + return observed + + +def _run_pipeline( + tmp_path: Path, + runner: FakeRunner, + state: dict[str, Any], + monkeypatch, + *, + env=None, + expected_failures=None, +): + catalog = _pipeline_catalog(expected_failures) + out = tmp_path / "out" + home = tmp_path / "home" + observed = _patch_pipeline_measurement(monkeypatch, state) + failures = harness.run_test( + catalog.get("implementation"), + catalog, + out=out, + home=home, + env=env if env is not None else _env(E2E_IMPLEMENTATION_ISSUE_URL=_IMPLEMENTATION_ISSUE), + runner=runner, + ) + return ( + failures, + json.loads((out / "result.json").read_text(encoding="utf-8")), + out, + home, + catalog, + observed, + ) + + def _run_recipe(tmp_path: Path, runner: FakeRunner, *, state: str = "open", env=None): catalog = _recipe_catalog(state) out = tmp_path / "out" @@ -1670,11 +1975,15 @@ def reject_login(argv: list[str]): assert ("autoskillit", "fleet", "run") not in prefixes def test_fleet_timeout_is_a_failure_and_cleanup_still_runs(self, tmp_path: Path) -> None: - timeout = subprocess.TimeoutExpired(["autoskillit"], 720) + timeout = subprocess.TimeoutExpired( + ["autoskillit"], 720, output=b"partial stdout", stderr=b"partial stderr" + ) runner = FakeRunner(_recipe_handler(fleet=timeout)) failures, result = _run_recipe(tmp_path, runner) assert result["passed"] is False assert any("fleet run: timed out" in failure for failure in failures) + assert (tmp_path / "out" / "fleet-run.stdout.log").read_text() == "partial stdout" + assert (tmp_path / "out" / "fleet-run.stderr.log").read_text() == "partial stderr" assert runner.calls[-1].argv[:4] == ["gh", "pr", "close", "8"] def test_unexpected_fleet_error_records_failure_and_still_cleans_up( @@ -1750,6 +2059,410 @@ def test_missing_sandbox_token_fails_before_any_launch(self, tmp_path: Path) -> assert runner.calls == [] +class TestPipelineRecipeFlow: + def test_pipeline_uses_the_seed_issue_and_cleans_only_its_owned_pr( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + unrelated = {**state["listed_prs"][0], "number": 9, "headRefName": "other/branch"} + state["listed_prs"].append(unrelated) + state["pr_details"][9] = _pipeline_pr(9, branch="other/branch") + failures, result, out, home, catalog, observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert failures == [] + assert result == { + "test": "implementation", + "passed": True, + "failures": [], + "outcome": "passed", + "expected_findings": [], + } + test = catalog.get("implementation") + assert test.pipeline is True + assert test.issue_url_env == "E2E_IMPLEMENTATION_ISSUE_URL" + assert test.required_changed_paths == ("sandbox/", "tests/") + assert test.test_command == ("task", "test-check") + + calls = runner.calls + config_call = _recorded_calls(calls, prefix=["autoskillit", "config", "show"])[0] + assert config_call.cwd == harness.SANDBOX_CLONE + assert config_call.env == harness.child_env( + _env(E2E_IMPLEMENTATION_ISSUE_URL=_IMPLEMENTATION_ISSUE) + ) + seed_call = _recorded_calls(calls, prefix=["gh", "issue", "view"])[0] + repo_call = _recorded_calls(calls, prefix=["gh", "repo", "view"])[0] + baseline_install = _recorded_calls(calls, exact=["task", "install-worktree"])[0] + checks = _recorded_calls(calls, exact=["task", "test-check"]) + baseline_list = _recorded_pr_list_calls(calls, baseline=True)[0] + fleet_call = _recorded_calls(calls, prefix=["autoskillit", "fleet", "run"])[0] + assert ( + calls.index(config_call) + < calls.index(seed_call) + < calls.index(repo_call) + < calls.index(baseline_install) + < calls.index(checks[0]) + < calls.index(baseline_list) + < calls.index(fleet_call) + ) + assert baseline_install.cwd == checks[0].cwd == harness.SANDBOX_CLONE + assert len(checks) == 2 and checks[1].cwd == harness.SANDBOX_CLONE + + listed_calls = _recorded_pr_list_calls(calls, baseline=False) + file_call = _recorded_calls(calls, contains="/files?per_page=100")[0] + fetch_call = _recorded_calls(calls, prefix=["git", "fetch"])[0] + oid_call = _recorded_calls(calls, prefix=["git", "rev-parse"])[0] + checkout_call = _recorded_calls(calls, prefix=["git", "checkout", "--detach"])[0] + install_calls = _recorded_calls(calls, exact=["task", "install-worktree"]) + workflow_call = _recorded_calls( + calls, prefix=["gh", "api"], contains="/actions/workflows?per_page=100" + )[0] + runs_call = _recorded_calls(calls, contains="/actions/workflows/1/runs?")[0] + jobs_call = _recorded_calls(calls, contains="/actions/runs/2/jobs?")[0] + pr_reads = _recorded_pr_detail_calls(calls, 8) + close = _recorded_calls(calls, prefix=["gh", "pr", "close"])[0] + branch_delete = _recorded_calls(calls, prefix=["gh", "api", "--method", "DELETE"])[0] + final_issue = _recorded_calls( + calls, prefix=["gh", "issue", "view"], contains="state,labels" + )[-1] + issue_edit = _recorded_calls(calls, prefix=["gh", "issue", "edit"])[0] + assert len(listed_calls) == 2 and len(install_calls) == 2 and len(pr_reads) == 3 + assert ( + calls.index(fleet_call) + < calls.index(listed_calls[0]) + < calls.index(file_call) + < calls.index(fetch_call) + < calls.index(oid_call) + < calls.index(checkout_call) + < calls.index(install_calls[1]) + < calls.index(checks[1]) + < calls.index(workflow_call) + < calls.index(runs_call) + < calls.index(jobs_call) + < calls.index(pr_reads[1]) + < calls.index(listed_calls[1]) + < calls.index(close) + < calls.index(branch_delete) + < calls.index(final_issue) + < calls.index(issue_edit) + ) + + ingredient_args = [ + fleet_call.argv[index + 1] + for index, arg in enumerate(fleet_call.argv[:-1]) + if arg == "-i" + ] + assert ingredient_args.count(f"issue_url={_IMPLEMENTATION_ISSUE}") == 1 + assert ( + ingredient_args.count("task=Add a greeting\n\nImplement the requested behavior.") == 1 + ) + ingredient_names = {key for key, _value in test.ingredients} + assert "issue_url" not in ingredient_names and "task" not in ingredient_names + assert fleet_call.timeout == test.timeout_sec + e2e_catalog.HARNESS_GRACE_SEC + assert fleet_call.cwd == harness.SANDBOX_CLONE + assert [call.input for call in calls if call.input] == [SANDBOX_TOKEN] + assert all(SANDBOX_TOKEN not in " ".join(call.argv) for call in calls) + assert all( + call.env == harness.child_env(_env(E2E_IMPLEMENTATION_ISSUE_URL=_IMPLEMENTATION_ISSUE)) + for call in calls + ) + + trace = out / "session-events.jsonl" + log_root = home / ".local" / "share" / "autoskillit" / "logs" + envelope = json.loads((out / "envelope.json").read_text(encoding="utf-8")) + assert observed["ownership"] == [ + (trace, envelope, log_root, _IMPLEMENTATION_ISSUE, SANDBOX, "in-progress") + ] + assert observed["summaries"] == [(trace, envelope, log_root)] + settings = json.loads((home / ".claude" / "settings.json").read_text(encoding="utf-8")) + assert settings["env"] == _EXPECTED_SETTINGS_ENV + assert all( + len(settings["hooks"][event]) == 1 + for event in ("SessionStart", "SessionEnd", "SubagentStart", "SubagentStop") + ) + + close = next(call for call in calls if call.argv[:3] == ["gh", "pr", "close"]) + assert close.argv == ["gh", "pr", "close", "8", "--repo", SANDBOX] + assert not any(call.argv[:4] == ["gh", "pr", "close", "9"] for call in calls) + assert any( + call.argv[:4] == ["gh", "api", "--method", "DELETE"] + and call.argv[4].endswith("codex%2Fissue-5233") + for call in calls + ) + assert any( + call.argv[:3] == ["gh", "issue", "edit"] + and call.argv[-2:] == ["--remove-label", "in-progress"] + for call in calls + ) + + @pytest.mark.parametrize( + ("problem", "expected"), + [ + ("malformed", "seed: expected the configured OPEN sandbox issue"), + ("closed", "seed: expected the configured OPEN sandbox issue"), + ("claimed", "seed: issue is already claimed"), + ], + ) + def test_invalid_seed_stops_before_baseline_or_fleet( + self, tmp_path: Path, monkeypatch, problem: str, expected: str + ) -> None: + runner, state = _pipeline_runner() + if problem == "malformed": + state["seed"].pop("title") + elif problem == "closed": + state["seed"]["state"] = "CLOSED" + else: + state["seed"]["labels"] = [{"name": "in-progress"}] + + failures, result, _out, _home, _catalog, observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert any(expected in failure for failure in failures) + assert result["outcome"] == "failed" + assert not any(call.argv[:2] == ["task", "install-worktree"] for call in runner.calls) + assert not any(call.argv[:2] == ["autoskillit", "fleet"] for call in runner.calls) + assert not any(call.argv[:3] == ["gh", "pr", "list"] for call in runner.calls) + assert observed["ownership"] == observed["summaries"] == [] + + def test_missing_seed_url_stops_before_github_issue_lookup( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, + runner, + state, + monkeypatch, + env=_env(E2E_IMPLEMENTATION_ISSUE_URL=""), + ) + + assert failures == ["preflight: seed: missing or noncanonical sandbox issue URL"] + assert result["outcome"] == "failed" + assert not any(call.argv[:3] == ["gh", "issue", "view"] for call in runner.calls) + assert not any(call.argv[:3] == ["autoskillit", "fleet", "run"] for call in runner.calls) + + @pytest.mark.parametrize("wrong_base", ["sandbox", "effective-config"]) + def test_seed_requires_the_main_branch_in_repository_and_config( + self, tmp_path: Path, monkeypatch, wrong_base: str + ) -> None: + runner, state = _pipeline_runner() + if wrong_base == "sandbox": + state["default_branch"] = "develop" + else: + state["config"]["branching"]["default_base_branch"] = "trunk" + + failures, _result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert any( + "sandbox and effective configured base branch must both be main" in item + for item in failures + ) + assert not any(call.argv[:2] == ["task", "install-worktree"] for call in runner.calls) + assert not any(call.argv[:3] == ["autoskillit", "fleet", "run"] for call in runner.calls) + + def test_linked_stable_finding_is_recorded_and_skips_pr_assertions( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["fleet_envelope"] = { + "success": False, + "dispatch_status": "retry_exhausted", + "dispatch_id": "dispatch-5233", + "dispatched_session_id": "session-5233", + "reason": "known orchestration failure", + } + state["listed_prs"] = [] + state["pr_details"] = {} + state["final_issue"] = {"state": "OPEN", "labels": []} + state["proof"] = {"branches": [], "claimed": False, "violations": []} + expected = { + "check": "fleet_envelope", + "message": "retry_exhausted: known orchestration failure", + "issue": _IMPLEMENTATION_EXPECTED_ISSUE, + } + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch, expected_failures=[expected] + ) + + assert failures == [] + assert result["outcome"] == "expected_failure" + assert result["passed"] is False + assert result["expected_findings"] == [expected] + assert not any(call.argv[:2] == ["git", "fetch"] for call in runner.calls) + assert not any( + "/pulls/" in " ".join(call.argv) and call.argv[0:2] == ["gh", "api"] + for call in runner.calls + ) + assert not any("actions/workflows" in " ".join(call.argv) for call in runner.calls) + + @pytest.mark.parametrize("envelope_kind", ["stale", "fatal"]) + def test_stale_or_fatal_pipeline_failure_cannot_be_allowlisted( + self, tmp_path: Path, monkeypatch, envelope_kind: str + ) -> None: + runner, state = _pipeline_runner() + expected = { + "check": "fleet_envelope", + "message": "retry_exhausted: known orchestration failure", + "issue": _IMPLEMENTATION_EXPECTED_ISSUE, + } + state["listed_prs"] = [] + state["pr_details"] = {} + state["final_issue"] = {"state": "OPEN", "labels": []} + state["proof"] = {"branches": [], "claimed": False, "violations": []} + if envelope_kind == "fatal": + state["fleet_envelope"] = { + "success": False, + "dispatch_status": "failed", + "dispatch_id": "dispatch-5233", + "dispatched_session_id": "session-5233", + "l3_payload": {}, + } + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch, expected_failures=[expected] + ) + + assert result["outcome"] == "failed" + assert result["expected_findings"] == [] + if envelope_kind == "stale": + assert any("expected recipe finding absent" in failure for failure in failures) + else: + assert any( + "unsuccessful envelope has no specific failure reason" in f for f in failures + ) + assert any("expected recipe finding absent" in failure for failure in failures) + assert not any(call.argv[:2] == ["git", "fetch"] for call in runner.calls) + + @pytest.mark.parametrize( + ("problem", "expected"), + [ + ("pr", "implementation PR: expected an open, unmerged sandbox PR"), + ("body", "implementation PR: body does not reference the canonical seed URL"), + ("paths", "implementation PR: missing non-removed changes under tests/"), + ("ci", "sandbox CI: CI run and test job must succeed"), + ("head", "implementation PR: fetched head differs from captured head"), + ("ci-job", "sandbox CI: CI run and test job must succeed"), + ("test", "pr-test: exit 1: tests failed"), + ], + ) + def test_pipeline_rejects_unverified_pr_or_validation_failure( + self, tmp_path: Path, monkeypatch, problem: str, expected: str + ) -> None: + runner, state = _pipeline_runner() + pr = state["pr_details"][8] + if problem == "pr": + pr["base"]["ref"] = "develop" + elif problem == "body": + pr["body"] = "No linked issue" + elif problem == "paths": + state["files"] = state["files"][:1] + elif problem == "ci": + state["workflow_conclusion"] = "failure" + elif problem == "ci-job": + state["job_conclusion"] = "failure" + elif problem == "head": + state["fetch_oid"] = "fetched-a-different-head" + else: + state["pr_test_returncode"] = 1 + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert any(expected in failure for failure in failures) + + def test_listed_pr_head_must_match_rest_head_before_validation( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["listed_prs"][0]["headRefOid"] = "stale-listed-head" + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert any("head" in failure.lower() or "oid" in failure.lower() for failure in failures) + assert not any("/files?" in " ".join(call.argv) for call in runner.calls) + assert not any("actions/workflows" in " ".join(call.argv) for call in runner.calls) + + @pytest.mark.parametrize( + ("problem", "expected"), + [ + ("coverage", "session measurement: incomplete lifecycle coverage"), + ("capacity", "session measurement: observed peak exceeds catalog capacity"), + ], + ) + def test_session_evidence_must_be_complete_and_within_catalog_capacity( + self, tmp_path: Path, monkeypatch, problem: str, expected: str + ) -> None: + runner, state = _pipeline_runner() + if problem == "coverage": + state["summary"]["coverage"]["complete"] = False + else: + state["summary"]["peak_sessions"] = 7 + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert any(expected in failure for failure in failures) + + def test_ownership_violations_fail_the_pipeline_result( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["proof"]["violations"] = ["transcript evidence is incomplete"] + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert "ownership: transcript evidence is incomplete" in failures + + def test_cleanup_errors_do_not_prevent_remaining_owned_cleanup( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["cleanup_errors"] = { + "pr_close": RuntimeError("close failed"), + "branch_delete": RuntimeError("branch delete failed"), + } + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert any("cleanup PR: close failed" in failure for failure in failures) + assert any("cleanup branch: branch delete failed" in failure for failure in failures) + assert any(call.argv[:3] == ["gh", "issue", "edit"] for call in runner.calls) + + def test_fleet_exception_and_cleanup_exception_are_both_preserved( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["fleet_exception"] = RuntimeError("fleet launch crashed") + state["cleanup_errors"] = {"branch_delete": RuntimeError("branch delete failed")} + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert any("harness error: fleet launch crashed" in failure for failure in failures) + assert any("cleanup branch: branch delete failed" in failure for failure in failures) + assert any(call.argv[:3] == ["gh", "issue", "edit"] for call in runner.calls) + + class TestCanaryFlow: def test_canary_runs_one_claude_prompt(self, tmp_path: Path, monkeypatch) -> None: runner = FakeRunner(lambda argv: _completed(argv, _CANARY_REPLY)) diff --git a/tests/infra/test_e2e_select.py b/tests/infra/test_e2e_select.py index ecec41bf95..1c2610d36b 100644 --- a/tests/infra/test_e2e_select.py +++ b/tests/infra/test_e2e_select.py @@ -23,6 +23,13 @@ REPOSITORY = "TalonT-Org/AutoSkillit" SHA_ZERO = "0" * 40 SYNTHETIC_SHAS = [hashlib.sha1(str(index).encode()).hexdigest() for index in range(2000)] +_IMPLEMENTATION_PATHS = ( + "src/autoskillit/recipes/", + "src/autoskillit/skills/", + "src/autoskillit/skills_extended/", + "src/autoskillit/server/", + "src/autoskillit/execution/", +) def _facts( @@ -188,6 +195,44 @@ def test_deleted_head_repository_is_not_selected(self) -> None: assert pr.head_repository is None assert not selection.selected + @pytest.mark.parametrize( + ("changed_files", "additions", "deletions"), + [(10, 0, 0), (1, 300, 0), (1, 100, 200)], + ) + def test_large_change_with_matching_root_selects_pipeline( + self, changed_files: int, additions: int, deletions: int + ) -> None: + pr = _facts( + head_sha=_sha_with_sample(5230, sampled=True), + changed_files=changed_files, + additions=additions, + deletions=deletions, + ) + selection = select.select_for_pull_request( + pr, + ["src/autoskillit/recipes/changed.py"], + _catalog(_pipeline_entry()), + REPOSITORY, + ) + assert selection.tests == ("implementation",) + + def test_label_does_not_make_a_small_change_pipeline_eligible(self) -> None: + pr = _facts(changed_files=1, additions=10, labels=("e2e",)) + selection = select.select_for_pull_request( + pr, + ["src/autoskillit/recipes/changed.py"], + _catalog(_pipeline_entry()), + REPOSITORY, + ) + assert not selection.selected + + def test_pipeline_is_not_an_unrelated_fallback(self) -> None: + pr = _facts(head_sha=_sha_with_sample(5230, sampled=True), changed_files=10) + selection = select.select_for_pull_request( + pr, ["README.md"], _catalog(_pipeline_entry()), REPOSITORY + ) + assert not selection.selected + class TestPickTests: _SEED = f"5230:{SHA_ZERO}" @@ -228,6 +273,25 @@ def test_star_matches_across_directories(self) -> None: catalog = _catalog(_entry("deep", ("src/*",)), _entry("other", ("docs/*",))) assert select.pick_tests(catalog, ["src/a/b/c.py"], self._SEED) == ("deep",) + @pytest.mark.parametrize("root", _IMPLEMENTATION_PATHS) + def test_pipeline_trigger_roots_match(self, root: str) -> None: + assert select.pick_tests( + _catalog(_pipeline_entry()), + [f"{root}changed.py"], + self._SEED, + large_change=True, + ) == ("implementation",) + + def test_at_most_one_pipeline_is_selected(self) -> None: + catalog = _catalog( + _pipeline_entry("pipeline-a", trigger_paths=["src/*"]), + _pipeline_entry("pipeline-b", trigger_paths=["src/*"]), + _entry("ordinary", ("src/*",)), + ) + picked = select.pick_tests(catalog, ["src/changed.py"], self._SEED, large_change=True) + assert len(picked) == select.MAX_TESTS + assert sum(catalog.get(name).pipeline for name in picked) == 1 + class TestSelectForEvent: def test_merge_group_is_never_selected(self) -> None: @@ -301,6 +365,18 @@ def test_matrix_includes_kind_from_each_catalog_entry() -> None: "ingredients": {"task": "Add a greeting"}, "expected_pull_request_state": "merged", } +_RECIPE_EXPECTED_FAILURE = { + "check": "implementation-check", + "message": "expected implementation failure", + "issue": "https://github.com/TalonT-Org/AutoSkillit/issues/123", +} +_IMPLEMENTATION_INGREDIENTS = { + "source_dir": "/workspace/sandbox", + "open_pr": "true", + "auto_merge": "false", + "arch_lenses": "false", + "audit_impl": "true", +} _MAX_TIMEOUT_SEC = ( e2e_catalog.MAX_JOB_MINUTES - e2e_catalog.JOB_OVERHEAD_MINUTES ) * 60 - e2e_catalog.HARNESS_GRACE_SEC @@ -311,6 +387,28 @@ def _recipe_entry_without(field: str) -> dict: return _entry("impl", kind="recipe", **fields) +def _pipeline_entry(name: str = "implementation", **overrides) -> dict: + entry = _entry( + name, + tuple(f"{path}*" for path in _IMPLEMENTATION_PATHS), + kind="recipe", + peak_sessions=8, + timeout_sec=7200, + recipe="implementation", + ingredients=dict(_IMPLEMENTATION_INGREDIENTS), + expected_pull_request_state="open", + pipeline=True, + issue_url_env="E2E_IMPLEMENTATION_ISSUE_URL", + required_changed_paths=list(_IMPLEMENTATION_PATHS), + test_command=["task", "test-check"], + peak_sessions_evidence=( + "configured capacity limit of eight concurrent sessions; no measured peak supplied" + ), + ) + entry.update(overrides) + return entry + + _INVALID_CATALOGS = { "duplicate-names": _raw_catalog(_entry("a"), _entry("a")), "bad-name": _raw_catalog(_entry("Bad_Name")), @@ -352,6 +450,9 @@ def _recipe_entry_without(field: str) -> dict: **_RECIPE_FIELDS, ) ), + "expected-failures-on-recipe": _raw_catalog( + _pipeline_entry(expected_failures=[_EXPECTED_FAILURE]) + ), "expected-failures-missing-field": _raw_catalog( _clean_install_entry( expected_failures=[ @@ -465,6 +566,7 @@ def test_recipe_entry_parses(self) -> None: "impl", kind="recipe", recipe_fixture="sandbox-smoke.yaml", + expected_failures=[{"severity": "error", **_RECIPE_EXPECTED_FAILURE}], **_RECIPE_FIELDS, ) ) @@ -473,6 +575,12 @@ def test_recipe_entry_parses(self) -> None: assert test.ingredients == (("task", "Add a greeting"),) assert test.expected_pull_request_state == "merged" assert test.recipe_fixture == "sandbox-smoke.yaml" + assert test.pipeline is False + assert test.issue_url_env is None + assert test.required_changed_paths == () + assert test.test_command == () + assert test.peak_sessions_evidence is None + assert test.expected_failures == ({"severity": "error", **_RECIPE_EXPECTED_FAILURE},) def test_recipe_without_fixture_defaults_to_none(self) -> None: test = _catalog(_entry("impl", kind="recipe", **_RECIPE_FIELDS)).get("impl") @@ -524,3 +632,111 @@ def test_unknown_test_name_raises_catalog_error(self) -> None: def test_invalid_catalog_raises(self, raw: dict) -> None: with pytest.raises(e2e_catalog.CatalogError): e2e_catalog.parse_catalog(raw) + + +class TestPipelineCatalog: + def test_shipped_implementation_targets_sandbox_source_and_tests_without_a_measurement( + self, + ) -> None: + test = e2e_catalog.load_catalog().get("implementation") + assert test.required_changed_paths == ("sandbox/", "tests/") + assert test.peak_sessions == 8 + assert test.peak_sessions_evidence is None + assert test.timeout_sec == 7200 + assert dict(test.ingredients) == _IMPLEMENTATION_INGREDIENTS + + def test_pipeline_recipe_parses_its_contract_and_expected_failures(self) -> None: + rows = [ + _RECIPE_EXPECTED_FAILURE, + {**_RECIPE_EXPECTED_FAILURE, "check": "another-check"}, + ] + test = _catalog(_pipeline_entry(expected_failures=rows)).get("implementation") + + assert test.pipeline is True + assert test.issue_url_env == "E2E_IMPLEMENTATION_ISSUE_URL" + assert test.required_changed_paths == _IMPLEMENTATION_PATHS + assert test.test_command == ("task", "test-check") + assert test.peak_sessions_evidence == ( + "configured capacity limit of eight concurrent sessions; no measured peak supplied" + ) + assert test.expected_failures == tuple(rows) + with pytest.raises(TypeError): + cast(MutableMapping[str, str], test.expected_failures[0])["check"] = "changed" + + def test_recipe_options_are_independently_optional(self) -> None: + entry = _pipeline_entry() + for key in ( + "issue_url_env", + "required_changed_paths", + "test_command", + "peak_sessions_evidence", + ): + entry.pop(key) + test = _catalog(entry).get("implementation") + assert test.pipeline is True + assert test.issue_url_env is None + assert test.required_changed_paths == () + assert test.test_command == () + assert test.peak_sessions_evidence is None + + def test_pipeline_rejects_unknown_keys(self) -> None: + with pytest.raises(e2e_catalog.CatalogError, match="unknown keys"): + _catalog(_pipeline_entry(unrecognized=True)) + + @pytest.mark.parametrize("ingredient", ["issue_url", "task"]) + def test_issue_url_env_conflicts_with_recipe_ingredient(self, ingredient: str) -> None: + ingredients = {**_IMPLEMENTATION_INGREDIENTS, ingredient: "value"} + with pytest.raises(e2e_catalog.CatalogError, match="cannot contain issue_url or task"): + _catalog(_pipeline_entry(ingredients=ingredients)) + + def test_recipe_options_do_not_require_pipeline_true(self) -> None: + entry = _pipeline_entry(pipeline=False) + assert _catalog(entry).get("implementation").pipeline is False + + @pytest.mark.parametrize("value", [None, "true", 1]) + def test_pipeline_flag_must_be_boolean(self, value: object) -> None: + with pytest.raises(e2e_catalog.CatalogError, match="pipeline must be a boolean"): + _catalog(_pipeline_entry(pipeline=value)) + + @pytest.mark.parametrize("value", ["", " ", 123]) + def test_issue_url_environment_name_must_be_nonempty(self, value: object) -> None: + with pytest.raises(e2e_catalog.CatalogError, match="issue_url_env"): + _catalog(_pipeline_entry(issue_url_env=value)) + + @pytest.mark.parametrize( + "prefix", + ["", " ", 123, "/absolute/path", "../parent", "src/../other", "C:\\absolute"], + ) + def test_required_changed_paths_must_be_relative_prefixes(self, prefix: object) -> None: + with pytest.raises(e2e_catalog.CatalogError): + _catalog(_pipeline_entry(required_changed_paths=[prefix])) + + @pytest.mark.parametrize("command", [[], [""], [1]]) + def test_pipeline_test_command_must_be_nonempty_argv(self, command: object) -> None: + with pytest.raises(e2e_catalog.CatalogError, match="test_command"): + _catalog(_pipeline_entry(test_command=command)) + + @pytest.mark.parametrize("evidence", ["", " ", 123]) + def test_pipeline_peak_sessions_evidence_must_be_nonempty(self, evidence: object) -> None: + with pytest.raises(e2e_catalog.CatalogError, match="peak_sessions_evidence"): + _catalog(_pipeline_entry(peak_sessions_evidence=evidence)) + + @pytest.mark.parametrize( + "row", + [ + {key: value for key, value in _RECIPE_EXPECTED_FAILURE.items() if key != "check"}, + {**_RECIPE_EXPECTED_FAILURE, "severity": "warning"}, + {**_RECIPE_EXPECTED_FAILURE, "issue": "https://github.com/other/repo/issues/123"}, + ], + ) + def test_recipe_expected_failures_have_exact_canonical_rows(self, row: dict) -> None: + with pytest.raises(e2e_catalog.CatalogError): + _catalog(_pipeline_entry(expected_failures=[row])) + + def test_recipe_expected_failures_reject_duplicate_check_and_message(self) -> None: + duplicate = { + **_RECIPE_EXPECTED_FAILURE, + "issue": "https://github.com/TalonT-Org/AutoSkillit/issues/124", + } + with pytest.raises(e2e_catalog.CatalogError, match="duplicate diagnostic"): + _catalog(_pipeline_entry(expected_failures=[_RECIPE_EXPECTED_FAILURE, duplicate])) diff --git a/tests/infra/test_e2e_select_cli.py b/tests/infra/test_e2e_select_cli.py index 25a7f2c1e2..0f0fb0f317 100644 --- a/tests/infra/test_e2e_select_cli.py +++ b/tests/infra/test_e2e_select_cli.py @@ -24,6 +24,9 @@ _HEADLESS_SMOKE_MATRIX = ( '{"include":[{"test":"headless-smoke","kind":"recipe","timeout_minutes":47}]}' ) +_IMPLEMENTATION_MATRIX = ( + '{"include":[{"test":"implementation","kind":"recipe","timeout_minutes":152}]}' +) def _run( @@ -112,6 +115,19 @@ def test_small_unlabelled_pull_request_selects_nothing(tmp_path: Path) -> None: assert outputs["matrix"] == '{"include":[]}' +def test_large_pull_request_selects_implementation_pipeline(tmp_path: Path) -> None: + changed = tmp_path / "changed-paths.txt" + changed.write_text("src/autoskillit/recipes/changed.py\n", encoding="utf-8") + payload = _pull_request(changed_files=10, additions=0, labels=("e2e",)) + + result = _select(tmp_path, changed, "pull_request", payload) + + assert result.returncode == 0, result.stderr + outputs = _outputs(result.stdout) + assert outputs["selected"] == "true" + assert outputs["matrix"] == _IMPLEMENTATION_MATRIX + + def test_merge_group_never_reads_the_changed_paths_file(tmp_path: Path) -> None: result = _select(tmp_path, tmp_path / "missing.txt", "merge_group", {"merge_group": {}}) assert result.returncode == 0, result.stderr @@ -151,6 +167,19 @@ def test_dispatch_selects_both_named_tests_in_request_order(tmp_path: Path) -> N ) +def test_dispatch_selects_named_pipeline_without_pull_request_eligibility(tmp_path: Path) -> None: + result = _select( + tmp_path, + tmp_path / "missing.txt", + "workflow_dispatch", + {"inputs": {"tests": "implementation"}}, + ) + assert result.returncode == 0, result.stderr + outputs = _outputs(result.stdout) + assert outputs["selected"] == "true" + assert outputs["matrix"] == _IMPLEMENTATION_MATRIX + + def test_dispatch_of_an_unknown_test_fails(tmp_path: Path) -> None: payload = {"inputs": {"tests": "nope"}} result = _select(tmp_path, tmp_path / "missing.txt", "workflow_dispatch", payload) diff --git a/tests/infra/test_e2e_sessions.py b/tests/infra/test_e2e_sessions.py new file mode 100644 index 0000000000..89ad47d9a9 --- /dev/null +++ b/tests/infra/test_e2e_sessions.py @@ -0,0 +1,580 @@ +"""Tests for the E2E session lifecycle and ownership evidence helper.""" + +from __future__ import annotations + +import json +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any + +import pytest + +from tests.infra._complexity_helpers import load_check_script + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] + +REPO_ROOT = Path(__file__).resolve().parents[2] +_SCRIPT = REPO_ROOT / "scripts" / "e2e" / "e2e_sessions.py" +sessions = load_check_script("_autoskillit_e2e_sessions", _SCRIPT) + +DISPATCH_ID = "dispatch-5233" +ROOT_ID = "session-root" +ISSUE_URL = "https://github.com/owner/repo/issues/5233" +REPOSITORY = "owner/repo" + + +def _jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + "".join(json.dumps(row, separators=(",", ":")) + "\n" for row in rows), + encoding="utf-8", + ) + + +def _session( + log_root: Path, + session_id: str, + *, + parent_session_id: str = "", + caller_session_id: str = "", + records: list[dict[str, Any]] | None = None, +) -> Path: + transcript = log_root / "transcripts" / f"{session_id}.jsonl" + _jsonl(transcript, records or []) + session_dir = log_root / "sessions" / session_id + session_dir.mkdir(parents=True, exist_ok=True) + (session_dir / "summary.json").write_text( + json.dumps( + { + "session_id": session_id, + "dispatch_id": DISPATCH_ID, + "execution_identity": {"parent_session_id": parent_session_id}, + } + ), + encoding="utf-8", + ) + return transcript + + +def _index(log_root: Path, entries: list[dict[str, Any]]) -> None: + _jsonl(log_root / "sessions.jsonl", entries) + + +def _entry( + session_id: str, + transcript: Path, + *, + parent_session_id: str = "", + caller_session_id: str = "", +) -> dict[str, Any]: + return { + "session_id": session_id, + "dir_name": session_id, + "dispatch_id": DISPATCH_ID, + "parent_session_id": parent_session_id, + "caller_session_id": caller_session_id, + "claude_code_log": str(transcript), + } + + +def _event( + name: str, + session_id: str, + timestamp: float, + *, + transcript: Path | None = None, + agent_id: str | None = None, + agent_transcript: Path | None = None, +) -> dict[str, Any]: + row: dict[str, Any] = { + "event": name, + "timestamp": timestamp, + "session_id": session_id, + } + if transcript is not None: + row["transcript_path"] = str(transcript) + if agent_id is not None: + row["agent_id"] = agent_id + if agent_transcript is not None: + row["agent_transcript_path"] = str(agent_transcript) + return row + + +def _trace(path: Path, events: list[dict[str, Any]]) -> None: + _jsonl(path, events) + + +def _envelope(session_id: str = ROOT_ID) -> dict[str, str]: + return {"dispatched_session_id": session_id, "dispatch_id": DISPATCH_ID} + + +def _base_sessions( + tmp_path: Path, + *, + child_id: str | None = None, + child_records: list[dict[str, Any]] | None = None, +) -> tuple[Path, Path, Path | None]: + log_root = tmp_path / "logs" + root_transcript = _session(log_root, ROOT_ID) + entries = [_entry(ROOT_ID, root_transcript)] + child_transcript = None + if child_id is not None: + child_transcript = _session( + log_root, + child_id, + parent_session_id=ROOT_ID, + records=child_records, + ) + entries.append( + _entry( + child_id, + child_transcript, + parent_session_id=ROOT_ID, + caller_session_id=ROOT_ID, + ) + ) + _index(log_root, entries) + trace = tmp_path / "run" / "session-lifecycle.jsonl" + return log_root, trace, child_transcript + + +def _tool_use(tool_id: str, name: str, tool_input: dict[str, Any]) -> dict[str, Any]: + return { + "type": "assistant", + "message": { + "content": [ + { + "type": "tool_use", + "id": tool_id, + "name": f"mcp__autoskillit__{name}", + "input": tool_input, + } + ] + }, + } + + +def _tool_result(tool_id: str, result: dict[str, Any]) -> dict[str, Any]: + return { + "type": "user", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": tool_id, + "content": json.dumps(result), + } + ] + }, + } + + +def test_hook_capture_appends_concurrent_allow_listed_records(tmp_path: Path) -> None: + trace = tmp_path / "events.jsonl" + with ThreadPoolExecutor(max_workers=8) as pool: + results = list( + pool.map( + lambda index: sessions.capture_hook_event( + trace, + { + "hook_event_name": "SessionStart", + "session_id": f"session-{index}", + "source": "startup", + "transcript_path": f"/tmp/{index}.jsonl", + "prompt": "must not be persisted", + "environment": {"TOKEN": "must not be persisted"}, + }, + ), + range(32), + ) + ) + + records = [json.loads(line) for line in trace.read_text(encoding="utf-8").splitlines()] + assert all(results) + assert len(records) == 32 + assert {record["session_id"] for record in records} == { + f"session-{index}" for index in range(32) + } + assert all(isinstance(record["timestamp"], (int, float)) for record in records) + assert all( + set(record) <= {"event", "timestamp", "session_id", "source", "transcript_path"} + for record in records + ) + assert ( + sessions.capture_hook_event(trace, {"event": "OtherHook", "session_id": "ignored"}) + is False + ) + assert len(trace.read_text(encoding="utf-8").splitlines()) == 32 + + +def test_summary_collapses_repeated_events_excludes_unrelated_and_sweeps_half_open( + tmp_path: Path, +) -> None: + log_root, trace, child_transcript = _base_sessions(tmp_path, child_id="session-child") + assert child_transcript is not None + unrelated = _session(log_root, "session-unrelated", parent_session_id="other-parent") + rows = json.loads((log_root / "sessions.jsonl").read_text(encoding="utf-8").splitlines()[0]) + entries = [ + rows, + _entry( + "session-child", child_transcript, parent_session_id=ROOT_ID, caller_session_id=ROOT_ID + ), + ] + entries.append( + _entry( + "session-unrelated", + unrelated, + parent_session_id="other-parent", + caller_session_id="other-parent", + ) + ) + _index(log_root, entries) + _trace( + trace, + [ + _event( + "SessionStart", + ROOT_ID, + 1, + transcript=log_root / "transcripts" / f"{ROOT_ID}.jsonl", + ), + _event( + "SessionStart", + ROOT_ID, + 1.5, + transcript=log_root / "transcripts" / f"{ROOT_ID}.jsonl", + ), + _event("SessionStart", "session-child", 3, transcript=child_transcript), + _event("SessionStart", "session-unrelated", 1, transcript=unrelated), + _event( + "SessionEnd", ROOT_ID, 2, transcript=log_root / "transcripts" / f"{ROOT_ID}.jsonl" + ), + _event( + "SessionEnd", ROOT_ID, 3, transcript=log_root / "transcripts" / f"{ROOT_ID}.jsonl" + ), + _event("SessionEnd", "session-child", 4, transcript=child_transcript), + _event("SessionEnd", "session-unrelated", 5, transcript=unrelated), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["peak_sessions"] == 1 + assert summary["coverage"]["complete"] is True + assert summary["coverage"]["managed_session_ids"] == ["session-child"] + assert [identity["session_id"] for identity in summary["identities"]] == [ + ROOT_ID, + "session-child", + ] + root_interval = summary["identities"][0] + assert (root_interval["start"], root_interval["end"]) == (1.0, 3.0) + assert (trace.parent / "session-concurrency.json").is_file() + + +@pytest.mark.parametrize( + ("child_event", "expected_violation"), + [("SessionStart", "missing_end"), ("SessionEnd", "orphan_end")], +) +def test_summary_does_not_infer_child_end( + tmp_path: Path, child_event: str, expected_violation: str +) -> None: + log_root, trace, child_transcript = _base_sessions(tmp_path, child_id="session-child") + assert child_transcript is not None + _trace( + trace, + [ + _event( + "SessionStart", + ROOT_ID, + 1, + transcript=log_root / "transcripts" / f"{ROOT_ID}.jsonl", + ), + _event(child_event, "session-child", 2, transcript=child_transcript), + _event( + "SessionEnd", ROOT_ID, 5, transcript=log_root / "transcripts" / f"{ROOT_ID}.jsonl" + ), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["peak_sessions"] == 1 + assert summary["coverage"]["complete"] is False + assert f"lifecycle:session:session-child:{expected_violation}" in summary["violations"] + + +def test_summary_enumerates_native_agent_transcripts_and_requires_endpoints( + tmp_path: Path, +) -> None: + log_root, trace, _child = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + agent_transcript = log_root / "transcripts" / ROOT_ID / "subagents" / "agent-agent-a.jsonl" + agent_transcript.parent.mkdir(parents=True) + agent_transcript.write_text("{}\n", encoding="utf-8") + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SubagentStart", ROOT_ID, 2, agent_id="agent-a"), + _event( + "SubagentStop", ROOT_ID, 4, agent_id="agent-a", agent_transcript=agent_transcript + ), + _event("SessionEnd", ROOT_ID, 5, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["complete"] is True + assert summary["peak_sessions"] == 2 + assert summary["coverage"]["native_agents"] == [ + { + "session_id": ROOT_ID, + "agent_id": "agent-a", + "transcript_path": str(agent_transcript.resolve()), + } + ] + + +def test_indexed_owned_child_without_hooks_is_incomplete(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path, child_id="missing-hooks") + transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=transcript), + _event("SessionEnd", ROOT_ID, 5, transcript=transcript), + ], + ) + summary = sessions.summarize(trace, _envelope(), log_root) + assert not summary["coverage"]["complete"] + assert "lifecycle:missing-hooks:missing_endpoints" in summary["violations"] + + +def test_resumed_identity_requires_a_final_end(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=transcript), + _event("SessionStart", ROOT_ID, 3, transcript=transcript), + ], + ) + summary = sessions.summarize(trace, _envelope(), log_root) + assert not summary["coverage"]["complete"] + assert f"lifecycle:session:{ROOT_ID}:missing_final_end" in summary["violations"] + + +def test_empty_half_open_interval_does_not_reduce_concurrent_peak() -> None: + assert ( + sessions._peak_sessions( + [{"start": 1, "end": 3}, {"start": 2, "end": 2}, {"start": 2, "end": 4}] + ) + == 2 + ) + + +def test_no_attempted_resource_calls_leave_cleanup_proof_empty(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=transcript), + ], + ) + proof = sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) + assert proof["branches"] == [] + assert proof["claimed"] is False + assert proof["violations"] == [] + + +def test_missing_root_hooks_do_not_discard_available_branch_proof(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + transcript, + [ + _tool_use( + "branch", + "create_and_publish_branch", + {"issue_number": "5233", "remote_url": "https://github.com/owner/repo.git"}, + ), + _tool_result("branch", {"merge_target": "impl/5233-fix", "was_unique": True}), + ], + ) + _trace(trace, []) + proof = sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) + assert proof["branches"] == ["impl/5233-fix"] + assert "trace:root_session_missing" in proof["violations"] + + +def test_missing_root_and_malformed_trace_are_incomplete(tmp_path: Path) -> None: + log_root = tmp_path / "logs" + trace = tmp_path / "run" / "session-lifecycle.jsonl" + trace.parent.mkdir(parents=True) + trace.write_text("{malformed\n", encoding="utf-8") + + summary = sessions.summarize(trace, None, log_root) + + assert summary["coverage"]["complete"] is False + assert "envelope:missing" in summary["violations"] + assert "sessions_index:unavailable" in summary["violations"] + assert "trace:invalid_json:1" in summary["violations"] + + +def test_summary_rejects_malformed_owned_index_summary_and_transcript( + tmp_path: Path, +) -> None: + log_root, trace, _child = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + index_path = log_root / "sessions.jsonl" + index_path.write_text( + index_path.read_text(encoding="utf-8") + "{malformed\n", encoding="utf-8" + ) + (log_root / "sessions" / ROOT_ID / "summary.json").write_text("{malformed\n", encoding="utf-8") + root_transcript.write_text("{malformed\n", encoding="utf-8") + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["complete"] is False + assert "sessions_index:invalid_json:2" in summary["violations"] + assert f"summary:{ROOT_ID}:missing_or_malformed" in summary["violations"] + assert f"transcript:{ROOT_ID}:invalid_json:1" in summary["violations"] + + +@pytest.mark.parametrize( + ("claim_input_label", "expected_claimed"), + [(None, True), ("in-progress", True), ("different-label", False)], +) +def test_ownership_pairs_root_branch_with_managed_claim_and_keeps_compact_proof( + tmp_path: Path, + claim_input_label: str | None, + expected_claimed: bool, +) -> None: + claim_input: dict[str, Any] = {"issue_url": ISSUE_URL, "allow_reentry": False} + if claim_input_label is not None: + claim_input["label"] = claim_input_label + claim_records = [ + _tool_use("claim-1", "claim_and_resolve_issue", claim_input), + _tool_result( + "claim-1", + { + "success": True, + "claimed": True, + "issue_number": 5233, + "issue_title": "private title", + }, + ), + ] + log_root, trace, child_transcript = _base_sessions( + tmp_path, + child_id="session-child", + child_records=claim_records, + ) + assert child_transcript is not None + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + root_transcript, + [ + _tool_use( + "branch-1", + "create_and_publish_branch", + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + ), + _tool_result("branch-1", {"merge_target": "impl/5233-fix", "was_unique": True}), + ], + ) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionStart", "session-child", 2, transcript=child_transcript), + _event("SessionEnd", "session-child", 4, transcript=child_transcript), + _event("SessionEnd", ROOT_ID, 5, transcript=root_transcript), + ], + ) + + proof = sessions.collect_ownership( + trace, _envelope(), log_root, ISSUE_URL, REPOSITORY, claim_label="in-progress" + ) + + assert proof["branches"] == ["impl/5233-fix"] + assert proof["claimed"] is expected_claimed + if expected_claimed: + assert proof["violations"] == [] + else: + assert "ownership:claim_result_failed" in proof["violations"] + assert {item["session_id"] for item in proof["transcripts"]} == {ROOT_ID, "session-child"} + + +def test_ownership_ignores_child_branch_and_rejects_reentry_and_unpaired_results( + tmp_path: Path, +) -> None: + child_records = [ + _tool_use( + "child-branch", + "create_and_publish_branch", + {"issue_number": "5233", "remote_url": "git@github.com:owner/repo.git"}, + ), + _tool_result("child-branch", {"merge_target": "wrong-owner/branch"}), + ] + log_root, trace, child_transcript = _base_sessions( + tmp_path, + child_id="session-child", + child_records=child_records, + ) + assert child_transcript is not None + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + root_transcript, + [ + _tool_use( + "root-branch", + "create_and_publish_branch", + {"issue_number": "5233", "remote_url": "git@github.com:owner/repo.git"}, + ), + _tool_use( + "claim-1", + "claim_and_resolve_issue", + {"issue_url": ISSUE_URL, "allow_reentry": False}, + ), + _tool_result("claim-1", {"success": True, "claimed": True, "issue_number": 5233}), + _tool_use( + "claim-2", + "claim_and_resolve_issue", + {"issue_url": ISSUE_URL, "allow_reentry": False}, + ), + _tool_result("claim-2", {"success": True, "claimed": True, "issue_number": 5233}), + _tool_result("wrong-branch-id", {"merge_target": "must-not-be-paired"}), + ], + ) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionStart", "session-child", 2, transcript=child_transcript), + _event("SessionEnd", "session-child", 4, transcript=child_transcript), + _event("SessionEnd", ROOT_ID, 5, transcript=root_transcript), + ], + ) + + proof = sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) + + assert proof["branches"] == [] + assert proof["claimed"] is False + assert "ownership:claim_reentry" in proof["violations"] + assert any("unpaired_tool_use:root-branch" in item for item in proof["violations"]) diff --git a/tests/infra/test_e2e_workflow.py b/tests/infra/test_e2e_workflow.py index ede505cd35..942f39f6de 100644 --- a/tests/infra/test_e2e_workflow.py +++ b/tests/infra/test_e2e_workflow.py @@ -382,9 +382,17 @@ def test_docker_run_forwards_secrets_by_name_only(self, workflow: dict, step_id: forwarded = re.findall(r"--env ([A-Z0-9_]+)", run) expected = ["MINIMAX_API_KEY", "E2E_SANDBOX_TOKEN"] if step_id == "e2e": - expected.append("E2E_SANDBOX_TOKEN") + expected.extend(["E2E_SANDBOX_TOKEN", "E2E_IMPLEMENTATION_ISSUE_URL"]) assert Counter(forwarded) == Counter(expected) + def test_seed_repository_variable_reaches_test_container(self, workflow: dict) -> None: + step = _step(workflow, "run", step_id="e2e") + assert ( + step["env"]["E2E_IMPLEMENTATION_ISSUE_URL"] + == "${{ vars.E2E_IMPLEMENTATION_ISSUE_URL }}" + ) + assert "--env E2E_IMPLEMENTATION_ISSUE_URL" in step["run"] + def test_redaction_keeps_required_secret_names(self, workflow: dict) -> None: run = _step(workflow, "run", step_id="redact")["run"] branch, outside = _model_branch(run) @@ -433,10 +441,10 @@ def test_clean_install_uses_the_common_user_image_and_artifacts( ] assert len(credential_lines) == 1 assert credential_lines[0].startswith("docker_args+=(") - assert re.findall(r"--env ([A-Z0-9_]+)", credential_lines[0]) == [ - "MINIMAX_API_KEY", - "E2E_SANDBOX_TOKEN", - ] + forwarded = ["MINIMAX_API_KEY", "E2E_SANDBOX_TOKEN"] + if step_id == "e2e": + forwarded.append("E2E_IMPLEMENTATION_ISSUE_URL") + assert re.findall(r"--env ([A-Z0-9_]+)", credential_lines[0]) == forwarded if step_id == "e2e": assert not re.search(r"--env MINIMAX_API_KEY\b", outside) assert re.findall(r"--env ([A-Z0-9_]+)", outside) == ["E2E_SANDBOX_TOKEN"] From e9866f58082125a2af9f6d118b2716976c2cc33f Mon Sep 17 00:00:00 2001 From: Trecek Date: Thu, 8 Oct 2026 07:35:47 -0700 Subject: [PATCH 11/20] fix(review): centralize E2E session log root construction --- scripts/e2e/e2e_harness.py | 4 ++-- scripts/e2e/e2e_sessions.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index 2e811479da..09955c2b4d 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -1481,7 +1481,7 @@ def finish_pipeline( ) -> list[str]: failures: list[str] = [] trace = out / "session-events.jsonl" - log_root = home / ".local" / "share" / "autoskillit" / "logs" + log_root = e2e_sessions.default_log_root(home) try: summary = e2e_sessions.summarize(trace, envelope, log_root) if not summary["coverage"]["complete"]: @@ -1525,7 +1525,7 @@ def launch_recipe( proof = save_ownership( out / "session-events.jsonl", envelope, - home / ".local" / "share" / "autoskillit" / "logs", + e2e_sessions.default_log_root(home), seed, repository, out, diff --git a/scripts/e2e/e2e_sessions.py b/scripts/e2e/e2e_sessions.py index bab3aec9be..86cfd89d35 100644 --- a/scripts/e2e/e2e_sessions.py +++ b/scripts/e2e/e2e_sessions.py @@ -190,13 +190,17 @@ def _read_summary(session_dir: Path, expected_session_id: str) -> dict[str, Any] return summary +def default_log_root(home: Path) -> Path: + return home / ".local" / "share" / "autoskillit" / "logs" + + def _discover_owned( trace_path: Path, envelope: dict[str, Any] | None, log_root: Path | None, ) -> dict[str, Any]: """Resolve only the dispatched session and fresh, linked managed descendants.""" - root = log_root or Path.home() / ".local" / "share" / "autoskillit" / "logs" + root = log_root or default_log_root(Path.home()) violations: list[str] = [] events = _load_trace(trace_path, violations) root_id, dispatch_id = _read_envelope(envelope, violations) From 59813e3ccaa67ab8e867d0f41b0a8e8c8814831a Mon Sep 17 00:00:00 2001 From: Trecek Date: Thu, 8 Oct 2026 07:36:13 -0700 Subject: [PATCH 12/20] fix(review): name the E2E session trace artifact --- scripts/e2e/e2e_harness.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index 09955c2b4d..e38ee2a3f6 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -43,6 +43,7 @@ _SCRUBBED_ENV = frozenset({"CLAUDE_CODE_OAUTH_TOKEN", *SECRET_ENV}) _STDERR_TAIL_CHARS = 2000 +_SESSION_TRACE_FILENAME = "session-events.jsonl" CLEAN_INSTALL_ALLOWED_WARNINGS: dict[tuple[str, str], str] = { ( "pytest_temp_capacity", @@ -1405,7 +1406,7 @@ def run_recipe( ingredients = dict(test.ingredients) label = "" home = home or Path.home() - trace = out / "session-events.jsonl" + trace = out / _SESSION_TRACE_FILENAME if test.pipeline or test.issue_url_env: try: seed, ingredients, label = prepare_seed(test, repository, out, env, runner) @@ -1480,7 +1481,7 @@ def finish_pipeline( proof: dict[str, Any] | None, ) -> list[str]: failures: list[str] = [] - trace = out / "session-events.jsonl" + trace = out / _SESSION_TRACE_FILENAME log_root = e2e_sessions.default_log_root(home) try: summary = e2e_sessions.summarize(trace, envelope, log_root) @@ -1523,7 +1524,7 @@ def launch_recipe( ) failures += fleet_failures proof = save_ownership( - out / "session-events.jsonl", + out / _SESSION_TRACE_FILENAME, envelope, e2e_sessions.default_log_root(home), seed, From 6feef276f6209eb56ec4f8b0fe1215329923f44b Mon Sep 17 00:00:00 2001 From: Trecek Date: Thu, 8 Oct 2026 07:36:40 -0700 Subject: [PATCH 13/20] fix(review): name sandbox CI polling settings --- scripts/e2e/e2e_harness.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index e38ee2a3f6..2ceb96ebc5 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -44,6 +44,8 @@ _SCRUBBED_ENV = frozenset({"CLAUDE_CODE_OAUTH_TOKEN", *SECRET_ENV}) _STDERR_TAIL_CHARS = 2000 _SESSION_TRACE_FILENAME = "session-events.jsonl" +_CI_POLL_ATTEMPTS = 6 +_CI_POLL_INTERVAL_SEC = 10 CLEAN_INSTALL_ALLOWED_WARNINGS: dict[tuple[str, str], str] = { ( "pytest_temp_capacity", @@ -1101,7 +1103,7 @@ def validate_sandbox_ci( ] if len(candidates) != 1: raise ValueError("sandbox CI: expected one active CI workflow") - for attempt in range(6): + for attempt in range(_CI_POLL_ATTEMPTS): pages = _api_pages( f"repos/{repository}/actions/workflows/{candidates[0]['id']}/runs?event=pull_request&per_page=100", env, @@ -1145,8 +1147,8 @@ def validate_sandbox_ci( ): raise ValueError("sandbox CI: CI run and test job must succeed") return - if attempt < 5: - time.sleep(10) + if attempt + 1 < _CI_POLL_ATTEMPTS: + time.sleep(_CI_POLL_INTERVAL_SEC) raise ValueError("sandbox CI: no completed CI run associated with the exact PR head") From 43139563347dc9a8f51d73efeb8b8a9d13726c9f Mon Sep 17 00:00:00 2001 From: Trecek Date: Thu, 8 Oct 2026 07:37:06 -0700 Subject: [PATCH 14/20] fix(review): clarify claim label predicate naming --- scripts/e2e/e2e_sessions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/e2e/e2e_sessions.py b/scripts/e2e/e2e_sessions.py index 86cfd89d35..3785066543 100644 --- a/scripts/e2e/e2e_sessions.py +++ b/scripts/e2e/e2e_sessions.py @@ -844,10 +844,10 @@ def _claim_succeeded( return False tool_input, result, is_error = claims[0] requested_label = tool_input.get("label") - effective_label = claim_label is None or requested_label in (None, "", claim_label) + label_matches_request = claim_label is None or requested_label in (None, "", claim_label) claimed = bool( not is_error - and effective_label + and label_matches_request and tool_input.get("allow_reentry") is not True and result is not None and result.get("success") is True From 5b69f5ef627201704d9d43bb5da7db19ea8a45ad Mon Sep 17 00:00:00 2001 From: Trecek Date: Thu, 8 Oct 2026 07:37:41 -0700 Subject: [PATCH 15/20] fix(review): ignore non-string transcript tool names --- scripts/e2e/e2e_sessions.py | 4 +++- tests/infra/test_e2e_sessions.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/scripts/e2e/e2e_sessions.py b/scripts/e2e/e2e_sessions.py index 3785066543..d483903417 100644 --- a/scripts/e2e/e2e_sessions.py +++ b/scripts/e2e/e2e_sessions.py @@ -709,7 +709,9 @@ def _remember_tool_use( violations: list[str], ) -> None: name = block.get("name") - name = name.removeprefix("mcp__autoskillit__") if isinstance(name, str) else name + if not isinstance(name, str): + return + name = name.removeprefix("mcp__autoskillit__") if name not in {"claim_and_resolve_issue", "create_and_publish_branch"}: return tool_id, tool_input = block.get("id"), block.get("input") diff --git a/tests/infra/test_e2e_sessions.py b/tests/infra/test_e2e_sessions.py index 89ad47d9a9..f2f2e72f20 100644 --- a/tests/infra/test_e2e_sessions.py +++ b/tests/infra/test_e2e_sessions.py @@ -521,6 +521,21 @@ def test_ownership_pairs_root_branch_with_managed_claim_and_keeps_compact_proof( assert {item["session_id"] for item in proof["transcripts"]} == {ROOT_ID, "session-child"} +def test_ownership_ignores_unhashable_tool_names(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + record = _tool_use("unrelated", "ignored", {}) + record["message"]["content"][0]["name"] = [] + _jsonl(root_transcript, [record]) + _trace(trace, [_event("SessionStart", ROOT_ID, 1, transcript=root_transcript)]) + + proof = sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) + + assert proof["branches"] == [] + assert proof["claimed"] is False + assert proof["violations"] == [] + + def test_ownership_ignores_child_branch_and_rejects_reentry_and_unpaired_results( tmp_path: Path, ) -> None: From 99531c1406954b07398e9095be3ea2628c4f60cd Mon Sep 17 00:00:00 2001 From: Trecek Date: Thu, 8 Oct 2026 07:38:14 -0700 Subject: [PATCH 16/20] fix(review): reject dot session directory components --- scripts/e2e/e2e_sessions.py | 2 +- tests/infra/test_e2e_sessions.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/e2e/e2e_sessions.py b/scripts/e2e/e2e_sessions.py index d483903417..7e7d79f5ac 100644 --- a/scripts/e2e/e2e_sessions.py +++ b/scripts/e2e/e2e_sessions.py @@ -172,7 +172,7 @@ def _load_index(log_root: Path, violations: list[str]) -> list[dict[str, Any]]: def _safe_session_dir(log_root: Path, row: Mapping[str, Any]) -> Path | None: name = row.get("dir_name") - if not isinstance(name, str) or not name or Path(name).name != name: + if not isinstance(name, str) or not name or name in {".", ".."} or Path(name).name != name: return None return log_root / "sessions" / name diff --git a/tests/infra/test_e2e_sessions.py b/tests/infra/test_e2e_sessions.py index f2f2e72f20..cb469cc107 100644 --- a/tests/infra/test_e2e_sessions.py +++ b/tests/infra/test_e2e_sessions.py @@ -426,6 +426,21 @@ def test_missing_root_and_malformed_trace_are_incomplete(tmp_path: Path) -> None assert "trace:invalid_json:1" in summary["violations"] +@pytest.mark.parametrize("dir_name", [".", ".."]) +def test_summary_rejects_dot_session_directories(tmp_path: Path, dir_name: str) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + entry = _entry(ROOT_ID, root_transcript) + entry["dir_name"] = dir_name + _index(log_root, [entry]) + _trace(trace, [_event("SessionStart", ROOT_ID, 1, transcript=root_transcript)]) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["complete"] is False + assert f"sessions_index:{ROOT_ID}:invalid_dir_name" in summary["violations"] + + def test_summary_rejects_malformed_owned_index_summary_and_transcript( tmp_path: Path, ) -> None: From d8a776ec241cd3563df8db2ca153e5f3a46d1a4e Mon Sep 17 00:00:00 2001 From: Trecek Date: Thu, 8 Oct 2026 07:39:33 -0700 Subject: [PATCH 17/20] test(review): isolate ownership proof scenarios --- tests/infra/test_e2e_sessions.py | 57 +++++++++++++++++--------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/tests/infra/test_e2e_sessions.py b/tests/infra/test_e2e_sessions.py index cb469cc107..c23e160532 100644 --- a/tests/infra/test_e2e_sessions.py +++ b/tests/infra/test_e2e_sessions.py @@ -551,8 +551,10 @@ def test_ownership_ignores_unhashable_tool_names(tmp_path: Path) -> None: assert proof["violations"] == [] -def test_ownership_ignores_child_branch_and_rejects_reentry_and_unpaired_results( +@pytest.mark.parametrize("scenario", ["child_branch", "unpaired_root_branch", "claim_reentry"]) +def test_ownership_isolates_branch_and_claim_scenarios( tmp_path: Path, + scenario: str, ) -> None: child_records = [ _tool_use( @@ -564,19 +566,12 @@ def test_ownership_ignores_child_branch_and_rejects_reentry_and_unpaired_results ] log_root, trace, child_transcript = _base_sessions( tmp_path, - child_id="session-child", - child_records=child_records, + child_id="session-child" if scenario == "child_branch" else None, + child_records=child_records if scenario == "child_branch" else None, ) - assert child_transcript is not None root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" - _jsonl( - root_transcript, - [ - _tool_use( - "root-branch", - "create_and_publish_branch", - {"issue_number": "5233", "remote_url": "git@github.com:owner/repo.git"}, - ), + if scenario == "claim_reentry": + root_records = [ _tool_use( "claim-1", "claim_and_resolve_issue", @@ -589,22 +584,30 @@ def test_ownership_ignores_child_branch_and_rejects_reentry_and_unpaired_results {"issue_url": ISSUE_URL, "allow_reentry": False}, ), _tool_result("claim-2", {"success": True, "claimed": True, "issue_number": 5233}), - _tool_result("wrong-branch-id", {"merge_target": "must-not-be-paired"}), - ], - ) - _trace( - trace, - [ - _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), - _event("SessionStart", "session-child", 2, transcript=child_transcript), - _event("SessionEnd", "session-child", 4, transcript=child_transcript), - _event("SessionEnd", ROOT_ID, 5, transcript=root_transcript), - ], - ) + ] + else: + root_records = [ + _tool_use( + "root-branch", + "create_and_publish_branch", + {"issue_number": "5233", "remote_url": "git@github.com:owner/repo.git"}, + ), + _tool_result( + "root-branch" if scenario == "child_branch" else "wrong-branch-id", + {"merge_target": "impl/5233-root"}, + ), + ] + _jsonl(root_transcript, root_records) + _trace(trace, [_event("SessionStart", ROOT_ID, 1, transcript=root_transcript)]) proof = sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) - assert proof["branches"] == [] + assert proof["branches"] == (["impl/5233-root"] if scenario == "child_branch" else []) assert proof["claimed"] is False - assert "ownership:claim_reentry" in proof["violations"] - assert any("unpaired_tool_use:root-branch" in item for item in proof["violations"]) + if scenario == "claim_reentry": + assert proof["violations"] == ["ownership:claim_reentry"] + elif scenario == "unpaired_root_branch": + assert len(proof["violations"]) == 1 + assert "unpaired_tool_use:root-branch" in proof["violations"][0] + else: + assert proof["violations"] == [] From c995db4ec40423476638d3b76612c9c754f110f7 Mon Sep 17 00:00:00 2001 From: Trecek Date: Thu, 8 Oct 2026 07:44:45 -0700 Subject: [PATCH 18/20] test(review): assert both unpaired branch diagnostics --- tests/infra/test_e2e_sessions.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/infra/test_e2e_sessions.py b/tests/infra/test_e2e_sessions.py index c23e160532..b2bce94d4e 100644 --- a/tests/infra/test_e2e_sessions.py +++ b/tests/infra/test_e2e_sessions.py @@ -607,7 +607,9 @@ def test_ownership_isolates_branch_and_claim_scenarios( if scenario == "claim_reentry": assert proof["violations"] == ["ownership:claim_reentry"] elif scenario == "unpaired_root_branch": - assert len(proof["violations"]) == 1 - assert "unpaired_tool_use:root-branch" in proof["violations"][0] + assert proof["violations"] == [ + "ownership:branch_result_failed", + f"transcript:{ROOT_ID}:unpaired_tool_use:root-branch", + ] else: assert proof["violations"] == [] From 4adccdb95438fae4b996fb01add9d6d6dcd8855a Mon Sep 17 00:00:00 2001 From: Talon Trecek <36818290+Trecek@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:58:11 -0700 Subject: [PATCH 19/20] fix(e2e): recover ownership evidence and reconcile sandbox branches (#5269) Real E2E transcripts used the plugin MCP namespace and row-level structured replies, so the collector missed the claim, published branch, and launched workers. Teardown could then silently leave a branch behind when no PR or claim label remained. This change decodes both canonical namespaces and authoritative structured receipts, adopts dispatch-valid workers through paired launch/acknowledgement/recovery receipts, and reports unexplained sessions. The harness records sandbox head refs before the run and after cleanup, preserving and failing cleanup for new branches without ownership proof. Sanitized captured CI evidence exercises ownership, concurrency, and teardown after a failed run. Stacked on #5255, targeting `impl-issue-5233-20261008-implementation`, where the collector was introduced. Validation: - All three phases passed `pre-commit run --all-files` and the configured `task test-local-gate` through the MCP test gate. - Final gate: 16,021 passed, 207 skipped, 12 xfailed. - All 15 planned mutation checks passed; temporary test selection was restored. - Captured replay adopts both workers and reports peak concurrency of five. The existing unmatched native stop still fails lifecycle coverage, as required. Live GitHub head-ref listing will be confirmed by the first E2E run after merge. Fixes #5267 --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .autoskillit/.gitignore | 1 + .autoskillit/test-filter-manifest.yaml | 3 + .github/workflows/e2e.yml | 2 +- docs/developer/diagnostics.md | 64 +- docs/operations/observability.md | 102 ++- pyproject.toml | 2 +- scripts/e2e/README.md | 72 +- scripts/e2e/e2e_harness.py | 144 +++- scripts/e2e/e2e_sessions.py | 91 ++- src/autoskillit/.claude-plugin/plugin.json | 2 +- src/autoskillit/cli/_init_helpers.py | 7 +- src/autoskillit/cli/doctor/_doctor_config.py | 8 +- src/autoskillit/cli/install/_marketplace.py | 5 +- src/autoskillit/config/__init__.py | 2 + src/autoskillit/config/_automation_config.py | 2 + src/autoskillit/config/_coercion.py | 60 +- src/autoskillit/config/_config_dataclasses.py | 3 + .../config/_dataclasses_execution.py | 7 +- src/autoskillit/config/settings.py | 4 + src/autoskillit/core/__init__.pyi | 4 + src/autoskillit/core/io/__init__.py | 6 + src/autoskillit/core/io/io.py | 63 +- src/autoskillit/core/io/markdown_sections.py | 29 +- src/autoskillit/core/io/paths.py | 8 + src/autoskillit/core/planset/allocation.py | 46 +- .../types/audit/_type_plan_set_authority.py | 21 + .../core/types/execution/_type_subprocess.py | 3 + .../core/types/results/_type_results.py | 2 + .../types/results/_type_results_records.py | 13 +- .../execution/evidence/linux_tracing.py | 25 +- .../execution/headless/_headless_execute.py | 34 +- .../execution/headless/_headless_launch.py | 32 +- .../execution/headless/_headless_terminal.py | 32 +- src/autoskillit/execution/process/__init__.py | 5 + .../execution/process/_lifecycle/runner.py | 5 + .../execution/recording/recording.py | 17 + .../_managed_headless_session_lineage.py | 7 +- ...anaged_headless_session_lineage_records.py | 5 + .../session_log/_session_log_recovery.py | 5 +- .../execution/session_log/session_log.py | 4 + src/autoskillit/fleet/_outcome.py | 34 +- .../_api_orchestration_match.py | 2 + src/autoskillit/recipe/helpers/_io_loading.py | 49 +- .../dataflow/rules_dataflow_multipart.py | 25 - .../recipes/diagrams/implementation-groups.md | 2 +- .../recipes/diagrams/implementation.md | 2 +- .../recipes/diagrams/remediation.md | 2 +- .../recipes/implementation-groups.json | 7 +- .../recipes/implementation-groups.yaml | 45 +- src/autoskillit/recipes/implementation.json | 6 +- src/autoskillit/recipes/implementation.yaml | 24 +- src/autoskillit/recipes/remediation.json | 10 +- src/autoskillit/recipes/remediation.yaml | 19 +- .../server/_plan_set_materializer.py | 74 ++- .../arch-constraint-catalog.md | 1 + .../skill_resources/plan-part-form.md | 24 + .../skills_extended/dry-walkthrough/SKILL.md | 7 + .../skills_extended/make-plan/SKILL.md | 38 +- .../skills_extended/rectify/SKILL.md | 12 +- tests/_ambient_env_surface.py | 2 +- tests/_helpers.py | 14 + tests/_retention_surface.py | 6 +- tests/_test_filter.py | 6 +- tests/arch/_acceptance_policy_surfaces.py | 21 +- tests/arch/_core_gateway_snapshot.py | 2 + ...st_autoskillit_root_children_registered.py | 159 +++++ tests/arch/test_e2e_sessions_contract.py | 27 + tests/arch/test_execution_source_split.py | 15 +- tests/arch/test_import_linter_contracts.py | 1 + tests/arch/test_layer_enforcement.py | 2 + .../test_mcp_prefix_literal_containment.py | 20 +- ..._new_exception_path_autoskillit_imports.py | 2 +- tests/cli/test_doctor.py | 25 + tests/cli/test_init.py | 29 +- tests/cli/test_init_helpers.py | 2 + tests/cli/test_init_helpers_state.py | 52 +- tests/cli/test_install.py | 4 +- tests/config/test_rollout_config.py | 158 +++++ tests/conftest.py | 7 + tests/contracts/config_key_ledger.txt | 1 + ..._dry_walkthrough_arch_catalog_reference.py | 9 +- tests/contracts/test_plan_part_contract.py | 72 ++ tests/contracts/test_protocol_satisfaction.py | 6 + .../test_skill_resource_derived_counts.py | 2 +- tests/core/io/test_markdown_sections.py | 64 +- tests/core/planset/test_allocation.py | 246 ++++++- tests/core/planset/test_coverage.py | 44 ++ .../test_ensure_project_temp_with_config.py | 71 +- .../core/test_grab_bag_split_completeness.py | 1 + tests/core/test_self_ignored.py | 45 ++ tests/core/test_session_index_schema.py | 9 + .../test_session_index_schema_version_lock.py | 2 +- tests/core/test_type_results_records.py | 1 + tests/core/types/test_plan_set_authority.py | 25 + tests/execution/conftest.py | 5 + .../headless/test_codex_attempt_retention.py | 6 +- .../headless/test_headless_recovery.py | 26 +- .../test_managed_headless_session_lineage.py | 50 ++ .../test_headless_capture_diagnostics.py | 1 + tests/execution/test_headless_dispatch.py | 3 +- .../execution/test_headless_enum_recovery.py | 9 +- tests/execution/test_headless_execute.py | 366 ++++++++++- tests/execution/test_linux_tracing.py | 63 +- .../test_model_backend_launch_contract.py | 42 +- tests/execution/test_process_run.py | 65 ++ tests/execution/test_recording.py | 10 + .../execution/test_session_index_roundtrip.py | 48 ++ .../test_session_log_backend_source.py | 1 + tests/execution/test_session_log_fields.py | 28 + tests/execution/test_session_log_retention.py | 47 +- .../test_dispatch_checkout_cleanliness.py | 74 +++ tests/fleet/test_dispatch_envelope_fields.py | 22 +- .../fleet/test_dispatch_outcome_classifier.py | 99 +++ tests/infra/_e2e_evidence.py | 49 ++ .../run_38170521660_attempt_1/PROVENANCE.json | 9 + .../logs/sessions.jsonl | 3 + .../summary.json | 23 + .../summary.json | 23 + .../summary.json | 23 + .../out/envelope.json | 114 ++++ .../out/session-events.jsonl | 16 + ...54f18144-1188-45d7-a392-17b4529f721e.jsonl | 5 + .../subagents/agent-a7e613554b849d6b2.jsonl | 2 + ...8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl | 5 + .../subagents/agent-a516cbae85151dc7c.jsonl | 2 + .../subagents/agent-a9ed51b7c02b828dd.jsonl | 2 + .../subagents/agent-aeb96cc0ddbdab294.jsonl | 2 + ...2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl | 23 + ...test_canonical_json_producer_convention.py | 4 +- tests/infra/test_e2e_harness.py | 476 +++++++++++++- tests/infra/test_e2e_sessions.py | 622 +++++++++++++++++- tests/infra/test_e2e_sessions_captured.py | 248 +++++++ tests/infra/test_e2e_workflow.py | 14 +- tests/infra/test_pretty_output_recipe.py | 8 +- tests/infra/test_release_sanity.py | 31 +- ...test_bundled_recipes_pipeline_structure.py | 111 +++- tests/recipe/test_implementation.py | 2 +- tests/recipe/test_io_json_precompile.py | 47 ++ tests/recipe/test_remediation_recipe.py | 10 +- .../recipe/test_rules_multipart_iteration.py | 36 +- .../server/test_delivery_e2e_verification.py | 8 +- tests/server/test_plan_set_materializer.py | 131 ++++ tests/server/test_tools_plan_set.py | 11 +- tests/test_test_filter_core_cascade.py | 7 +- .../test_clone_gate_runtime_state.py | 128 ++++ ...test_project_local_overrides_resolution.py | 4 +- uv.lock | 2 +- 147 files changed, 5243 insertions(+), 405 deletions(-) create mode 100644 src/autoskillit/skill_resources/plan-part-form.md create mode 100644 tests/arch/test_autoskillit_root_children_registered.py create mode 100644 tests/arch/test_e2e_sessions_contract.py create mode 100644 tests/config/test_rollout_config.py create mode 100644 tests/contracts/test_plan_part_contract.py create mode 100644 tests/core/test_self_ignored.py create mode 100644 tests/fleet/test_dispatch_checkout_cleanliness.py create mode 100644 tests/infra/_e2e_evidence.py create mode 100644 tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/PROVENANCE.json create mode 100644 tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions.jsonl create mode 100644 tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/2292658a-34cf-44a0-a0b1-f7a20218874b/summary.json create mode 100644 tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/54f18144-1188-45d7-a392-17b4529f721e/summary.json create mode 100644 tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/8ef8f993-8caa-47dd-83b0-26f9e5b51139/summary.json create mode 100644 tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/out/envelope.json create mode 100644 tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/out/session-events.jsonl create mode 100644 tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl create mode 100644 tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e/subagents/agent-a7e613554b849d6b2.jsonl create mode 100644 tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl create mode 100644 tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a516cbae85151dc7c.jsonl create mode 100644 tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a9ed51b7c02b828dd.jsonl create mode 100644 tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-aeb96cc0ddbdab294.jsonl create mode 100644 tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl create mode 100644 tests/infra/test_e2e_sessions_captured.py create mode 100644 tests/workspace/test_clone_gate_runtime_state.py diff --git a/.autoskillit/.gitignore b/.autoskillit/.gitignore index e2a673f82f..2bf53cfb58 100644 --- a/.autoskillit/.gitignore +++ b/.autoskillit/.gitignore @@ -1,4 +1,5 @@ temp/ +.state.yaml .secrets.yaml .onboarded sync_manifest.json diff --git a/.autoskillit/test-filter-manifest.yaml b/.autoskillit/test-filter-manifest.yaml index 093b3344f7..9a12476995 100644 --- a/.autoskillit/test-filter-manifest.yaml +++ b/.autoskillit/test-filter-manifest.yaml @@ -243,6 +243,9 @@ tests/execution/backends/fixtures/codex_ndjson/*.json: tests/execution/fixtures/*.json: - execution/ +tests/infra/fixtures/e2e_sessions/**: + - infra/ + tests/config/golden_configs/*.yaml: - config/ diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 6ad180bb68..aedb28d507 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -108,7 +108,7 @@ jobs: TEST: ${{ matrix.test }} run: | set -euo pipefail - docker_args=() + docker_args=(--shm-size=3g) if [[ '${{ matrix.kind }}' != 'clean-install' ]]; then docker_args+=(--env MINIMAX_API_KEY --env E2E_SANDBOX_TOKEN --env E2E_IMPLEMENTATION_ISSUE_URL) docker_args+=(--volume "$RUNNER_TEMP/e2e/data:/home/autoskillit/.local/share/autoskillit") diff --git a/docs/developer/diagnostics.md b/docs/developer/diagnostics.md index 2d373df397..3609f46a28 100644 --- a/docs/developer/diagnostics.md +++ b/docs/developer/diagnostics.md @@ -67,6 +67,68 @@ Logs are stored in a **global** directory (not per-project), so they persist acr `summary.json` contains: `session_id`, `dir_name`, `pid`, `cwd`, `evidence_worktree_path`, `evidence_worktree_source`, `evidence_worktree_detail`, `skill_command`, `success`, `subtype`, `exit_code`, `start_ts`, `snapshot_count`, `anomaly_count`, `peak_rss_kb`, `peak_oom_score`, `peak_fd_ratio`, `session_type`, `session_errors`, `child_outcomes` (see [Child Terminal Reasons](#child-terminal-reasons)). +### Rollout declarations + +`summary.json.rollout` and `sessions.jsonl.rollout` store the same optional +declaration. Configure it in `.autoskillit/config.yaml`: + +```yaml +rollout: + active: + change_id: "prompt-cost-study" + effective_at: "2026-11-01T18:00:00Z" + scope: + skills: ["make-plan"] + backends: ["claude-code", "codex"] + roles: ["planner"] + expected_effect: "Reduce planning input tokens while preserving requirement coverage." +``` + +The stored object has these same fields, with `effective_at` normalized to UTC +ISO-8601 (`2026-11-01T18:00:00+00:00` here). An active declaration must be complete; +`change_id` and `expected_effect` must be nonblank strings. Scope has exactly the +`skills`, `backends`, and `roles` lists of nonblank strings; empty lists are valid. +Scope describes the change and does not filter launches. All admitted Claude Code +and Codex headless runs carry the active marker, including launched failures. +Missing `rollout`, absent/null `active`, or clearing `active` means inactive. +New inactive records store null; older rows without the field remain unassigned. + +Capture occurs after final launch admission, immediately before the first runner +call. The marker and logical start remain fixed through the same-run nudge and +terminal logging even if configuration changes. Summary `start_ts`, index +`timestamp`, and trace enrollment `logical_run_start_ts` share that instant; +operation timing, nudge-turn timing, and enrollment `enrolled_at` remain separate. +Enrollment schema 3 preserves the marker/start for crash recovery. Older sidecars +recover without a marker and retain the historical trace-mtime fallback. + +Declare the prediction and stable change id **before** cutover, launch marked +baseline work, apply the change at the stated UTC boundary, and retain the same +declaration for after runs. Clear `active` when observation ends. A revised +prediction, scope, or cutover requires a new `change_id`; stability across +independent runs is the operator's responsibility. A pre-cutover marked row is +evidence that its prediction was recorded beforehand. A current config or first +after row alone cannot prove that timing; file mtimes do not provide declaration +history. Keeping the prediction allows comparison with observations without +replacing it with an explanation developed after seeing outcomes. + +Use the stored run start and cutover as parsed timezone-aware instants, preserving +fractional seconds. The canonical comparison is: + +```python +started_at = datetime.fromisoformat(row["timestamp"]) +cutover_at = datetime.fromisoformat(rollout["effective_at"]) +if started_at < cutover_at: + cohort = "before" +elif started_at >= cutover_at: + cohort = "after" +``` + +A before/after delta is a prompt to investigate, not proof that the change caused it. +Work difficulty, repository state, and model versions can differ across cohorts. +See the [streaming retained-index query](../operations/observability.md#rollout-cohorts) +for validation and bounded aggregation. Never infer marker membership from hashes, +repository revisions, mtimes, or guessed dates. + ### Execution-candidate manifests Before evaluating a primary binding or configured alternative, `run_skill` @@ -178,7 +240,7 @@ Completed sessions with per-request evidence publish `turn_usage.jsonl` before version-4 token descriptor carries `turn_usage_file`, `turn_usage_count`, and `turn_usage_schema_version`. The file reference is `null` with count zero when no series was observed. The JSONL row schema is version 2; the retained -`sessions.jsonl` index uses schema version 16. +`sessions.jsonl` index uses schema version 17. Each token-bearing row carries non-empty `backend` and `provider_used`, plus nullable source `message_id`, `request_id`, `timestamp`, and observed `model`. diff --git a/docs/operations/observability.md b/docs/operations/observability.md index 9f2561d09e..a8cad35f2e 100644 --- a/docs/operations/observability.md +++ b/docs/operations/observability.md @@ -58,7 +58,7 @@ a named normalization contract. A ratio with an `unknown` required measure renders `unknown`; `unavailable` and `not_applicable` scopes do not contribute to its denominator, and an observed zero denominator renders `—`. -Token descriptor schema v4, turn sidecar schema v2, session index schema v14, +Token descriptor schema v4, turn sidecar schema v2, session index schema v17, fleet campaign state v13, and campaign summary v2 retain these records. Legacy numeric zero is decoded as `unknown` because older formats did not prove an observation. `context_window_tokens` is positive capacity metadata, not a @@ -140,6 +140,106 @@ rows by `dir_name`. Archive-only history excludes the current live survivor window. The archive retains index metadata, not pruned session artifact directories, and operational consumers continue to read only `sessions.jsonl`. +## Rollout cohorts + +Declare an evaluated change in `.autoskillit/config.yaml` before applying it: + +```yaml +rollout: + active: + change_id: "prompt-cost-study" + effective_at: "2026-11-01T18:00:00Z" + scope: + skills: ["make-plan"] + backends: ["claude-code", "codex"] + roles: ["planner"] + expected_effect: "Reduce planning input tokens while preserving requirement coverage." +``` + +`change_id` identifies an unchanged declaration across the observation window. +`effective_at` is the intentional UTC cutover, not a date inferred from code or +configuration history. `scope` names affected skills, backends, and roles in a +queryable form; its lists describe the change and never govern admission or marker +attachment. `expected_effect` preserves the required, non-null, nonblank prediction +with every run so readers can compare intended effects with subsequent observations. +Empty scope lists are valid. Missing/null `active` is inactive; incomplete or +otherwise invalid active declarations are rejected before generic string coercion. + +Declare the prediction and stable id before cutover, launch marked baseline work, +apply the change at the stated UTC boundary, and keep the same declaration active +while observing after runs. Clear `active` when the observation period ends. Use a +new `change_id` for a revised prediction, scope, or cutover. Code validates each +declaration and preserves each admitted run's snapshot; operators maintain stability +across independent runs. Recording the prediction beforehand distinguishes intended +effects from explanations developed after seeing outcomes. A pre-cutover marked +row supplies that evidence; a current configuration or first after row alone does +not establish preregistration. + +Summary and index records store the same nested `rollout` object with normalized +UTC `effective_at`. Capture happens immediately before the first admitted launch; +all headless runs then carry it, including failures and runs outside the descriptive +scope. Same-run nudges and recovery keep that snapshot and logical start despite +later configuration changes. The index `timestamp` and summary `start_ts` identify +the admitted logical run. Operation, nudge-turn, and trace-enrollment times have +different purposes. A subsequent independent run captures the current declaration. +New unmarked rows contain null; older absent fields never imply cohort membership. + +This query counts one explicit change id while streaming the live retained index. +Set `index_path` to `linux_tracing.log_dir / "sessions.jsonl"` if overriding the log +root. It skips reader non-row entries and absent/null markers, and raises a visible +error for malformed selected declarations or timestamps. It covers retained rows +and excludes archived history; memory and printed output do not grow with row count. + +```python +from datetime import datetime, timedelta + +from autoskillit.core import default_log_dir +from autoskillit.execution.session_log.session_index import iter_tolerant_session_index_lines + +index_path = default_log_dir() / "sessions.jsonl" +change_id = "prompt-cost-study" +counts = {"before": 0, "after": 0} +for end_offset, row in iter_tolerant_session_index_lines(index_path): + if row is None: + continue + rollout = row.get("rollout") + if rollout is None: + continue + if not isinstance(rollout, dict): + raise ValueError(f"Malformed rollout at byte {end_offset}") + if rollout["change_id"] != change_id: + continue + if set(rollout) != {"change_id", "effective_at", "scope", "expected_effect"}: + raise ValueError(f"Incomplete rollout at byte {end_offset}") + prediction = rollout["expected_effect"] + scope = rollout["scope"] + if not isinstance(prediction, str) or not prediction.strip(): + raise ValueError(f"Invalid prediction at byte {end_offset}") + if not isinstance(scope, dict) or set(scope) != {"skills", "backends", "roles"}: + raise ValueError(f"Invalid scope at byte {end_offset}") + if any( + not isinstance(values, list) + or any(not isinstance(value, str) or not value.strip() for value in values) + for values in scope.values() + ): + raise ValueError(f"Invalid scope values at byte {end_offset}") + started_at = datetime.fromisoformat(row["timestamp"]) + cutover_at = datetime.fromisoformat(rollout["effective_at"]) + if started_at.utcoffset() is None or cutover_at.utcoffset() != timedelta(0): + raise ValueError(f"Expected aware run start and UTC cutover at byte {end_offset}") + if started_at < cutover_at: + counts["before"] += 1 + elif started_at >= cutover_at: + counts["after"] += 1 +print(counts) +``` + +A before/after delta is a prompt to investigate, not proof that the change caused it. +Work difficulty, repository state, and model versions can differ between cohorts. +Compare parsed instants, including fractional seconds: equality is after cutover. +Do not manufacture cohorts from content hashes, repository revisions, file mtimes, +or guessed dates. Existing aggregate trends do not consume rollout markers. + ## Native headless OTLP capture Every headless executor invocation starts one best-effort OTLP/HTTP-JSON sink diff --git a/pyproject.toml b/pyproject.toml index 46a89ed7d7..92de103f32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "autoskillit" -version = "0.10.1266" +version = "0.10.1271" description = "MCP server for orchestrating automated skill-driven workflows with Claude Code" readme = "README.md" license = {text = "MIT"} diff --git a/scripts/e2e/README.md b/scripts/e2e/README.md index dc041abb0a..a93f10dad4 100644 --- a/scripts/e2e/README.md +++ b/scripts/e2e/README.md @@ -202,9 +202,19 @@ The implementation pipeline installs synchronous command hooks for `SessionStart settings. These hooks follow the [Claude lifecycle schema](https://code.claude.com/docs/en/hooks). Each identity spans its first start through its final end, so compaction and resume yield a conservative upper bound. Half-open intervals end before simultaneous starts. The -summary reconciles the dispatched root, managed lineage and enumerable native transcripts; -missing endpoints, identities or owned transcripts fail coverage. Transcript activity is -never used to infer a lifetime. A peak exceeding capacity fails validation. +summary reconciles the dispatched root, managed lineage and enumerable native transcripts. +AutoSkillit writes `logs/sessions.jsonl` and each session's `summary.json`; their dispatch +and parent/caller fields are AutoSkillit provenance, separate from Claude Code transcript +and lifecycle hook fields. Empty links do not imply a Claude Code schema guarantee. +Managed sessions are adopted through declared parent fields or receipted `run_skill`, +`complete_run_skill_result` or `recover_run_skill_result` replies in an owned transcript. +Each must have exactly one index row with the run's `dispatch_id`. Parentless sessions +without receipts, missing or ambiguous index rows, and receipt-named sessions absent +from both trace and index fail coverage. Shared dispatch IDs or overlapping lifetimes +never adopt a session. Missing endpoints, identities or owned transcripts fail coverage; +transcript activity never infers a lifetime. An auto-compacted root can leave an unmatched +`SubagentStop`, which still fails coverage and is tracked separately. A peak exceeding +capacity fails validation. Commission with a successful MiniMax CI run and repeat against the same seed. Inspect planning, dry-walkthrough, implementation, test, integration, audit GO and open-PR evidence, @@ -226,9 +236,19 @@ and are often `0600`, so a second container run copies `out/` and `data/logs/` i fails if any secret survives. Only `upload/` is uploaded, and only when redaction succeeded. +When a test does not pass, the harness preserves plans from each pipeline clone under +`out/pipeline-artifacts///` and Claude Code JSONL under +`out/transcripts/`, retaining relative paths. The transcripts include the orchestrator, +skill sessions and their subagents; the orchestrator's transcript holds each raw +`bind_plan_set` response and its `plan_parts` argument. These files go through the same +redaction and upload steps as the other output artifacts. Collection also runs for an +expected failure or a harness exception. A container killed before or during collection +may leave no evidence or only partial evidence. + Pipeline evidence includes `session-events.jsonl`, `session-concurrency.json`, compact `ownership.json`, the seed, effective configuration, fleet stdout/stderr/envelope, captured -PR metadata/files, detached-head test commands and associated CI runs/jobs. Collection +PR metadata/files, detached-head test commands, associated CI runs/jobs, and sandbox head-ref +listings before the run and after cleanup (`head-refs-before.*`, `head-refs-after.*`). Collection occurs inside the container before transcript paths disappear. Incomplete evidence is retained and fails the scenario; redaction and upload still run after failure. @@ -302,7 +322,19 @@ must be `main`. No caller `base_branch` ingredient is sent. Sandbox setup and ba must succeed before launch. Paired root transcript tool calls/results prove the created branch; paired root/managed -claim results prove any added claim label. Discovery paginates through the PR baseline +claim results prove any added claim label. The collector accepts the direct +`mcp__autoskillit__` and plugin `mcp__plugin_autoskillit_autoskillit__` namespaces; +a tracked tool under any other namespace fails the proof. Claude Code persisted +transcripts carry replies in row-level `mcpMeta.structuredContent` when `mcpMeta` +is present, otherwise in JSON block content. Malformed metadata or metadata on a +row with several results fails the paired resource call. Hook display text never +replaces unusable metadata. The pretty-output hook emits `updatedMCPToolOutput`; +`_result_object` remains the shared envelope decoder, and metadata presence alone +does not identify whether the hook ran. These shapes are pinned by +`tests/infra/fixtures/e2e_sessions/` from Claude Code 2.1.280; re-capture them when +an otherwise successful call reports a failed branch or claim result. + +Discovery paginates through the PR baseline and leaves unrelated PRs untouched. Require exactly one owned open, unmerged sandbox PR targeting `main`, without auto-merge, with the canonical seed URL and non-removed changes under both `sandbox/` and `tests/`. Fetch and detach at the captured head before setup and @@ -316,6 +348,17 @@ removes only the effective claim label proven added by this run. The seed remain and other labels are preserved. Missing ownership proof preserves uncertain resources and fails cleanup. Cleanup errors remain fatal alongside the original failure. +Branch reconciliation lists the sandbox's branches before the run and after cleanup. +A branch created during the run that `ownership.json` does not explain is preserved +and fails cleanup; baseline branches are excluded from unexplained-branch reports. +Branches named by the ownership proof retain the existing cleanup rule. The original +API stdout saved as `head-refs-before.stdout.txt` and `head-refs-after.stdout.txt` supplies +the in-memory snapshots; accompanying stderr and command records diagnose failed requests. +This check only detects unexplained branches: the creating run records the failure once, +and the branch and any PR on it must be removed by hand. The next run includes it in +its baseline. A manual run overlapping CI can trigger the check because CI serialization +does not cover local runs. + ## Catalog schema ```json @@ -411,6 +454,25 @@ docker run --rm \ python3 /opt/e2e/e2e_harness.py run --test clean-install --out /artifacts ``` +### Check a real plan against the plan-set gate before dispatching a pipeline scenario + +After the plan-set changes have merged, perform this check before CI commissioning: + +1. Check out the #5255 branch revision containing both the `implementation` E2E scenario + and the evidence collector. Build and run that revision with the Docker + invocation above, selecting `--test implementation`. Use the same revision for + commissioning and record the model. +2. If the run reaches implementation, the first bind succeeded. If the test does not + pass, inspect the orchestrator's `bind_plan_set` results in `out/transcripts/`. + An observed `success: true` on the first or second bind is the pass condition. + A `coverage_failed` result explains the rejection in `coverage_gaps`, and + `out/pipeline-artifacts/` retains the rejected plan. Missing evidence is inconclusive + and does not satisfy this gate: require an observed successful bind. When a run + replans, also confirm the second bind's `plan_parts` names only the replacement plan. +3. Record the model, plan path and bind result in issue #5259's body using + `gh issue edit --body-file`, never as a comment. Only an observed successful bind + unblocks `gh workflow run e2e.yml -f tests=implementation`. + ### Sandbox smoke with teardown Set `MINIMAX_API_KEY` and `E2E_SANDBOX_TOKEN` in the local environment first. Build the same diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index a8df99d646..36aee365fa 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -5,6 +5,7 @@ import argparse import ast +import fnmatch import json import os import re @@ -38,6 +39,7 @@ REDACTED = b"[REDACTED]" CONFIG_TEMPLATE = Path(__file__).with_name("autoskillit-config.yaml") SANDBOX_CLONE = Path("/workspace/sandbox") +PIPELINE_RUNS_DIR = "autoskillit-runs" _SCRUBBED_ENV = frozenset({"CLAUDE_CODE_OAUTH_TOKEN", *SECRET_ENV}) _STDERR_TAIL_CHARS = 2000 @@ -971,6 +973,31 @@ def _api_pages(endpoint: str, env: Mapping[str, str], runner: Runner, evidence: return pages +def list_head_refs( + repository: str, out: Path, env: Mapping[str, str], runner: Runner, stage: str +) -> frozenset[str]: + """Return the branch names currently present in the sandbox repository.""" + pages = _api_pages( + f"repos/{repository}/git/matching-refs/heads/", env, runner, out / f"head-refs-{stage}" + ) + if not pages: + raise ValueError("head-ref listing must contain a page") + refs: set[str] = set() + for page in pages: + if not isinstance(page, list): + raise ValueError("head-ref listing page must be an array") + for row in page: + ref = row.get("ref") if isinstance(row, dict) else None + if ( + not isinstance(ref, str) + or not ref.startswith("refs/heads/") + or ref == "refs/heads/" + ): + raise ValueError("head-ref listing must contain nonempty refs/heads/ objects") + refs.add(ref.removeprefix("refs/heads/")) + return frozenset(refs) + + def prepare_seed( test: e2e_catalog.CatalogTest, repository: str, @@ -1327,9 +1354,29 @@ def release_owned_claim( return failures +def unexplained_branch_failures( + repository: str, + head_refs: frozenset[str], + proof: Mapping[str, Any], + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + """Report branches created during the run that the ownership proof does not explain.""" + try: + current = list_head_refs(repository, out, env, runner, "after") + except Exception as exc: + return [f"cleanup refs: {exc}"] + return [ + f"cleanup: branch {branch} was created during this run without ownership proof" + for branch in sorted(current - head_refs - set(proof["branches"])) + ] + + def cleanup_implementation( repository: str, baseline: int, + head_refs: frozenset[str], proof: Mapping[str, Any], seed: Mapping[str, Any], label: str, @@ -1354,6 +1401,7 @@ def cleanup_implementation( delete_owned_branch(repository, branch, env, runner) except Exception as exc: failures.append(f"cleanup branch: {exc}") + failures += unexplained_branch_failures(repository, head_refs, proof, out, env, runner) try: failures += release_owned_claim(proof, seed, label, out, env, runner) except Exception as exc: @@ -1403,6 +1451,7 @@ def run_recipe( if test.recipe_fixture is not None: return run_smoke_recipe(test, repository, out, env, runner) seed: dict[str, Any] = {} + head_refs: frozenset[str] = frozenset() ingredients = dict(test.ingredients) label = "" home = home or Path.home() @@ -1411,6 +1460,7 @@ def run_recipe( try: seed, ingredients, label = prepare_seed(test, repository, out, env, runner) if test.pipeline: + head_refs = list_head_refs(repository, out, env, runner, "before") trace.write_text("", encoding="utf-8") install_session_hooks(home, trace) except Exception as exc: @@ -1421,7 +1471,7 @@ def run_recipe( if not test.pipeline: return launch_regular_recipe(test, repository, baseline, ingredients, out, env, runner) return launch_recipe( - test, repository, baseline, seed, ingredients, label, out, home, env, runner + test, repository, baseline, head_refs, seed, ingredients, label, out, home, env, runner ) @@ -1471,6 +1521,7 @@ def finish_pipeline( test: e2e_catalog.CatalogTest, repository: str, baseline: int, + head_refs: frozenset[str], seed: Mapping[str, Any], label: str, out: Path, @@ -1498,7 +1549,9 @@ def finish_pipeline( except Exception as exc: failures.append(f"cleanup ownership: {exc}") proof = proof or {"branches": [], "claimed": False} - failures += cleanup_implementation(repository, baseline, proof, seed, label, out, env, runner) + failures += cleanup_implementation( + repository, baseline, head_refs, proof, seed, label, out, env, runner + ) return failures @@ -1506,6 +1559,7 @@ def launch_recipe( test: e2e_catalog.CatalogTest, repository: str, baseline: int, + head_refs: frozenset[str], seed: Mapping[str, Any], ingredients: Mapping[str, str], label: str, @@ -1546,7 +1600,18 @@ def launch_recipe( failures.append(f"harness error: {exc}") finally: failures += finish_pipeline( - test, repository, baseline, seed, label, out, home, env, runner, envelope, proof + test, + repository, + baseline, + head_refs, + seed, + label, + out, + home, + env, + runner, + envelope, + proof, ) return match_recipe_findings(failures, findings, test.expected_failures) @@ -1665,6 +1730,67 @@ def _run_test( return run_recipe(test, catalog.sandbox_repository, token, out, scrubbed, runner, home=home) +def _copy_failure_files( + source: Path, dest: Path, boundary: Path, pattern: str, errors: list[str] +) -> int: + relative = source.relative_to(boundary) + try: + if any( + boundary.joinpath(*relative.parts[:depth]).is_symlink() + for depth in range(len(relative.parts) + 1) + ): + return 0 + if not source.exists(): + return 0 + except OSError as exc: + errors.append(f"{source}: {exc}") + return 0 + copied = 0 + for root, dirs, files in os.walk( + source, followlinks=False, onerror=lambda exc: errors.append(f"{source}: {exc}") + ): + dirs.sort() + for name in sorted(fnmatch.filter(files, pattern)): + path = Path(root) / name + try: + if path.is_symlink() or not path.is_file(): + continue + target = dest / path.relative_to(source) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(path, target) + copied += 1 + except OSError as exc: + errors.append(f"{path}: {exc}") + return copied + + +def collect_failure_evidence(out: Path, home: Path) -> dict[str, Any]: + """Keep pipeline plans and Claude transcripts for an unsuccessful E2E run.""" + errors: list[str] = [] + runs = SANDBOX_CLONE.parent / PIPELINE_RUNS_DIR + try: + clones = [] if runs.is_symlink() else sorted(runs.iterdir()) + except FileNotFoundError: + clones = [] + except OSError as exc: + errors.append(f"{runs}: {exc}") + clones = [] + plan_files = 0 + for clone in clones: + for kind in ("make-plan", "rectify"): + plan_files += _copy_failure_files( + clone / ".autoskillit" / "temp" / kind, + out / "pipeline-artifacts" / clone.name / kind, + clone, + "*", + errors, + ) + transcripts = _copy_failure_files( + home / ".claude" / "projects", out / "transcripts", home, "*.jsonl", errors + ) + return {"plan_files": plan_files, "transcripts": transcripts, "errors": errors} + + def run_test( test: e2e_catalog.CatalogTest, catalog: e2e_catalog.Catalog, @@ -1691,6 +1817,18 @@ def run_test( result.update( outcome=outcome, passed=outcome == "passed", expected_findings=matched_findings ) + if not result["passed"]: + try: + evidence = collect_failure_evidence(out, home) + except Exception as exc: + evidence = { + "plan_files": 0, + "transcripts": 0, + "errors": [f"{type(exc).__name__}: {exc}"], + "exception": traceback.format_exc(), + } + if any(evidence.values()): + result["evidence"] = evidence (out / "result.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") return failures diff --git a/scripts/e2e/e2e_sessions.py b/scripts/e2e/e2e_sessions.py index 7e7d79f5ac..85c8552dd9 100644 --- a/scripts/e2e/e2e_sessions.py +++ b/scripts/e2e/e2e_sessions.py @@ -17,6 +17,13 @@ _SESSION_EVENTS = frozenset({"SessionStart", "SessionEnd"}) _AGENT_EVENTS = frozenset({"SubagentStart", "SubagentStop"}) _LIFECYCLE_EVENTS = _SESSION_EVENTS | _AGENT_EVENTS +# Pinned to product prefixes by tests/arch/test_e2e_sessions_contract.py. +_AUTOSKILLIT_SERVERS = frozenset( + {("mcp", "autoskillit"), ("mcp", "plugin_autoskillit_autoskillit")} +) +_RESOURCE_TOOLS = frozenset({"claim_and_resolve_issue", "create_and_publish_branch"}) +_LAUNCH_TOOLS = frozenset({"run_skill", "complete_run_skill_result", "recover_run_skill_result"}) +_TRACKED_TOOLS = _RESOURCE_TOOLS | _LAUNCH_TOOLS _MAX_HOOK_INPUT_BYTES = 65_536 _MAX_TRACE_RECORD_BYTES = 8_192 @@ -224,8 +231,13 @@ def _discover_owned( if not dispatch_id or root_row.get("dispatch_id") != dispatch_id: violations.append("sessions_index:root_dispatch_mismatch") _add_owned_session(root, root_row, root_id, violations, owned) - candidates = _fresh_candidates(root, root_id, dispatch_id, trace_session_ids, rows_by_id) + candidates = _fresh_candidates( + root, root_id, dispatch_id, trace_session_ids, rows_by_id, violations + ) _add_linked_descendants(root, dispatch_id, candidates, owned, violations) + observed = trace_session_ids | set(rows_by_id) + for session_id in sorted(_launched_session_ids(owned) - observed): + violations.append(f"lineage:{session_id}:launched_session_unobserved") return { "root_id": root_id, @@ -244,6 +256,7 @@ def _fresh_candidates( dispatch_id: str, trace_session_ids: set[str], rows_by_id: Mapping[str, list[dict[str, Any]]], + violations: list[str], ) -> dict[str, tuple[dict[str, Any], dict[str, Any] | None]]: candidates: dict[str, tuple[dict[str, Any], dict[str, Any] | None]] = {} candidate_ids = trace_session_ids | { @@ -256,6 +269,7 @@ def _fresh_candidates( continue matching_rows = rows_by_id.get(session_id, []) if len(matching_rows) != 1: + violations.append(f"sessions_index:{session_id}:missing_or_ambiguous") continue row = matching_rows[0] session_dir = _safe_session_dir(log_root, row) @@ -264,6 +278,31 @@ def _fresh_candidates( return candidates +def _launched_session_ids(owned: Mapping[str, Mapping[str, Any]]) -> set[str]: + """Return the session ids named by receipted launch replies in owned transcripts.""" + launched: set[str] = set() + for item in owned.values(): + for tool, _tool_input, reply, is_error in item["calls"]: + if tool not in _LAUNCH_TOOLS or is_error or reply is None: + continue + session_id = reply.get("session_id") + if reply.get("receipt_id") and isinstance(session_id, str) and session_id: + launched.add(session_id) + return launched + + +def _report_unattributed( + remaining: Mapping[str, tuple[dict[str, Any], dict[str, Any] | None]], violations: list[str] +) -> None: + """Report sessions of the run that declare no parent and were named by no launch receipt.""" + for session_id, (row, summary) in sorted(remaining.items()): + declared = [row.get("parent_session_id"), row.get("caller_session_id")] + if summary: + declared.append(summary["execution_identity"]["parent_session_id"]) + if not any(isinstance(value, str) and value for value in declared): + violations.append(f"lineage:{session_id}:unattributed") + + def _add_linked_descendants( log_root: Path, dispatch_id: str, @@ -275,6 +314,7 @@ def _add_linked_descendants( while remaining: changed = False owned_ids = set(owned) + launched = _launched_session_ids(owned) for session_id, (row, summary) in list(remaining.items()): parents = { value @@ -282,7 +322,9 @@ def _add_linked_descendants( if isinstance((value := row.get(key)), str) and value } summary_parent = summary["execution_identity"]["parent_session_id"] if summary else "" - if not parents.intersection(owned_ids) and summary_parent not in owned_ids: + by_receipt = session_id in launched + linked = by_receipt or parents.intersection(owned_ids) or summary_parent in owned_ids + if not linked: continue if not dispatch_id or row.get("dispatch_id") != dispatch_id: violations.append(f"sessions_index:{session_id}:dispatch_mismatch") @@ -290,7 +332,7 @@ def _add_linked_descendants( continue if summary is None: violations.append(f"summary:{session_id}:missing_or_malformed") - elif summary_parent not in owned_ids: + elif summary_parent not in owned_ids and (summary_parent or not by_receipt): violations.append(f"summary:{session_id}:parent_mismatch") if parents and not parents.intersection(owned_ids): violations.append(f"sessions_index:{session_id}:parent_mismatch") @@ -299,6 +341,7 @@ def _add_linked_descendants( changed = True if not changed: break + _report_unattributed(remaining, violations) def _add_owned_session( @@ -338,15 +381,18 @@ def _add_owned_session( transcript = log_root / transcript else: violations.append(f"sessions_index:{session_id}:missing_transcript") + calls = [] if transcript is None or not transcript.is_file(): violations.append(f"transcript:{session_id}:missing") else: _read_jsonl(transcript, f"transcript:{session_id}", violations, collect=False) + calls = _transcript_tool_pairs(transcript, session_id, violations) owned[session_id] = { "row": dict(row), "summary": summary, "session_dir": session_dir, "transcript": transcript, + "calls": calls, } @@ -510,6 +556,9 @@ def _verify_native_transcript( if transcript is None: violations.append(f"{label}:transcript_unenumerated") return None + if not transcript.is_file(): + violations.append(f"{label}:transcript_missing") + return None expected_path = str(transcript.resolve()) paths = { str(Path(event["agent_transcript_path"]).resolve()) @@ -520,9 +569,6 @@ def _verify_native_transcript( violations.append(f"{label}:transcript_mismatch") if not paths: violations.append(f"{label}:transcript_path_missing") - if not transcript.is_file(): - violations.append(f"{label}:transcript_missing") - return None _read_jsonl(transcript, f"transcript:{session_id}:{agent_id}", violations, collect=False) return {"session_id": session_id, "agent_id": agent_id, "transcript_path": expected_path} @@ -673,6 +719,23 @@ def _transcript_tool_pairs( return paired +def _receipt( + row: Mapping[str, Any], content: list[Any], block: Mapping[str, Any] +) -> dict[str, Any] | None: + """Decode row metadata when present, otherwise decode the JSON block carrier.""" + meta = row.get("mcpMeta") + if "mcpMeta" in row: + results = [ + item + for item in content + if isinstance(item, dict) and item.get("type") == "tool_result" + ] + if len(results) != 1 or not isinstance(meta, dict): + return None + return _result_object(meta.get("structuredContent")) + return _result_object(block.get("content")) + + def _pair_record( row: dict[str, Any], line_number: int, @@ -697,7 +760,7 @@ def _pair_record( tool_id = block.get("tool_use_id") if isinstance(tool_id, str) and tool_id in pending: name, tool_input = pending.pop(tool_id) - result = _result_object(block.get("content")) + result = _receipt(row, content, block) paired.append((name, tool_input, result, block.get("is_error") is True)) @@ -711,8 +774,12 @@ def _remember_tool_use( name = block.get("name") if not isinstance(name, str): return - name = name.removeprefix("mcp__autoskillit__") - if name not in {"claim_and_resolve_issue", "create_and_publish_branch"}: + parts = tuple(name.split("__")) + tool = parts[-1] + if tool not in _TRACKED_TOOLS: + return + if len(parts) != 3 or parts[:2] not in _AUTOSKILLIT_SERVERS: + violations.append(f"transcript:{prefix}:unrecognised_tool_namespace:{line_number}") return tool_id, tool_input = block.get("id"), block.get("input") if not isinstance(tool_id, str) or not tool_id or not isinstance(tool_input, dict): @@ -720,7 +787,7 @@ def _remember_tool_use( return if tool_id in pending: violations.append(f"transcript:{prefix}:duplicate_tool_use:{line_number}") - pending[tool_id] = (name, tool_input) + pending[tool_id] = (tool, tool_input) def collect_ownership( @@ -779,9 +846,7 @@ def _owned_tool_proofs( if not isinstance(transcript, Path) or not transcript.is_file(): continue transcripts.append({"session_id": session_id, "path": str(transcript)}) - for name, tool_input, result, is_error in _transcript_tool_pairs( - transcript, session_id, violations - ): + for name, tool_input, result, is_error in item["calls"]: target = _branch_target( session_id, root_id, diff --git a/src/autoskillit/.claude-plugin/plugin.json b/src/autoskillit/.claude-plugin/plugin.json index f78c5ea284..6ed7a71df9 100644 --- a/src/autoskillit/.claude-plugin/plugin.json +++ b/src/autoskillit/.claude-plugin/plugin.json @@ -1,5 +1,5 @@ { "name": "autoskillit", - "version": "0.10.1266", + "version": "0.10.1271", "description": "Orchestrated skill-driven workflows using Claude Code headless sessions" } diff --git a/src/autoskillit/cli/_init_helpers.py b/src/autoskillit/cli/_init_helpers.py index 010d5d03cb..f3e94607a9 100644 --- a/src/autoskillit/cli/_init_helpers.py +++ b/src/autoskillit/cli/_init_helpers.py @@ -278,8 +278,10 @@ def _log_secret_scan_bypass(project_dir: Path) -> None: logger.warning("corrupt_state_yaml", path=str(state_path), error=str(exc)) msg = f"Cannot update {state_path}: file has YAML syntax errors. Delete the file to reset." raise SystemExit(msg) from exc + from autoskillit.core import ensure_project_gitignore + data.setdefault("safety", {})["secret_scan_bypass_accepted"] = datetime.now(UTC).isoformat() - state_path.parent.mkdir(exist_ok=True) + ensure_project_gitignore(project_dir) atomic_write(state_path, dump_yaml_str(data, default_flow_style=False, allow_unicode=True)) @@ -640,7 +642,7 @@ def _register_all( import autoskillit.core.paths as _core_paths from autoskillit.cli._hooks import sweep_all_scopes_for_orphans from autoskillit.config import load_config - from autoskillit.core import ensure_project_temp + from autoskillit.core import ensure_project_gitignore, ensure_project_temp from autoskillit.execution import get_backend # Refuse to register from inside the autoskillit source tree — this would @@ -669,6 +671,7 @@ def _register_all( colors = _colors() ensure_project_temp(project_dir) + ensure_project_gitignore(project_dir) _cfg = None try: diff --git a/src/autoskillit/cli/doctor/_doctor_config.py b/src/autoskillit/cli/doctor/_doctor_config.py index f20a0b4580..bceccf8a5a 100644 --- a/src/autoskillit/cli/doctor/_doctor_config.py +++ b/src/autoskillit/cli/doctor/_doctor_config.py @@ -157,7 +157,11 @@ def _collect_uncovered_gitignore_entries( autoskillit_dir: Path, gitignore_content: str, ) -> list[str]: - from autoskillit.core import _AUTOSKILLIT_GITIGNORE_ENTRIES, _COMMITTED_BY_DESIGN + from autoskillit.core import ( + _AUTOSKILLIT_GITIGNORE_ENTRIES, + _COMMITTED_BY_DESIGN, + is_self_ignored, + ) uncovered: list[str] = [] for item in sorted(autoskillit_dir.iterdir()): @@ -165,6 +169,8 @@ def _collect_uncovered_gitignore_entries( continue if item.name in _COMMITTED_BY_DESIGN: continue + if item.is_dir() and is_self_ignored(item): + continue check_name = item.name + "/" if item.is_dir() else item.name if check_name not in gitignore_content: uncovered.append(item.name) diff --git a/src/autoskillit/cli/install/_marketplace.py b/src/autoskillit/cli/install/_marketplace.py index a6ebdfe824..cdcccabefa 100644 --- a/src/autoskillit/cli/install/_marketplace.py +++ b/src/autoskillit/cli/install/_marketplace.py @@ -151,11 +151,12 @@ def _ensure_workspace_ready(*, cwd: Path | None = None) -> None: Called after the CLAUDECODE guard — only when the actual install proceeds. Idempotent: safe to call on any project state. """ - from autoskillit.core import ensure_project_temp + from autoskillit.core import ensure_project_gitignore, ensure_project_temp project_dir = Path.cwd() if cwd is None else Path(cwd) - # Repair .autoskillit/.gitignore and ensure temp/ exists + # Repair project runtime ignores and ensure temp/ exists. if (project_dir / ".autoskillit").is_dir(): + ensure_project_gitignore(project_dir) ensure_project_temp(project_dir) # Migrate legacy .autoskillit/scripts/ to .autoskillit/recipes/ if present diff --git a/src/autoskillit/config/__init__.py b/src/autoskillit/config/__init__.py index 734d847d24..79834a006a 100644 --- a/src/autoskillit/config/__init__.py +++ b/src/autoskillit/config/__init__.py @@ -55,6 +55,7 @@ ResetWorkspaceConfig, RetiredConfigKeyDef, ReviewConfig, + RolloutConfig, RunSkillConfig, SafetyConfig, SkillsConfig, @@ -104,6 +105,7 @@ "RetiredConfigKeyDef", "ReviewConfig", "RunSkillConfig", + "RolloutConfig", "SafetyConfig", "SkillsConfig", "SubsetsConfig", diff --git a/src/autoskillit/config/_automation_config.py b/src/autoskillit/config/_automation_config.py index 149e067eae..33a9835b31 100644 --- a/src/autoskillit/config/_automation_config.py +++ b/src/autoskillit/config/_automation_config.py @@ -44,6 +44,7 @@ from autoskillit.config._dataclasses_execution import ( CodexRuntimeConfig, QuotaGuardConfig, + RolloutConfig, RunSkillConfig, ) from autoskillit.config._dataclasses_fleet import FleetConfig, ProcessTetherConfig @@ -181,6 +182,7 @@ class AutomationConfig: safety: SafetyConfig = field(default_factory=SafetyConfig) read_db: ReadDbConfig = field(default_factory=ReadDbConfig) run_skill: RunSkillConfig = field(default_factory=RunSkillConfig) + rollout: RolloutConfig = field(default_factory=RolloutConfig) model: CoreRunConfig = field(default_factory=CoreRunConfig) worktree_setup: WorktreeSetupConfig = field(default_factory=WorktreeSetupConfig) migration: MigrationConfig = field(default_factory=MigrationConfig) diff --git a/src/autoskillit/config/_coercion.py b/src/autoskillit/config/_coercion.py index 9797322f70..76978c9226 100644 --- a/src/autoskillit/config/_coercion.py +++ b/src/autoskillit/config/_coercion.py @@ -19,9 +19,12 @@ import dataclasses import types -from collections.abc import Callable +from collections.abc import Callable, Mapping +from datetime import UTC, datetime, timedelta from typing import TYPE_CHECKING, Any, TypeVar, Union, get_args, get_origin, get_type_hints +import regex as re + from autoskillit.config._config_loader import ( _build_packs_config, _build_subsets_config, @@ -29,6 +32,7 @@ ) from autoskillit.config._dataclasses_errors import ConfigSchemaError from autoskillit.config._dataclasses_test_gating import _COMMAND_UNSET +from autoskillit.core import RolloutMarker if TYPE_CHECKING: from _typeshed import DataclassInstance @@ -190,10 +194,64 @@ def _coerce_value(value: Any, target_type: type, context: str) -> Any: } +def _rollout_cutover_utc(cutover: Any) -> str: + if not isinstance(cutover, str) or re.search(r"(?:Z|[+-]00:00)$", cutover) is None: + raise ConfigSchemaError( + "rollout.active.effective_at must be an explicit UTC ISO-8601 timestamp" + ) + try: + instant = datetime.fromisoformat(cutover.replace("Z", "+00:00")) + except ValueError as exc: + raise ConfigSchemaError( + "rollout.active.effective_at must be an explicit UTC ISO-8601 timestamp" + ) from exc + if instant.utcoffset() != timedelta(0): + raise ConfigSchemaError("rollout.active.effective_at must be UTC") + return instant.astimezone(UTC).isoformat() + + +def _build_rollout_active( + section: dict[str, Any], defaults: dict[str, Any] +) -> RolloutMarker | None: + raw = section.get("active", defaults.get("active")) + if raw is None: + return None + keys = {"change_id", "effective_at", "scope", "expected_effect"} + if not isinstance(raw, Mapping) or set(raw) != keys: + raise ConfigSchemaError( + "rollout.active must contain exactly change_id, effective_at, scope, expected_effect" + ) + for name in ("change_id", "expected_effect"): + if not isinstance(raw[name], str) or not raw[name].strip(): + raise ConfigSchemaError(f"rollout.active.{name} must be a nonblank string") + effective_at = _rollout_cutover_utc(raw["effective_at"]) + scope = raw["scope"] + if not isinstance(scope, Mapping) or set(scope) != {"skills", "backends", "roles"}: + raise ConfigSchemaError( + "rollout.active.scope must contain exactly skills, backends, roles" + ) + copied_scope: dict[str, list[str]] = {} + for axis, values in scope.items(): + if not isinstance(values, list) or any( + not isinstance(value, str) or not value.strip() for value in values + ): + raise ConfigSchemaError( + f"rollout.active.scope.{axis} must be a list of nonblank strings" + ) + copied_scope[axis] = list(values) + return RolloutMarker( + change_id=raw["change_id"], + effective_at=effective_at, + scope=copied_scope, + expected_effect=raw["expected_effect"], + ) + + # Custom field builders that bypass _coerce_value. # Signature: (section_dict, defaults_dict) -> coerced_value # The override is responsible for its own key lookup from section_dict. _FIELD_OVERRIDES: dict[tuple[str, str], Callable[[dict[str, Any], dict[str, Any]], Any]] = { + ("rollout", "active"): _build_rollout_active, # YAML key "default" with None-means-unset semantic. # ``defs.get(...)`` (no explicit fallback) is structurally defensive — if the # dataclass field is ever removed without updating this lambda, the .get() diff --git a/src/autoskillit/config/_config_dataclasses.py b/src/autoskillit/config/_config_dataclasses.py index a0de1cc138..e1723d1f03 100644 --- a/src/autoskillit/config/_config_dataclasses.py +++ b/src/autoskillit/config/_config_dataclasses.py @@ -59,6 +59,9 @@ from autoskillit.config._dataclasses_execution import ( QuotaGuardConfig as QuotaGuardConfig, ) +from autoskillit.config._dataclasses_execution import ( + RolloutConfig as RolloutConfig, +) from autoskillit.config._dataclasses_execution import ( RunSkillConfig as RunSkillConfig, ) diff --git a/src/autoskillit/config/_dataclasses_execution.py b/src/autoskillit/config/_dataclasses_execution.py index d2cd83d168..941152874d 100644 --- a/src/autoskillit/config/_dataclasses_execution.py +++ b/src/autoskillit/config/_dataclasses_execution.py @@ -14,7 +14,12 @@ from dataclasses import dataclass, field from typing import ClassVar, Literal, cast -from autoskillit.core import CodexRuntimeSpec, OutputFormat +from autoskillit.core import CodexRuntimeSpec, OutputFormat, RolloutMarker + + +@dataclass +class RolloutConfig: + active: RolloutMarker | None = None @dataclass diff --git a/src/autoskillit/config/settings.py b/src/autoskillit/config/settings.py index 4ae954cb80..69c906d05f 100644 --- a/src/autoskillit/config/settings.py +++ b/src/autoskillit/config/settings.py @@ -149,6 +149,9 @@ from autoskillit.config._config_dataclasses import ( ReviewConfig as ReviewConfig, ) +from autoskillit.config._config_dataclasses import ( + RolloutConfig as RolloutConfig, +) from autoskillit.config._config_dataclasses import ( RunSkillConfig as RunSkillConfig, ) @@ -271,6 +274,7 @@ "RetiredConfigKeyDef", "ReviewConfig", "RunSkillConfig", + "RolloutConfig", "SafetyConfig", "SkillsConfig", "SubsetsConfig", diff --git a/src/autoskillit/core/__init__.pyi b/src/autoskillit/core/__init__.pyi index 25ccba53eb..6acedcaa42 100644 --- a/src/autoskillit/core/__init__.pyi +++ b/src/autoskillit/core/__init__.pyi @@ -145,7 +145,9 @@ from .io import default_log_dir as default_log_dir from .io import destination_location as destination_location from .io import directory_tree_digest as directory_tree_digest from .io import dump_yaml_str as dump_yaml_str +from .io import ensure_project_gitignore as ensure_project_gitignore from .io import ensure_project_temp as ensure_project_temp +from .io import ensure_self_ignored as ensure_self_ignored from .io import extract_section as extract_section from .io import fast_dumps as fast_dumps from .io import fast_loads as fast_loads @@ -156,6 +158,7 @@ from .io import is_git_main_checkout as is_git_main_checkout from .io import is_git_worktree as is_git_worktree from .io import is_in_git_repo as is_in_git_repo from .io import is_python_bytecode_path as is_python_bytecode_path +from .io import is_self_ignored as is_self_ignored from .io import is_yaml_mapping_node as is_yaml_mapping_node from .io import load_yaml as load_yaml from .io import main_checkout_root as main_checkout_root @@ -1266,6 +1269,7 @@ from .types import ReviewOperationState as ReviewOperationState from .types import ReviewReconciliationResult as ReviewReconciliationResult from .types import ReviewResponseClass as ReviewResponseClass from .types import RollbackAdmissionEvent as RollbackAdmissionEvent +from .types import RolloutMarker as RolloutMarker from .types import RolloverEpochEvent as RolloverEpochEvent from .types import RunSkillCompletionAuthority as RunSkillCompletionAuthority from .types import RuntimeAdmission as RuntimeAdmission diff --git a/src/autoskillit/core/io/__init__.py b/src/autoskillit/core/io/__init__.py index 82eb3d54b4..c3f1e19f4a 100644 --- a/src/autoskillit/core/io/__init__.py +++ b/src/autoskillit/core/io/__init__.py @@ -37,7 +37,9 @@ atomic_write, decode_versioned_json_bytes, directory_tree_digest, + ensure_project_gitignore, ensure_project_temp, + ensure_self_ignored, is_python_bytecode_path, read_versioned_json, resolve_skill_temp_dir, @@ -81,6 +83,7 @@ is_git_main_checkout, is_git_worktree, is_in_git_repo, + is_self_ignored, main_checkout_root, parse_worktree_porcelain, pkg_root, @@ -129,7 +132,9 @@ "destination_location", "directory_tree_digest", "dump_yaml_str", + "ensure_project_gitignore", "ensure_project_temp", + "ensure_self_ignored", "extract_section", "fast_dumps", "fast_loads", @@ -140,6 +145,7 @@ "is_git_worktree", "is_in_git_repo", "is_python_bytecode_path", + "is_self_ignored", "is_yaml_mapping_node", "load_yaml", "main_checkout_root", diff --git a/src/autoskillit/core/io/io.py b/src/autoskillit/core/io/io.py index cff2d59191..7fa095d094 100644 --- a/src/autoskillit/core/io/io.py +++ b/src/autoskillit/core/io/io.py @@ -48,7 +48,9 @@ def __init__(self, path: Path, cause: OSError) -> None: "TreeVanishedError", "atomic_write", "decode_versioned_json_bytes", + "ensure_project_gitignore", "ensure_project_temp", + "ensure_self_ignored", "is_python_bytecode_path", "read_versioned_json", "resolve_skill_temp_dir", @@ -706,6 +708,7 @@ def _reset_schema_drift_logged_for_tests() -> None: _AUTOSKILLIT_GITIGNORE_ENTRIES = [ "temp/", + ".state.yaml", ".secrets.yaml", ".onboarded", "sync_manifest.json", @@ -718,49 +721,47 @@ def _reset_schema_drift_logged_for_tests() -> None: _COMMITTED_BY_DESIGN: frozenset[str] = frozenset( { "config.yaml", + "evidence", + "experiment-types", + "methodology-traditions", "recipes", + "skills", "test-source-map.json", + "waivers", } ) -def ensure_project_temp(project_dir: Path, override: str | None = None) -> Path: - """Create the autoskillit temp directory with a self-gitignore; idempotent. +def ensure_self_ignored(directory: Path) -> None: + """Write a ``*`` marker if absent; preserve an existing marker.""" + marker = directory / ".gitignore" + if not marker.exists(): + atomic_write(marker, "# Created by autoskillit automatically.\n*\n") - Uses the pytest/mypy self-gitignoring directory pattern: the temp directory - owns a ``.gitignore`` containing ``*`` — no mutation of the project root - ``.gitignore``. Works identically for default, custom-relative, and absolute - external overrides. - Also maintains ``.autoskillit/.gitignore`` covering session artefacts that - land alongside ``config.yaml`` (``.secrets.yaml``, ``.onboarded``, - ``sync_manifest.json``) when the project uses the canonical ``.autoskillit`` - directory. - """ +def ensure_project_temp(project_dir: Path, override: str | None = None) -> Path: + """Create the project temp directory with its own gitignore.""" temp_dir = resolve_temp_dir(project_dir, override) temp_dir.mkdir(parents=True, exist_ok=True) - # Race-safe ordering: .gitignore is the FIRST file written after mkdir, - # before any session content lands. See pytest #12167 / mypy #12442. - gitignore_path = temp_dir / ".gitignore" - if not gitignore_path.exists(): + ensure_self_ignored(temp_dir) + return temp_dir + + +def ensure_project_gitignore(project_dir: Path) -> None: + """Create or update the canonical project runtime-artifact ignore file.""" + autoskillit_dir = project_dir / ".autoskillit" + autoskillit_dir.mkdir(parents=True, exist_ok=True) + autoskillit_gitignore = autoskillit_dir / ".gitignore" + if not autoskillit_gitignore.exists(): atomic_write( - gitignore_path, - "# Created by autoskillit automatically.\n*\n", + autoskillit_gitignore, + "\n".join(_AUTOSKILLIT_GITIGNORE_ENTRIES) + "\n", ) - autoskillit_dir = project_dir / ".autoskillit" - if autoskillit_dir.is_dir(): - autoskillit_gitignore = autoskillit_dir / ".gitignore" - if not autoskillit_gitignore.exists(): + else: + existing = autoskillit_gitignore.read_text(encoding="utf-8") + missing = [e for e in _AUTOSKILLIT_GITIGNORE_ENTRIES if e not in existing.splitlines()] + if missing: atomic_write( autoskillit_gitignore, - "\n".join(_AUTOSKILLIT_GITIGNORE_ENTRIES) + "\n", + existing.rstrip("\n") + "\n" + "\n".join(missing) + "\n", ) - else: - existing = autoskillit_gitignore.read_text(encoding="utf-8") - missing = [e for e in _AUTOSKILLIT_GITIGNORE_ENTRIES if e not in existing.splitlines()] - if missing: - atomic_write( - autoskillit_gitignore, - existing.rstrip("\n") + "\n" + "\n".join(missing) + "\n", - ) - return temp_dir diff --git a/src/autoskillit/core/io/markdown_sections.py b/src/autoskillit/core/io/markdown_sections.py index 003039e709..80225fe996 100644 --- a/src/autoskillit/core/io/markdown_sections.py +++ b/src/autoskillit/core/io/markdown_sections.py @@ -18,7 +18,9 @@ def extract_section(markdown: str, heading: str) -> str: pattern = re.compile(rf"^## {re.escape(heading)}[ \t]*$", re.MULTILINE) matches = tuple(pattern.finditer(markdown)) if len(matches) != 1: - raise ValueError(f"expected exactly one ## {heading} section") + raise ValueError( + f"expected exactly one ## {heading} section; found {len(matches)} matching headings" + ) start = matches[0].end() next_heading = re.search(r"^##\s+", markdown[start:], re.MULTILINE) end = start + next_heading.start() if next_heading is not None else len(markdown) @@ -29,7 +31,7 @@ def split_table_row(line: str) -> tuple[str, ...]: """Parse one strictly pipe-delimited row.""" stripped = line.strip() if not stripped.startswith("|") or not stripped.endswith("|"): - raise ValueError("table rows must be pipe-delimited") + raise ValueError(f'table rows must be pipe-delimited; offending line: "{stripped[:120]}"') return tuple(cell.strip() for cell in stripped[1:-1].split("|")) @@ -41,15 +43,24 @@ def parse_pipe_table( lines = tuple(line for line in section.splitlines() if line.strip()) header = tuple(expected_header) if len(lines) < 2: - raise ValueError("table must contain a header and separator") + line = lines[0].strip()[:120] if lines else "" + raise ValueError(f'table must contain a header and separator; offending line: "{line}"') if split_table_row(lines[0]) != header: - raise ValueError(f"table header must be {header!r}") + raise ValueError( + f'table header must be {header!r}; offending line: "{lines[0].strip()[:120]}"' + ) separator = split_table_row(lines[1]) if len(separator) != len(header) or any( re.fullmatch(r":?-{3,}:?", cell) is None for cell in separator ): - raise ValueError("table separator is invalid") - rows = tuple(split_table_row(line) for line in lines[2:]) - if any(len(row) != len(header) for row in rows): - raise ValueError(f"table rows must have exactly {len(header)} columns") - return rows + raise ValueError(f'table separator is invalid; offending line: "{lines[1].strip()[:120]}"') + rows = [] + for line in lines[2:]: + row = split_table_row(line) + if len(row) != len(header): + raise ValueError( + f"table rows must have exactly {len(header)} columns; " + f'offending line: "{line.strip()[:120]}"' + ) + rows.append(row) + return tuple(rows) diff --git a/src/autoskillit/core/io/paths.py b/src/autoskillit/core/io/paths.py index 627b2aff45..8d1cbe8fb8 100644 --- a/src/autoskillit/core/io/paths.py +++ b/src/autoskillit/core/io/paths.py @@ -357,6 +357,14 @@ def is_generated_path(file_path: str) -> bool: return False +def is_self_ignored(directory: Path) -> bool: + """Recognize a regular, non-symlink marker with a standalone ``*`` rule.""" + marker = directory / ".gitignore" + if marker.is_symlink() or not marker.is_file(): + return False + return "*" in marker.read_text(encoding="utf-8", errors="replace").splitlines() + + def _coerce_str_or_path(name: str, value: object) -> str | Path: """Reject non-str/Path inputs with a uniform error message. diff --git a/src/autoskillit/core/planset/allocation.py b/src/autoskillit/core/planset/allocation.py index af085f3e47..2becede91f 100644 --- a/src/autoskillit/core/planset/allocation.py +++ b/src/autoskillit/core/planset/allocation.py @@ -21,16 +21,29 @@ def parse_part_allocation(part_markdown: str, *, part_key: str) -> tuple[Allocat rows = parse_pipe_table(section, _HEADER) result: list[AllocationRowDef] = [] seen: set[str] = set() - for requirement_id, allocation, implementation_step in rows: + raw_rows = tuple(line for line in section.splitlines() if line.strip())[2:] + for (requirement_id, allocation, implementation_step), raw_row in zip( + rows, raw_rows, strict=True + ): + if not requirement_id: + raise ValueError( + f'Requirement ID is empty in row "{raw_row.strip()[:120]}"; ' + "use a non-empty bare Requirement ID" + ) if requirement_id in seen: raise ValueError(f"duplicate allocated requirement {requirement_id}") seen.add(requirement_id) if not implementation_step: - raise ValueError("allocation implementation step is empty") + raise ValueError( + f"{requirement_id}: allocation implementation step is empty; " + "use one matching heading key such as Step 2.1" + ) try: kind = AllocationKind(allocation.lower()) except ValueError as exc: - raise ValueError(f"unknown allocation {allocation!r}") from exc + raise ValueError( + f'{requirement_id}: unknown allocation "{allocation}"; use owned or shared' + ) from exc result.append( AllocationRowDef( requirement_id=requirement_id, @@ -45,7 +58,7 @@ def parse_part_allocation(part_markdown: str, *, part_key: str) -> tuple[Allocat def verify_allocation_evidence( part_markdown: str, rows: tuple[AllocationRowDef, ...], -) -> tuple[tuple[str, PlanSetRejectReason], ...]: +) -> tuple[tuple[str, PlanSetRejectReason, str], ...]: """Ensure an allocation names a real step whose own text names the requirement.""" matches = tuple(STEP_HEADING_RE.finditer(part_markdown)) blocks: dict[str, str] = {} @@ -57,11 +70,30 @@ def verify_allocation_evidence( ) end = min(next_step, section_end) blocks[match.group(1)] = part_markdown[match.start() : end] - errors: list[tuple[str, PlanSetRejectReason]] = [] + errors: list[tuple[str, PlanSetRejectReason, str]] = [] for row in rows: block = blocks.get(row.implementation_step) if block is None: - errors.append((row.requirement_id, PlanSetRejectReason.ALLOCATION_STEP_MISSING)) + found = ( + f"step keys found: {', '.join(blocks)}" if blocks else "no step heading was found" + ) + errors.append( + ( + row.requirement_id, + PlanSetRejectReason.ALLOCATION_STEP_MISSING, + f'{row.requirement_id}: Implementation Step cell "{row.implementation_step}" ' + f"does not match a step; {found}. Expected exactly one key such as Step 2.1, " + "copied from a ### Step N: title heading (### Step N.M: title for sub-steps).", + ) + ) elif row.requirement_id not in block: - errors.append((row.requirement_id, PlanSetRejectReason.ALLOCATION_STEP_UNREFERENCED)) + errors.append( + ( + row.requirement_id, + PlanSetRejectReason.ALLOCATION_STEP_UNREFERENCED, + f'Requirement ID cell "{row.requirement_id}" is not referenced in ' + f"{row.implementation_step}; the bare ID must appear in that step's own " + "text, on the line under its heading.", + ) + ) return tuple(errors) diff --git a/src/autoskillit/core/types/audit/_type_plan_set_authority.py b/src/autoskillit/core/types/audit/_type_plan_set_authority.py index 2a0ef475e4..fb13163c49 100644 --- a/src/autoskillit/core/types/audit/_type_plan_set_authority.py +++ b/src/autoskillit/core/types/audit/_type_plan_set_authority.py @@ -440,12 +440,25 @@ def _validate_collections(self) -> None: object.__setattr__(self, "unparsed_marker_lines", tuple(self.unparsed_marker_lines)) if not isinstance(self.coverage, CoverageResultDef): raise ValueError("PlanSetAuthority.coverage is invalid") + part_keys = {part.part_key for part in self.parts} + if not set(self.coverage.parts_without_obligations) <= part_keys: + raise ValueError("coverage.parts_without_obligations contains an unknown part key") if self.state is PlanSetState.SEALED and self.coverage.status is not CoverageStatus.PASS: raise ValueError("sealed authority requires passing coverage") if tuple(part.ordinal for part in self.parts) != tuple(range(1, len(self.parts) + 1)): raise ValueError("PlanSetAuthority.parts must have consecutive ordinals") if len({Path(part.locator).resolve() for part in self.parts}) != len(self.parts): raise ValueError("PlanSetAuthority.parts contain duplicate locators") + if self.inventory_mode in {InventoryMode.UNENUMERATED, InventoryMode.NO_ISSUE}: + empty_parts = part_keys - {row.part_key for row in self.allocations} + if set(self.coverage.parts_without_obligations) != empty_parts: + raise ValueError( + "coverage.parts_without_obligations must match actual empty parts" + ) + elif self.coverage.parts_without_obligations: + raise ValueError( + "coverage.parts_without_obligations is invalid for enumerated inventory" + ) @classmethod def create( @@ -622,6 +635,14 @@ class PlanSetBindResult: first_part_path: str = "" coverage_gaps: str = "" + def __post_init__(self) -> None: + if self.success != (self.reason is None): + raise ValueError("successful results require no reason; failures require a reason") + if self.reason is not PlanSetRejectReason.COVERAGE_FAILED and self.coverage_gaps: + raise ValueError("only coverage_failed results may contain coverage_gaps") + if self.reason is PlanSetRejectReason.COVERAGE_FAILED and not self.coverage_gaps.strip(): + raise ValueError("coverage_failed requires non-empty coverage_gaps") + def to_dict(self) -> dict[str, Any]: result = {name: getattr(self, name) for name in self.__dataclass_fields__} result["reason"] = self.reason.value if self.reason else "" diff --git a/src/autoskillit/core/types/execution/_type_subprocess.py b/src/autoskillit/core/types/execution/_type_subprocess.py index 392b52680e..abcc9ab1b8 100644 --- a/src/autoskillit/core/types/execution/_type_subprocess.py +++ b/src/autoskillit/core/types/execution/_type_subprocess.py @@ -12,6 +12,7 @@ from typing import Any, Protocol, runtime_checkable from ..foundation._type_enums import ChannelConfirmation, KillReason, TerminationReason +from ..results._type_results_records import RolloutMarker from ._type_inspector import InspectorCallback, InspectorVerdict __all__ = [ @@ -258,6 +259,8 @@ def __call__( completion_drain_timeout: float = 5.0, natural_exit_grace_seconds: float = 3.0, linux_tracing_config: Any | None = None, + rollout: RolloutMarker | None = None, + logical_run_start_ts: str = "", idle_output_timeout: float | None = None, max_suppression_seconds: float | None = None, on_process_spawned: Callable[[int, int], None] | None = None, diff --git a/src/autoskillit/core/types/results/_type_results.py b/src/autoskillit/core/types/results/_type_results.py index aa4bf37df5..277bbd6a97 100644 --- a/src/autoskillit/core/types/results/_type_results.py +++ b/src/autoskillit/core/types/results/_type_results.py @@ -51,6 +51,7 @@ RecipePathValidationInputError, RecipePathValidationReport, RecipePathValidationResult, + RolloutMarker, SessionIndexEntry, SkillUnavailabilityPayload, SkillUnavailabilityRecord, @@ -126,6 +127,7 @@ "ModelTotalEntry", "TokenUsageFileEntry", "SessionIndexEntry", + "RolloutMarker", "SESSION_INDEX_SCHEMA_VERSION", "TOKEN_USAGE_SCHEMA_VERSION", "TURN_USAGE_SCHEMA_VERSION", diff --git a/src/autoskillit/core/types/results/_type_results_records.py b/src/autoskillit/core/types/results/_type_results_records.py index ec8f007a90..c6afeec2fd 100644 --- a/src/autoskillit/core/types/results/_type_results_records.py +++ b/src/autoskillit/core/types/results/_type_results_records.py @@ -45,13 +45,14 @@ "EvidenceWorktreeSource", "ValidatedAddDir", "ValidatedWorktreePath", + "RolloutMarker", "TokenUsageFileEntry", "SessionIndexEntry", ] T = TypeVar("T") -SESSION_INDEX_SCHEMA_VERSION: Final[int] = 16 +SESSION_INDEX_SCHEMA_VERSION: Final[int] = 17 TOKEN_USAGE_SCHEMA_VERSION: Final[int] = 4 TURN_USAGE_SCHEMA_VERSION: Final[int] = 2 @@ -361,12 +362,22 @@ class TokenUsageFileEntry(TypedDict): schema_version: int +class RolloutMarker(TypedDict): + """Run declaration; scope axes are skills, backends, and roles.""" + + change_id: str + effective_at: str + scope: dict[str, list[str]] + expected_effect: str + + class SessionIndexEntry(TypedDict): """Schema contract for sessions.jsonl entries written by flush_session_log.""" session_id: str dir_name: str timestamp: str + rollout: RolloutMarker | None cwd: str evidence_worktree_path: str evidence_worktree_source: str diff --git a/src/autoskillit/execution/evidence/linux_tracing.py b/src/autoskillit/execution/evidence/linux_tracing.py index 25114067b7..8748efa288 100644 --- a/src/autoskillit/execution/evidence/linux_tracing.py +++ b/src/autoskillit/execution/evidence/linux_tracing.py @@ -32,8 +32,8 @@ import anyio.abc import psutil +from autoskillit.core import RolloutMarker, get_logger from autoskillit.core import fast_dumps as _fast_dumps -from autoskillit.core import get_logger from autoskillit.core import is_pid_zombie as is_pid_zombie from autoskillit.core import read_boot_id as read_boot_id from autoskillit.core import read_starttime_ticks as read_starttime_ticks @@ -199,7 +199,7 @@ def trace_target_from_pid(pid: int) -> TraceTarget: # --------------------------------------------------------------------------- -# TraceEnrollmentRecord: schema version 2 (adds comm field) +# TraceEnrollmentRecord: schema version 3 (adds rollout and logical run start) # --------------------------------------------------------------------------- @@ -213,9 +213,10 @@ class TraceEnrollmentRecord: Schema version 2 adds 'comm' to the enrollment record so that crash recovery can verify the process identity (not just its PID) and reject alien trace files. + Schema version 3 preserves the admitted rollout declaration and logical run start. """ - schema_version: int # 2 for post-#806 records; 1 for pre-fix records + schema_version: int pid: int boot_id: str | None # read_boot_id(); None if /proc unavailable starttime_ticks: int | None # read_starttime_ticks(pid); None if unavailable @@ -224,6 +225,8 @@ class TraceEnrollmentRecord: kitchen_id: str # "" order_id: str # "" comm: str = "" # /proc/{pid}/comm value; "" for pre-fix (schema_version=1) records + rollout: RolloutMarker | None = None + logical_run_start_ts: str = "" def _write_enrollment_atomic(path: Path, record: TraceEnrollmentRecord) -> None: @@ -256,8 +259,15 @@ def read_enrollment(path: Path) -> TraceEnrollmentRecord | None: kitchen_id=data.get("kitchen_id", ""), order_id=data.get("order_id", ""), comm=data.get("comm", ""), # "" for pre-fix schema_version=1 records + rollout=data.get("rollout"), + logical_run_start_ts=data.get("logical_run_start_ts", ""), + ) + except FileNotFoundError: + return None + except (OSError, KeyError, TypeError, ValueError) as exc: + logger.warning( + "Failed to read enrollment sidecar %s (%s): %s", path, type(exc).__name__, exc ) - except (OSError, KeyError, TypeError, ValueError, json.JSONDecodeError): return None @@ -578,6 +588,8 @@ def start_linux_tracing( session_id: str = "", kitchen_id: str = "", order_id: str = "", + rollout: RolloutMarker | None = None, + logical_run_start_ts: str = "", ) -> LinuxTracingHandle | None: """Start Linux tracing if all gates pass. Returns handle or None. @@ -614,11 +626,10 @@ def start_linux_tracing( handle._trace_file = None # Write enrollment sidecar atomically for crash-recovery identity contract - # Schema version 2: includes comm field for alien-file rejection in recovery enrollment_path = tmpfs / f"autoskillit_enrollment_{pid}.json" try: record = TraceEnrollmentRecord( - schema_version=2, + schema_version=3, pid=pid, boot_id=read_boot_id(), starttime_ticks=target.starttime_ticks, @@ -627,6 +638,8 @@ def start_linux_tracing( kitchen_id=kitchen_id, order_id=order_id, comm=target.comm, + rollout=rollout, + logical_run_start_ts=logical_run_start_ts, ) _write_enrollment_atomic(enrollment_path, record) handle._enrollment_path = enrollment_path diff --git a/src/autoskillit/execution/headless/_headless_execute.py b/src/autoskillit/execution/headless/_headless_execute.py index aebe7be4d6..92d3a014b7 100644 --- a/src/autoskillit/execution/headless/_headless_execute.py +++ b/src/autoskillit/execution/headless/_headless_execute.py @@ -7,6 +7,7 @@ import time import traceback from collections.abc import Awaitable, Callable, Mapping, Sequence +from copy import deepcopy from datetime import UTC, datetime, timedelta from pathlib import Path from types import TracebackType @@ -32,6 +33,7 @@ PluginLoadMode, ResolvedLaunchContract, RetryReason, + RolloutMarker, SkillResult, WriteBehaviorSpec, collect_version_snapshot, @@ -191,6 +193,17 @@ async def _execute_claude_headless( _start_ts = datetime.now(UTC).isoformat() _start_mono = time.monotonic() _start_epoch = time.time() + _rollout_snapshot: RolloutMarker | None = None + _logical_run_start_ts: str | None = None + + def capture_rollout_on_launch() -> tuple[RolloutMarker | None, str]: + """Capture marker and UTC start once after admission; reuse across retries and nudges.""" + nonlocal _rollout_snapshot, _logical_run_start_ts + if _logical_run_start_ts is None: + _rollout_snapshot = deepcopy(ctx.config.rollout.active) + _logical_run_start_ts = datetime.now(UTC).isoformat() + return _rollout_snapshot, _logical_run_start_ts + _versions = collect_version_snapshot(_step_backend) # type: ignore[arg-type] _readonly_skill = readonly_skill @@ -346,6 +359,7 @@ def observe_execution_started() -> None: completion_drain_timeout=cfg.completion_drain_timeout, natural_exit_grace_seconds=natural_exit_grace_seconds, linux_tracing_config=linux_tracing_cfg, + capture_rollout_on_launch=capture_rollout_on_launch, idle_output_timeout=base_effective_idle, max_suppression_seconds=cfg.max_suppression_seconds, child_deferral_ceiling=cfg.completion_child_deferral_ceiling_seconds, @@ -506,6 +520,8 @@ def observe_execution_started() -> None: mark_execution_started=observe_execution_started, on_session_id_resolved=capture_resolved_session_id, natural_exit_grace_seconds=natural_exit_grace_seconds, + capture_rollout_on_launch=capture_rollout_on_launch, + linux_tracing_config=linux_tracing_cfg, nudge_timeout=min(60.0, nudge_deadline_remaining), attempt=physical_attempt, ceiling_seconds=ceiling_seconds, @@ -654,21 +670,7 @@ def observe_execution_started() -> None: execution_selection=terminal_selection, ) - if terminal_selection is not None: - from autoskillit.execution.session_log.session_log import ( - write_execution_candidate_manifest, - ) - - try: - write_execution_candidate_manifest( - terminal_selection, - ctx.config.linux_tracing.log_dir, - max_sessions=ctx.config.linux_tracing.max_sessions, - project_dir=str(ctx.project_dir), - build_protected_campaign_ids=ctx.build_protected_campaign_ids, - ) - except Exception: - logger.debug("execution_candidate_manifest_write_failed", exc_info=True) + _terminal.persist_terminal_selection(ctx, terminal_selection) terminal_capture_diagnostic = _diag.capture(managed_lineage_observer) if _diag.should_flush(result, skill_result, step_name, terminal_capture_diagnostic): @@ -692,6 +694,8 @@ def observe_execution_started() -> None: skill_command=skill_command, step_name=step_name, start_ts=_start_ts, + rollout=_rollout_snapshot, + logical_run_start_ts=_logical_run_start_ts, termination_reason=terminal_reason_override, exception_text=terminal_exception_text, versions=_versions, diff --git a/src/autoskillit/execution/headless/_headless_launch.py b/src/autoskillit/execution/headless/_headless_launch.py index d4f7d18316..35409d651d 100644 --- a/src/autoskillit/execution/headless/_headless_launch.py +++ b/src/autoskillit/execution/headless/_headless_launch.py @@ -20,6 +20,7 @@ PluginLoadMode, ResolvedLaunchContract, RetryReason, + RolloutMarker, SkillContractView, SkillResult, StreamParser, @@ -64,21 +65,6 @@ def _report_plugin_binding_close_failure(error: BaseException, _: BaseException) logger.warning("plugin_launch_binding_close_failed", primary_error=repr(error), exc_info=True) -def _plugin_launch_binding( - *, - authority: PluginArtifactAuthority | None, - backend: CodingAgentBackend, - load_mode: PluginLoadMode, -) -> AbstractContextManager[PluginLaunchBinding | None]: - """Own one exact artifact binding for the complete child attempt.""" - return plugin_launch_binding_scope( - authority=authority, - backend=backend, - load_mode=load_mode, - on_suppressed_close_error=_report_plugin_binding_close_failure, - ) - - async def _run_headless_attempt( build_spec: _BuildSpec, *, @@ -97,6 +83,7 @@ async def _run_headless_attempt( completion_drain_timeout: float, natural_exit_grace_seconds: float, linux_tracing_config: Any, + capture_rollout_on_launch: Callable[[], tuple[RolloutMarker | None, str]], idle_output_timeout: float | None, max_suppression_seconds: float, child_deferral_ceiling: float, @@ -126,10 +113,11 @@ async def _run_headless_attempt( binding_scope: AbstractContextManager[PluginLaunchBinding | None] = ( nullcontext(retained_binding) if retained_binding is not None - else _plugin_launch_binding( + else plugin_launch_binding_scope( authority=plugin_authority, backend=backend, load_mode=plugin_load_mode, + on_suppressed_close_error=_report_plugin_binding_close_failure, ) ) with binding_scope as binding: @@ -187,6 +175,7 @@ async def _run_headless_attempt( on_spec_built(spec) if mark_execution_started is not None: mark_execution_started() + rollout, logical_run_start_ts = capture_rollout_on_launch() result = await runner( list(spec.cmd), cwd=Path(spec.cwd), @@ -201,6 +190,8 @@ async def _run_headless_attempt( completion_drain_timeout=completion_drain_timeout, natural_exit_grace_seconds=natural_exit_grace_seconds, linux_tracing_config=linux_tracing_config, + rollout=rollout, + logical_run_start_ts=logical_run_start_ts, idle_output_timeout=idle_output_timeout, max_suppression_seconds=max_suppression_seconds, child_deferral_ceiling=child_deferral_ceiling, @@ -234,6 +225,8 @@ async def _attempt_contract_nudge( cwd: str, runner: SubprocessRunner, *, + capture_rollout_on_launch: Callable[[], tuple[RolloutMarker | None, str]], + linux_tracing_config: Any | None = None, backend: CodingAgentBackend | None = None, result_parser: ResultParser | None = None, provider_extras: Mapping[str, str] | None = None, @@ -326,10 +319,11 @@ async def _attempt_contract_nudge( nudge_binding_scope: AbstractContextManager[PluginLaunchBinding | None] = ( nullcontext(retained_binding) if retained_binding is not None - else _plugin_launch_binding( + else plugin_launch_binding_scope( authority=plugin_authority, backend=backend, load_mode=plugin_load_mode, + on_suppressed_close_error=_report_plugin_binding_close_failure, ) ) try: @@ -430,6 +424,7 @@ def build_nudge_spec( nudge_start_ts = datetime.now(UTC).isoformat() if mark_execution_started is not None: mark_execution_started() + rollout, logical_run_start_ts = capture_rollout_on_launch() nudge_result = await runner( list(spec.cmd), cwd=Path(spec.cwd), @@ -443,6 +438,9 @@ def build_nudge_spec( on_process_reaped=handle.record_reaped if handle is not None else None, ceiling_seconds=ceiling_seconds, natural_exit_grace_seconds=natural_exit_grace_seconds, + rollout=rollout, + logical_run_start_ts=logical_run_start_ts, + linux_tracing_config=linux_tracing_config, line_driver=backend.line_driver(spec), ) nudge_end_ts = datetime.now(UTC).isoformat() diff --git a/src/autoskillit/execution/headless/_headless_terminal.py b/src/autoskillit/execution/headless/_headless_terminal.py index 49e0c42a0f..df19d770b0 100644 --- a/src/autoskillit/execution/headless/_headless_terminal.py +++ b/src/autoskillit/execution/headless/_headless_terminal.py @@ -5,7 +5,7 @@ import dataclasses import time from collections.abc import Callable -from typing import Any +from typing import TYPE_CHECKING, Any from autoskillit.core import ( CodingAgentBackend, @@ -20,12 +20,37 @@ RecipeIdentity, ResolvedLaunchContract, RetryReason, + RolloutMarker, SessionTelemetry, SessionType, SkillResult, SubprocessResult, + get_logger, ) +if TYPE_CHECKING: + from autoskillit.pipeline.context import ToolContext + +logger = get_logger(__name__) + + +def persist_terminal_selection(ctx: ToolContext, selection: ExecutionSelection | None) -> None: + """Publish the final candidate manifest without interrupting session logging.""" + if selection is None: + return + from autoskillit.execution.session_log.session_log import write_execution_candidate_manifest + + try: + write_execution_candidate_manifest( + selection, + ctx.config.linux_tracing.log_dir, + max_sessions=ctx.config.linux_tracing.max_sessions, + project_dir=str(ctx.project_dir), + build_protected_campaign_ids=ctx.build_protected_campaign_ids, + ) + except Exception: + logger.debug("execution_candidate_manifest_write_failed", exc_info=True) + def reconcile_token_evidence( skill_result: SkillResult, @@ -290,6 +315,8 @@ def build_terminal_flush_kwargs( skill_command: str, step_name: str, start_ts: str, + rollout: RolloutMarker | None, + logical_run_start_ts: str | None, termination_reason: str, exception_text: str, versions: dict[str, Any], @@ -332,7 +359,8 @@ def build_terminal_flush_kwargs( "is_error": skill_result.is_error, "subtype": skill_result.subtype, "exit_code": skill_result.exit_code, - "start_ts": result.start_ts if result is not None else start_ts, + "start_ts": logical_run_start_ts if logical_run_start_ts is not None else start_ts, + "rollout": rollout, "proc_snapshots": result.proc_snapshots if result is not None else None, "termination_reason": ( result.termination.value if result is not None else termination_reason diff --git a/src/autoskillit/execution/process/__init__.py b/src/autoskillit/execution/process/__init__.py index 57959232d1..115be7083e 100644 --- a/src/autoskillit/execution/process/__init__.py +++ b/src/autoskillit/execution/process/__init__.py @@ -24,6 +24,7 @@ from autoskillit.core import ( ChannelBStatus, ChannelConfirmation, + RolloutMarker, SessionEvent, SubprocessResult, TerminationReason, @@ -249,6 +250,8 @@ async def run_managed_async( completion_drain_timeout: float = 5.0, natural_exit_grace_seconds: float = 3.0, linux_tracing_config: LinuxTracingConfig | None = None, + rollout: RolloutMarker | None = None, + logical_run_start_ts: str = "", idle_output_timeout: float | None = None, max_suppression_seconds: float | None = None, on_process_spawned: Callable[[int, int], None] | None = None, @@ -542,6 +545,8 @@ async def run_managed_async( target=_target, config=linux_tracing_config, tg=tg, + rollout=rollout, + logical_run_start_ts=logical_run_start_ts, ) timeout_scope = await _await_race_and_drain( tg, diff --git a/src/autoskillit/execution/process/_lifecycle/runner.py b/src/autoskillit/execution/process/_lifecycle/runner.py index 0a43cc0b04..8e274a3e58 100644 --- a/src/autoskillit/execution/process/_lifecycle/runner.py +++ b/src/autoskillit/execution/process/_lifecycle/runner.py @@ -10,6 +10,7 @@ from autoskillit.core import ( ChannelConfirmation, + RolloutMarker, SubprocessResult, TerminationReason, get_logger, @@ -149,6 +150,8 @@ async def __call__( completion_drain_timeout: float = 5.0, natural_exit_grace_seconds: float = 3.0, linux_tracing_config: LinuxTracingConfig | None = None, + rollout: RolloutMarker | None = None, + logical_run_start_ts: str = "", idle_output_timeout: float | None = None, max_suppression_seconds: float | None = None, on_process_spawned: Callable[[int, int], None] | None = None, @@ -188,6 +191,8 @@ async def __call__( completion_drain_timeout=completion_drain_timeout, natural_exit_grace_seconds=natural_exit_grace_seconds, linux_tracing_config=linux_tracing_config, + rollout=rollout, + logical_run_start_ts=logical_run_start_ts, idle_output_timeout=idle_output_timeout, max_suppression_seconds=max_suppression_seconds, on_process_spawned=on_process_spawned, diff --git a/src/autoskillit/execution/recording/recording.py b/src/autoskillit/execution/recording/recording.py index a22e9615fe..d4f7519ae1 100644 --- a/src/autoskillit/execution/recording/recording.py +++ b/src/autoskillit/execution/recording/recording.py @@ -18,6 +18,7 @@ CLAUDE_CODE_CAPABILITIES, BackendCapabilities, InspectorCallback, + RolloutMarker, SubprocessResult, SubprocessRunner, TerminationReason, @@ -136,6 +137,8 @@ async def __call__( completion_drain_timeout: float = 5.0, natural_exit_grace_seconds: float = 3.0, linux_tracing_config: Any | None = None, + rollout: RolloutMarker | None = None, + logical_run_start_ts: str = "", idle_output_timeout: float | None = None, max_suppression_seconds: float | None = None, on_process_spawned: Callable[[int, int], None] | None = None, @@ -193,6 +196,8 @@ async def __call__( completion_drain_timeout=completion_drain_timeout, natural_exit_grace_seconds=natural_exit_grace_seconds, linux_tracing_config=linux_tracing_config, + rollout=rollout, + logical_run_start_ts=logical_run_start_ts, idle_output_timeout=idle_output_timeout, max_suppression_seconds=max_suppression_seconds, on_process_spawned=on_process_spawned, @@ -231,6 +236,8 @@ async def __call__( completion_drain_timeout=completion_drain_timeout, natural_exit_grace_seconds=natural_exit_grace_seconds, linux_tracing_config=linux_tracing_config, + rollout=rollout, + logical_run_start_ts=logical_run_start_ts, idle_output_timeout=idle_output_timeout, max_suppression_seconds=max_suppression_seconds, on_process_spawned=on_process_spawned, @@ -278,6 +285,8 @@ async def __call__( completion_drain_timeout=completion_drain_timeout, natural_exit_grace_seconds=natural_exit_grace_seconds, linux_tracing_config=linux_tracing_config, + rollout=rollout, + logical_run_start_ts=logical_run_start_ts, idle_output_timeout=idle_output_timeout, max_suppression_seconds=max_suppression_seconds, on_process_spawned=on_process_spawned, @@ -387,6 +396,8 @@ async def _record_non_pty_session( completion_drain_timeout: float, natural_exit_grace_seconds: float, linux_tracing_config: Any | None, + rollout: RolloutMarker | None, + logical_run_start_ts: str, idle_output_timeout: float | None, max_suppression_seconds: float | None, on_process_spawned: Callable[[int, int], None] | None, @@ -424,6 +435,8 @@ async def _record_non_pty_session( completion_drain_timeout=completion_drain_timeout, natural_exit_grace_seconds=natural_exit_grace_seconds, linux_tracing_config=linux_tracing_config, + rollout=rollout, + logical_run_start_ts=logical_run_start_ts, idle_output_timeout=idle_output_timeout, max_suppression_seconds=max_suppression_seconds, on_process_spawned=on_process_spawned, @@ -542,6 +555,8 @@ async def __call__( completion_drain_timeout: float = 5.0, natural_exit_grace_seconds: float = 3.0, linux_tracing_config: Any | None = None, + rollout: RolloutMarker | None = None, + logical_run_start_ts: str = "", idle_output_timeout: float | None = None, max_suppression_seconds: float | None = None, on_process_spawned: Callable[[int, int], None] | None = None, @@ -580,6 +595,8 @@ async def __call__( ceiling_seconds, systemd_scope_enabled, natural_exit_grace_seconds, + rollout, + logical_run_start_ts, ) step_name = (env or {}).get(SCENARIO_STEP_NAME_ENV, "") diff --git a/src/autoskillit/execution/session/_managed_headless_session_lineage.py b/src/autoskillit/execution/session/_managed_headless_session_lineage.py index 436b25e719..f1d5de8060 100644 --- a/src/autoskillit/execution/session/_managed_headless_session_lineage.py +++ b/src/autoskillit/execution/session/_managed_headless_session_lineage.py @@ -1,4 +1,9 @@ -"""Crash-durable managed headless session lineage persistence.""" +"""Crash-durable managed headless session lineage persistence. + +Records live in /.autoskillit/managed-headless-session-lineage/, a +self-ignoring runtime directory at the .autoskillit root. Records are durable +across resume; keeping the namespace here repairs existing stores in place. +""" from __future__ import annotations diff --git a/src/autoskillit/execution/session/_managed_headless_session_lineage_records.py b/src/autoskillit/execution/session/_managed_headless_session_lineage_records.py index c76c01bf4b..6a5adf7277 100644 --- a/src/autoskillit/execution/session/_managed_headless_session_lineage_records.py +++ b/src/autoskillit/execution/session/_managed_headless_session_lineage_records.py @@ -22,6 +22,8 @@ NativeShellCaptureDecision, acquire_flock_with_timeout, atomic_write, + ensure_self_ignored, + is_self_ignored, ) from autoskillit.execution.session._managed_headless_session_lineage import ( _DISPATCH_INDEX, @@ -137,6 +139,9 @@ def _prepare_root(anchor: Path) -> Path: current.mkdir(mode=0o700, exist_ok=True) if current.is_symlink() or not current.is_dir(): raise ValueError("Managed lineage namespace is not a regular directory") + ensure_self_ignored(current) + if not is_self_ignored(current): + raise OSError("Lineage namespace requires a self-ignoring .gitignore") for relative in ( Path(_RECORDS_DIR), Path(_INDEXES_DIR) / _FINAL_NATIVE_INDEX, diff --git a/src/autoskillit/execution/session_log/_session_log_recovery.py b/src/autoskillit/execution/session_log/_session_log_recovery.py index 1e041112ef..12e62808e7 100644 --- a/src/autoskillit/execution/session_log/_session_log_recovery.py +++ b/src/autoskillit/execution/session_log/_session_log_recovery.py @@ -131,6 +131,7 @@ def _finalize_crashed_trace( build_protected_campaign_ids: CampaignProtector | None, pid: int, snapshots: list[dict[str, object]], + enrollment: TraceEnrollmentRecord, ) -> bool: """Write one decoded crash trace into the durable session log.""" try: @@ -160,7 +161,8 @@ def _finalize_crashed_trace( is_error=True, subtype="crashed", exit_code=-1, - start_ts=mtime_ts, + start_ts=enrollment.logical_run_start_ts or mtime_ts, + rollout=enrollment.rollout, proc_snapshots=snapshots if snapshots else None, termination_reason="CRASHED", provider_outcome=ProviderOutcome.none_used(), @@ -245,6 +247,7 @@ def recover_crashed_sessions( build_protected_campaign_ids=build_protected_campaign_ids, pid=pid, snapshots=snapshots, + enrollment=enrollment, ): _delete_trace_pair(trace_file, enrollment_path) count += 1 diff --git a/src/autoskillit/execution/session_log/session_log.py b/src/autoskillit/execution/session_log/session_log.py index 0652034458..ee46fc73a8 100644 --- a/src/autoskillit/execution/session_log/session_log.py +++ b/src/autoskillit/execution/session_log/session_log.py @@ -31,6 +31,7 @@ EvidenceWorktree, EvidenceWorktreeSource, ModelIdentity, + RolloutMarker, SessionType, atomic_write, default_log_dir, @@ -158,6 +159,7 @@ def flush_session_log( subtype: str, exit_code: int, start_ts: str, + rollout: RolloutMarker | None = None, proc_snapshots: list[dict[str, object]] | None, end_ts: str = "", elapsed_seconds: float | None = None, @@ -413,6 +415,7 @@ def flush_session_log( ) summary = { + "rollout": rollout, "session_id": session_id, "dir_name": dir_name, "pid": pid, @@ -582,6 +585,7 @@ def flush_session_log( refresh_summary_child_outcomes(summary_path, list(telemetry.child_outcomes)) index_entry = { + "rollout": rollout, "session_id": session_id, "dir_name": dir_name, "timestamp": start_ts, diff --git a/src/autoskillit/fleet/_outcome.py b/src/autoskillit/fleet/_outcome.py index 8651dd22dc..66ec770214 100644 --- a/src/autoskillit/fleet/_outcome.py +++ b/src/autoskillit/fleet/_outcome.py @@ -5,6 +5,8 @@ from pathlib import Path from typing import Any, TypedDict +import regex as re + from autoskillit.core import ( FleetErrorCode, InfraExitCategory, @@ -49,6 +51,23 @@ class _DispatchCommonFields(TypedDict): "attempt_id", ) +_MANAGED_LINEAGE_PATH_RE = re.compile( + r"(?|,;:(){}\[\]]+/)*)|" + r"(?:[^/\s\\\r\n\"'<>|,;:(){}\[\]]+/)+)?" + r"\.autoskillit/managed-headless-session-lineage" + r"(?=$|[/\s\"'<>|,;:(){}\[\]])(?:/[^\\\s\"'<>|,;:(){}\[\]]*)?", + re.IGNORECASE, +) +_MANAGED_CAPTURE_FIELD_ASSIGNMENT_RE = re.compile( + r"(?[\"']?)" + r"(?:attempt_id|launch_id|requested_mode|effective_mode|lineage[-_ ]status|" + r"native[-_ ]shell[-_ ]capture|managed[-_ ]headless[-_ ]session[-_ ]lineage|" + r"managed[-_ ]lineage|lineage[-_ ]validation)" + r"(?P=quote)\s*[=:]\s*(?:\"[^\"]*\"|'[^']*'|[^\s,;]+)", + re.IGNORECASE, +) +_REDACTED_ONLY_LINE_RE = re.compile(r"(?:|[\s\p{P}])*", re.IGNORECASE) + def _is_managed_capture_diagnostic_text(value: str) -> bool: """Return whether text belongs only in managed-lineage diagnostics.""" @@ -67,6 +86,17 @@ def _sanitize_managed_capture_diagnostics(value: str) -> str: ) +def _sanitize_failed_managed_capture_reason(value: str) -> str: + """Redact private lineage details while retaining failure causes.""" + normalized = _MANAGED_LINEAGE_PATH_RE.sub("", value) + normalized = _MANAGED_CAPTURE_FIELD_ASSIGNMENT_RE.sub("", normalized) + return "".join( + line + for line in normalized.splitlines(keepends=True) + if not _REDACTED_ONLY_LINE_RE.fullmatch(line) + ) + + def _is_abandon_reason(skill_result: SkillResult) -> bool: """Return True when the kill reason indicates resume would be futile.""" if skill_result.retry_reason in _ABANDON_REASONS: @@ -153,7 +183,9 @@ def classify_dispatch_outcome( return DispatchStatus.RESUMABLE, FleetErrorCode.FLEET_QUOTA_EXHAUSTED if parsed.outcome == "completed_clean": raw_reason = str(parsed.payload.get("reason", "")) if parsed.payload else "" - reason = _sanitize_managed_capture_diagnostics(raw_reason) + reason = _sanitize_managed_capture_diagnostics( + _sanitize_failed_managed_capture_reason(raw_reason) + ) if raw_reason and not reason: reason = FleetErrorCode.FLEET_L3_STARTUP_OR_CRASH return DispatchStatus.FAILURE, reason diff --git a/src/autoskillit/recipe/api_orchestration/_api_orchestration_match.py b/src/autoskillit/recipe/api_orchestration/_api_orchestration_match.py index 90153a047e..a2cf3b1ee1 100644 --- a/src/autoskillit/recipe/api_orchestration/_api_orchestration_match.py +++ b/src/autoskillit/recipe/api_orchestration/_api_orchestration_match.py @@ -17,6 +17,7 @@ _LoadPipelineInputs, _ValidationResult, ) +from autoskillit.recipe.helpers._io_loading import _substitute_recipe_notes_in_yaml from autoskillit.recipe.io import ( RecipeInfo, find_recipe_by_name, @@ -45,6 +46,7 @@ def _resolve_recipe_match( raw_declared = match.content if match.content is not None else match.path.read_text() raw = substitute_temp_placeholder_yaml_safe(raw_declared, pipeline_inputs.temp_dir_relpath) raw = substitute_scripts_placeholder_yaml_safe(raw) + raw = _substitute_recipe_notes_in_yaml(raw) if match.source == RecipeSource.BUILTIN: recipes_dir = _orch.pkg_root() / "recipes" diff --git a/src/autoskillit/recipe/helpers/_io_loading.py b/src/autoskillit/recipe/helpers/_io_loading.py index ffd8f6190b..9b6f6c72d7 100644 --- a/src/autoskillit/recipe/helpers/_io_loading.py +++ b/src/autoskillit/recipe/helpers/_io_loading.py @@ -13,8 +13,10 @@ LoadReport, LoadResult, RecipeSource, + compose_yaml, fast_loads, get_logger, + is_yaml_mapping_node, load_yaml, pkg_root, ) @@ -25,7 +27,7 @@ # Canonical recipe step note prose, single-sourced here so a doctrinal update -# lands in one place instead of eight recipe YAML/JSON copies. +# lands in one place instead of repeated recipe YAML/JSON copies. ANALYZE_PIPELINE_HEALTH_ERROR_NOTE: str = ( "Preserve the originating failure response separately. Inspect the diagnostic " "response and include its non-empty result verbatim as supplementary evidence. " @@ -39,16 +41,57 @@ _RECIPE_NOTE_TABLE: dict[str, str] = { _RECIPE_NOTE_ANALYZE_PIPELINE_HEALTH_ERROR: ANALYZE_PIPELINE_HEALTH_ERROR_NOTE, + "__RECIPE_NOTE_PLAN_SET_RENEWAL_FAILURE__": ( + "coverage_failed here is terminal by decision. The part passed the plan-set gate " + "before the walkthrough edited it, and a walkthrough cannot be re-run until its " + "edits are renewed. Relay the result's coverage_gaps in the failure diagnostic: " + "it names the rows the edit broke." + ), + "__RECIPE_NOTE_PLAN_PARTS_BINDING__": ( + "Bind exactly the plan_parts value captured from the producer that just ran. " + "A successful replan replaces the earlier list with its returned list: do not " + "append rejected paths or rebuild the list from a directory listing. When " + "salvage_plan is the producer, bind its returned list unchanged under the " + "existing salvage recovery and strict coverage checks." + ), } def _substitute_recipe_note_placeholders(value: Any) -> Any: - """Substitute recipe note placeholders. Only strings reach this layer.""" + """Expand note placeholders in strings, leaving non-string values unchanged.""" if isinstance(value, str): - return _RECIPE_NOTE_TABLE.get(value, value) + for placeholder, note in _RECIPE_NOTE_TABLE.items(): + value = value.replace(placeholder, note) return value +def _substitute_recipe_notes_in_yaml(text: str) -> str: + """Expand shared step notes without changing the surrounding recipe source.""" + if "__RECIPE_NOTE_" not in text: + return text + root = compose_yaml(text) + if not is_yaml_mapping_node(root): + return text + steps = next((node for key, node in root.value if key.value == "steps"), None) + if not is_yaml_mapping_node(steps): + return text + edits: dict[tuple[int, int], str] = {} + for _, step in steps.value: + if not is_yaml_mapping_node(step): + continue + for field, node in step.value: + if field.value != "note": + continue + expanded = _substitute_recipe_note_placeholders(node.value) + if expanded != node.value: + start, end = node.start_mark.index, node.end_mark.index + newline = "\n" if text[start:end].endswith("\n") else "" + edits[start, end] = json.dumps(expanded, ensure_ascii=False) + newline + for (start, end), value in sorted(edits.items(), reverse=True): + text = text[:start] + value + text[end:] + return text + + _TEMP_PLACEHOLDER = "{{AUTOSKILLIT_TEMP}}" _SCRIPTS_PLACEHOLDER = "{{AUTOSKILLIT_SCRIPTS}}" diff --git a/src/autoskillit/recipe/rules/dataflow/rules_dataflow_multipart.py b/src/autoskillit/recipe/rules/dataflow/rules_dataflow_multipart.py index 397728ff1e..112a356d5a 100644 --- a/src/autoskillit/recipe/rules/dataflow/rules_dataflow_multipart.py +++ b/src/autoskillit/recipe/rules/dataflow/rules_dataflow_multipart.py @@ -25,31 +25,6 @@ def _check_multipart_iteration_notes(ctx: ValidationContext) -> list[RuleFinding if not has_multipart_step: return [] - plan_step = wf.steps.get("plan") - plan_note = (plan_step.note or "") if plan_step is not None else "" - # Also accept the glob pattern from the note of whichever step invokes the multipart skill - multipart_step_notes = [ - (step.note or "") - for step in wf.steps.values() - if step.tool in SKILL_TOOLS - and extract_skill_name(step.with_args.get("skill_command", "")) in MULTIPART_SKILL_NAMES - ] - if "*_part_*.md" not in plan_note and not any( - "*_part_*.md" in note for note in multipart_step_notes - ): - findings.append( - make_finding( - rule_name="multipart-iteration-notes", - step_name="plan", - message="Recipe uses make-plan or rectify but neither the 'plan' step note nor " - "the planning step's own note contains '*_part_*.md'. Agents will not " - "know to glob for multi-part plan files. Add: " - "'Glob plan_dir for *_part_*.md or single plan file.' to the plan " - "step's note (or to the make-plan/rectify step's note if no separate " - "'plan' step exists).", - ) - ) - sequential_keywords = ("SEQUENTIAL EXECUTION", "full cycle", "Never run verify for all parts") rules_text = " ".join(wf.kitchen_rules) if not any(kw in rules_text for kw in sequential_keywords): diff --git a/src/autoskillit/recipes/diagrams/implementation-groups.md b/src/autoskillit/recipes/diagrams/implementation-groups.md index 7dad121550..5614952a9e 100644 --- a/src/autoskillit/recipes/diagrams/implementation-groups.md +++ b/src/autoskillit/recipes/diagrams/implementation-groups.md @@ -1,4 +1,4 @@ - + ## implementation-groups diff --git a/src/autoskillit/recipes/diagrams/implementation.md b/src/autoskillit/recipes/diagrams/implementation.md index 52cd41d81c..98733d2e27 100644 --- a/src/autoskillit/recipes/diagrams/implementation.md +++ b/src/autoskillit/recipes/diagrams/implementation.md @@ -1,4 +1,4 @@ - + ## implementation diff --git a/src/autoskillit/recipes/diagrams/remediation.md b/src/autoskillit/recipes/diagrams/remediation.md index 7a3ad79f51..fe3ca00d1d 100644 --- a/src/autoskillit/recipes/diagrams/remediation.md +++ b/src/autoskillit/recipes/diagrams/remediation.md @@ -1,4 +1,4 @@ - + ## remediation diff --git a/src/autoskillit/recipes/implementation-groups.json b/src/autoskillit/recipes/implementation-groups.json index cab0d3507b..a4e9b58e9a 100644 --- a/src/autoskillit/recipes/implementation-groups.json +++ b/src/autoskillit/recipes/implementation-groups.json @@ -256,7 +256,7 @@ "on_failure": "release_issue_failure", "on_rate_limit": "release_issue_failure", "on_context_limit": "salvage_plan", - "note": "Produces plan in {{AUTOSKILLIT_TEMP}}/make-plan/. Glob plan_dir for *_part_*.md or single plan file. plan_parts context key contains the full ordered list of part files (sorted alphabetically). For a single-part plan, plan_parts has one entry equal to plan_path. GROUPS MODE: Set skill_inputs.task to the current group's file path selected from context.group_files — do NOT pass the entire list or read the file; pass the path directly to the skill. REMEDIATION MODE: on re-entry, pass the exact current audit_cycle_path; make-plan emits the matching plan_disposition_path. ACCUMULATION: After each plan step execution in groups mode, the agent MUST append the captured plan_path to context.all_plan_paths using a comma separator instead of letting the capture block overwrite the accumulated value. For multi-group runs, the accumulated value looks like \"/path/groupA_plan.md,/path/groupB_plan.md\". ISSUE CONTEXT: issue_url is passed via skill_inputs.issue_url — do NOT append it to skill_command. The make-plan skill reads skill_inputs.issue_url directly and calls fetch_github_issue internally when non-empty. ADVERSARIAL REVIEW LEVEL: adversarial_review_level is passed via skill_inputs.adversarial_review_level — do NOT append it to skill_command. The make-plan skill reads it directly from skill_inputs when non-empty. verdict=plan is the only successful result.\n" + "note": "Produces plan in {{AUTOSKILLIT_TEMP}}/make-plan/. For a single-part plan, plan_parts has one entry equal to plan_path. GROUPS MODE: Set skill_inputs.task to the current group's file path selected from context.group_files — do NOT pass the entire list or read the file; pass the path directly to the skill. REMEDIATION MODE: on re-entry, pass the exact current audit_cycle_path; make-plan emits the matching plan_disposition_path. ACCUMULATION: After each plan step execution in groups mode, the agent MUST append the captured plan_path to context.all_plan_paths using a comma separator instead of letting the capture block overwrite the accumulated value. For multi-group runs, the accumulated value looks like \"/path/groupA_plan.md,/path/groupB_plan.md\". ISSUE CONTEXT: issue_url is passed via skill_inputs.issue_url — do NOT append it to skill_command. The make-plan skill reads skill_inputs.issue_url directly and calls fetch_github_issue internally when non-empty. ADVERSARIAL REVIEW LEVEL: adversarial_review_level is passed via skill_inputs.adversarial_review_level — do NOT append it to skill_command. The make-plan skill reads it directly from skill_inputs when non-empty. verdict=plan is the only successful result." }, "salvage_plan": { "tool": "run_python", @@ -318,7 +318,8 @@ "on_failure": "release_issue_failure", "optional_context_refs": [ "plan_set_authority_path" - ] + ], + "note": "__RECIPE_NOTE_PLAN_PARTS_BINDING__ coverage_failed here is terminal by decision. Two producers (plan, gap_replan) share this bind, and a correct replan must return to the one that wrote the rejected part. Relay the result's coverage_gaps in the failure diagnostic: it names the part, the rows and the cells." }, "check_group_plan_iteration": { "tool": "run_python", @@ -529,7 +530,7 @@ } ], "on_failure": "release_issue_failure", - "note": "Rebinds current part bytes after walkthrough edits before implementation." + "note": "Rebinds current part bytes after walkthrough edits before implementation. __RECIPE_NOTE_PLAN_SET_RENEWAL_FAILURE__" }, "create_impl_worktree": { "tool": "run_cmd", diff --git a/src/autoskillit/recipes/implementation-groups.yaml b/src/autoskillit/recipes/implementation-groups.yaml index 925411eea1..de87cc3da0 100644 --- a/src/autoskillit/recipes/implementation-groups.yaml +++ b/src/autoskillit/recipes/implementation-groups.yaml @@ -254,25 +254,22 @@ steps: on_failure: release_issue_failure on_rate_limit: release_issue_failure on_context_limit: salvage_plan - note: "Produces plan in {{AUTOSKILLIT_TEMP}}/make-plan/. Glob plan_dir for *_part_*.md\ - \ or single plan file. plan_parts context key contains the full ordered list\ - \ of part files (sorted alphabetically). For a single-part plan, plan_parts\ - \ has one entry equal to plan_path. GROUPS MODE: Set skill_inputs.task to the\ - \ current group's file path selected from context.group_files \u2014 do NOT\ - \ pass the entire list or read the file; pass the path directly to the skill.\ - \ REMEDIATION MODE: on re-entry, pass the exact current audit_cycle_path; make-plan\ - \ emits the matching plan_disposition_path. ACCUMULATION: After each plan step\ - \ execution in groups mode, the agent MUST append the captured plan_path to\ - \ context.all_plan_paths using a comma separator instead of letting the capture\ - \ block overwrite the accumulated value. For multi-group runs, the accumulated\ - \ value looks like \"/path/groupA_plan.md,/path/groupB_plan.md\". ISSUE CONTEXT:\ - \ issue_url is passed via skill_inputs.issue_url \u2014 do NOT append it to skill_command.\ - \ The make-plan skill reads skill_inputs.issue_url directly and calls fetch_github_issue\ - \ internally when non-empty. ADVERSARIAL REVIEW LEVEL: adversarial_review_level\ - \ is passed via skill_inputs.adversarial_review_level \u2014 do NOT append it to\ - \ skill_command. The make-plan skill reads it directly from skill_inputs when\ - \ non-empty. verdict=plan\ - \ is the only successful result.\n" + note: >- + Produces plan in {{AUTOSKILLIT_TEMP}}/make-plan/. For a single-part plan, plan_parts has + one entry equal to plan_path. GROUPS MODE: Set skill_inputs.task to the current group's + file path selected from context.group_files — do NOT pass the entire list or read the + file; pass the path directly to the skill. REMEDIATION MODE: on re-entry, pass the exact + current audit_cycle_path; make-plan emits the matching plan_disposition_path. + ACCUMULATION: After each plan step execution in groups mode, the agent MUST append the + captured plan_path to context.all_plan_paths using a comma separator instead of letting + the capture block overwrite the accumulated value. For multi-group runs, the accumulated + value looks like "/path/groupA_plan.md,/path/groupB_plan.md". ISSUE CONTEXT: issue_url + is passed via skill_inputs.issue_url — do NOT append it to skill_command. The make-plan + skill reads skill_inputs.issue_url directly and calls fetch_github_issue internally when + non-empty. ADVERSARIAL REVIEW LEVEL: adversarial_review_level is passed via + skill_inputs.adversarial_review_level — do NOT append it to skill_command. The make-plan + skill reads it directly from skill_inputs when non-empty. verdict=plan is the only + successful result. salvage_plan: tool: run_python with: @@ -321,6 +318,12 @@ steps: on_failure: release_issue_failure optional_context_refs: - plan_set_authority_path + note: >- + __RECIPE_NOTE_PLAN_PARTS_BINDING__ + coverage_failed here is terminal by decision. Two producers (plan, gap_replan) share + this bind, and a correct replan must return to the one that wrote the rejected part. + Relay the result's coverage_gaps in the failure diagnostic: it names the part, the rows + and the cells. check_group_plan_iteration: tool: run_python with: @@ -475,7 +478,9 @@ steps: route: create_impl_worktree - route: release_issue_failure on_failure: release_issue_failure - note: Rebinds current part bytes after walkthrough edits before implementation. + note: >- + Rebinds current part bytes after walkthrough edits before implementation. + __RECIPE_NOTE_PLAN_SET_RENEWAL_FAILURE__ create_impl_worktree: tool: run_cmd with: diff --git a/src/autoskillit/recipes/implementation.json b/src/autoskillit/recipes/implementation.json index 0847f38d44..583014e08e 100644 --- a/src/autoskillit/recipes/implementation.json +++ b/src/autoskillit/recipes/implementation.json @@ -524,7 +524,7 @@ "audit_cycle_path", "plan_set_coverage_gaps" ], - "note": "Produces an ordinary plan when no audit_cycle_path is present. On NO GO re-entry, consumes that exact current authority and emits both plan_path and plan_disposition_path. all_plan_paths appends result.plan_path across re-entries; verdict=plan is the only successful result. Glob plan_dir for *_part_*.md or a single plan file and preserve alphabetical part order." + "note": "Produces an ordinary plan when no audit_cycle_path is present. On NO GO re-entry, consumes that exact current authority and emits both plan_path and plan_disposition_path. all_plan_paths appends result.plan_path across re-entries; verdict=plan is the only successful result." }, "salvage_plan": { "tool": "run_python", @@ -587,7 +587,7 @@ } ], "on_failure": "release_issue_failure", - "note": "Aggregate plan-set gate before every walkthrough.\n\nSingle-shot plans omit parent_authority_path intentionally: there is no prior\nmultipart authority to thread through. Multi-shot bind_group_parts in\nimplementation-groups.yaml supplies one because group iteration threads the\nparent digest from the previous cycle's bind step.\n" + "note": "__RECIPE_NOTE_PLAN_PARTS_BINDING__ Aggregate plan-set gate before every walkthrough. Single-shot plans omit parent_authority_path intentionally: there is no prior multipart authority to thread through. Multi-shot bind_group_parts in implementation-groups.yaml supplies one because group iteration threads the parent digest from the previous cycle's bind step." }, "check_replan_iteration": { "tool": "run_python", @@ -692,7 +692,7 @@ } ], "on_failure": "release_issue_failure", - "note": "Renews plan-set bytes after a walkthrough before implementation." + "note": "Renews plan-set bytes after a walkthrough before implementation. __RECIPE_NOTE_PLAN_SET_RENEWAL_FAILURE__" }, "create_impl_worktree": { "tool": "run_cmd", diff --git a/src/autoskillit/recipes/implementation.yaml b/src/autoskillit/recipes/implementation.yaml index 492628cce9..8192890687 100644 --- a/src/autoskillit/recipes/implementation.yaml +++ b/src/autoskillit/recipes/implementation.yaml @@ -451,11 +451,11 @@ steps: optional_context_refs: - audit_cycle_path - plan_set_coverage_gaps - note: Produces an ordinary plan when no audit_cycle_path is present. On NO GO - re-entry, consumes that exact current authority and emits both plan_path and + note: >- + Produces an ordinary plan when no audit_cycle_path is present. On NO GO re-entry, + consumes that exact current authority and emits both plan_path and plan_disposition_path. all_plan_paths appends result.plan_path across re-entries; - verdict=plan is the only successful result. Glob plan_dir for *_part_*.md or - a single plan file and preserve alphabetical part order. + verdict=plan is the only successful result. salvage_plan: tool: run_python with: @@ -503,13 +503,12 @@ steps: route: check_replan_iteration - route: release_issue_failure on_failure: release_issue_failure - note: | - Aggregate plan-set gate before every walkthrough. - - Single-shot plans omit parent_authority_path intentionally: there is no prior - multipart authority to thread through. Multi-shot bind_group_parts in - implementation-groups.yaml supplies one because group iteration threads the - parent digest from the previous cycle's bind step. + note: >- + __RECIPE_NOTE_PLAN_PARTS_BINDING__ + Aggregate plan-set gate before every walkthrough. Single-shot plans omit + parent_authority_path intentionally: there is no prior multipart authority to thread + through. Multi-shot bind_group_parts in implementation-groups.yaml supplies one because + group iteration threads the parent digest from the previous cycle's bind step. check_replan_iteration: tool: run_python with: @@ -591,7 +590,8 @@ steps: route: create_impl_worktree - route: release_issue_failure on_failure: release_issue_failure - note: Renews plan-set bytes after a walkthrough before implementation. + note: >- + Renews plan-set bytes after a walkthrough before implementation. __RECIPE_NOTE_PLAN_SET_RENEWAL_FAILURE__ create_impl_worktree: tool: run_cmd with: diff --git a/src/autoskillit/recipes/remediation.json b/src/autoskillit/recipes/remediation.json index 75d3fc48f8..bf645b685a 100644 --- a/src/autoskillit/recipes/remediation.json +++ b/src/autoskillit/recipes/remediation.json @@ -603,7 +603,7 @@ "on_failure": "release_issue_failure", "on_rate_limit": "release_issue_failure", "on_context_limit": "salvage_rectify_plan", - "note": "Glob plan_dir for *_part_*.md or single plan file. Sort alphabetically into plan_parts[].\n" + "note": "plan_parts is exactly the plan_parts value of this step's result, in the order the skill emitted it." }, "salvage_rectify_plan": { "tool": "run_python", @@ -659,7 +659,8 @@ "route": "release_issue_failure" } ], - "on_failure": "release_issue_failure" + "on_failure": "release_issue_failure", + "note": "__RECIPE_NOTE_PLAN_PARTS_BINDING__" }, "check_replan_iteration": { "tool": "run_python", @@ -766,7 +767,8 @@ "route": "release_issue_failure" } ], - "on_failure": "release_issue_failure" + "on_failure": "release_issue_failure", + "note": "__RECIPE_NOTE_PLAN_SET_RENEWAL_FAILURE__" }, "retry_walkthrough": { "tool": "run_skill", @@ -821,7 +823,7 @@ } ], "on_failure": "release_issue_failure", - "note": "Bounded one-hop re-invoke of dry-walkthrough after a context-limit stumble on the primary dry_walkthrough step. Never routes back to dry_walkthrough — on_failure and on_context_limit both fall through to the original failure destination." + "note": "Bounded one-hop re-invoke of dry-walkthrough after a context-limit stumble on the primary dry_walkthrough step. Never routes back to dry_walkthrough — on_failure and on_context_limit both fall through to the original failure destination. __RECIPE_NOTE_PLAN_SET_RENEWAL_FAILURE__" }, "create_impl_worktree": { "tool": "run_cmd", diff --git a/src/autoskillit/recipes/remediation.yaml b/src/autoskillit/recipes/remediation.yaml index 9d7dfb829b..37693bc758 100644 --- a/src/autoskillit/recipes/remediation.yaml +++ b/src/autoskillit/recipes/remediation.yaml @@ -499,10 +499,9 @@ steps: on_failure: release_issue_failure on_rate_limit: release_issue_failure on_context_limit: salvage_rectify_plan - note: 'Glob plan_dir for *_part_*.md or single plan file. Sort alphabetically - into plan_parts[]. - - ' + note: >- + plan_parts is exactly the plan_parts value of this step's result, in the order the skill + emitted it. salvage_rectify_plan: tool: run_python with: @@ -540,6 +539,8 @@ steps: route: check_replan_iteration - route: release_issue_failure on_failure: release_issue_failure + note: >- + __RECIPE_NOTE_PLAN_PARTS_BINDING__ check_replan_iteration: tool: run_python with: @@ -622,6 +623,8 @@ steps: route: retry_walkthrough - route: release_issue_failure on_failure: release_issue_failure + note: >- + __RECIPE_NOTE_PLAN_SET_RENEWAL_FAILURE__ retry_walkthrough: tool: run_skill model: '' @@ -664,10 +667,10 @@ steps: route: create_impl_worktree - route: release_issue_failure on_failure: release_issue_failure - note: "Bounded one-hop re-invoke of dry-walkthrough after a context-limit stumble\ - \ on the primary dry_walkthrough step. Never routes back to dry_walkthrough\ - \ \u2014 on_failure and on_context_limit both fall through to the original failure\ - \ destination." + note: >- + Bounded one-hop re-invoke of dry-walkthrough after a context-limit stumble on the + primary dry_walkthrough step. Never routes back to dry_walkthrough — on_failure and + on_context_limit both fall through to the original failure destination. __RECIPE_NOTE_PLAN_SET_RENEWAL_FAILURE__ create_impl_worktree: tool: run_cmd with: diff --git a/src/autoskillit/server/_plan_set_materializer.py b/src/autoskillit/server/_plan_set_materializer.py index 10883b5571..8e990a41fb 100644 --- a/src/autoskillit/server/_plan_set_materializer.py +++ b/src/autoskillit/server/_plan_set_materializer.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Sequence from datetime import UTC, datetime from pathlib import Path @@ -11,6 +12,7 @@ PLAN_SET_AUTHORITY_ID_DOMAIN, PLAN_SET_MAX_ISSUE_BYTES, PLAN_SET_MAX_PART_BYTES, + AllocationRowDef, CoverageResultDef, CoverageStatus, GitHubFetcher, @@ -44,8 +46,10 @@ _ISSUE_NUMBER = re.compile(r"(?:#|/issues/)(\d+)(?:$|[/?#])") -def _failure(reason: PlanSetRejectReason, error: str) -> PlanSetBindResult: - return PlanSetBindResult(False, reason, error) +def _failure( + reason: PlanSetRejectReason, error: str, coverage_gaps: str = "" +) -> PlanSetBindResult: + return PlanSetBindResult(False, reason, error, coverage_gaps=coverage_gaps) def _write_or_verify(path: Path, data: bytes, allowed_root: Path) -> None: @@ -66,7 +70,11 @@ def _part_suffix(path: Path, *, only_part: bool) -> str: return "A" if only_part else path.stem.upper() -def _coverage_gaps(coverage: CoverageResultDef, requirements: tuple[RequirementDef, ...]) -> str: +def _coverage_gaps( + coverage: CoverageResultDef, + requirements: tuple[RequirementDef, ...], + parts: Sequence[PlanPartRef], +) -> str: requirement_text = {item.requirement_id: item.text for item in requirements} gaps: list[str] = [] for reason, values in ( @@ -79,9 +87,45 @@ def _coverage_gaps(coverage: CoverageResultDef, requirements: tuple[RequirementD gaps.extend( f"{item} [{reason}]: {requirement_text.get(item, '')}".rstrip() for item in values ) + locators = {part.part_key: part.locator for part in parts} + gaps.extend( + f"{key} [part_without_obligations]: {locators[key]} allocates no requirement; " + "every part needs at least one allocation row" + for key in coverage.parts_without_obligations + ) return "\n".join(gaps) +def _inspect_parts( + parts: Sequence[PlanPartRef], loaded_by_locator: dict[str, bytes] +) -> tuple[list[AllocationRowDef], list[str], list[str]]: + allocations: list[AllocationRowDef] = [] + errors: list[str] = [] + blocks: list[str] = [] + for part in parts: + if part.locator not in loaded_by_locator: + continue + try: + text = loaded_by_locator[part.locator].decode("utf-8") + rows = parse_part_allocation(text, part_key=part.part_key) + except (UnicodeDecodeError, ValueError) as exc: + errors.append(str(exc)) + details = [str(exc)] + else: + allocations.extend(rows) + defects = verify_allocation_evidence(text, rows) + errors.extend(f"{identifier}: {reason.value}" for identifier, reason, _ in defects) + details = [detail for _, _, detail in defects] + if details: + blocks.append( + f"Plan part rejected by the plan-set gate: {part.locator}\n" + + "\n".join(f"- {detail}" for detail in details) + + "\nWrite a corrected plan part to a new file in the form the planning skill's " + "plan template shows; keep the same requirements allocated." + ) + return allocations, errors, blocks + + def _result(authority: PlanSetAuthority, path: Path) -> PlanSetBindResult: coverage = authority.coverage return PlanSetBindResult( @@ -102,7 +146,7 @@ def _result(authority: PlanSetAuthority, path: Path) -> PlanSetBindResult: container_allocated=coverage.container_allocated, parts_without_obligations=coverage.parts_without_obligations, first_part_path=authority.parts[0].locator if authority.parts else "", - coverage_gaps=_coverage_gaps(coverage, authority.requirements), + coverage_gaps=_coverage_gaps(coverage, authority.requirements, authority.parts), ) @@ -354,23 +398,13 @@ def _assemble_authority( content_digest=compute_bytes_hash(data), ) ) - errors: list[str] = [] loaded_by_locator = {str(path): data for path, data in loaded} - for part in parts: - if part.locator not in loaded_by_locator: - continue - try: - text = loaded_by_locator[part.locator].decode("utf-8") - rows = parse_part_allocation(text, part_key=part.part_key) - except (UnicodeDecodeError, ValueError) as exc: - return _failure(PlanSetRejectReason.COVERAGE_FAILED, str(exc)) - allocations.extend(rows) - errors.extend( - f"{identifier}: {reason.value}" - for identifier, reason in verify_allocation_evidence(text, rows) - ) + rows, errors, blocks = _inspect_parts(parts, loaded_by_locator) + allocations.extend(rows) if errors: - return _failure(PlanSetRejectReason.COVERAGE_FAILED, "; ".join(errors)) + return _failure( + PlanSetRejectReason.COVERAGE_FAILED, "; ".join(errors), "\n\n".join(blocks) + ) requirements: tuple[RequirementDef, ...] unparsed: tuple[int, ...] @@ -421,7 +455,7 @@ def _assemble_authority( unknown=coverage.unknown, container_allocated=coverage.container_allocated, parts_without_obligations=coverage.parts_without_obligations, - coverage_gaps=_coverage_gaps(coverage, requirements), + coverage_gaps=_coverage_gaps(coverage, requirements, parts), ) authority = PlanSetAuthority.create( binding_mode=request.binding_mode, diff --git a/src/autoskillit/skill_resources/arch-constraint-catalog.md b/src/autoskillit/skill_resources/arch-constraint-catalog.md index 94938ff0ef..f3e01d15d7 100644 --- a/src/autoskillit/skill_resources/arch-constraint-catalog.md +++ b/src/autoskillit/skill_resources/arch-constraint-catalog.md @@ -26,6 +26,7 @@ classified `REJECT` with `category: "arch_violation"`. | Canonical maintenance argv | `test_maintenance_install_argv_contract.py` | Hand-built list or tuple argv containing `--maintenance-update` outside `MaintenanceInstallArgv.to_argv()` | | step_name in run_cmd | `test_anti_pattern_guards.py` | Missing `step_name` in `run_cmd` `with:` blocks in recipe YAML | | No hardcoded temp paths | `test_python_no_hardcoded_temp.py` | Hardcoded temp-root path strings in Python outside whitelist | +| AutoSkillit root children | `test_autoskillit_root_children_registered.py` | Naming an unclassified `.autoskillit/` child in production Python; runtime directories need a self-ignore marker and a real-Git proof | | run_python path resolution | `test_run_python_path_resolution.py` | smoke_utils callable with relative-path fallback or path-like param not in `_PATH_LIKE_ARGS` | | SkillResult kill_reason | `test_skill_result_construction_guard.py` | `SkillResult()` without `kill_reason=` kwarg | | Never-raises contracts | `test_never_raises_contracts.py` | `mcp.tool()` handlers without top-level `try/except` and "Never raises" docstring | diff --git a/src/autoskillit/skill_resources/plan-part-form.md b/src/autoskillit/skill_resources/plan-part-form.md new file mode 100644 index 0000000000..50a40535bc --- /dev/null +++ b/src/autoskillit/skill_resources/plan-part-form.md @@ -0,0 +1,24 @@ +--- +id: plan-part-form +title: Plan-part authoring form +summary: Shared authoring rules for plan parts accepted by the plan-set allocation verifier. +--- +**Plan-part form.** `bind_plan_set` deterministically checks the allocation table, +resolves each row's step key and checks that its ID cell occurs in that step's text. +Follow this authoring form; the verifier's existing acceptance rules stay unchanged: +- Write each step as a level-3 heading `### Step N: title` (sub-steps + `### Step N.M: title`). Other heading levels, and a dash in place of the colon, are + not recognised. Do not repeat a step heading anywhere in the file, quoted examples + included. +- On the line under each heading, name every Requirement ID the step delivers + (`Requirements: P-1, P-2`). The ID must appear in that step's own text, which ends at + the next step heading or the next line starting with `# ` or `## `, code fences + included. +- The line `## Issue Requirement Allocation` appears once in the file, and its section + holds only the table: no sentence before, between or after the rows. +- Each row is the bare Requirement ID with no description, then `owned` or `shared`, + then exactly one step key copied from a heading (`Step 2.1`): not a list, a range, a + backticked key or a key with its title. One row per ID. +- Use the issue's own labels as IDs when it enumerates items; otherwise number your own + `P-1`, `P-2`. `owned` means this part alone delivers the item; `shared` means another + part allocates it too. diff --git a/src/autoskillit/skills_extended/dry-walkthrough/SKILL.md b/src/autoskillit/skills_extended/dry-walkthrough/SKILL.md index da3e435558..fea19f2614 100644 --- a/src/autoskillit/skills_extended/dry-walkthrough/SKILL.md +++ b/src/autoskillit/skills_extended/dry-walkthrough/SKILL.md @@ -4,6 +4,7 @@ write_paths: unrestricted uses_capabilities: [] requires_resources: - arch-constraint-catalog +- plan-part-form description: Plan validation executor. ALWAYS invoke this skill when instructed to validate or dry-walkthrough a plan. Do not read the plan or trace changes directly — use this skill first to load the validation workflow. hooks: @@ -338,6 +339,12 @@ or reconstruct any inventory, remediation, authority, or report. ### Step 5: Fix the Plan +Every edit must leave the part in the plan-part form. When a `### Step` heading cited +by an allocation row is renamed, renumbered, re-levelled or removed, update that row's +`Implementation Step` cell to the surviving key. Keep each allocated Requirement ID +in its step's text, and add no prose inside the allocation section. The recipe +re-verifies the part after this skill; a nonconforming part stops the pipeline. + For each issue found: 1. Directly edit the plan file to fix it 2. Do NOT add any "gap analysis" or "issues" sections to the plan diff --git a/src/autoskillit/skills_extended/make-plan/SKILL.md b/src/autoskillit/skills_extended/make-plan/SKILL.md index 25c3fdcc46..5d2e114334 100644 --- a/src/autoskillit/skills_extended/make-plan/SKILL.md +++ b/src/autoskillit/skills_extended/make-plan/SKILL.md @@ -1,5 +1,7 @@ --- name: make-plan +requires_resources: +- plan-part-form write_paths: - '{{AUTOSKILLIT_TEMP}}/make-plan/' uses_capabilities: @@ -68,9 +70,12 @@ Create focused, actionable implementation plans that recommend the technically b not prose flags or ambient files, activates remediation mode. Before reading any referenced artifact, verify that this authority is the server-published current `NO GO` head and that its generation, plan set, scope, part, round, parent, and audited-plan lineage match this run. -- `plan_set_coverage_gaps` (optional) — A bounded list of requirement gaps returned by a - failed sealed plan-set bind. On re-entry, allocate every listed requirement in the new - plan set; do not absorb the gaps into Part A by default. In grouped planning, the +- `plan_set_coverage_gaps` (optional) — Feedback from a failed plan-set bind, never empty + on re-entry. Requirement-gap lines such as `R2 [unassigned]: …` mean allocate every + listed requirement in the new plan set; do not absorb gaps into Part A by default. + A block starting `Plan part rejected by the plan-set gate: ` lists what the parser + rejected in that part: write a corrected replacement to a new file in the plan-part + form, keeping its requirements allocated. In grouped planning, the dedicated gap-replan route adds a group to the still-open set. In conflict-resolution examples, `{branch}` is the source PR branch and @@ -427,6 +432,9 @@ plan_parts = {path_to_part_a} plan_disposition_path = {absolute_path_when_in_remediation_mode} ``` +**Plan-part form.** Follow the shared `plan-part-form` resource when writing +each part; the template below applies those rules. + **Plan structure (single-part):** ```markdown # Implementation Plan: {Task Name} @@ -438,7 +446,14 @@ plan_disposition_path = {absolute_path_when_in_remediation_mode} {Tests to write first — should fail now, pass after implementation} ## Implementation Steps -{Ordered steps, each making one or more of the above tests pass} + +### Step 1: {step title} +Requirements: P-1 +{What this step changes; it makes one or more of the above tests pass} + +### Step 2: {step title} +Requirements: P-2 +{What this step changes} ## Verification {How to verify the implementation is correct} @@ -446,7 +461,8 @@ plan_disposition_path = {absolute_path_when_in_remediation_mode} ## Issue Requirement Allocation | Requirement ID | Allocation | Implementation Step | |---|---|---| -| {label from issue, or P-1 without enumerated items} | owned \| shared | Step {N.M} | +| P-1 | owned | Step 1 | +| P-2 | owned | Step 2 | ## Requirements Map | Requirement (from prose) | Implementation Step | @@ -467,7 +483,14 @@ plan_disposition_path = {absolute_path_when_in_remediation_mode} {Tests for THIS part only — should fail now, pass after THIS part's implementation} ## Implementation Steps -{Steps for THIS part only} + +### Step 1: {step title} +Requirements: P-1 +{What this step changes; it makes one or more of the above tests pass} + +### Step 2: {step title} +Requirements: P-2 +{What this step changes} ## Verification {How to verify THIS part's implementation is correct} @@ -475,7 +498,8 @@ plan_disposition_path = {absolute_path_when_in_remediation_mode} ## Issue Requirement Allocation | Requirement ID | Allocation | Implementation Step | |---|---|---| -| {label from issue, or P-1 without enumerated items} | owned \| shared | Step {N.M} | +| P-1 | owned | Step 1 | +| P-2 | owned | Step 2 | ## Requirements Map | Requirement (from prose) | Implementation Step | diff --git a/src/autoskillit/skills_extended/rectify/SKILL.md b/src/autoskillit/skills_extended/rectify/SKILL.md index 62c7183def..cc4bc0b57d 100644 --- a/src/autoskillit/skills_extended/rectify/SKILL.md +++ b/src/autoskillit/skills_extended/rectify/SKILL.md @@ -1,5 +1,7 @@ --- name: rectify +requires_resources: +- plan-part-form write_paths: - '{{AUTOSKILLIT_TEMP}}/rectify/' uses_capabilities: [] @@ -273,6 +275,12 @@ plan_parts = {path_to_part_a} {path_to_part_c} ``` +**Plan-part form.** Follow the shared `plan-part-form` resource when writing +each part; the template below applies those rules. + +Sub-steps are further level-3 headings (`### Step 2.1: …`); an allocation row cites +the sub-step key. + **Plan structure:** ```markdown # Rectify: {Topic} @@ -295,9 +303,11 @@ plan_parts = {path_to_part_a} ## Immunity Plan ### Step 1: Failing Tests +Requirements: P-1 {Tests that reproduce the issue and capture the gap — these must be written first} ### Step 2: Implementation +Requirements: P-1 {The architectural solution that makes this class of bug impossible or instantly caught, structured so each change makes a failing test pass} ## Verification @@ -306,5 +316,5 @@ plan_parts = {path_to_part_a} ## Issue Requirement Allocation | Requirement ID | Allocation | Implementation Step | |---|---|---| -| {label from issue, or P-1 without enumerated items} | owned \| shared | Step {N.M} | +| P-1 | owned | Step 2 | ``` diff --git a/tests/_ambient_env_surface.py b/tests/_ambient_env_surface.py index 947b7371a1..9ea1ef46bf 100644 --- a/tests/_ambient_env_surface.py +++ b/tests/_ambient_env_surface.py @@ -992,7 +992,7 @@ class AmbientEnvDisposition: } FORWARDING_SITES: dict[str, str] = { - "cli/install/_marketplace.py:357": ( + "cli/install/_marketplace.py:358": ( "Unfiltered dict(os.environ) snapshot (`ambient_env`) used as the base for an" "install/publish subprocess env; intentional wholesale forward for a maintenance-style" "operation." diff --git a/tests/_helpers.py b/tests/_helpers.py index a75b0fddd7..43e5e6f5c8 100644 --- a/tests/_helpers.py +++ b/tests/_helpers.py @@ -20,6 +20,7 @@ from autoskillit.core import ( InstructionExtractionMode, OrchestratorSurfaceDef, + RolloutMarker, ToolParamRole, get_tool_def, ) @@ -38,6 +39,19 @@ INVALID_PATH_INPUTS: tuple[object, ...] = (None, 0, b"", object(), MagicMock()) +def make_rollout_marker(*, backends: tuple[str, ...] = ("claude-code", "codex")) -> RolloutMarker: + return { + "change_id": "prompt-cost-study", + "effective_at": "2026-11-01T18:00:00+00:00", + "scope": { + "skills": ["make-plan"], + "backends": list(backends), + "roles": ["planner"], + }, + "expected_effect": "Reduce planning input tokens while preserving requirement coverage.", + } + + def load_core_stub_generator() -> ModuleType: """Load a fresh generator module so callers can independently patch its paths.""" script = Path(__file__).resolve().parent.parent / "scripts" / "generate_core_stub.py" diff --git a/tests/_retention_surface.py b/tests/_retention_surface.py index b0fc2bcc76..47a8d6b96b 100644 --- a/tests/_retention_surface.py +++ b/tests/_retention_surface.py @@ -1435,7 +1435,7 @@ def assert_second_pass_is_quiet( # the #4623 pass's own child_outcomes import to inside the try block (issue # #4672 decomposition — module-level would circularly import back through # the evidence/ gateway that now wraps this file). - f"{_SR}::L231": _retries_after_input_changes( + f"{_SR}::L233": _retries_after_input_changes( "The configured trace root is absent, so no crash candidate can be discovered yet." ), f"{_SRE}::L45": _retries_after_input_changes( @@ -1474,10 +1474,10 @@ def assert_second_pass_is_quiet( f"{_SRD}::L121": _self_limiting( "An alien-command trace and its enrollment are deleted as a terminal safety disposition." ), - f"{_SRF}::L139": _retries_after_input_changes( + f"{_SRF}::L140": _retries_after_input_changes( "A second stat failure keeps the trace retryable until the filesystem becomes available." ), - f"{_SRF}::L180": _retries_after_input_changes( + f"{_SRF}::L182": _retries_after_input_changes( "Flush or output-index failure retains both files until output infrastructure recovers." ), } diff --git a/tests/_test_filter.py b/tests/_test_filter.py index 5a8cfcb557..763f1714a7 100644 --- a/tests/_test_filter.py +++ b/tests/_test_filter.py @@ -150,7 +150,9 @@ class SourceMapRejection(enum.StrEnum): "tests/arch/_rules.py": _ARCH_HELPER_TEST_TARGETS, "tests/fleet/_reaper_test_support.py": frozenset({"fleet"}), "tests/hooks/_interactive_guard_harness.py": frozenset({"hooks", "skills"}), - "tests/_git_topology.py": frozenset({"core", "execution", "server", "integration"}), + "tests/_git_topology.py": frozenset( + {"core", "execution", "server", "integration", "workspace", "fleet"} + ), "tests/fleet/_codex_mcp_env.py": frozenset({"fleet", "integration"}), "tests/fleet/_descendant_worker.py": frozenset({"fleet"}), } @@ -550,7 +552,7 @@ class SourceMapRejection(enum.StrEnum): "tool_sequence_analysis": frozenset({"core", "execution", "fleet", "server", "cli"}), "_type_checkpoint": frozenset({"core", "execution", "fleet", "server"}), "_type_results": _TYPE_RESULTS_CASCADE, - "_type_results_records": _TYPE_RESULTS_CASCADE, + "_type_results_records": _TYPE_RESULTS_CASCADE | {"config"}, "_type_results_execution": frozenset({"core", "execution", "server", "pipeline"}), "_type_execution_identity": frozenset( {"core", "execution", "server", "cli", "fleet", "smoke_utils"} diff --git a/tests/arch/_acceptance_policy_surfaces.py b/tests/arch/_acceptance_policy_surfaces.py index 3abfefbdd2..22d85cd0a6 100644 --- a/tests/arch/_acceptance_policy_surfaces.py +++ b/tests/arch/_acceptance_policy_surfaces.py @@ -2,11 +2,12 @@ # automated repair sessions must not. Every approval needs a tracking issue and code-owner review. """Declared acceptance-policy surfaces and the human approval ledger. -A policy surface is a value the repository treats as the accepted limit for -something it measures. Because a surface and the measurement it gates live in -the same working tree, raising one is indistinguishable from satisfying it -unless a trusted revision is consulted; scripts/check_policy_relaxation.py -does that consulting and reads this file by AST from both revisions. +A policy surface holds accepted limits or numeric exemption flags. Because a +surface and the check it gates live in the same working tree, raising a limit +or adding an exemption is indistinguishable from satisfying the check unless +a trusted revision is consulted; scripts/check_policy_relaxation.py does that +consulting and reads this file by AST from both revisions. Classification flags +use int_map with default=0: enabling a flag with 1 is a policy relaxation. """ from __future__ import annotations @@ -21,13 +22,13 @@ @dataclasses.dataclass(frozen=True) class PolicySurface: - """One module-level assignment holding an accepted limit.""" + """One module-level assignment holding accepted limits or exemption flags.""" path: str symbol: str kind: SurfaceKind default: int | None = None - """int_map only: the limit an absent key already implies.""" + """int_map only: the value an absent key already implies.""" @dataclasses.dataclass(frozen=True) @@ -59,6 +60,12 @@ class PolicyRelaxationApproval: "int_map", default=12, ), + PolicySurface( + "tests/arch/test_autoskillit_root_children_registered.py", + "_CLASSIFIED_ELSEWHERE", + "int_map", + default=0, + ), PolicySurface( "tests/arch/test_pyright_suppression_allowlist.py", "TYPE_IGNORE_BUDGET", "int_scalar" ), diff --git a/tests/arch/_core_gateway_snapshot.py b/tests/arch/_core_gateway_snapshot.py index 8ec74d5d3c..bab33a9846 100644 --- a/tests/arch/_core_gateway_snapshot.py +++ b/tests/arch/_core_gateway_snapshot.py @@ -762,6 +762,7 @@ "ReviewResponseClass", "Revocability", "RollbackAdmissionEvent", + "RolloutMarker", "RolloverEpochEvent", "RunSkillCompletionAuthority", "RuntimeAdmission", @@ -984,6 +985,7 @@ "due_retiring_records", "dump_yaml_str", "encode_stored_context_admission_envelope", + "ensure_project_gitignore", "ensure_project_temp", "entrypoint_shim_path", "executable_binding_matches_current_file", diff --git a/tests/arch/test_autoskillit_root_children_registered.py b/tests/arch/test_autoskillit_root_children_registered.py new file mode 100644 index 0000000000..c865e74668 --- /dev/null +++ b/tests/arch/test_autoskillit_root_children_registered.py @@ -0,0 +1,159 @@ +"""Architectural invariant: every source-named .autoskillit child is classified.""" + +from __future__ import annotations + +import ast +import re +from collections.abc import Iterator +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("arch"), pytest.mark.small] + +_STR_RX = re.compile(r"(?:^|[/\"'`\s(~])\.autoskillit/([A-Za-z0-9._-]+)") + +# These names are accounted for elsewhere: home-level artifacts, a read-only +# legacy path, and a runtime directory with a real-git proof. +_CLASSIFIED_ELSEWHERE: dict[str, int] = { + # Home-level hook state: hooks/guards/mcp_health_advisor.py:36. + "active_kitchens.json": 1, + # Home-level marketplace shape: core/types/constants/_type_constants_retirements.py:309. + "marketplace": 1, + # Home-level plugin storage: cli/doctor/_doctor_install.py:355. + "plugin-generations": 1, + # Home-level hook projection repair: workspace/_projected_artifact/_hook_repair.py:538. + "plugin-projections": 1, + # Home-level dismissal state: cli/update/_update_checks.py:79. + "update_check.json": 1, + # Legacy path is read only for migration: cli/install/_marketplace.py:162. + "scripts": 1, + # Runtime self-ignore is proven by tests/workspace/test_clone_gate_runtime_state.py. + "managed-headless-session-lineage": 1, +} + + +def _str_const(node: ast.AST) -> str | None: + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + return None + + +def _assignment_targets(node: ast.AST) -> Iterator[ast.expr]: + if isinstance(node, ast.Assign): + yield from node.targets + elif isinstance(node, ast.AnnAssign) and node.value is not None: + yield node.target + + +def _is_root(node: ast.AST, roots: frozenset[str]) -> bool: + if _str_const(node) == ".autoskillit": + return True + if isinstance(node, ast.Name): + return node.id in roots + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Div): + return _str_const(node.right) == ".autoskillit" + if isinstance(node, ast.Call) and node.args: + return _str_const(node.args[-1]) == ".autoskillit" + return False + + +def _children_named_in(tree: ast.Module) -> set[str]: + """Collect children from literal paths and simple syntactic path expressions. + + Recognize direct root assignments, slash joins, and adjacent literal + arguments or sequence elements. Root names are matched across the module + without scope analysis; dynamic strings and indirect aliases are not traced. + """ + roots = { + target.id + for node in ast.walk(tree) + for target in _assignment_targets(node) + if isinstance(node, (ast.Assign, ast.AnnAssign)) + and node.value is not None + and isinstance(target, ast.Name) + and _is_root(node.value, frozenset()) + } + names: set[str] = set() + for node in ast.walk(tree): + text = _str_const(node) + if text is not None: + names.update(match.group(1) for match in _STR_RX.finditer(text)) + if ( + isinstance(node, ast.BinOp) + and isinstance(node.op, ast.Div) + and _is_root(node.left, roots) + ): + right = _str_const(node.right) + if right: + names.add(right.split("/")[0]) + seq = ( + node.elts + if isinstance(node, (ast.Tuple, ast.List)) + else node.args + if isinstance(node, ast.Call) + else () + ) + for first, second in zip(seq, seq[1:]): + if _str_const(first) == ".autoskillit" and _str_const(second): + names.add(_str_const(second).split("/")[0]) + return {name.rstrip(".") for name in names} - {""} + + +def _src_root() -> Path: + return Path(__file__).resolve().parent.parent.parent / "src" / "autoskillit" + + +def test_every_autoskillit_child_named_in_src_is_classified() -> None: + from autoskillit.core.io import _AUTOSKILLIT_GITIGNORE_ENTRIES, _COMMITTED_BY_DESIGN + + src = _src_root() + assert src.is_dir(), f"src/autoskillit not found at {src}" + + named = { + child + for path in src.rglob("*.py") + for child in _children_named_in(ast.parse(path.read_text(encoding="utf-8"))) + } + ignored_names = {entry.rstrip("/") for entry in _AUTOSKILLIT_GITIGNORE_ENTRIES} + classified_names = { + name for name, classification in _CLASSIFIED_ELSEWHERE.items() if classification == 1 + } + unclassified = sorted( + named - set(_COMMITTED_BY_DESIGN) - ignored_names - {".gitignore"} - classified_names + ) + + assert not unclassified, ( + f"Unclassified .autoskillit children named in src/autoskillit: {unclassified}. " + f"Commit a persistent child by adding it to _COMMITTED_BY_DESIGN; list a file " + f"in _AUTOSKILLIT_GITIGNORE_ENTRIES only when explicit setup writes it through " + f"ensure_project_gitignore; runtime ensure_project_temp does not maintain the " + f"tracked parent .gitignore. For a runtime " + f"directory, call ensure_self_ignored when creating it, prove the behavior with " + f"a real-git test, and name that test in an adjacent justification comment." + ) + + +def test_zero_classification_flag_is_reported(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(_CLASSIFIED_ELSEWHERE, "managed-headless-session-lineage", 0) + + with pytest.raises(AssertionError, match="managed-headless-session-lineage"): + test_every_autoskillit_child_named_in_src_is_classified() + + +def test_classification_map_matches_policy_checker_contract() -> None: + from tests.arch.test_acceptance_policy_relaxation_gate import check + + surface = check.PolicySurface( + "tests/arch/test_autoskillit_root_children_registered.py", + "_CLASSIFIED_ELSEWHERE", + "int_map", + default=0, + ) + set_source = "_CLASSIFIED_ELSEWHERE: set[str] = {'managed-headless-session-lineage'}" + with pytest.raises(check.UnsupportedSurfaceShape): + check.extract_surface_values(set_source, surface) + + actual_source = Path(__file__).read_text(encoding="utf-8") + actual = check.extract_surface_values(actual_source, surface) + assert {name: value.limit for name, value in actual.items()} == _CLASSIFIED_ELSEWHERE diff --git a/tests/arch/test_e2e_sessions_contract.py b/tests/arch/test_e2e_sessions_contract.py new file mode 100644 index 0000000000..ff8eb67a1d --- /dev/null +++ b/tests/arch/test_e2e_sessions_contract.py @@ -0,0 +1,27 @@ +"""Pin the standalone collector's identities and tools to product definitions.""" + +from pathlib import Path + +import pytest + +from autoskillit.core import DIRECT_PREFIX, PLUGIN_PREFIX +from autoskillit.core.types import GATED_TOOLS, UNGATED_TOOLS +from tests.infra._complexity_helpers import load_check_script + +pytestmark = pytest.mark.medium + +sessions = load_check_script( + "_autoskillit_e2e_sessions_contract", + Path(__file__).resolve().parents[2] / "scripts" / "e2e" / "e2e_sessions.py", +) + + +def test_server_identities_match_canonical_prefixes() -> None: + assert sessions._AUTOSKILLIT_SERVERS == frozenset( + tuple(prefix.split("__")[:-1]) for prefix in (DIRECT_PREFIX, PLUGIN_PREFIX) + ) + + +def test_tracked_tools_are_registered_product_tools() -> None: + assert sessions._TRACKED_TOOLS + assert sessions._TRACKED_TOOLS <= GATED_TOOLS | UNGATED_TOOLS diff --git a/tests/arch/test_execution_source_split.py b/tests/arch/test_execution_source_split.py index 68b1595ca8..781cbc1252 100644 --- a/tests/arch/test_execution_source_split.py +++ b/tests/arch/test_execution_source_split.py @@ -1,5 +1,6 @@ """Arch guards for execution layer source splits (P8-F1, P8-F3, P8-F4 audit fixes).""" +import ast from pathlib import Path import pytest @@ -85,12 +86,22 @@ def test_headless_facade_does_not_define_recovery_functions(): def test_plugin_bound_launch_lifecycle_is_centralized(): launch = (SRC_EXECUTION / "headless" / "_headless_launch.py").read_text() + functions = { + node.name: node + for node in ast.parse(launch).body + if isinstance(node, ast.AsyncFunctionDef) + } for fn in ( - "_plugin_launch_binding", "_run_headless_attempt", "_attempt_contract_nudge", ): - assert f"def {fn}" in launch, f"{fn} must live in _headless_launch.py" + assert fn in functions, f"{fn} must live in _headless_launch.py" + assert any( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "plugin_launch_binding_scope" + for node in ast.walk(functions[fn]) + ), f"{fn} must own its plugin binding scope" def test_headless_facade_does_not_define_path_token_functions(): diff --git a/tests/arch/test_import_linter_contracts.py b/tests/arch/test_import_linter_contracts.py index f30a2a3303..6a39d9c320 100644 --- a/tests/arch/test_import_linter_contracts.py +++ b/tests/arch/test_import_linter_contracts.py @@ -42,6 +42,7 @@ "execution/headless/_headless_outcome.py": frozenset({"recipe"}), "execution/headless/_headless_prepare.py": frozenset({"pipeline"}), "execution/headless/_headless_result.py": frozenset({"recipe"}), + "execution/headless/_headless_terminal.py": frozenset({"pipeline"}), "execution/headless/_managed/_executor.py": frozenset({"pipeline", "recipe"}), "execution/evidence/linux_tracing.py": frozenset({"config"}), "execution/process/__init__.py": frozenset({"config"}), diff --git a/tests/arch/test_layer_enforcement.py b/tests/arch/test_layer_enforcement.py index 0fdabf1f51..51c8dea33c 100644 --- a/tests/arch/test_layer_enforcement.py +++ b/tests/arch/test_layer_enforcement.py @@ -2002,6 +2002,8 @@ def test_default_classes_only_instantiated_inside_factory_or_allowlist() -> None "tests/execution/test_process_tether.py": frozenset({"autoskillit.config"}), # workspace tests "tests/workspace/test_clone_ci_contract.py": frozenset({"autoskillit.execution"}), + # Clone cleanliness is exercised against the real execution-layer lineage writer. + "tests/workspace/test_clone_gate_runtime_state.py": frozenset({"autoskillit.execution"}), # the frontmatter write scope round-trips through the stdlib-only write-scope # authority and the hook-side manifest decoder it feeds "tests/workspace/test_skill_write_boundary_contract.py": frozenset({"autoskillit.hooks"}), diff --git a/tests/arch/test_mcp_prefix_literal_containment.py b/tests/arch/test_mcp_prefix_literal_containment.py index aba6020538..01b6ff01b7 100644 --- a/tests/arch/test_mcp_prefix_literal_containment.py +++ b/tests/arch/test_mcp_prefix_literal_containment.py @@ -8,6 +8,7 @@ import pytest from autoskillit.core import find_qualified_autoskillit_tool_names, paths +from tests.arch._helpers import SRC_ROOT pytestmark = [pytest.mark.layer("arch"), pytest.mark.small] @@ -42,14 +43,17 @@ def test_mcp_prefix_literals_confined_to_canonical_module() -> None: """The string literals for both MCP prefix forms appear only in _plugin_ids.py.""" src_root = paths.pkg_root() violations: list[str] = [] - for py_file in sorted(src_root.rglob("*.py")): - rel = py_file.relative_to(src_root).as_posix() - if rel == _CANONICAL_MODULE: - continue - for lineno, value in _string_literals_in_module(py_file): - for literal in _CONFINED_LITERALS: - if literal in value: - violations.append(f"{rel}:{lineno}: contains {literal!r} in {value!r}") + for root in (src_root, SRC_ROOT.parents[1] / "scripts"): + files = sorted(path for path in root.rglob("*.py") if "__pycache__" not in path.parts) + assert files, f"{root} scan universe must not be empty" + for py_file in files: + rel = py_file.relative_to(root).as_posix() + if root == src_root and rel == _CANONICAL_MODULE: + continue + for lineno, value in _string_literals_in_module(py_file): + for literal in _CONFINED_LITERALS: + if literal in value: + violations.append(f"{rel}:{lineno}: contains {literal!r} in {value!r}") assert not violations, ( "MCP prefix literals must appear only in " f"{_CANONICAL_MODULE}. Violations:\n" + "\n".join(violations) diff --git a/tests/arch/test_no_new_exception_path_autoskillit_imports.py b/tests/arch/test_no_new_exception_path_autoskillit_imports.py index 3f29b3f42d..646a1e92fa 100644 --- a/tests/arch/test_no_new_exception_path_autoskillit_imports.py +++ b/tests/arch/test_no_new_exception_path_autoskillit_imports.py @@ -28,7 +28,7 @@ # the failure it is inside of. ("pipeline/background.py", 81), # Config re-read on a config-load failure path. - ("cli/_init_helpers.py", 687), + ("cli/_init_helpers.py", 690), } ) diff --git a/tests/cli/test_doctor.py b/tests/cli/test_doctor.py index 375fca71ad..5f844e3861 100644 --- a/tests/cli/test_doctor.py +++ b/tests/cli/test_doctor.py @@ -1086,6 +1086,31 @@ def test_doctor_gitignore_ok_when_all_covered( assert result.severity == Severity.OK +def test_doctor_gitignore_exempts_only_self_ignoring_directories( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from autoskillit.core.io import _AUTOSKILLIT_GITIGNORE_ENTRIES + + autoskillit_dir = tmp_path / ".autoskillit" + autoskillit_dir.mkdir() + (autoskillit_dir / ".gitignore").write_text( + "\n".join(_AUTOSKILLIT_GITIGNORE_ENTRIES) + "\n", encoding="utf-8" + ) + runtime_dir = autoskillit_dir / "runtime-a" + runtime_dir.mkdir() + (runtime_dir / ".gitignore").write_text("*\n", encoding="utf-8") + (autoskillit_dir / "runtime-b").mkdir() + + monkeypatch.chdir(tmp_path) + from autoskillit.cli.doctor import _check_gitignore_completeness + from autoskillit.core import Severity + + result = _check_gitignore_completeness(tmp_path) + assert result.severity == Severity.WARNING + assert "runtime-b" in result.message + assert "runtime-a" not in result.message + + # SS-DOC-1 def test_doctor_includes_secret_scanning_hook_check( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py index f933d68f07..fb3cfb495a 100644 --- a/tests/cli/test_init.py +++ b/tests/cli/test_init.py @@ -423,10 +423,10 @@ def test_ensure_project_temp_writes_self_gitignore(self, tmp_path): assert temp_gitignore.exists() assert "*" in temp_gitignore.read_text() - def test_ensure_project_temp_writes_autoskillit_gitignore_with_secrets(self, tmp_path): - from autoskillit.core.io import ensure_project_temp + def test_ensure_project_gitignore_writes_autoskillit_gitignore_with_secrets(self, tmp_path): + from autoskillit.core.io import ensure_project_gitignore - ensure_project_temp(tmp_path) + ensure_project_gitignore(tmp_path) gitignore = tmp_path / ".autoskillit" / ".gitignore" content = gitignore.read_text() assert ".secrets.yaml" in content @@ -438,15 +438,15 @@ def test_ensure_project_temp_is_idempotent(self, tmp_path): ensure_project_temp(tmp_path) # second call must not raise assert (tmp_path / ".autoskillit" / "temp").is_dir() - def test_ensure_project_temp_backfills_secrets_into_existing_gitignore(self, tmp_path): - from autoskillit.core.io import ensure_project_temp + def test_ensure_project_gitignore_backfills_secrets_into_existing_gitignore(self, tmp_path): + from autoskillit.core.io import ensure_project_gitignore # Simulate a pre-fix .gitignore without .secrets.yaml autoskillit_dir = tmp_path / ".autoskillit" autoskillit_dir.mkdir() (autoskillit_dir / ".gitignore").write_text(".onboarded\n") - ensure_project_temp(tmp_path) + ensure_project_gitignore(tmp_path) content = (autoskillit_dir / ".gitignore").read_text() assert ".secrets.yaml" in content assert ".onboarded" in content @@ -537,9 +537,15 @@ def test_init_all_created_files_covered_by_gitignore(tmp_path: Path) -> None: be in the committed allowlist. """ from autoskillit.cli._init_helpers import _create_secrets_template - from autoskillit.core.io import _COMMITTED_BY_DESIGN, ensure_project_temp + from autoskillit.core.io import ( + _COMMITTED_BY_DESIGN, + ensure_project_gitignore, + ensure_project_temp, + is_self_ignored, + ) ensure_project_temp(tmp_path) + ensure_project_gitignore(tmp_path) _create_secrets_template(tmp_path) autoskillit_dir = tmp_path / ".autoskillit" @@ -552,10 +558,7 @@ def test_init_all_created_files_covered_by_gitignore(tmp_path: Path) -> None: continue if item.is_dir() and (item / ".gitignore").exists(): # Self-gitignoring directory — its own .gitignore covers its contents. - self_ignore = (item / ".gitignore").read_text() - assert "*" in self_ignore, ( - f"{item.name}/ is a directory without a self-gitignore covering '*'" - ) + assert is_self_ignored(item) continue check_name = item.name + "/" if item.is_dir() else item.name assert check_name in gitignore_content, ( @@ -569,9 +572,9 @@ def test_init_all_created_files_covered_by_gitignore(tmp_path: Path) -> None: def test_secrets_template_gitignore_comment_is_true(tmp_path: Path) -> None: """The comment in .secrets.yaml claiming it is gitignored must be actually true.""" from autoskillit.cli._init_helpers import _create_secrets_template - from autoskillit.core.io import ensure_project_temp + from autoskillit.core.io import ensure_project_gitignore - ensure_project_temp(tmp_path) + ensure_project_gitignore(tmp_path) _create_secrets_template(tmp_path) autoskillit_dir = tmp_path / ".autoskillit" diff --git a/tests/cli/test_init_helpers.py b/tests/cli/test_init_helpers.py index cdc3902ac7..f4d5aa37b7 100644 --- a/tests/cli/test_init_helpers.py +++ b/tests/cli/test_init_helpers.py @@ -86,6 +86,8 @@ def test_register_all_accepts_backend_kwarg( mock_backend = MagicMock() mock_backend.capabilities.mcp_config_capable = False _register_all("user", tmp_path, backend=mock_backend) + ignore_lines = (tmp_path / ".autoskillit" / ".gitignore").read_text().splitlines() + assert ".state.yaml" in ignore_lines class TestInitBackendResolution: diff --git a/tests/cli/test_init_helpers_state.py b/tests/cli/test_init_helpers_state.py index 0b43deac8d..7186dfc9ff 100644 --- a/tests/cli/test_init_helpers_state.py +++ b/tests/cli/test_init_helpers_state.py @@ -28,15 +28,61 @@ def test_log_secret_scan_bypass_refuses_overwrite_on_corrupt_yaml( _log_secret_scan_bypass(tmp_path) assert state_path.read_text() == corrupt_content + assert not (state_path.parent / ".gitignore").exists() - def test_log_secret_scan_bypass_works_on_missing_file(self, tmp_path: Path) -> None: - from autoskillit.cli._init_helpers import _log_secret_scan_bypass + def test_log_secret_scan_bypass_works_on_missing_file( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from autoskillit.cli import _init_helpers state_path = tmp_path / ".autoskillit" / ".state.yaml" assert not state_path.exists() - _log_secret_scan_bypass(tmp_path) + write_state = _init_helpers.atomic_write + + def write_after_ignore(path: Path, content: str) -> None: + assert ".state.yaml" in (state_path.parent / ".gitignore").read_text().splitlines() + write_state(path, content) + + monkeypatch.setattr(_init_helpers, "atomic_write", write_after_ignore) + _init_helpers._log_secret_scan_bypass(tmp_path) assert state_path.exists() content = state_path.read_text() assert "secret_scan_bypass_accepted" in content + ignore_lines = (state_path.parent / ".gitignore").read_text().splitlines() + assert ".state.yaml" in ignore_lines + + def test_log_secret_scan_bypass_preserves_unknown_state_and_config( + self, tmp_path: Path + ) -> None: + from autoskillit.cli._init_helpers import _log_secret_scan_bypass + from autoskillit.core import load_yaml + + state_path = tmp_path / ".autoskillit" / ".state.yaml" + state_path.parent.mkdir(parents=True) + state_path.write_text("custom: keep\nsafety:\n other: keep\n", encoding="utf-8") + config_path = state_path.parent / "config.yaml" + config_path.write_bytes(b"config: unchanged\n") + + _log_secret_scan_bypass(tmp_path) + + state = load_yaml(state_path) + assert state["custom"] == "keep" + assert state["safety"]["other"] == "keep" + assert config_path.read_bytes() == b"config: unchanged\n" + + def test_log_secret_scan_bypass_does_not_update_ignore_on_scalar_safety( + self, tmp_path: Path + ) -> None: + from autoskillit.cli._init_helpers import _log_secret_scan_bypass + + state_path = tmp_path / ".autoskillit" / ".state.yaml" + scalar_safety = b"safety: scalar\n" + _write_state(state_path, scalar_safety.decode()) + + with pytest.raises(TypeError): + _log_secret_scan_bypass(tmp_path) + + assert state_path.read_bytes() == scalar_safety + assert not (state_path.parent / ".gitignore").exists() diff --git a/tests/cli/test_install.py b/tests/cli/test_install.py index 8b72816059..b83177fc78 100644 --- a/tests/cli/test_install.py +++ b/tests/cli/test_install.py @@ -1137,7 +1137,7 @@ def test_install_sweeps_all_scopes_for_orphans( def test_install_creates_autoskillit_gitignore( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """After install(), .autoskillit/.gitignore must exist (ensure_project_temp was called).""" + """install() ensures runtime artifacts are ignored in the project directory.""" import importlib as _importlib from autoskillit.cli.install._marketplace import install as _install @@ -1163,6 +1163,8 @@ def test_install_creates_autoskillit_gitignore( assert (tmp_path / ".autoskillit" / ".gitignore").exists(), ( ".autoskillit/.gitignore must be created by install(), not just by init()" ) + ignore_lines = (tmp_path / ".autoskillit" / ".gitignore").read_text().splitlines() + assert ".state.yaml" in ignore_lines def test_install_calls_upgrade_when_scripts_dir_exists( diff --git a/tests/config/test_rollout_config.py b/tests/config/test_rollout_config.py new file mode 100644 index 0000000000..ca20456f82 --- /dev/null +++ b/tests/config/test_rollout_config.py @@ -0,0 +1,158 @@ +"""Validation for the optional rollout declaration in configuration.""" + +from __future__ import annotations + +import copy + +import pytest + +from autoskillit.config import ConfigSchemaError, load_config +from autoskillit.core import dump_yaml_str + +pytestmark = [pytest.mark.layer("config"), pytest.mark.small] + +_VALID_ACTIVE: dict[str, object] = { + "change_id": "prompt-cost-study", + "effective_at": "2026-11-01T18:00:00Z", + "scope": { + "skills": ["make-plan"], + "backends": ["claude-code", "codex"], + "roles": ["planner"], + }, + "expected_effect": "Reduce planning input tokens while preserving requirement coverage.", +} + + +_MISSING = object() + + +def _write_config(tmp_path, config: dict) -> None: + config_dir = tmp_path / ".autoskillit" + config_dir.mkdir() + (config_dir / "config.yaml").write_text(dump_yaml_str(config)) + + +@pytest.mark.parametrize( + ("effective_at", "expected"), + [ + ("2099-11-01T18:00:00Z", "2099-11-01T18:00:00+00:00"), + ("2000-01-01T00:00:00+00:00", "2000-01-01T00:00:00+00:00"), + ], + ids=["future-cutover", "past-cutover"], +) +def test_loads_complete_rollout_active_and_normalizes_utc(tmp_path, effective_at, expected): + active = copy.deepcopy(_VALID_ACTIVE) + active["effective_at"] = effective_at + active["expected_effect"] = " Preserve this prediction text. " + _write_config(tmp_path, {"rollout": {"active": active}}) + + marker = load_config(tmp_path).rollout.active + assert marker is not None + + assert marker == { + **active, + "effective_at": expected, + } + assert marker["expected_effect"] == " Preserve this prediction text. " + assert marker["scope"] == _VALID_ACTIVE["scope"] + + +def test_rollout_validator_copies_raw_scope_lists() -> None: + from autoskillit.config._coercion import _build_rollout_active + + raw_active = copy.deepcopy(_VALID_ACTIVE) + section = {"active": raw_active} + + marker = _build_rollout_active(section, {}) + assert marker is not None + raw_scope = raw_active["scope"] + assert isinstance(raw_scope, dict) + raw_skills = raw_scope["skills"] + raw_backends = raw_scope["backends"] + assert isinstance(raw_skills, list) + assert isinstance(raw_backends, list) + raw_skills.append("later-skill") + raw_backends.clear() + + assert marker["scope"] == { + "skills": ["make-plan"], + "backends": ["claude-code", "codex"], + "roles": ["planner"], + } + + +@pytest.mark.parametrize( + "config", + [ + {}, + {"rollout": {}}, + {"rollout": {"active": None}}, + ], + ids=["missing-section", "missing-active", "null-active"], +) +def test_missing_or_null_rollout_active_is_inactive(tmp_path, config): + _write_config(tmp_path, config) + + assert load_config(tmp_path).rollout.active is None + + +def _invalid_active_cases() -> list[tuple[str, object]]: + cases: list[tuple[str, object]] = [] + + def add(name: str, **changes: object) -> None: + active: dict[str, object] = copy.deepcopy(_VALID_ACTIVE) + for key, value in changes.items(): + if value is _MISSING: + active.pop(key, None) + else: + active[key] = value + cases.append((name, active)) + + cases.append(("empty-active", {})) + cases.append(("partial-active", {"change_id": "only-an-id"})) + add("missing-change-id", change_id=_MISSING) + add("blank-change-id", change_id=" \t") + add("non-string-change-id", change_id=7) + add("missing-expected-effect", expected_effect=_MISSING) + add("null-expected-effect", expected_effect=None) + add("blank-expected-effect", expected_effect=" \t") + add("non-string-expected-effect", expected_effect=7) + add("missing-effective-at", effective_at=_MISSING) + add("malformed-effective-at", effective_at="not-a-date") + add("naive-effective-at", effective_at="2026-11-01T18:00:00") + add("non-utc-effective-at", effective_at="2026-11-01T18:00:00-07:00") + add("non-string-effective-at", effective_at=1700000000) + add("missing-scope", scope=_MISSING) + add("null-scope", scope=None) + add("non-mapping-scope", scope=["make-plan"]) + add("missing-scope-axis", scope={"skills": [], "backends": []}) + add( + "unknown-scope-axis", + scope={"skills": [], "backends": [], "roles": [], "repositories": ["repo"]}, + ) + add("non-list-scope-value", scope={"skills": "make-plan", "backends": [], "roles": []}) + add("non-string-scope-item", scope={"skills": [None], "backends": [], "roles": []}) + add("blank-scope-item", scope={"skills": [" "], "backends": [], "roles": []}) + add("unknown-marker-key", cohort="not-supported") + cases.extend( + [ + ("non-mapping-active-list", []), + ("non-mapping-active-string", "enabled"), + ] + ) + return cases + + +_INVALID_ACTIVE_CASES = _invalid_active_cases() + + +@pytest.mark.parametrize( + ("case", "active"), + _INVALID_ACTIVE_CASES, + ids=[case for case, _active in _INVALID_ACTIVE_CASES], +) +def test_invalid_rollout_active_declarations_raise_schema_error(tmp_path, case, active): + _write_config(tmp_path, {"rollout": {"active": active}}) + + with pytest.raises(ConfigSchemaError): + load_config(tmp_path) diff --git a/tests/conftest.py b/tests/conftest.py index 4be839ca04..4c193a50fe 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -279,6 +279,13 @@ def exploration_snapshot_service() -> MagicMock: return service +@pytest.fixture +def isolated_git_config(monkeypatch: pytest.MonkeyPatch) -> None: + """Keep real-Git visibility tests independent of user and system config.""" + monkeypatch.setenv("GIT_CONFIG_GLOBAL", os.devnull) + monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1") + + @pytest.fixture def remote_only_base_clone(tmp_path: _Path, monkeypatch: pytest.MonkeyPatch) -> tuple[_Path, str]: """Create an isolated ``feat`` clone with only remote ``develop`` history.""" diff --git a/tests/contracts/config_key_ledger.txt b/tests/contracts/config_key_ledger.txt index 1c4515e7be..9765be4594 100644 --- a/tests/contracts/config_key_ledger.txt +++ b/tests/contracts/config_key_ledger.txt @@ -110,6 +110,7 @@ report_bug.timeout reset_workspace.command reset_workspace.preserve_dirs review.local_review_rounds +rollout.active run_skill.completion_child_deferral_ceiling_seconds run_skill.completion_drain_timeout run_skill.completion_marker diff --git a/tests/contracts/test_dry_walkthrough_arch_catalog_reference.py b/tests/contracts/test_dry_walkthrough_arch_catalog_reference.py index 7075da5f2d..8570478bf3 100644 --- a/tests/contracts/test_dry_walkthrough_arch_catalog_reference.py +++ b/tests/contracts/test_dry_walkthrough_arch_catalog_reference.py @@ -23,7 +23,10 @@ def test_dry_walkthrough_declares_arch_constraint_catalog_resource(): """The catalog is provided data, not a source-tree prose dependency.""" frontmatter = read_skill_frontmatter(_skill_path()) assert frontmatter.data is not None, frontmatter.error - assert frontmatter.data.get("requires_resources") == ["arch-constraint-catalog"], ( - "dry-walkthrough must declare requires_resources: [arch-constraint-catalog] " - "so the architectural constraint catalog is delivered with the projected skill" + assert frontmatter.data.get("requires_resources") == [ + "arch-constraint-catalog", + "plan-part-form", + ], ( + "dry-walkthrough must deliver the architectural constraint catalog and canonical " + "plan-part authoring form with the projected skill" ) diff --git a/tests/contracts/test_plan_part_contract.py b/tests/contracts/test_plan_part_contract.py new file mode 100644 index 0000000000..a46b08f348 --- /dev/null +++ b/tests/contracts/test_plan_part_contract.py @@ -0,0 +1,72 @@ +"""Plan-part templates stay compatible with the allocation verifier.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("contracts"), pytest.mark.medium] + +_ALLOCATION_HEADING = "## Issue Requirement Allocation" + + +def _allocation_blocks(content: str, skill_path: Path) -> list[str]: + blocks: list[str] = [] + block_lines: list[str] | None = None + for line_number, line in enumerate(content.splitlines(), start=1): + if line.startswith("```"): + if block_lines is None: + block_lines = [] + else: + if _ALLOCATION_HEADING in block_lines: + blocks.append("\n".join(block_lines)) + block_lines = None + else: + if line == _ALLOCATION_HEADING: + assert block_lines is not None, ( + f"{skill_path}:{line_number} has an allocation heading outside a fenced block" + ) + if block_lines is not None: + block_lines.append(line) + if block_lines is not None and _ALLOCATION_HEADING in block_lines: + blocks.append("\n".join(block_lines)) + return blocks + + +def test_bundled_plan_part_templates_pass_the_allocation_verifier() -> None: + from autoskillit.core import parse_part_allocation, verify_allocation_evidence + from autoskillit.workspace.skills import bundled_skills_dir, bundled_skills_extended_dir + + expected_skills: set[str] = set() + verified_skills: set[str] = set() + verified_block_count = 0 + for skills_root in (bundled_skills_dir(), bundled_skills_extended_dir()): + for skill_path in skills_root.rglob("SKILL.md"): + content = skill_path.read_text(encoding="utf-8") + if "**Plan-part form.**" in content: + expected_skills.add(skill_path.parent.name) + assert "Step {N.M} |" not in content, f"{skill_path} has a stale step placeholder" + assert r"owned \| shared" not in content, ( + f"{skill_path} escapes the allocation separator" + ) + + blocks = _allocation_blocks(content, skill_path) + for block_index, block in enumerate(blocks, start=1): + try: + rows = parse_part_allocation(block, part_key="P1") + except ValueError as exc: + pytest.fail( + f"{skill_path} allocation block {block_index} is not parseable: {exc}" + ) + assert rows, f"{skill_path} allocation block {block_index} has no rows" + defects = verify_allocation_evidence(block, rows) + assert not defects, ( + f"{skill_path} allocation block {block_index} has evidence defects: {defects}" + ) + verified_block_count += 1 + verified_skills.add(skill_path.parent.name) + + assert verified_block_count > 0, "no bundled plan-part allocation template was checked" + assert expected_skills, "no bundled plan-part authoring skill was found" + assert expected_skills <= verified_skills diff --git a/tests/contracts/test_protocol_satisfaction.py b/tests/contracts/test_protocol_satisfaction.py index 3eeecb0778..03b62916f3 100644 --- a/tests/contracts/test_protocol_satisfaction.py +++ b/tests/contracts/test_protocol_satisfaction.py @@ -535,6 +535,8 @@ def test_req_api_001_public_params_present(self): "lifecycle_observation_enabled", "ceiling_seconds", "systemd_scope_enabled", + "rollout", + "logical_run_start_ts", } assert expected == public_params, ( f"run_managed_async public params changed.\n" @@ -560,6 +562,8 @@ def test_req_api_001_public_param_defaults(self): assert p["completion_marker"].default == "" assert p["stale_threshold"].default == 1200 assert p["completion_drain_timeout"].default == 5.0 + assert p["rollout"].default is None + assert p["logical_run_start_ts"].default == "" # ------------------------------------------------------------------ # REQ-API-002: DefaultSubprocessRunner satisfies SubprocessRunner protocol @@ -618,6 +622,8 @@ def test_req_api_002_call_params_match_protocol(self): "lifecycle_observation_enabled", "ceiling_seconds", "systemd_scope_enabled", + "rollout", + "logical_run_start_ts", } assert expected == actual, ( f"DefaultSubprocessRunner.__call__ params changed.\n" diff --git a/tests/contracts/test_skill_resource_derived_counts.py b/tests/contracts/test_skill_resource_derived_counts.py index 29a27d2106..00b4b0aa73 100644 --- a/tests/contracts/test_skill_resource_derived_counts.py +++ b/tests/contracts/test_skill_resource_derived_counts.py @@ -19,7 +19,7 @@ @pytest.mark.parametrize( ("resource_id", "expected_count"), [ - ("arch-constraint-catalog", 71), + ("arch-constraint-catalog", 72), ("review-approach-criteria", None), ], ) diff --git a/tests/core/io/test_markdown_sections.py b/tests/core/io/test_markdown_sections.py index 3057d91a96..a57f43e86d 100644 --- a/tests/core/io/test_markdown_sections.py +++ b/tests/core/io/test_markdown_sections.py @@ -4,7 +4,7 @@ import pytest -from autoskillit.core import STEP_HEADING_RE, extract_section, split_table_row +from autoskillit.core import STEP_HEADING_RE, extract_section, parse_pipe_table, split_table_row pytestmark = [pytest.mark.layer("core"), pytest.mark.small] @@ -13,10 +13,68 @@ def test_extract_section_requires_one_matching_heading() -> None: markdown = "# Title\n\n## Allocation\nbody\n\n## Next\nmore\n" assert extract_section(markdown, "Allocation") == "\nbody\n\n" - with pytest.raises(ValueError, match="exactly one"): + with pytest.raises(ValueError, match="exactly one") as missing: extract_section(markdown, "Missing") - with pytest.raises(ValueError, match="exactly one"): + assert str(missing.value).startswith("expected exactly one ## Missing section") + assert "; found 0 matching headings" in str(missing.value) + with pytest.raises(ValueError, match="exactly one") as duplicate: extract_section("## Allocation\na\n## Allocation\nb", "Allocation") + assert str(duplicate.value).startswith("expected exactly one ## Allocation section") + assert "; found 2 matching headings" in str(duplicate.value) + + +@pytest.mark.parametrize( + ("section", "offending_line", "message_part"), + ( + pytest.param( + "| Wrong | Allocation | Implementation Step |\n| --- | --- | --- |", + "| Wrong | Allocation | Implementation Step |", + "header", + id="wrong-header", + ), + pytest.param( + "| Requirement ID | Allocation | Implementation Step |\n| -- | --- | --- |", + "| -- | --- | --- |", + "separator", + id="bad-separator", + ), + pytest.param( + "| Requirement ID | Allocation | Implementation Step |\n" + "| --- | --- | --- |\nnot a pipe row", + "not a pipe row", + "pipe", + id="non-pipe-line", + ), + pytest.param( + "| Requirement ID | Allocation | Implementation Step |\n" + "| --- | --- | --- |\n| R1 | owned | Step 1.1 | extra |", + "| R1 | owned | Step 1.1 | extra |", + "3 columns", + id="four-cell-row", + ), + pytest.param( + f"| {'x' * 150} | Allocation | Implementation Step |\n| --- | --- | --- |", + f"| {'x' * 150} | Allocation | Implementation Step |", + "header", + id="long-line-is-truncated", + ), + ), +) +def test_parse_pipe_table_errors_quote_stripped_bounded_line( + section: str, offending_line: str, message_part: str +) -> None: + with pytest.raises(ValueError) as error: + parse_pipe_table( + section, + ("Requirement ID", "Allocation", "Implementation Step"), + ) + + message = str(error.value) + stripped = offending_line.strip() + assert message_part in message.lower() + assert stripped[:120] in message + if len(stripped) > 120: + assert stripped[120:] not in message def test_pipe_row_and_step_pattern_are_strict() -> None: diff --git a/tests/core/planset/test_allocation.py b/tests/core/planset/test_allocation.py index 7f1c867652..767e21f02c 100644 --- a/tests/core/planset/test_allocation.py +++ b/tests/core/planset/test_allocation.py @@ -9,37 +9,249 @@ pytestmark = [pytest.mark.layer("core"), pytest.mark.small] -def test_allocation_requires_step_that_mentions_requirement() -> None: - markdown = """### Step 1.1: implementation +def _plan_with_allocation(body: str, requirement_id: str, implementation_step: str) -> str: + return f"""{body} -The step omits the identifier. +## Issue Requirement Allocation +| Requirement ID | Allocation | Implementation Step | +| --- | --- | --- | +| {requirement_id} | owned | {implementation_step} | +""" + + +@pytest.mark.parametrize( + ("body", "requirement_id", "implementation_step", "reason", "detail_parts"), + ( + pytest.param( + "### Step 1: R1\nImplement the requirement.", + "R1", + "Step 1", + None, + (), + id="bare-id-in-step", + ), + pytest.param( + "### Step 1.1: t\nImplement R1.", + "R1", + "Step 1.1", + None, + (), + id="decimal-heading", + ), + pytest.param( + "### Step 3\nImplement R1.", + "R1", + "Step 3", + None, + (), + id="heading-without-title", + ), + pytest.param( + "1. Step 1: Implement R1.", + "R1", + "Step 1", + PlanSetRejectReason.ALLOCATION_STEP_MISSING, + ("Step 1", "no step heading", "found"), + id="numbered-list-is-not-a-heading", + ), + pytest.param( + "### Step 1: t\n\n- Step 1.1: Implement R1.", + "R1", + "Step 1.1", + PlanSetRejectReason.ALLOCATION_STEP_MISSING, + ("Step 1.1", "Step 1"), + id="listed-substep-is-not-a-heading", + ), + pytest.param( + "#### Step 1.1: R1", + "R1", + "Step 1.1", + PlanSetRejectReason.ALLOCATION_STEP_MISSING, + (), + id="four-hash-heading", + ), + pytest.param( + "### Step 1 — R1", + "R1", + "Step 1", + PlanSetRejectReason.ALLOCATION_STEP_MISSING, + (), + id="em-dash-title-separator", + ), + pytest.param( + "### Step 1.1 - R1", + "R1", + "Step 1.1", + PlanSetRejectReason.ALLOCATION_STEP_MISSING, + (), + id="hyphen-title-separator", + ), + pytest.param( + "### Step 1.1: R1\nImplement R1.\n### Step 1.2: R1\nAlso implement R1.", + "R1", + "Steps 1, 2", + PlanSetRejectReason.ALLOCATION_STEP_MISSING, + ("Steps 1, 2", "Step 1.1", "Step 1.2"), + id="step-range-cell", + ), + pytest.param( + "### Step 1.1: R1\nImplement R1.\n### Step 1.2: R1\nAlso implement R1.", + "R1", + "Step 1.1, Step 1.2", + PlanSetRejectReason.ALLOCATION_STEP_MISSING, + ("Step 1.1, Step 1.2", "Step 1.1", "Step 1.2"), + id="comma-separated-keys", + ), + pytest.param( + "### Step 1.1: R1\nImplement R1.", + "R1", + "1.1", + PlanSetRejectReason.ALLOCATION_STEP_MISSING, + ("1.1", "Step 1.1"), + id="bare-decimal-key", + ), + pytest.param( + "### Step 1.1: R1\nImplement R1.", + "R1", + "`Step 1.1`", + PlanSetRejectReason.ALLOCATION_STEP_MISSING, + ("`Step 1.1`", "Step 1.1"), + id="backticked-key", + ), + pytest.param( + "### Step 1.1: R1\nImplement R1.", + "R1", + "Step 99.9", + PlanSetRejectReason.ALLOCATION_STEP_MISSING, + ("Step 99.9", "Step 1.1"), + id="unknown-step-key", + ), + pytest.param( + "### Step 1.1: implementation\nThe step omits the identifier.", + "R1", + "Step 1.1", + PlanSetRejectReason.ALLOCATION_STEP_UNREFERENCED, + ("R1", "Step 1.1", "step's own text"), + id="step-omits-requirement-id", + ), + pytest.param( + "### Step 1.1: P-1\nImplement P-1.", + "P-1 (Add fn)", + "Step 1.1", + PlanSetRejectReason.ALLOCATION_STEP_UNREFERENCED, + ("P-1 (Add fn)", "Step 1.1"), + id="annotated-id-parentheses", + ), + pytest.param( + "### Step 1.1: P-1\nImplement P-1.", + "P-1: Add fn", + "Step 1.1", + PlanSetRejectReason.ALLOCATION_STEP_UNREFERENCED, + ("P-1: Add fn", "Step 1.1"), + id="annotated-id-colon", + ), + ), +) +def test_allocation_evidence_reports_cells_and_found_step_keys( + body: str, + requirement_id: str, + implementation_step: str, + reason: PlanSetRejectReason | None, + detail_parts: tuple[str, ...], +) -> None: + markdown = _plan_with_allocation(body, requirement_id, implementation_step) + rows = parse_part_allocation(markdown, part_key="P1") + + defects = verify_allocation_evidence(markdown, rows) + + if reason is None: + assert defects == () + return + assert len(defects) == 1 + found_id, found_reason, detail = defects[0] + assert found_id == requirement_id + assert found_reason is reason + for part in detail_parts: + assert part in detail + if "no step heading" in detail_parts: + lowered = detail.lower() + assert "no step heading" in lowered + assert "found" in lowered + + +def test_allocation_rejects_prose_before_table_with_line_context() -> None: + sentence = "The implementation plan starts below." + markdown = f"""### Step 1.1: R1 +Implement R1. ## Issue Requirement Allocation +{sentence} | Requirement ID | Allocation | Implementation Step | | --- | --- | --- | | R1 | owned | Step 1.1 | """ - rows = parse_part_allocation(markdown, part_key="P1") - assert verify_allocation_evidence(markdown, rows) == ( - ("R1", PlanSetRejectReason.ALLOCATION_STEP_UNREFERENCED), - ) - with pytest.raises(ValueError, match="exactly one"): - parse_part_allocation("# no allocation", part_key="P1") + with pytest.raises(ValueError) as error: + parse_part_allocation(markdown, part_key="P1") + assert "pipe" in str(error.value).lower() + assert sentence in str(error.value) -def test_allocation_flags_step_not_present_in_markdown() -> None: - markdown = """### Step 1.1: implementation -The step exists. +def test_allocation_rejects_escaped_pipe_and_quotes_raw_row() -> None: + raw_row = r"| R1 | owned \| shared | Step 1.1 |" + markdown = f"""### Step 1.1: R1 +Implement R1. ## Issue Requirement Allocation | Requirement ID | Allocation | Implementation Step | | --- | --- | --- | -| R1 | owned | Step 99.9 | +{raw_row} """ - rows = parse_part_allocation(markdown, part_key="P1") - assert verify_allocation_evidence(markdown, rows) == ( - ("R1", PlanSetRejectReason.ALLOCATION_STEP_MISSING), - ) + with pytest.raises(ValueError) as error: + parse_part_allocation(markdown, part_key="P1") + + assert "3 columns" in str(error.value) + assert raw_row in str(error.value) + + +@pytest.mark.parametrize( + ("row", "row_identifier"), + ( + ("| R1 | partial | Step 1.1 |", "R1"), + ("| R1 | owned | |", "R1"), + ("| | owned | Step 1.1 |", "| | owned | Step 1.1 |"), + ), +) +def test_allocation_rejects_invalid_rows_with_row_identity(row: str, row_identifier: str) -> None: + markdown = f"""### Step 1.1: R1 +Implement R1. + +## Issue Requirement Allocation +| Requirement ID | Allocation | Implementation Step | +| --- | --- | --- | +{row} +""" + + with pytest.raises(ValueError) as error: + parse_part_allocation(markdown, part_key="P1") + + assert row_identifier in str(error.value) + + +def test_allocation_rejects_duplicate_requirement_id() -> None: + markdown = """### Step 1.1: R1 +Implement R1. + +## Issue Requirement Allocation +| Requirement ID | Allocation | Implementation Step | +| --- | --- | --- | +| R1 | owned | Step 1.1 | +| R1 | shared | Step 1.1 | +""" + + with pytest.raises(ValueError) as error: + parse_part_allocation(markdown, part_key="P1") + + assert "R1" in str(error.value) diff --git a/tests/core/planset/test_coverage.py b/tests/core/planset/test_coverage.py index 376883a097..88c85c7ab9 100644 --- a/tests/core/planset/test_coverage.py +++ b/tests/core/planset/test_coverage.py @@ -9,6 +9,8 @@ from autoskillit.core import ( CoverageStatus, InventoryMode, + PlanSetBindResult, + PlanSetRejectReason, evaluate_coverage, extract_requirement_inventory, parse_part_allocation, @@ -45,3 +47,45 @@ def test_unenumerated_parts_require_planner_obligations() -> None: assert evaluate_coverage( InventoryMode.UNENUMERATED, (), {"P1": (), "P2": ()} ).parts_without_obligations == ("P1", "P2") + + +@pytest.mark.parametrize("coverage_gaps", ("", " \n\t")) +def test_coverage_failure_requires_nonblank_feedback(coverage_gaps: str) -> None: + with pytest.raises(ValueError): + PlanSetBindResult( + False, + PlanSetRejectReason.COVERAGE_FAILED, + "plan-set coverage has gaps", + coverage_gaps=coverage_gaps, + ) + + +@pytest.mark.parametrize( + ("success", "reason", "coverage_gaps"), + [ + (True, PlanSetRejectReason.NO_PARTS, ""), + (False, None, ""), + (True, None, "R2 [unassigned]"), + (False, PlanSetRejectReason.NO_PARTS, "R2 [unassigned]"), + ], +) +def test_bind_result_rejects_inconsistent_success_fields( + success: bool, reason: PlanSetRejectReason | None, coverage_gaps: str +) -> None: + with pytest.raises(ValueError): + PlanSetBindResult(success, reason, "", coverage_gaps=coverage_gaps) + + +def test_coverage_feedback_invariant_applies_only_to_coverage_failures() -> None: + failed = PlanSetBindResult( + False, + PlanSetRejectReason.COVERAGE_FAILED, + "plan-set coverage has gaps", + coverage_gaps="R2 [unassigned]", + ) + other_failure = PlanSetBindResult(False, PlanSetRejectReason.NO_PARTS, "no plan parts") + success = PlanSetBindResult(True, None, "") + + assert failed.coverage_gaps == "R2 [unassigned]" + assert other_failure.coverage_gaps == "" + assert success.coverage_gaps == "" diff --git a/tests/core/test_ensure_project_temp_with_config.py b/tests/core/test_ensure_project_temp_with_config.py index f13b6a297b..9543c65336 100644 --- a/tests/core/test_ensure_project_temp_with_config.py +++ b/tests/core/test_ensure_project_temp_with_config.py @@ -2,18 +2,21 @@ from __future__ import annotations +import subprocess from pathlib import Path import pytest -from autoskillit.core.io import ensure_project_temp +from autoskillit.core.io import ( + _AUTOSKILLIT_GITIGNORE_ENTRIES, + ensure_project_gitignore, + ensure_project_temp, +) -pytestmark = [pytest.mark.layer("core"), pytest.mark.small] +pytestmark = [pytest.mark.layer("core"), pytest.mark.medium] def test_repository_profile_artifact_is_project_gitignored() -> None: - from autoskillit.core.io import _AUTOSKILLIT_GITIGNORE_ENTRIES - assert "repository-profile.v1.json" in _AUTOSKILLIT_GITIGNORE_ENTRIES @@ -69,5 +72,63 @@ def test_ensure_project_temp_idempotent_on_existing_gitignore(tmp_path: Path) -> ensure_project_temp(tmp_path) ensure_project_temp(tmp_path) - assert (target / ".gitignore").read_text() == custom + + +@pytest.mark.parametrize("existing", [None, "# keep this comment\ncustom/\n"]) +def test_ensure_project_temp_leaves_project_gitignore_untouched( + tmp_path: Path, existing: str | None +) -> None: + autoskillit_dir = tmp_path / ".autoskillit" + autoskillit_dir.mkdir() + gitignore = autoskillit_dir / ".gitignore" + if existing is not None: + gitignore.write_text(existing, encoding="utf-8") + root_gitignore = tmp_path / ".gitignore" + root_gitignore.write_text("root rules\n", encoding="utf-8") + + ensure_project_temp(tmp_path) + + actual = gitignore.read_text(encoding="utf-8") if gitignore.exists() else None + assert actual == existing + assert root_gitignore.read_text(encoding="utf-8") == "root rules\n" + + +def test_ensure_project_gitignore_preserves_rules_and_is_idempotent(tmp_path: Path) -> None: + ensure_project_gitignore(tmp_path) + gitignore = tmp_path / ".autoskillit" / ".gitignore" + assert set(_AUTOSKILLIT_GITIGNORE_ENTRIES).issubset(gitignore.read_text().splitlines()) + + gitignore.write_text("# user comment\ncustom/\ntemp/\n", encoding="utf-8") + ensure_project_gitignore(tmp_path) + content = gitignore.read_text(encoding="utf-8") + assert content.startswith("# user comment\ncustom/\ntemp/\n") + assert set(_AUTOSKILLIT_GITIGNORE_ENTRIES).issubset(content.splitlines()) + + ensure_project_gitignore(tmp_path) + assert gitignore.read_text(encoding="utf-8") == content + + +def test_runtime_state_is_ignored_in_a_git_checkout(tmp_path: Path) -> None: + from tests._git_topology import commit_file, init_checkout + + repo = init_checkout(tmp_path / "repo") + ensure_project_gitignore(repo) + gitignore = repo / ".autoskillit" / ".gitignore" + commit_file( + repo, + ".autoskillit/.gitignore", + gitignore.read_text(), + "ignore runtime state", + ) + (gitignore.parent / ".state.yaml").write_text("runtime: true\n", encoding="utf-8") + + status = subprocess.run( + ["git", "status", "--porcelain", "--untracked-files=all"], + cwd=repo, + check=True, + capture_output=True, + text=True, + timeout=10, + ) + assert status.stdout == "" diff --git a/tests/core/test_grab_bag_split_completeness.py b/tests/core/test_grab_bag_split_completeness.py index 048593d0e4..1f36e350c2 100644 --- a/tests/core/test_grab_bag_split_completeness.py +++ b/tests/core/test_grab_bag_split_completeness.py @@ -533,6 +533,7 @@ def test_no_unintended_new_test_files_under_tests_core() -> None: "test_resolve_main_worktree.py", "test_resolve_temp_dir.py", "test_session_checkpoint.py", + "test_self_ignored.py", "test_session_env_specs.py", "test_session_hook_root.py", "test_session_index_schema.py", diff --git a/tests/core/test_self_ignored.py b/tests/core/test_self_ignored.py new file mode 100644 index 0000000000..942a150d98 --- /dev/null +++ b/tests/core/test_self_ignored.py @@ -0,0 +1,45 @@ +"""Tests for self-ignore markers in runtime directories.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from autoskillit.core.io import ensure_self_ignored, is_self_ignored + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + + +def test_ensure_self_ignored_writes_marker_only_when_absent(tmp_path: Path) -> None: + target = tmp_path / "runtime" + target.mkdir() + marker = target / ".gitignore" + ensure_self_ignored(target) + assert marker.is_file() and is_self_ignored(target) + custom = "# custom user content\n*\n!keep.txt\n" + marker.write_text(custom, encoding="utf-8") + + ensure_self_ignored(target) + + assert marker.read_text(encoding="utf-8") == custom + + +def test_is_self_ignored_requires_a_star_line(tmp_path: Path) -> None: + target = tmp_path / "runtime" + target.mkdir() + assert not is_self_ignored(target) + marker = target / ".gitignore" + marker.write_text("*.pyc\n", encoding="utf-8") + assert not is_self_ignored(target) + marker.write_text("# generated\n*\n", encoding="utf-8") + assert is_self_ignored(target) + + +def test_is_self_ignored_rejects_symlinked_marker(tmp_path: Path) -> None: + target = tmp_path / "runtime" + target.mkdir() + external_marker = tmp_path / "external-ignore" + external_marker.write_text("*\n", encoding="utf-8") + (target / ".gitignore").symlink_to(external_marker) + assert not is_self_ignored(target) diff --git a/tests/core/test_session_index_schema.py b/tests/core/test_session_index_schema.py index 00678c66d0..88b5e6042e 100644 --- a/tests/core/test_session_index_schema.py +++ b/tests/core/test_session_index_schema.py @@ -27,6 +27,7 @@ "backend", "skill_command", "session_type", + "rollout", "subagent_model_outcomes", "evidence_worktree_path", "evidence_worktree_source", @@ -71,6 +72,14 @@ def test_has_nullable_session_type(self): hints = get_type_hints(SessionIndexEntry) assert hints["session_type"] == str | None + def test_has_nullable_rollout_marker(self): + from typing import get_type_hints + + from autoskillit.core.types.results._type_results import RolloutMarker, SessionIndexEntry + + hints = get_type_hints(SessionIndexEntry) + assert hints["rollout"] == RolloutMarker | None + def test_canonical_cache_fields(self): """SessionIndexEntry must use canonical cache field names, not v1 API names.""" from autoskillit.core.types.results._type_results import SessionIndexEntry diff --git a/tests/core/test_session_index_schema_version_lock.py b/tests/core/test_session_index_schema_version_lock.py index edd97e7001..b530feadd5 100644 --- a/tests/core/test_session_index_schema_version_lock.py +++ b/tests/core/test_session_index_schema_version_lock.py @@ -12,7 +12,7 @@ pytestmark = [pytest.mark.layer("core"), pytest.mark.small] -_EXPECTED = (16, "9584465d5e3ca791680c05a0698482e23561ef03533c209a40380d8506580eb3") +_EXPECTED = (17, "02755757afa42542ffde3c6ac1be6d3e7a2116d57069447932eac4380dd1fdb1") def test_session_index_schema_version_matches_field_digest() -> None: diff --git a/tests/core/test_type_results_records.py b/tests/core/test_type_results_records.py index c232fe5a97..c105b96afe 100644 --- a/tests/core/test_type_results_records.py +++ b/tests/core/test_type_results_records.py @@ -39,6 +39,7 @@ "EvidenceWorktreeSource", "ValidatedAddDir", "ValidatedWorktreePath", + "RolloutMarker", ) INTERNAL_INDEX_NAMES = ( "TokenUsageFileEntry", diff --git a/tests/core/types/test_plan_set_authority.py b/tests/core/types/test_plan_set_authority.py index acc943f9a5..aadc66bcf7 100644 --- a/tests/core/types/test_plan_set_authority.py +++ b/tests/core/types/test_plan_set_authority.py @@ -77,6 +77,23 @@ def test_create_digest_and_canonical_round_trip(tmp_path: Path) -> None: "execution generation", ), ({"binding_mode": "foreign"}, "binding_mode"), + ( + { + "coverage": CoverageResultDef( + CoverageStatus.PASS, parts_without_obligations=("P2",) + ) + }, + "unknown part key", + ), + ( + { + "inventory_mode": InventoryMode.ENUMERATED, + "coverage": CoverageResultDef( + CoverageStatus.PASS, parts_without_obligations=("P1",) + ), + }, + "invalid for enumerated inventory", + ), ], ) def test_invalid_authority_states_fail_closed( @@ -86,6 +103,12 @@ def test_invalid_authority_states_fail_closed( _authority(tmp_path, **overrides) +@pytest.mark.parametrize("mode", (InventoryMode.NO_ISSUE, InventoryMode.UNENUMERATED)) +def test_authority_rejects_omitted_empty_parts(tmp_path: Path, mode: InventoryMode) -> None: + with pytest.raises(ValueError, match="must match actual empty parts"): + _authority(tmp_path, inventory_mode=mode, allocations=()) + + def test_part_ordinals_and_resolved_locators_are_unique(tmp_path: Path) -> None: authority = _authority(tmp_path) first = authority.parts[0] @@ -101,6 +124,8 @@ def test_part_ordinals_and_resolved_locators_are_unique(tmp_path: Path) -> None: valid = _authority( tmp_path, parts=(first, replace(alias, locator=str(second))), + allocations=authority.allocations + + (replace(authority.allocations[0], requirement_id="P-2", part_key="P2"),), ) assert [part.part_suffix for part in valid.parts] == ["A", "A"] diff --git a/tests/execution/conftest.py b/tests/execution/conftest.py index d286cc74e4..83315773d6 100644 --- a/tests/execution/conftest.py +++ b/tests/execution/conftest.py @@ -40,6 +40,11 @@ from tests._helpers import make_tracing_config from tests.fixtures.codex import FLAT_ERROR_MODEL_CAPACITY, TURN_FAILED_MODEL_CAPACITY + +def _unmarked_rollout_snapshot() -> tuple[None, str]: + return None, "2000-01-01T00:00:00+00:00" + + # Strip regex metacharacters (e.g. \b word boundaries) to yield plain test signal strings. CODEX_API_ERROR_SIGNAL_STRINGS: tuple[str, ...] = tuple( p.pattern.replace(r"\b", "") for p in _CODEX_API_ERROR_PATTERNS diff --git a/tests/execution/headless/test_codex_attempt_retention.py b/tests/execution/headless/test_codex_attempt_retention.py index 204b8b851a..31e3ac8064 100644 --- a/tests/execution/headless/test_codex_attempt_retention.py +++ b/tests/execution/headless/test_codex_attempt_retention.py @@ -27,7 +27,7 @@ from autoskillit.execution.backends import codex as codex_module from autoskillit.execution.backends._codex_session_storage import CodexSessionStore from autoskillit.execution.backends.codex import CodexBackend -from tests.execution.conftest import _launch_inputs, _mock_backend +from tests.execution.conftest import _launch_inputs, _mock_backend, _unmarked_rollout_snapshot from tests.fakes import FakePluginArtifactAuthority pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] @@ -189,6 +189,7 @@ async def _run_generated_attempt( completion_marker="%%DONE%%", stale_threshold=1.0, completion_drain_timeout=1.0, + capture_rollout_on_launch=_unmarked_rollout_snapshot, natural_exit_grace_seconds=1.0, linux_tracing_config=None, idle_output_timeout=None, @@ -303,6 +304,7 @@ def build_resume_cmd(**kwargs: object) -> CmdSpec: session_home=str(first_home), launch_resolver=nudge_resolver, launch_preparation=nudge_preparation, + capture_rollout_on_launch=_unmarked_rollout_snapshot, natural_exit_grace_seconds=1.0, attempt=2, ceiling_seconds=_EXECUTION_CEILING_SECONDS, @@ -415,6 +417,7 @@ def build_with_retained_binding(binding, _extras, _attempt_id=None) -> CmdSpec: completion_marker="%%DONE%%", stale_threshold=1.0, completion_drain_timeout=1.0, + capture_rollout_on_launch=_unmarked_rollout_snapshot, natural_exit_grace_seconds=1.0, linux_tracing_config=None, idle_output_timeout=None, @@ -488,6 +491,7 @@ def build_resume_cmd(**kwargs: object) -> CmdSpec: session_home=str(home), launch_resolver=nudge_resolver, launch_preparation=nudge_preparation, + capture_rollout_on_launch=_unmarked_rollout_snapshot, natural_exit_grace_seconds=1.0, attempt=2, ceiling_seconds=_EXECUTION_CEILING_SECONDS, diff --git a/tests/execution/headless/test_headless_recovery.py b/tests/execution/headless/test_headless_recovery.py index fd2e28f495..b395e7107a 100644 --- a/tests/execution/headless/test_headless_recovery.py +++ b/tests/execution/headless/test_headless_recovery.py @@ -27,7 +27,7 @@ from autoskillit.core.types.results._type_results import WriteEvidence from autoskillit.execution.headless._managed import _ManagedLineageObserver from autoskillit.execution.session.turn_usage import build_turn_token_entry -from tests.execution.conftest import _launch_inputs, _mock_backend +from tests.execution.conftest import _launch_inputs, _mock_backend, _unmarked_rollout_snapshot from tests.fakes import FakeManagedHeadlessSessionLineageStore pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] @@ -138,6 +138,7 @@ async def _run_turn_usage_nudge( retry_reason=RetryReason.EARLY_STOP, launch_resolver=launch_resolver, launch_preparation=launch_preparation, + capture_rollout_on_launch=_unmarked_rollout_snapshot, natural_exit_grace_seconds=3.0, ) assert result is not None @@ -175,6 +176,7 @@ class TestNudgePtyMode: @pytest.mark.anyio async def test_nudge_passes_pty_mode_for_claude_backend(self, tmp_path: Path) -> None: """_attempt_contract_nudge passes pty_mode=True to runner for ClaudeCode backend.""" + from autoskillit.config import LinuxTracingConfig from autoskillit.execution.headless._headless_launch import _attempt_contract_nudge from tests.execution.conftest import _mock_backend from tests.fakes import MockSubprocessRunner @@ -226,6 +228,15 @@ async def test_nudge_passes_pty_mode_for_claude_backend(self, tmp_path: Path) -> termination=TerminationReason.NATURAL_EXIT, pid=0, ) + rollout_marker = { + "change_id": "rollout-nudge", + "effective_at": "2026-10-10T12:00:00+00:00", + "scope": {"skills": ["audit"], "backends": ["claude-code"], "roles": []}, + "expected_effect": "Nudge retains the admitted snapshot", + } + logical_start = "2026-10-10T12:00:01+00:00" + capture_rollout = Mock(return_value=(rollout_marker, logical_start)) + linux_tracing_config = LinuxTracingConfig(tmpfs_path=str(tmp_path)) await _attempt_contract_nudge( skill_result=skill_result, @@ -239,6 +250,8 @@ async def test_nudge_passes_pty_mode_for_claude_backend(self, tmp_path: Path) -> retry_reason=RetryReason.EARLY_STOP, launch_resolver=launch_resolver, launch_preparation=launch_preparation, + capture_rollout_on_launch=capture_rollout, + linux_tracing_config=linux_tracing_config, natural_exit_grace_seconds=3.0, ) @@ -247,6 +260,11 @@ async def test_nudge_passes_pty_mode_for_claude_backend(self, tmp_path: Path) -> f"Expected last_pty_mode=True for ClaudeCode backend (pty_required=True), " f"got {mock_runner.last_pty_mode!r}" ) + nudge_kwargs = mock_runner.call_args_list[0][3] + capture_rollout.assert_called_once_with() + assert nudge_kwargs["rollout"] == rollout_marker + assert nudge_kwargs["logical_run_start_ts"] == logical_start + assert nudge_kwargs["linux_tracing_config"] is linux_tracing_config @pytest.mark.anyio async def test_nudge_respects_pty_override_false(self, tmp_path: Path) -> None: @@ -315,6 +333,7 @@ async def test_nudge_respects_pty_override_false(self, tmp_path: Path) -> None: pty_override=False, launch_resolver=launch_resolver, launch_preparation=launch_preparation, + capture_rollout_on_launch=_unmarked_rollout_snapshot, natural_exit_grace_seconds=3.0, ) @@ -429,6 +448,7 @@ async def __call__(self, *args, **kwargs): managed_lineage_observer=lineage_observer, launch_resolver=launch_resolver, launch_preparation=launch_preparation, + capture_rollout_on_launch=_unmarked_rollout_snapshot, natural_exit_grace_seconds=3.0, ) @@ -514,6 +534,7 @@ async def test_nudge_passes_backend_line_driver_to_runner(self, tmp_path: Path) retry_reason=RetryReason.EARLY_STOP, launch_resolver=launch_resolver, launch_preparation=launch_preparation, + capture_rollout_on_launch=_unmarked_rollout_snapshot, natural_exit_grace_seconds=3.0, ) @@ -603,6 +624,7 @@ def acquire_launch_binding(self, *, backend, load_mode): retained_binding=retained_binding, launch_resolver=launch_resolver, launch_preparation=launch_preparation, + capture_rollout_on_launch=_unmarked_rollout_snapshot, natural_exit_grace_seconds=3.0, ) @@ -680,6 +702,7 @@ async def test_nudge_passes_managed_skill_catalog_to_build_resume_cmd( managed_skill_catalog=catalog, launch_resolver=launch_resolver, launch_preparation=launch_preparation, + capture_rollout_on_launch=_unmarked_rollout_snapshot, natural_exit_grace_seconds=3.0, ) @@ -809,6 +832,7 @@ async def test_nudge_skips_for_block_delimiter_patterns(tmp_path: Path) -> None: result_parser=result_parser, launch_resolver=launch_resolver, launch_preparation=launch_preparation, + capture_rollout_on_launch=_unmarked_rollout_snapshot, natural_exit_grace_seconds=3.0, ) diff --git a/tests/execution/session/test_managed_headless_session_lineage.py b/tests/execution/session/test_managed_headless_session_lineage.py index 7793cbcf4d..ec9f657609 100644 --- a/tests/execution/session/test_managed_headless_session_lineage.py +++ b/tests/execution/session/test_managed_headless_session_lineage.py @@ -363,6 +363,56 @@ def test_corrupt_unsupported_and_digest_mismatched_records_fail_closed( store.load(lineage_anchor=anchor, launch_id=lineage.launch_id) +def test_create_fails_closed_when_namespace_cannot_be_self_ignored( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + anchor = tmp_path / "project" + anchor.mkdir() + from autoskillit.execution.session import ( + _managed_headless_session_lineage_records as lineage_module, + ) + + launch_id = "1" * 32 + + def fail_self_ignore(_namespace: Path) -> None: + raise OSError("disk unavailable") + + monkeypatch.setattr(lineage_module, "ensure_self_ignored", fail_self_ignore) + with pytest.raises(OSError, match="disk unavailable"): + _create(DefaultManagedHeadlessSessionLineageStore(), anchor, launch_id=launch_id) + + record = ( + anchor + / ".autoskillit" + / "managed-headless-session-lineage" + / "records" + / f"{launch_id}.json" + ) + assert not record.exists() + assert not list((anchor / ".autoskillit" / "managed-headless-session-lineage").rglob("*.json")) + + +def test_create_rejects_existing_non_self_ignoring_marker( + tmp_path: Path, +) -> None: + anchor = tmp_path / "project" + anchor.mkdir() + namespace = anchor / ".autoskillit" / "managed-headless-session-lineage" + namespace.mkdir(parents=True) + marker = namespace / ".gitignore" + malformed = "# not a self-ignoring marker\nmanaged-data/\n" + marker.write_text(malformed, encoding="utf-8") + + with pytest.raises( + OSError, + match="Lineage namespace requires a self-ignoring .gitignore", + ): + _create(DefaultManagedHeadlessSessionLineageStore(), anchor) + + assert marker.read_text(encoding="utf-8") == malformed + assert not (namespace / "records").exists() + + def test_cross_process_conflicting_creation_has_one_winner(tmp_path: Path) -> None: anchor = tmp_path / "project" anchor.mkdir() diff --git a/tests/execution/test_headless_capture_diagnostics.py b/tests/execution/test_headless_capture_diagnostics.py index 73d0101e3a..9b8c43fc34 100644 --- a/tests/execution/test_headless_capture_diagnostics.py +++ b/tests/execution/test_headless_capture_diagnostics.py @@ -99,6 +99,7 @@ def __init__(self, *, runner: Any, log_dir: str) -> None: completion_child_deferral_ceiling_seconds=1.0, ), providers=SimpleNamespace(provider_retry_limit=0), + rollout=SimpleNamespace(active=None), linux_tracing=SimpleNamespace( log_dir=log_dir, max_sessions=20, diff --git a/tests/execution/test_headless_dispatch.py b/tests/execution/test_headless_dispatch.py index 3e8562382a..f349369435 100644 --- a/tests/execution/test_headless_dispatch.py +++ b/tests/execution/test_headless_dispatch.py @@ -183,10 +183,11 @@ def close(self) -> None: monkeypatch.setattr(headless_launch, "logger", logger) with pytest.raises(RuntimeError) as caught: - with headless_launch._plugin_launch_binding( + with headless_launch.plugin_launch_binding_scope( authority=authority, backend=ClaudeCodeBackend(), load_mode=PluginLoadMode.EXPLICIT_PLUGIN_DIR, + on_suppressed_close_error=headless_launch._report_plugin_binding_close_failure, ): raise primary diff --git a/tests/execution/test_headless_enum_recovery.py b/tests/execution/test_headless_enum_recovery.py index b9944f5963..96c243507d 100644 --- a/tests/execution/test_headless_enum_recovery.py +++ b/tests/execution/test_headless_enum_recovery.py @@ -23,7 +23,12 @@ from autoskillit.recipe._contracts_types import SkillContract, SkillOutput from autoskillit.recipe.contracts import get_skill_contract, load_bundled_manifest from tests.conftest import _make_result -from tests.execution.conftest import _launch_inputs, _mock_backend, _success_session_json +from tests.execution.conftest import ( + _launch_inputs, + _mock_backend, + _success_session_json, + _unmarked_rollout_snapshot, +) pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] @@ -366,6 +371,7 @@ async def test_enum_nudge_success_requires_pattern_match(self, tmp_path): skill_contract=contract, launch_resolver=launch_resolver, launch_preparation=launch_preparation, + capture_rollout_on_launch=_unmarked_rollout_snapshot, natural_exit_grace_seconds=3.0, ) assert patched is not None @@ -388,6 +394,7 @@ async def test_enum_nudge_success_requires_pattern_match(self, tmp_path): skill_contract=contract, launch_resolver=launch_resolver, launch_preparation=launch_preparation, + capture_rollout_on_launch=_unmarked_rollout_snapshot, natural_exit_grace_seconds=3.0, ) assert rejected is None diff --git a/tests/execution/test_headless_execute.py b/tests/execution/test_headless_execute.py index b93cbe5e75..211554fa19 100644 --- a/tests/execution/test_headless_execute.py +++ b/tests/execution/test_headless_execute.py @@ -4,6 +4,7 @@ import json import os +from datetime import UTC, datetime, tzinfo from pathlib import Path from unittest.mock import patch @@ -11,7 +12,7 @@ import structlog import autoskillit.execution.headless._headless_execute as _patch_headless__headless_execute -from autoskillit.core import CmdSpec, ValidatedAddDir +from autoskillit.core import CmdSpec, RolloutMarker, ValidatedAddDir from autoskillit.core.types import SubprocessResult, TerminationReason from tests.execution.conftest import _sink_env @@ -38,6 +39,369 @@ def _success_result() -> SubprocessResult: ) +class _ControlledDateTime(datetime): + current: datetime = datetime(2026, 10, 10, 12, 0, 0, 499999, tzinfo=UTC) + + @classmethod + def now(cls, tz: tzinfo | None = None) -> datetime: + value = cls.current.replace(tzinfo=None) if tz is None else cls.current.astimezone(tz) + return cls.fromisoformat(value.isoformat()) + + +def _rollout_marker(change_id: str) -> RolloutMarker: + return { + "change_id": change_id, + "effective_at": "2026-10-10T12:00:00.500000+00:00", + "scope": {"skills": ["audit"], "backends": ["claude-code", "codex"], "roles": []}, + "expected_effect": f"Expected effect for {change_id}", + } + + +async def _run_rollout_execution( + ctx, + tmp_path: Path, + *, + backend_name: str = "claude-code", + pre_spawn_admission=None, +): + from autoskillit.execution.headless import _execute_claude_headless + from tests.execution.conftest import _launch_preparation, _mock_backend + + backend = _mock_backend(pty_required=True, channel_b_capable=True) + backend.name = backend_name + ctx.backend = backend + return await _execute_claude_headless( + lambda _binding, extras, _attempt_id=None: CmdSpec( + cmd=(backend_name, "--test"), env=dict(extras or {}) + ), + str(tmp_path), + ctx, + timeout=30.0, + stale_threshold=5.0, + step_name="rollout-test", + launch_resolver=ctx.launch_resolver, + launch_preparation=_launch_preparation(ctx, cwd=str(tmp_path)), + pre_spawn_admission=pre_spawn_admission, + ) + + +def _capture_rollout_flushes(monkeypatch, ctx, tmp_path: Path) -> list[dict[str, object]]: + import autoskillit.execution as execution + import autoskillit.execution.headless._headless_execute as execute_module + import autoskillit.execution.session_log.session_log as session_log + + log_dir = str(tmp_path / "rollout-logs") + ctx.config.linux_tracing.log_dir = log_dir + _install_fake_sink( + monkeypatch, + execute_module, + [], + log_dir, + close_raises=False, + ) + flushed: list[dict[str, object]] = [] + original_flush = session_log.flush_session_log + + def capture_flush(**kwargs): + flushed.append(kwargs) + original_flush(**kwargs) + + monkeypatch.setattr(session_log, "flush_session_log", capture_flush) + monkeypatch.setattr(execution, "flush_session_log", capture_flush) + return flushed + + +def _assert_rollout_artifacts(ctx, rollout: RolloutMarker | None, start_ts: str) -> None: + log_root = Path(ctx.config.linux_tracing.log_dir) + summary_paths = list(log_root.rglob("summary.json")) + assert len(summary_paths) == 1 + summary = json.loads(summary_paths[0].read_text()) + index_rows = [ + json.loads(line) + for line in (log_root / "sessions.jsonl").read_text().splitlines() + if line.strip() + ] + assert len(index_rows) == 1 + assert summary["rollout"] == rollout + assert summary["start_ts"] == start_ts + assert index_rows[0]["rollout"] == rollout + assert index_rows[0]["timestamp"] == start_ts + + +@pytest.mark.anyio +@pytest.mark.parametrize("backend_name", ["claude-code", "codex"]) +async def test_rollout_snapshot_is_taken_after_admission_for_both_backends( + minimal_ctx, tmp_path: Path, monkeypatch, backend_name: str +) -> None: + import autoskillit.execution.headless._headless_execute as execute_module + from tests.fakes import MockSubprocessRunner + + before_cutover = datetime.fromisoformat("2026-10-10T12:00:00.499999+00:00") + after_cutover = datetime.fromisoformat("2026-10-10T12:00:00.500001+00:00") + marker_a = _rollout_marker("rollout-a") + marker_b = _rollout_marker("rollout-b") + minimal_ctx.config.rollout.active = marker_a + monkeypatch.setattr(_ControlledDateTime, "current", before_cutover) + monkeypatch.setattr(execute_module, "datetime", _ControlledDateTime) + + runner = MockSubprocessRunner() + runner.set_default(_success_result()) + minimal_ctx.runner = runner + flushed = _capture_rollout_flushes(monkeypatch, minimal_ctx, tmp_path) + admission_calls = 0 + + async def admit_after_cutover(_contract): + nonlocal admission_calls + admission_calls += 1 + minimal_ctx.config.rollout.active = marker_b + monkeypatch.setattr(_ControlledDateTime, "current", after_cutover) + return None + + await _run_rollout_execution( + minimal_ctx, + tmp_path, + backend_name=backend_name, + pre_spawn_admission=admit_after_cutover, + ) + + assert admission_calls == 1 + assert len(runner.call_args_list) == 1 + runner_kwargs = runner.call_args_list[0][3] + assert runner_kwargs["rollout"] == marker_b + assert runner_kwargs["logical_run_start_ts"] == after_cutover.isoformat() + assert datetime.fromisoformat(runner_kwargs["logical_run_start_ts"]) >= datetime.fromisoformat( + marker_b["effective_at"] + ) + assert flushed[0]["rollout"] == marker_b + assert flushed[0]["start_ts"] == after_cutover.isoformat() + _assert_rollout_artifacts(minimal_ctx, marker_b, after_cutover.isoformat()) + + +@pytest.mark.anyio +async def test_admitted_inactive_run_uses_logical_start_over_later_result_start( + minimal_ctx, tmp_path: Path, monkeypatch +) -> None: + import autoskillit.execution.headless._headless_execute as execute_module + import autoskillit.execution.headless._headless_terminal as terminal + from tests.fakes import MockSubprocessRunner + + operation_start = datetime.fromisoformat("2026-10-10T12:00:00.499999+00:00") + logical_start = datetime.fromisoformat("2026-10-10T12:00:00.500001+00:00") + later_result_start = datetime.fromisoformat("2026-10-10T12:00:00.500003+00:00") + minimal_ctx.config.rollout.active = None + monkeypatch.setattr(_ControlledDateTime, "current", operation_start) + monkeypatch.setattr(execute_module, "datetime", _ControlledDateTime) + + runner = MockSubprocessRunner() + runner.set_default(_success_result()) + minimal_ctx.runner = runner + flushed = _capture_rollout_flushes(monkeypatch, minimal_ctx, tmp_path) + original_builder = terminal.build_terminal_flush_kwargs + observed_result_starts: list[str] = [] + + def set_later_result_start(**kwargs): + result = kwargs["result"] + assert result is not None + observed_result_starts.append(result.start_ts) + result.start_ts = later_result_start.isoformat() + return original_builder(**kwargs) + + monkeypatch.setattr(terminal, "build_terminal_flush_kwargs", set_later_result_start) + + async def admit(_contract): + monkeypatch.setattr(_ControlledDateTime, "current", logical_start) + return None + + await _run_rollout_execution(minimal_ctx, tmp_path, pre_spawn_admission=admit) + + runner_kwargs = runner.call_args_list[0][3] + assert runner_kwargs["rollout"] is None + assert runner_kwargs["logical_run_start_ts"] == logical_start.isoformat() + assert observed_result_starts == [operation_start.isoformat()] + assert later_result_start > logical_start + assert flushed[0]["rollout"] is None + assert flushed[0]["start_ts"] == logical_start.isoformat() + + +@pytest.mark.anyio +async def test_rollout_snapshot_survives_midrun_config_mutation_and_independent_run( + minimal_ctx, tmp_path: Path, monkeypatch +) -> None: + from copy import deepcopy + + import autoskillit.execution.headless._headless_execute as execute_module + from tests.fakes import MockSubprocessRunner + + marker_a = _rollout_marker("rollout-a") + marker_b = _rollout_marker("rollout-b") + minimal_ctx.config.rollout.active = deepcopy(marker_a) + monkeypatch.setattr( + _ControlledDateTime, + "current", + datetime.fromisoformat("2026-10-10T12:00:01+00:00"), + ) + monkeypatch.setattr(execute_module, "datetime", _ControlledDateTime) + + class MutatingRunner(MockSubprocessRunner): + rollout_calls: list[dict[str, object]] + + def __init__(self) -> None: + super().__init__() + self.rollout_calls = [] + self.set_default(_success_result()) + + async def __call__(self, *args, **kwargs): + self.rollout_calls.append(kwargs) + if len(self.rollout_calls) == 1: + configured = minimal_ctx.config.rollout.active + assert configured is not None + configured["scope"]["skills"].clear() + minimal_ctx.config.rollout.active = None + monkeypatch.setattr( + _ControlledDateTime, + "current", + datetime.fromisoformat("2026-10-10T12:00:02+00:00"), + ) + return await super().__call__(*args, **kwargs) + + runner = MutatingRunner() + minimal_ctx.runner = runner + flushed = _capture_rollout_flushes(monkeypatch, minimal_ctx, tmp_path) + + await _run_rollout_execution(minimal_ctx, tmp_path) + + assert runner.rollout_calls[0]["rollout"] == marker_a + assert runner.rollout_calls[0]["logical_run_start_ts"] == "2026-10-10T12:00:01+00:00" + assert flushed[0]["rollout"] == marker_a + assert flushed[0]["start_ts"] == runner.rollout_calls[0]["logical_run_start_ts"] + + minimal_ctx.config.rollout.active = marker_b + await _run_rollout_execution(minimal_ctx, tmp_path) + + assert runner.rollout_calls[1]["rollout"] == marker_b + assert runner.rollout_calls[1]["logical_run_start_ts"] == "2026-10-10T12:00:02+00:00" + assert flushed[1]["rollout"] == marker_b + assert flushed[1]["start_ts"] == runner.rollout_calls[1]["logical_run_start_ts"] + + +@pytest.mark.anyio +async def test_contract_nudge_reuses_executor_rollout_snapshot_after_config_changes( + minimal_ctx, tmp_path: Path, monkeypatch +) -> None: + import autoskillit.execution.headless._headless_execute as execute_module + from autoskillit.core import RetryReason, SkillResult + from tests.fakes import MockSubprocessRunner + + marker_a = _rollout_marker("rollout-a") + marker_b = _rollout_marker("rollout-b") + minimal_ctx.config.rollout.active = marker_a + monkeypatch.setattr( + _ControlledDateTime, + "current", + datetime.fromisoformat("2026-10-10T12:00:03+00:00"), + ) + monkeypatch.setattr(execute_module, "datetime", _ControlledDateTime) + + class MutatingRunner(MockSubprocessRunner): + observed_kwargs: dict[str, object] | None = None + + async def __call__(self, *args, **kwargs): + self.observed_kwargs = kwargs + minimal_ctx.config.rollout.active = marker_b + monkeypatch.setattr( + _ControlledDateTime, + "current", + datetime.fromisoformat("2026-10-10T12:00:04+00:00"), + ) + return await super().__call__(*args, **kwargs) + + runner = MutatingRunner() + runner.set_default(_success_result()) + minimal_ctx.runner = runner + flushed = _capture_rollout_flushes(monkeypatch, minimal_ctx, tmp_path) + retry_result = SkillResult( + success=False, + result="missing completion marker", + session_id="nudge-session", + subtype="contract_recovery", + is_error=False, + exit_code=1, + needs_retry=True, + retry_reason=RetryReason.CONTRACT_RECOVERY, + stderr="", + ) + nudge_snapshots: list[tuple[RolloutMarker | None, str]] = [] + nudge_configs: list[object] = [] + + monkeypatch.setattr( + execute_module, + "_build_skill_result", + lambda *_args, **_kwargs: retry_result, + ) + + async def nudge_with_shared_snapshot(*_args, **kwargs): + nudge_snapshots.append(kwargs["capture_rollout_on_launch"]()) + nudge_configs.append(kwargs["linux_tracing_config"]) + return SkillResult( + success=True, + result="recovered", + session_id="nudge-session", + subtype="success", + is_error=False, + exit_code=0, + needs_retry=False, + retry_reason=RetryReason.NONE, + stderr="", + ) + + monkeypatch.setattr(execute_module, "_attempt_contract_nudge", nudge_with_shared_snapshot) + + await _run_rollout_execution(minimal_ctx, tmp_path) + + assert runner.observed_kwargs is not None + main_snapshot = ( + runner.observed_kwargs["rollout"], + runner.observed_kwargs["logical_run_start_ts"], + ) + assert main_snapshot == (marker_a, "2026-10-10T12:00:03+00:00") + assert nudge_snapshots == [main_snapshot] + assert nudge_configs == [minimal_ctx.config.linux_tracing] + assert flushed[0]["rollout"] == marker_a + assert flushed[0]["start_ts"] == main_snapshot[1] + + +@pytest.mark.anyio +async def test_launched_runner_failure_keeps_rollout_marker_in_flush( + minimal_ctx, tmp_path: Path, monkeypatch +) -> None: + from tests.fakes import MockSubprocessRunner + + marker = _rollout_marker("rollout-failure") + minimal_ctx.config.rollout.active = marker + + class FailingRunner(MockSubprocessRunner): + observed_kwargs: dict[str, object] | None = None + + async def __call__(self, *args, **kwargs): + self.observed_kwargs = kwargs + raise RuntimeError("runner failed after launch") + + runner = FailingRunner() + minimal_ctx.runner = runner + flushed = _capture_rollout_flushes(monkeypatch, minimal_ctx, tmp_path) + + result = await _run_rollout_execution(minimal_ctx, tmp_path) + + assert result.subtype == "crashed" + assert runner.observed_kwargs is not None + assert runner.observed_kwargs["rollout"] == marker + logical_start = runner.observed_kwargs["logical_run_start_ts"] + assert isinstance(logical_start, str) and logical_start + assert flushed[0]["rollout"] == marker + assert flushed[0]["start_ts"] == logical_start + _assert_rollout_artifacts(minimal_ctx, marker, logical_start) + + def _install_fake_sink( monkeypatch, execute_module, diff --git a/tests/execution/test_linux_tracing.py b/tests/execution/test_linux_tracing.py index 032851e2ee..d480126f82 100644 --- a/tests/execution/test_linux_tracing.py +++ b/tests/execution/test_linux_tracing.py @@ -6,10 +6,12 @@ import os import sys from datetime import datetime +from unittest.mock import Mock import anyio import pytest +from tests._helpers import make_rollout_marker from tests.conftest import production_interpreter_env pytestmark = [ @@ -450,21 +452,36 @@ async def test_start_linux_tracing_writes_enrollment_sidecar(tmp_path): import anyio from autoskillit.execution.evidence.linux_tracing import ( + read_enrollment, start_linux_tracing, trace_target_from_pid, ) from tests._helpers import make_tracing_config cfg = make_tracing_config(enabled=True, proc_interval=0.1, tmpfs_path=str(tmp_path)) + rollout = make_rollout_marker() + logical_run_start_ts = "2026-10-10T19:00:00+00:00" async with anyio.create_task_group() as tg: proc = await anyio.open_process(["sleep", "2"]) - handle = start_linux_tracing(trace_target_from_pid(proc.pid), cfg, tg) + handle = start_linux_tracing( + trace_target_from_pid(proc.pid), + cfg, + tg, + rollout=rollout, + logical_run_start_ts=logical_run_start_ts, + ) assert handle is not None enrollment = tmp_path / f"autoskillit_enrollment_{proc.pid}.json" assert enrollment.exists() data = json.loads(enrollment.read_text()) - assert data["schema_version"] == 2 + assert data["schema_version"] == 3 assert data["pid"] == proc.pid + assert data["rollout"] == rollout + assert data["logical_run_start_ts"] == logical_run_start_ts + record = read_enrollment(enrollment) + assert record is not None + assert record.rollout == rollout + assert record.logical_run_start_ts == logical_run_start_ts try: handle.stop() finally: @@ -472,6 +489,48 @@ async def test_start_linux_tracing_writes_enrollment_sidecar(tmp_path): assert not enrollment.exists(), "Enrollment must be deleted by stop()" +@pytest.mark.parametrize("schema_version", [1, 2]) +def test_read_enrollment_defaults_rollout_fields_for_historical_schemas(tmp_path, schema_version): + from autoskillit.execution.evidence.linux_tracing import read_enrollment + + data = {"schema_version": schema_version, "pid": 12345} + if schema_version == 2: + data["comm"] = "claude" + enrollment = tmp_path / "enrollment.json" + enrollment.write_text(json.dumps(data)) + + record = read_enrollment(enrollment) + + assert record is not None + assert record.schema_version == schema_version + assert record.rollout is None + assert record.logical_run_start_ts == "" + + +@pytest.mark.parametrize("contents", [None, "{"]) +def test_read_enrollment_logs_invalid_sidecar_but_not_missing(tmp_path, monkeypatch, contents): + from autoskillit.execution.evidence import linux_tracing + + enrollment = tmp_path / "enrollment.json" + if contents is not None: + enrollment.write_text(contents) + logger = Mock() + monkeypatch.setattr(linux_tracing, "logger", logger) + + assert linux_tracing.read_enrollment(enrollment) is None + + if contents is None: + logger.warning.assert_not_called() + else: + logger.warning.assert_called_once() + message, path, error_type, error = logger.warning.call_args.args + assert "Failed to read enrollment sidecar" in message + assert path == enrollment + assert error_type == "JSONDecodeError" + assert isinstance(error, json.JSONDecodeError) + assert str(error) + + # --- LinuxTracingConfig guard tests --- diff --git a/tests/execution/test_model_backend_launch_contract.py b/tests/execution/test_model_backend_launch_contract.py index a7e46f5125..173b2f1843 100644 --- a/tests/execution/test_model_backend_launch_contract.py +++ b/tests/execution/test_model_backend_launch_contract.py @@ -9,7 +9,7 @@ from __future__ import annotations from typing import Any -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, Mock, patch import pytest @@ -17,7 +17,7 @@ from autoskillit.core import LaunchValueSourceKind from autoskillit.core.types import LaunchContractError, RetryReason, SkillResult -from .conftest import _backend_authority, _launch_inputs, _mock_backend +from .conftest import _backend_authority, _launch_inputs, _mock_backend, _unmarked_rollout_snapshot pytestmark = [pytest.mark.layer("execution"), pytest.mark.small, pytest.mark.model_contract] @@ -166,7 +166,7 @@ async def test_retired_codex_model_fails_during_real_command_preparation_before_ """The real Codex command builder rejects retired targets before runner invocation.""" from autoskillit.execution.headless._headless_launch import _run_headless_attempt - kwargs, runner = _build_real_codex_attempt_kwargs_for_retired_model(model_id, tmp_path) + kwargs, runner = _build_codex_attempt_kwargs(model_id, tmp_path) with pytest.raises(ValueError): await _run_headless_attempt(kwargs.pop("build_spec"), runner=runner, **kwargs) @@ -174,15 +174,38 @@ async def test_retired_codex_model_fails_during_real_command_preparation_before_ runner.assert_not_awaited() -def _build_real_codex_attempt_kwargs_for_retired_model( +@pytest.mark.anyio +async def test_rejected_admission_does_not_capture_rollout_or_invoke_runner(tmp_path) -> None: + from autoskillit.core import CandidatePreSpawnRejection + from autoskillit.execution.headless._headless_launch import _run_headless_attempt + + kwargs, runner = _build_codex_attempt_kwargs("gpt-6-sol", tmp_path) + capture = Mock(side_effect=_unmarked_rollout_snapshot) + kwargs["capture_rollout_on_launch"] = capture + + async def reject(contract): + return CandidatePreSpawnRejection( + reason="candidate is not admitted", + attempted_contract=contract, + ) + + kwargs["pre_spawn_admission"] = reject + build_spec = kwargs.pop("build_spec") + result = await _run_headless_attempt(build_spec, runner=runner, **kwargs) + + assert isinstance(result, CandidatePreSpawnRejection) + capture.assert_not_called() + runner.assert_not_awaited() + + +def _build_codex_attempt_kwargs( model_id: str, tmp_path, ) -> tuple[dict[str, Any], AsyncMock]: - """Build the kwargs dict for one retired-Codex-model launch attempt. + """Build the kwargs dict for one real Codex launch attempt. A new required parameter on ``_run_headless_attempt`` will break this helper - at its single definition site, which is the desired locality for a contract - test that pins the function's full parameter surface. + at this single fixture site. """ from autoskillit.core import PluginLoadMode from autoskillit.execution.backends import CodexBackend @@ -192,7 +215,9 @@ def _build_real_codex_attempt_kwargs_for_retired_model( runner = AsyncMock() def build_spec(_binding, _extras, _managed_attempt_id=None): - return backend.build_headless_cmd("test prompt", model=model_id) + return backend.build_headless_cmd( + "test prompt", model=model_id, generated_home=tmp_path / "codex-home" + ) return ( { @@ -201,6 +226,7 @@ def build_spec(_binding, _extras, _managed_attempt_id=None): "launch_resolver": resolver, "launch_preparation": preparation, "expected_launch_contract": None, + "capture_rollout_on_launch": _unmarked_rollout_snapshot, "plugin_authority": None, "plugin_load_mode": PluginLoadMode.NONE, "provider_extras": None, diff --git a/tests/execution/test_process_run.py b/tests/execution/test_process_run.py index f81a966e48..ff80039375 100644 --- a/tests/execution/test_process_run.py +++ b/tests/execution/test_process_run.py @@ -338,6 +338,42 @@ def test_runner_pass_fds_defaults_match_protocol(self): == () ) + @pytest.mark.anyio + async def test_default_runner_forwards_rollout_and_logical_run_start( + self, monkeypatch, tmp_path + ): + from autoskillit.execution.process import DefaultSubprocessRunner + from tests.conftest import _make_result + + rollout = { + "change_id": "prompt-cost-study", + "effective_at": "2026-11-01T18:00:00+00:00", + "scope": {"skills": ["make-plan"], "backends": ["codex"], "roles": ["planner"]}, + "expected_effect": "Reduce planning input tokens while preserving coverage.", + } + logical_run_start_ts = "2026-10-10T19:00:00+00:00" + captured: dict[str, object] = {} + expected = _make_result(returncode=0) + + async def fake_run_managed_async(_cmd, **kwargs): + captured.update(kwargs) + return expected + + import autoskillit.execution.process as process + + monkeypatch.setattr(process, "run_managed_async", fake_run_managed_async) + result = await DefaultSubprocessRunner()( + ["python", "-c", "pass"], + cwd=tmp_path, + timeout=10, + rollout=rollout, + logical_run_start_ts=logical_run_start_ts, + ) + + assert result is expected + assert captured["rollout"] is rollout + assert captured["logical_run_start_ts"] == logical_run_start_ts + class TestTracingStopOnException: """Verify tracing_handle.stop() is called on BaseException in run_managed_async.""" @@ -384,6 +420,35 @@ def patched_stop(self_handle: LinuxTracingHandle) -> list: assert len(stop_called) >= 1 +@pytest.mark.anyio +@pytest.mark.skipif(sys.platform != "linux", reason="Linux-only tracing") +async def test_run_managed_async_forwards_rollout_to_trace_enrollment(monkeypatch, tmp_path): + from autoskillit.execution.evidence import linux_tracing + from tests._helpers import make_rollout_marker, make_tracing_config + + rollout = make_rollout_marker(backends=("codex",)) + logical_run_start_ts = "2026-10-10T19:00:00+00:00" + forwarded: list[tuple[object, str]] = [] + + def capture_enrollment(*, rollout=None, logical_run_start_ts="", **_kwargs): + forwarded.append((rollout, logical_run_start_ts)) + + monkeypatch.setattr(linux_tracing, "start_linux_tracing", capture_enrollment) + cfg = make_tracing_config(enabled=True, proc_interval=0.1, tmpfs_path=str(tmp_path)) + + result = await run_managed_async( + [sys.executable, "-c", "pass"], + cwd=tmp_path, + timeout=10, + linux_tracing_config=cfg, + rollout=rollout, + logical_run_start_ts=logical_run_start_ts, + ) + + assert result.returncode == 0 + assert forwarded == [(rollout, logical_run_start_ts)] + + class TestManagedProcessOwnership: """The subprocess boundary, not a server supervisor, owns process cleanup.""" diff --git a/tests/execution/test_recording.py b/tests/execution/test_recording.py index b9a00887ad..dcd6f25130 100644 --- a/tests/execution/test_recording.py +++ b/tests/execution/test_recording.py @@ -27,6 +27,7 @@ _detect_backend_format, _extract_model, ) +from tests._helpers import make_rollout_marker from tests.conftest import _make_result from tests.execution.backends._plugin_binding import plugin_binding from tests.fakes import MockSubprocessRunner @@ -1036,6 +1037,8 @@ async def test_recording_runner_forwards_process_lifecycle_callbacks(tmp_path: P ) on_spawned = Mock() on_reaped = Mock() + rollout = make_rollout_marker(backends=("codex",)) + logical_run_start_ts = "2026-10-10T19:00:00+00:00" await runner( ["codex", "exec", "do something"], @@ -1044,11 +1047,15 @@ async def test_recording_runner_forwards_process_lifecycle_callbacks(tmp_path: P env={"SCENARIO_STEP_NAME": "recorded"}, on_process_spawned=on_spawned, on_process_reaped=on_reaped, + rollout=rollout, + logical_run_start_ts=logical_run_start_ts, ) forwarded = inner.call_args_list[0][3] assert forwarded["on_process_spawned"] is on_spawned assert forwarded["on_process_reaped"] is on_reaped + assert forwarded["rollout"] is rollout + assert forwarded["logical_run_start_ts"] == logical_run_start_ts @pytest.mark.anyio @@ -1098,6 +1105,7 @@ async def test_replay_runner_never_invokes_process_lifecycle_callbacks(tmp_path: ) on_spawned = Mock() on_reaped = Mock() + rollout = make_rollout_marker(backends=("codex",)) await runner( ["codex", "exec", "do something"], @@ -1106,6 +1114,8 @@ async def test_replay_runner_never_invokes_process_lifecycle_callbacks(tmp_path: env={"SCENARIO_STEP_NAME": "recorded"}, on_process_spawned=on_spawned, on_process_reaped=on_reaped, + rollout=rollout, + logical_run_start_ts="2026-10-10T19:00:00+00:00", ) on_spawned.assert_not_called() diff --git a/tests/execution/test_session_index_roundtrip.py b/tests/execution/test_session_index_roundtrip.py index c1f85baf51..8eb6196790 100644 --- a/tests/execution/test_session_index_roundtrip.py +++ b/tests/execution/test_session_index_roundtrip.py @@ -3,9 +3,11 @@ from __future__ import annotations import json +from datetime import datetime import pytest +from autoskillit.execution.session_log.session_index import iter_tolerant_session_index_lines from tests.execution.conftest import _flush pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] @@ -38,6 +40,7 @@ def test_appending_current_schema_row_preserves_retained_v8_row_bytes(self, tmp_ index_path = tmp_path / "sessions.jsonl" retained = json.loads(index_path.read_text().strip()) retained["schema_version"] = 8 + retained.pop("rollout") retained.pop("subagent_model_outcomes") retained_line = json.dumps(retained, sort_keys=True, separators=(",", ":")) index_path.write_text(retained_line + "\n") @@ -59,6 +62,51 @@ def test_appending_current_schema_row_preserves_retained_v8_row_bytes(self, tmp_ assert lines[0] == retained_line old_row, new_row = map(json.loads, lines) assert old_row["schema_version"] == 8 + assert "rollout" not in old_row assert "subagent_model_outcomes" not in old_row assert new_row["schema_version"] == SESSION_INDEX_SCHEMA_VERSION assert new_row["subagent_model_outcomes"] == [outcome] + + def test_streaming_reader_classifies_exact_rollout_cutover_boundary(self, tmp_path): + index_path = tmp_path / "sessions.jsonl" + effective_at = "2026-11-01T18:00:00.500000+00:00" + starts = ( + ("before", "2026-11-01T18:00:00.499999+00:00"), + ("equal", "2026-11-01T18:00:00.500000+00:00"), + ("after", "2026-11-01T18:00:00.500001+00:00"), + ) + marker = { + "change_id": "prompt-cost-study", + "effective_at": effective_at, + "scope": {"skills": [], "backends": [], "roles": []}, + "expected_effect": "Reduce planning input tokens.", + } + rows = [{"timestamp": timestamp, "rollout": marker} for _name, timestamp in starts] + rows.extend( + [ + {"timestamp": "2026-11-01T18:00:00.500000+00:00"}, + {"timestamp": "2026-11-01T18:00:00.500000+00:00", "rollout": None}, + ] + ) + index_path.write_text("".join(json.dumps(row) + "\n" for row in rows)) + + cohorts = [] + unassigned = 0 + for _end_offset, row in iter_tolerant_session_index_lines(index_path): + if row is None: + continue + rollout = row.get("rollout") + if rollout is None: + unassigned += 1 + continue + started_at = datetime.fromisoformat(row["timestamp"]) + cutover_at = datetime.fromisoformat(rollout["effective_at"]) + if started_at < cutover_at: + cohorts.append("before") + elif started_at >= cutover_at: + cohorts.append("after") + else: + raise AssertionError("timezone-aware timestamps must have an ordering") + + assert cohorts == ["before", "after", "after"] + assert unassigned == 2 diff --git a/tests/execution/test_session_log_backend_source.py b/tests/execution/test_session_log_backend_source.py index 1660484855..b3b821fd4b 100644 --- a/tests/execution/test_session_log_backend_source.py +++ b/tests/execution/test_session_log_backend_source.py @@ -111,6 +111,7 @@ def test_session_index_entry_default_value(self) -> None: "adjudication_verdict": None, "native_shell_capture": None, "session_type": None, + "rollout": None, "subagent_model_outcomes": [], "child_outcomes": [], "schema_version": SESSION_INDEX_SCHEMA_VERSION, diff --git a/tests/execution/test_session_log_fields.py b/tests/execution/test_session_log_fields.py index ca588bbb56..4558bc6059 100644 --- a/tests/execution/test_session_log_fields.py +++ b/tests/execution/test_session_log_fields.py @@ -21,6 +21,7 @@ SessionType, ) from autoskillit.core.types.results._type_results import ModelIdentity +from tests._helpers import make_rollout_marker from tests.execution.conftest import ( _flush, _make_cc_jsonl_record, @@ -32,6 +33,7 @@ pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] _SYNTHETIC_MODEL_ID = "gpt-synthetic" +_ROLLOUT_MARKER = make_rollout_marker() class _FakeLocator: @@ -2103,3 +2105,29 @@ def test_missing_evidence_worktree_persists_not_observed_defaults(tmp_path): assert row["evidence_worktree_path"] == "" assert row["evidence_worktree_source"] == "not_observed" assert row["evidence_worktree_detail"] == "" + + +def test_rollout_marker_is_identical_in_summary_and_index(tmp_path): + _flush( + tmp_path, + session_id="marked-rollout", + rollout=_ROLLOUT_MARKER, + proc_snapshots=None, + ) + + summary = json.loads((tmp_path / "sessions" / "marked-rollout" / "summary.json").read_text()) + index = json.loads((tmp_path / "sessions.jsonl").read_text().strip()) + + assert summary["rollout"] == _ROLLOUT_MARKER + assert index["rollout"] == _ROLLOUT_MARKER + assert summary["rollout"] == index["rollout"] + + +def test_inactive_rollout_is_null_in_summary_and_index(tmp_path): + _flush(tmp_path, session_id="unmarked-rollout", proc_snapshots=None) + + summary = json.loads((tmp_path / "sessions" / "unmarked-rollout" / "summary.json").read_text()) + index = json.loads((tmp_path / "sessions.jsonl").read_text().strip()) + + assert summary["rollout"] is None + assert index["rollout"] is None diff --git a/tests/execution/test_session_log_retention.py b/tests/execution/test_session_log_retention.py index 11bb39a506..7355fef8d4 100644 --- a/tests/execution/test_session_log_retention.py +++ b/tests/execution/test_session_log_retention.py @@ -27,6 +27,7 @@ ) from autoskillit.execution.session_log.session_index import read_tolerant_session_index_rows from autoskillit.fleet import FLEET_STATE_SCHEMA_VERSION, build_protected_campaign_ids +from tests._helpers import make_rollout_marker from tests._retention_surface import ( RECLAIMER_CONVERGENCE_CASES, assert_second_pass_is_quiet, @@ -206,11 +207,12 @@ def test_recover_crashed_sessions_finalizes_orphaned_file(tmp_path): """recover_crashed_sessions reads tmpfs file and writes permanent session dir.""" tmpfs = tmp_path / "shm" tmpfs.mkdir() - _write_old_trace( + trace = _write_old_trace( tmpfs, "autoskillit_trace_12345.jsonl", json.dumps({"vm_rss_kb": 500, "captured_at": "2026-03-03T10:00:00+00:00"}) + "\n", ) + expected_start_ts = datetime.fromtimestamp(trace.stat().st_mtime, tz=UTC).isoformat() count = recover_crashed_sessions(tmpfs_path=str(tmpfs), log_dir=str(tmp_path / "logs")) assert count == 1 sessions = list((tmp_path / "logs" / "sessions").iterdir()) @@ -219,6 +221,7 @@ def test_recover_crashed_sessions_finalizes_orphaned_file(tmp_path): assert (sessions[0] / "summary.json").exists() summary = json.loads((sessions[0] / "summary.json").read_text()) assert summary["termination_reason"] == "CRASHED" + assert summary["start_ts"] == expected_start_ts assert summary["success"] is False assert summary["infra_exit_category"] == "unclassified" assert summary["infra_cleanup_incomplete"] is False @@ -228,10 +231,52 @@ def test_recover_crashed_sessions_finalizes_orphaned_file(tmp_path): assert len(index_rows) == 1 index = json.loads(index_rows[0]) assert index["infra_exit_category"] == "unclassified" + assert index["timestamp"] == expected_start_ts assert index["infra_cleanup_incomplete"] is False assert index["infra_fault_domain"] == "infrastructure" +def test_recovery_preserves_interrupted_nudge_rollout_and_logical_start(tmp_path): + tmpfs = tmp_path / "shm" + tmpfs.mkdir() + pid = 99992 + trace = _write_old_trace( + tmpfs, + f"autoskillit_trace_{pid}.jsonl", + json.dumps( + { + "vm_rss_kb": 500, + "captured_at": "2026-03-03T10:00:00+00:00", + "comm": "claude", + } + ) + + "\n", + ) + enrollment = tmpfs / f"autoskillit_enrollment_{pid}.json" + sidecar = json.loads(enrollment.read_text()) + rollout = make_rollout_marker() + logical_run_start_ts = "2000-01-01T00:00:00+00:00" + sidecar.update( + schema_version=3, + comm="claude", + rollout=rollout, + logical_run_start_ts=logical_run_start_ts, + ) + enrollment.write_text(json.dumps(sidecar)) + trace_mtime_ts = datetime.fromtimestamp(trace.stat().st_mtime, tz=UTC).isoformat() + + assert recover_crashed_sessions(tmpfs_path=str(tmpfs), log_dir=str(tmp_path / "logs")) == 1 + + session_dir = next((tmp_path / "logs" / "sessions").iterdir()) + summary = json.loads((session_dir / "summary.json").read_text()) + index = json.loads((tmp_path / "logs" / "sessions.jsonl").read_text().splitlines()[0]) + assert logical_run_start_ts != trace_mtime_ts + assert summary["start_ts"] == logical_run_start_ts + assert index["timestamp"] == logical_run_start_ts + assert summary["rollout"] == rollout + assert index["rollout"] == rollout + + def test_recovery_replay_upserts_without_rewriting_committed_directory( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/fleet/test_dispatch_checkout_cleanliness.py b/tests/fleet/test_dispatch_checkout_cleanliness.py new file mode 100644 index 0000000000..d11aa6bd55 --- /dev/null +++ b/tests/fleet/test_dispatch_checkout_cleanliness.py @@ -0,0 +1,74 @@ +"""Ensure a real fleet dispatch adds no git-visible AutoSkillit state.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from autoskillit.core.io import _AUTOSKILLIT_GITIGNORE_ENTRIES +from autoskillit.fleet import execute_dispatch +from tests._git_topology import commit_file, init_checkout +from tests.fleet._helpers import ( + _noop_quota_refresher, + _setup_dispatch, + _simple_prompt_builder, +) + +pytestmark = [ + pytest.mark.layer("fleet"), + pytest.mark.medium, + pytest.mark.feature("fleet"), + pytest.mark.usefixtures("isolated_git_config"), +] + + +def _autoskillit_status(project_dir: Path) -> set[str]: + output = subprocess.run( + [ + "git", + "status", + "--porcelain=v1", + "--untracked-files=all", + "--", + ".autoskillit", + ], + cwd=project_dir, + check=True, + capture_output=True, + text=True, + ).stdout + return {line for line in output.splitlines() if line.strip()} + + +@pytest.mark.anyio +async def test_dispatch_adds_nothing_git_visible_under_autoskillit( + tool_ctx, monkeypatch: pytest.MonkeyPatch +) -> None: + project_dir = tool_ctx.project_dir + init_checkout(project_dir) + commit_file( + project_dir, + ".autoskillit/.gitignore", + "\n".join(_AUTOSKILLIT_GITIGNORE_ENTRIES) + "\n", + "add AutoSkillit ignore entries", + ) + _setup_dispatch(tool_ctx, monkeypatch) + before = _autoskillit_status(project_dir) + + await execute_dispatch( + tool_ctx=tool_ctx, + recipe="test-recipe", + task="task", + ingredients=None, + dispatch_name="dispatch", + timeout_sec=None, + prompt_builder=_simple_prompt_builder, + quota_refresher=_noop_quota_refresher, + ) + + assert len(tool_ctx.executor.dispatch_calls) == 1 + records_dir = project_dir / ".autoskillit" / "managed-headless-session-lineage" / "records" + assert records_dir.is_dir() and any(records_dir.iterdir()) + assert _autoskillit_status(project_dir) - before == set() diff --git a/tests/fleet/test_dispatch_envelope_fields.py b/tests/fleet/test_dispatch_envelope_fields.py index e87992f5fb..2f409b6f67 100755 --- a/tests/fleet/test_dispatch_envelope_fields.py +++ b/tests/fleet/test_dispatch_envelope_fields.py @@ -87,7 +87,14 @@ async def test_failure_payload_retains_exact_optional_evidence(self, tool_ctx, m _setup_dispatch(tool_ctx, monkeypatch) payload = { "success": False, - "reason": "invariant violated: exact write evidence was not retained", + "reason": ( + "uncommitted_changes: bootstrap_clone failed at " + "/tmp/project/.autoskillit/managed-headless-session-lineage/records/" + "0123456789abcdef0123456789abcdef.json " + "attempt_id=deaddeaddeaddeaddeaddeaddeaddead " + "launch_id=feedfacefeedfacefeedfacefeedface " + "requested_mode=direct effective_mode=capture lineage_status=corrupt" + ), "summary": "Outcome invariant failed.", "reason_kind": "outcome_invariant", "outcome_fields": {"write_count": 0, "required_writes": 1}, @@ -110,6 +117,19 @@ async def test_failure_payload_retains_exact_optional_evidence(self, tool_ctx, m assert result["success"] is False assert result["dispatch_status"] == "failure" assert result["l3_parse_source"] == "stdout" + assert "uncommitted_changes" in result["reason"] + assert "bootstrap_clone" in result["reason"] + for private_detail in ( + "managed-headless-session-lineage", + "0123456789abcdef", + "attempt_id", + "launch_id", + "requested_mode", + "effective_mode", + "lineage_status", + ): + assert private_detail not in result["reason"] + assert private_detail not in result["error"] # Independent assertions: a silent drop or rename of any single # evidence field names the regression, instead of passing through a # wholesale equality check that lets transforms hide. diff --git a/tests/fleet/test_dispatch_outcome_classifier.py b/tests/fleet/test_dispatch_outcome_classifier.py index d72b006de6..0b316ebd5c 100644 --- a/tests/fleet/test_dispatch_outcome_classifier.py +++ b/tests/fleet/test_dispatch_outcome_classifier.py @@ -215,6 +215,105 @@ def test_diagnostic_payload_reason_uses_standard_fleet_failure_wording(self): assert reason == FleetErrorCode.FLEET_L3_STARTUP_OR_CRASH assert "feedface" not in reason + @pytest.mark.parametrize( + "private_record", + [ + ".autoskillit/managed-headless-session-lineage/", + "/tmp/project/.autoskillit/managed-headless-session-lineage/records/" + "0123456789abcdef0123456789abcdef.json", + ], + ) + def test_failed_reason_preserves_uncommitted_changes_cause(self, private_record): + diagnostic = ( + f"uncommitted_changes: bootstrap_clone failed at {private_record}; " + "cache path /tmp/project/cache/clone.lock" + ) + parsed = L3ParseResult( + outcome="completed_clean", + payload={"success": False, "reason": diagnostic}, + raw_body=None, + parse_error=None, + source="stdout", + ) + + status, reason = classify_dispatch_outcome( + parsed, + dataclasses.replace(_DEFAULT_SKILL_RESULT, success=False), + ) + + assert status == DispatchStatus.FAILURE + assert reason == ( + "uncommitted_changes: bootstrap_clone failed at ; " + "cache path /tmp/project/cache/clone.lock" + ) + assert private_record not in reason + + def test_failed_reason_redacts_inline_capture_fields_but_keeps_cause(self): + private_record = ( + "/tmp/project/.autoskillit/managed-headless-session-lineage/records/" + "abcdef0123456789abcdef0123456789.json" + ) + diagnostic = ( + f"bootstrap_clone failed at {private_record} " + "attempt_id=deaddeaddeaddeaddeaddeaddeaddead " + "launch_id=feedfacefeedfacefeedfacefeedface " + "requested_mode=direct effective_mode=capture " + "lineage_status=corrupt native_shell_capture=enabled" + ) + parsed = L3ParseResult( + outcome="completed_clean", + payload={"success": False, "reason": diagnostic}, + raw_body=None, + parse_error=None, + source="stdout", + ) + + status, reason = classify_dispatch_outcome( + parsed, + dataclasses.replace(_DEFAULT_SKILL_RESULT, success=False), + ) + + assert status == DispatchStatus.FAILURE + assert reason == ( + "bootstrap_clone failed at " + " " + ) + for private_value in ( + private_record, + "deaddead", + "feedface", + "direct", + "capture", + "corrupt", + "enabled", + ): + assert private_value not in reason + + @pytest.mark.parametrize( + "private_record", + [ + ".autoskillit/managed-headless-session-lineage/", + "/tmp/project/.autoskillit/managed-headless-session-lineage/records/" + "0123456789abcdef0123456789abcdef.json", + ], + ) + def test_private_path_only_uses_standard_startup_failure(self, private_record): + parsed = L3ParseResult( + outcome="completed_clean", + payload={"success": False, "reason": private_record}, + raw_body=None, + parse_error=None, + source="stdout", + ) + + status, reason = classify_dispatch_outcome( + parsed, + dataclasses.replace(_DEFAULT_SKILL_RESULT, success=False), + ) + + assert status == DispatchStatus.FAILURE + assert reason == FleetErrorCode.FLEET_L3_STARTUP_OR_CRASH + class TestReasonAwareResumePolicy: def test_idle_stall_with_progress_is_failure(self): diff --git a/tests/infra/_e2e_evidence.py b/tests/infra/_e2e_evidence.py new file mode 100644 index 0000000000..b847eee377 --- /dev/null +++ b/tests/infra/_e2e_evidence.py @@ -0,0 +1,49 @@ +"""Load and materialize captured E2E session evidence fixtures.""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import Any + +_PROJECTS_TOKEN = "@CLAUDE_PROJECTS@" + + +def _fixture_dir(name: str) -> Path: + return Path(__file__).with_name("fixtures") / "e2e_sessions" / name + + +def load_provenance(name: str) -> dict[str, Any]: + """Read the provenance record for a captured E2E run.""" + return json.loads((_fixture_dir(name) / "PROVENANCE.json").read_text(encoding="utf-8")) + + +def _replace_projects_token(path: Path, projects: Path) -> None: + text = path.read_text(encoding="utf-8") + path.write_text(text.replace(_PROJECTS_TOKEN, str(projects)), encoding="utf-8") + + +def materialize_run( + name: str, + *, + trace: Path, + log_root: Path, + projects: Path, +) -> dict[str, Any]: + """Copy a captured run to the given locations and return its envelope.""" + fixture = _fixture_dir(name) + trace.parent.mkdir(parents=True, exist_ok=True) + log_root.mkdir(parents=True, exist_ok=True) + projects.mkdir(parents=True, exist_ok=True) + + shutil.copyfile(fixture / "out/session-events.jsonl", trace) + shutil.copytree(fixture / "logs", log_root, dirs_exist_ok=True) + shutil.copytree(fixture / "projects", projects, dirs_exist_ok=True) + + _replace_projects_token(trace, projects) + _replace_projects_token(log_root / "sessions.jsonl", projects) + for summary in (log_root / "sessions").glob("*/summary.json"): + _replace_projects_token(summary, projects) + + return json.loads((fixture / "out/envelope.json").read_text(encoding="utf-8")) diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/PROVENANCE.json b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/PROVENANCE.json new file mode 100644 index 0000000000..77b3700720 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/PROVENANCE.json @@ -0,0 +1,9 @@ +{ + "producer": "claude-code", + "cli_version": "2.1.280", + "source_run": "38170521660", + "issue_url": "https://github.com/TalonT-Org/autoskillit-e2e-sandbox/issues/1", + "repository": "TalonT-Org/autoskillit-e2e-sandbox", + "claim_label": "in-progress", + "sanitization": "Derived from the retained run artifact. The root transcript retains rows 1-3, 54-55, 61-62, 66-67, 81, 84, 88-89, 93-94, 547-548, 552-553, 569, 679, 680, and 682; each retained row keeps every source key. Worker transcripts retain their first three and final two rows; native subagent transcripts retain their first and final rows. Prose strings longer than 400 characters are truncated with a removed-character marker, while the claim and branch tool inputs remain complete. Serialized JSON carriers, including mcpMeta.structuredContent.result, are parsed, recursively sanitized, and reserialized so structural and ownership fields remain intact. The container prefix /home/autoskillit/.claude/projects is replaced with @CLAUDE_PROJECTS@ in the event trace, session index, and summaries; other /home/ paths are tokenized. Index and summary data retain identity fields needed to resolve the captured lineage. logs/skill-session-contracts/ and logs/otlp.jsonl are excluded." +} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions.jsonl new file mode 100644 index 0000000000..52f67da240 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions.jsonl @@ -0,0 +1,3 @@ +{"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","dir_name":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","dispatch_id":"4666c3fd-486f-4ce1-b51f-9d9d660a5560","parent_session_id":"","caller_session_id":"","claude_code_log":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl","session_type":"skill","step_name":"plan","order_id":"4666c3fd-486f-4ce1-b51f-9d9d660a5560","campaign_id":"48915f19-d220-4069-8d55-b81efff84c05","kitchen_id":"48915f19-d220-4069-8d55-b81efff84c05","timestamp":"2026-10-11T20:05:22.655547+00:00","claude_code_version":"2.1.280 (Claude Code)","autoskillit_version":"0.10.1271","backend":"claude-code","provider_used":"anthropic","model_identifier":"MiniMax-M3","configured_model":"sonnet","success":true,"schema_version":17,"recipe_name":"implementation","recipe_version":"1.0.0","requested_parent_backend":"claude-code","requested_parent_model":"sonnet"} +{"session_id":"54f18144-1188-45d7-a392-17b4529f721e","dir_name":"54f18144-1188-45d7-a392-17b4529f721e","dispatch_id":"4666c3fd-486f-4ce1-b51f-9d9d660a5560","parent_session_id":"","caller_session_id":"","claude_code_log":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl","session_type":"skill","step_name":"verify","order_id":"4666c3fd-486f-4ce1-b51f-9d9d660a5560","campaign_id":"48915f19-d220-4069-8d55-b81efff84c05","kitchen_id":"48915f19-d220-4069-8d55-b81efff84c05","timestamp":"2026-10-11T20:11:57.256584+00:00","claude_code_version":"2.1.280 (Claude Code)","autoskillit_version":"0.10.1271","backend":"claude-code","provider_used":"anthropic","model_identifier":"MiniMax-M3","configured_model":"sonnet","success":false,"schema_version":17,"recipe_name":"implementation","recipe_version":"1.0.0","requested_parent_backend":"claude-code","requested_parent_model":"sonnet"} +{"session_id":"2292658a-34cf-44a0-a0b1-f7a20218874b","dir_name":"2292658a-34cf-44a0-a0b1-f7a20218874b","dispatch_id":"4666c3fd-486f-4ce1-b51f-9d9d660a5560","parent_session_id":"","caller_session_id":"","claude_code_log":"@CLAUDE_PROJECTS@/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl","session_type":"orchestrator","step_name":"","order_id":"4666c3fd-486f-4ce1-b51f-9d9d660a5560","campaign_id":"48915f19-d220-4069-8d55-b81efff84c05","kitchen_id":"48915f19-d220-4069-8d55-b81efff84c05","timestamp":"2026-10-11T20:04:36.539164+00:00","claude_code_version":"2.1.280 (Claude Code)","autoskillit_version":"0.10.1271","backend":"claude-code","provider_used":"anthropic","model_identifier":"MiniMax-M3","configured_model":"sonnet","success":true,"schema_version":17,"recipe_name":"","recipe_version":"","requested_parent_backend":"","requested_parent_model":""} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/2292658a-34cf-44a0-a0b1-f7a20218874b/summary.json b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/2292658a-34cf-44a0-a0b1-f7a20218874b/summary.json new file mode 100644 index 0000000000..d13696603e --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/2292658a-34cf-44a0-a0b1-f7a20218874b/summary.json @@ -0,0 +1,23 @@ +{ + "backend": "claude-code", + "campaign_id": "48915f19-d220-4069-8d55-b81efff84c05", + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "end_ts": "2026-10-11T20:16:23.869672+00:00", + "execution_identity": { + "children": [], + "cli_version": "", + "effective_parent_backend": "", + "effective_parent_effort": "", + "effective_parent_model": "", + "override_key_path": "", + "override_tier": "", + "parent_session_id": "", + "requested_parent_backend": "", + "requested_parent_effort": "", + "requested_parent_model": "" + }, + "provider_used": "anthropic", + "session_id": "2292658a-34cf-44a0-a0b1-f7a20218874b", + "session_type": "orchestrator", + "start_ts": "2026-10-11T20:04:36.539164+00:00" +} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/54f18144-1188-45d7-a392-17b4529f721e/summary.json b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/54f18144-1188-45d7-a392-17b4529f721e/summary.json new file mode 100644 index 0000000000..38000961d4 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/54f18144-1188-45d7-a392-17b4529f721e/summary.json @@ -0,0 +1,23 @@ +{ + "backend": "claude-code", + "campaign_id": "48915f19-d220-4069-8d55-b81efff84c05", + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "end_ts": "2026-10-11T20:14:46.893573+00:00", + "execution_identity": { + "children": [], + "cli_version": "", + "effective_parent_backend": "", + "effective_parent_effort": "", + "effective_parent_model": "", + "override_key_path": "", + "override_tier": "", + "parent_session_id": "", + "requested_parent_backend": "claude-code", + "requested_parent_effort": "", + "requested_parent_model": "sonnet" + }, + "provider_used": "anthropic", + "session_id": "54f18144-1188-45d7-a392-17b4529f721e", + "session_type": "skill", + "start_ts": "2026-10-11T20:11:57.256584+00:00" +} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/8ef8f993-8caa-47dd-83b0-26f9e5b51139/summary.json b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/8ef8f993-8caa-47dd-83b0-26f9e5b51139/summary.json new file mode 100644 index 0000000000..e2c1ce5cc3 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/8ef8f993-8caa-47dd-83b0-26f9e5b51139/summary.json @@ -0,0 +1,23 @@ +{ + "backend": "claude-code", + "campaign_id": "48915f19-d220-4069-8d55-b81efff84c05", + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "end_ts": "2026-10-11T20:07:16.866414+00:00", + "execution_identity": { + "children": [], + "cli_version": "", + "effective_parent_backend": "", + "effective_parent_effort": "", + "effective_parent_model": "", + "override_key_path": "", + "override_tier": "", + "parent_session_id": "", + "requested_parent_backend": "claude-code", + "requested_parent_effort": "", + "requested_parent_model": "sonnet" + }, + "provider_used": "anthropic", + "session_id": "8ef8f993-8caa-47dd-83b0-26f9e5b51139", + "session_type": "skill", + "start_ts": "2026-10-11T20:05:22.655547+00:00" +} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/out/envelope.json b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/out/envelope.json new file mode 100644 index 0000000000..ed1e998ea1 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/out/envelope.json @@ -0,0 +1,114 @@ +{ + "kind": "completed", + "success": false, + "dispatch_status": "resumable", + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "dispatched_session_id": "2292658a-34cf-44a0-a0b1-f7a20218874b", + "reason": "fleet_l3_no_result_block", + "effect_provenance": { + "operation_id": "b6908330d53b40b98656d73e3a8fdb05", + "aggregate_phase": "started", + "retry_disposition": "resume_by_identity", + "effects": [ + { + "name": "campaign_state_write", + "phase": "confirmed", + "effect_id": "b6908330d53b40b98656d73e3a8fdb05:campaign_state_write", + "retry_relevant": true, + "confirmation_receipt": "per-dispatch state and captures persisted", + "known_downstream_identities": { + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "state_path": "/workspace/sandbox/.autoskillit/temp/dispatches/4666c3fd-486f-4ce1-b51f-9d9d660a5560.json" + }, + "ambiguity": "" + }, + { + "name": "child_discovery", + "phase": "confirmed", + "effect_id": "b6908330d53b40b98656d73e3a8fdb05:child_discovery", + "retry_relevant": true, + "confirmation_receipt": "captured one process identity tuple", + "known_downstream_identities": { + "boot_id": "95f60de8-f823-4b7a-abc9-9e3f15310bc6", + "create_time": "1791749076.16", + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "identity_degraded": "False", + "pid": "106", + "starttime_ticks": "15116" + }, + "ambiguity": "" + }, + { + "name": "dispatch_allocation", + "phase": "confirmed", + "effect_id": "b6908330d53b40b98656d73e3a8fdb05:dispatch_allocation", + "retry_relevant": true, + "confirmation_receipt": "per-dispatch state identity persisted", + "known_downstream_identities": { + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "state_path": "/workspace/sandbox/.autoskillit/temp/dispatches/4666c3fd-486f-4ce1-b51f-9d9d660a5560.json" + }, + "ambiguity": "" + }, + { + "name": "process_spawn", + "phase": "confirmed", + "effect_id": "b6908330d53b40b98656d73e3a8fdb05:process_spawn", + "retry_relevant": true, + "confirmation_receipt": "executor reported spawned process and authoritative session identity", + "known_downstream_identities": { + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "dispatched_session_id": "2292658a-34cf-44a0-a0b1-f7a20218874b", + "pid": "106", + "starttime_ticks": "15116" + }, + "ambiguity": "" + } + ], + "cancel_requested": false, + "local_cleanup": null, + "state_cleanup_confirmed": false, + "labels_cleanup_confirmed": false + }, + "token_usage": { + "backend": "claude-code", + "provider_used": "anthropic", + "input_tokens": { + "state": "measured", + "value": 205063 + }, + "output_tokens": { + "state": "measured", + "value": 39864 + }, + "cache_read_tokens": { + "state": "measured", + "value": 11571251 + }, + "cache_write_tokens": { + "state": "measured_zero", + "value": 0 + }, + "peak_context": { + "state": "unknown", + "value": null + } + }, + "l3_payload": null, + "l3_parse_source": "stdout", + "lifespan_started": true, + "stderr": "", + "elapsed_seconds": 708.0250108242035, + "error": "fleet_l3_no_result_block", + "user_visible_message": "fleet_l3_no_result_block", + "resume_checkpoint": { + "completed_items": [ + "plan" + ], + "step_name": "plan", + "progress_pct": 0.007407407407407408, + "ts": "2026-10-11T20:07:18.776245+00:00", + "backend_name": "claude-code", + "skill_name": "implementation" + } +} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/out/session-events.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/out/session-events.jsonl new file mode 100644 index 0000000000..54806953b0 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/out/session-events.jsonl @@ -0,0 +1,16 @@ +{"event":"SessionStart","timestamp":1791749076.9067402,"session_id":"2292658a-34cf-44a0-a0b1-f7a20218874b","source":"startup","transcript_path":"@CLAUDE_PROJECTS@/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl"} +{"event":"SessionStart","timestamp":1791749123.0673993,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","source":"startup","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl"} +{"event":"SubagentStart","timestamp":1791749157.9630604,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","agent_id":"a9ed51b7c02b828dd","agent_type":"autoskillit:plan-interface-mapper","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl"} +{"event":"SubagentStart","timestamp":1791749158.0156765,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","agent_id":"aeb96cc0ddbdab294","agent_type":"autoskillit:plan-foundation-auditor","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl"} +{"event":"SubagentStart","timestamp":1791749158.1193779,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","agent_id":"a516cbae85151dc7c","agent_type":"autoskillit:plan-registry-tracer","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl"} +{"event":"SubagentStop","timestamp":1791749183.3175266,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","agent_id":"a516cbae85151dc7c","agent_type":"autoskillit:plan-registry-tracer","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl","agent_transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a516cbae85151dc7c.jsonl"} +{"event":"SubagentStop","timestamp":1791749183.933718,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","agent_id":"a9ed51b7c02b828dd","agent_type":"autoskillit:plan-interface-mapper","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl","agent_transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a9ed51b7c02b828dd.jsonl"} +{"event":"SubagentStop","timestamp":1791749190.3404713,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","agent_id":"aeb96cc0ddbdab294","agent_type":"autoskillit:plan-foundation-auditor","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl","agent_transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-aeb96cc0ddbdab294.jsonl"} +{"event":"SessionEnd","timestamp":1791749233.8509,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl"} +{"event":"SessionStart","timestamp":1791749517.653047,"session_id":"54f18144-1188-45d7-a392-17b4529f721e","source":"startup","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl"} +{"event":"SubagentStart","timestamp":1791749551.111115,"session_id":"54f18144-1188-45d7-a392-17b4529f721e","agent_id":"a7e613554b849d6b2","agent_type":"general-purpose","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl"} +{"event":"SubagentStop","timestamp":1791749679.8737457,"session_id":"54f18144-1188-45d7-a392-17b4529f721e","agent_id":"a7e613554b849d6b2","agent_type":"general-purpose","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl","agent_transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e/subagents/agent-a7e613554b849d6b2.jsonl"} +{"event":"SessionEnd","timestamp":1791749686.7281744,"session_id":"54f18144-1188-45d7-a392-17b4529f721e","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl"} +{"event":"SubagentStop","timestamp":1791749722.6609342,"session_id":"2292658a-34cf-44a0-a0b1-f7a20218874b","agent_id":"a610a4bda1f28ca35","agent_type":"","transcript_path":"@CLAUDE_PROJECTS@/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl","agent_transcript_path":"@CLAUDE_PROJECTS@/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b/subagents/agent-a610a4bda1f28ca35.jsonl"} +{"event":"SessionStart","timestamp":1791749722.700151,"session_id":"2292658a-34cf-44a0-a0b1-f7a20218874b","source":"compact","transcript_path":"@CLAUDE_PROJECTS@/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl"} +{"event":"SessionEnd","timestamp":1791749780.901994,"session_id":"2292658a-34cf-44a0-a0b1-f7a20218874b","transcript_path":"@CLAUDE_PROJECTS@/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl"} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl new file mode 100644 index 0000000000..5fc748643b --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl @@ -0,0 +1,5 @@ +{"type":"queue-operation","operation":"enqueue","timestamp":"2026-10-11T20:12:00.782Z","sessionId":"54f18144-1188-45d7-a392-17b4529f721e","content":"Use the /dry-walkthrough skill AUTOSKILLIT_BOUND_INVOCATION_V1\n{\"skill_inputs\":[{\"name\":\"plan_path\",\"value\":\"/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md\"},{\"name\":\"issue_url\",\"value\":\"https://github.com/TalonT-Org/autoskillit-e2e-sandbox/issues/1\"},{\"name\":\"review_path\",\"value\":\"\"},{\"name\":\"audit_cycle_path\",\"va...[fixture-truncated 2001 chars]"} +{"type":"queue-operation","operation":"dequeue","timestamp":"2026-10-11T20:12:00.783Z","sessionId":"54f18144-1188-45d7-a392-17b4529f721e"} +{"parentUuid":null,"isSidechain":false,"promptId":"6ec22124-b069-42f2-a3d7-361204649ceb","type":"user","message":{"role":"user","content":"Use the /dry-walkthrough skill AUTOSKILLIT_BOUND_INVOCATION_V1\n{\"skill_inputs\":[{\"name\":\"plan_path\",\"value\":\"/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md\"},{\"name\":\"issue_url\",\"value\":\"https://github.com/TalonT-Org/autoskillit-e2e-sandbox/issues/1\"},{\"name\":\"review_path\",\"value\":\"\"},{\"name\":\"audit_cycle_path\",\"va...[fixture-truncated 2001 chars]"},"uuid":"1fa895db-0f63-4e4d-9319-5e0a1ea3bd2e","timestamp":"2026-10-11T20:12:00.823Z","permissionMode":"bypassPermissions","promptSource":"sdk","turnOrigin":"sdk","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"54f18144-1188-45d7-a392-17b4529f721e","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} +{"type":"last-prompt","lastPrompt":"Use the /dry-walkthrough skill AUTOSKILLIT_BOUND_INVOCATION_V1 {\"skill_inputs\":[{\"name\":\"plan_path\",\"value\":\"/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normali…","leafUuid":"9a7737fa-95fb-4950-9f81-a0f69f85140a","sessionId":"54f18144-1188-45d7-a392-17b4529f721e"} +{"type":"cost-state","sessionId":"54f18144-1188-45d7-a392-17b4529f721e","totalCostUSD":0.4940016000000001,"totalAPIDuration":149446,"totalAPIDurationWithoutRetries":149406,"totalToolDuration":130687,"totalLinesAdded":3,"totalLinesRemoved":1,"totalDuration":169414,"startTime":1791749517277,"modelUsage":{"claude-sonnet-5":{"inputTokens":93799,"outputTokens":19776,"thinkingTokens":13280,"cacheReadInputTokens":543218,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.4940016000000001}},"hasUnknownModelCost":false} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e/subagents/agent-a7e613554b849d6b2.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e/subagents/agent-a7e613554b849d6b2.jsonl new file mode 100644 index 0000000000..9b09bf65bd --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e/subagents/agent-a7e613554b849d6b2.jsonl @@ -0,0 +1,2 @@ +{"parentUuid":null,"isSidechain":true,"promptId":"6ec22124-b069-42f2-a3d7-361204649ceb","agentId":"a7e613554b849d6b2","type":"user","message":{"role":"user","content":"You are validating a proposed implementation plan by performing a dry walkthrough of each change without implementing. Your role is technical validation, not strategic decision-making. Fix factual inaccuracies (wrong file paths, nonexistent functions, incorrect line numbers). Preserve all goals and scope.\n\n## Plan to Validate\nRead the exact plan file at:\n`/workspace/autoskillit-runs/impl-20261011-...[fixture-truncated 7212 chars]"},"uuid":"bd262550-a49c-499e-8443-0dad17d21c1a","timestamp":"2026-10-11T20:12:31.074Z","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"54f18144-1188-45d7-a392-17b4529f721e","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} +{"parentUuid":"1bee3141-ddeb-4c5a-8e05-5c397c01fc75","isSidechain":true,"agentId":"a7e613554b849d6b2","message":{"id":"071b212510333ffa9b724744f66211b1","type":"message","role":"assistant","content":[{"type":"text","text":"# Dry Walkthrough Report\n\n## Plan Validation Summary\n\n**File**: `/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md`\n**Stamp**: `Dry-walkthrough verified = TRUE` (verified present on line 1)\n\n---\n\n## Step 1: Multi-Part Detection\n- Filename has no `_part_` suffix. PASS\n- Plan contains no PART scope warning block. PASS\n\n...[fixture-truncated 4672 chars]"}],"model":"MiniMax-M3","stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":188,"cache_creation_input_tokens":0,"cache_read_input_tokens":45709,"output_tokens":1882,"output_tokens_details":{"thinking_tokens":532},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionAgent":"general-purpose","attributionSkill":"dry-walkthrough","type":"assistant","uuid":"3819a31b-bbba-46b9-83ca-61def8b3f88f","timestamp":"2026-10-11T20:14:39.838Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"54f18144-1188-45d7-a392-17b4529f721e","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl new file mode 100644 index 0000000000..3f16a49374 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl @@ -0,0 +1,5 @@ +{"type":"queue-operation","operation":"enqueue","timestamp":"2026-10-11T20:05:26.169Z","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","content":"Use the /make-plan skill AUTOSKILLIT_BOUND_INVOCATION_V1\n{\"skill_inputs\":[{\"name\":\"task\",\"value\":\"E2E implementation fixture: normalize whitespace\\n\\n# Reusable implementation E2E fixture\\n\\nAdd `normalize_whitespace(text: str) -> str` to `sandbox/text.py`.\\n\\nThe function must remove leading and trailing whitespace and replace each run of\\nwhitespace (including spaces, tabs, and newlines) with on...[fixture-truncated 2035 chars]"} +{"type":"queue-operation","operation":"dequeue","timestamp":"2026-10-11T20:05:26.169Z","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139"} +{"parentUuid":null,"isSidechain":false,"promptId":"b304e3c8-f9a9-4e0e-8d13-8e7d6617cc4e","type":"user","message":{"role":"user","content":"Use the /make-plan skill AUTOSKILLIT_BOUND_INVOCATION_V1\n{\"skill_inputs\":[{\"name\":\"task\",\"value\":\"E2E implementation fixture: normalize whitespace\\n\\n# Reusable implementation E2E fixture\\n\\nAdd `normalize_whitespace(text: str) -> str` to `sandbox/text.py`.\\n\\nThe function must remove leading and trailing whitespace and replace each run of\\nwhitespace (including spaces, tabs, and newlines) with on...[fixture-truncated 2035 chars]"},"uuid":"0b38c9ec-2c12-4619-8935-2121f8c249ad","timestamp":"2026-10-11T20:05:26.210Z","permissionMode":"bypassPermissions","promptSource":"sdk","turnOrigin":"sdk","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} +{"type":"last-prompt","lastPrompt":"Use the /make-plan skill AUTOSKILLIT_BOUND_INVOCATION_V1 {\"skill_inputs\":[{\"name\":\"task\",\"value\":\"E2E implementation fixture: normalize whitespace\\n\\n# Reusable implementation E2E fixture\\n\\nAdd `norm…","leafUuid":"f50148a8-7b86-4926-a4d2-961755148f52","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139"} +{"type":"cost-state","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","totalCostUSD":0.41991880000000004,"totalAPIDuration":123555,"totalAPIDurationWithoutRetries":123501,"totalToolDuration":91414,"totalLinesAdded":199,"totalLinesRemoved":0,"totalDuration":111137,"startTime":1791749122675,"modelUsage":{"claude-sonnet-5":{"inputTokens":79993,"outputTokens":16833,"thinkingTokens":8494,"cacheReadInputTokens":458014,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.41991880000000004}},"hasUnknownModelCost":false} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a516cbae85151dc7c.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a516cbae85151dc7c.jsonl new file mode 100644 index 0000000000..e37cb2a70e --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a516cbae85151dc7c.jsonl @@ -0,0 +1,2 @@ +{"parentUuid":null,"isSidechain":true,"promptId":"b304e3c8-f9a9-4e0e-8d13-8e7d6617cc4e","agentId":"a516cbae85151dc7c","type":"user","message":{"role":"user","content":"You are the plan-registry-tracer. Perform a registry and artifact trace audit of the following draft implementation plan.\n\nContrastive frame: \"A junior reviewer found this plan's registry coverage complete — what did they miss?\"\n\nDo NOT suggest scope expansion. Identify only gaps in what the plan already claims to do.\n\nCodebase root: /workspace/autoskillit-runs/impl-20261011-200500-587134\n\nWorking...[fixture-truncated 715 chars]"},"uuid":"0a0a8820-6e0f-43bd-9b90-5a9dadd38fb9","timestamp":"2026-10-11T20:05:58.067Z","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} +{"parentUuid":"8349df76-a174-48ed-87fa-55fcd3bb3647","isSidechain":true,"agentId":"a516cbae85151dc7c","message":{"id":"071b1f367fc2617d89e17e0512b28e11","type":"message","role":"assistant","content":[{"type":"text","text":"# Registry Trace Findings\n\n## Touched Symbols\n- `normalize_whitespace` — new function in `sandbox/text.py` (no rename, no removal)\n\n## Grep/LSP Results\n\n| Symbol | File Found | Reference Type | Updated by Plan? |\n|--------|-----------|----------------|------------------|\n| normalize_whitespace | (none) | N/A — net-new symbol, no prior references anywhere in repo | N/A |\n\nA recursive grep across `/...[fixture-truncated 1847 chars]"}],"model":"MiniMax-M3","stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":1894,"cache_creation_input_tokens":0,"cache_read_input_tokens":9755,"output_tokens":1025,"output_tokens_details":{"thinking_tokens":474},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionAgent":"autoskillit:plan-registry-tracer","attributionSkill":"make-plan","attributionPlugin":"autoskillit","type":"assistant","uuid":"27b5b29b-5719-4134-b996-7009d7bee590","timestamp":"2026-10-11T20:06:23.283Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a9ed51b7c02b828dd.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a9ed51b7c02b828dd.jsonl new file mode 100644 index 0000000000..b2c3fede89 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a9ed51b7c02b828dd.jsonl @@ -0,0 +1,2 @@ +{"parentUuid":null,"isSidechain":true,"promptId":"b304e3c8-f9a9-4e0e-8d13-8e7d6617cc4e","agentId":"a9ed51b7c02b828dd","type":"user","message":{"role":"user","content":"You are the plan-interface-mapper. Perform a variable and data-flow trace audit of the following draft implementation plan.\n\nContrastive frame: \"A junior reviewer found this plan's variable usage correct — what did they miss?\"\n\nDo NOT suggest scope expansion. Identify only gaps in what the plan already claims to do.\n\nCodebase root: /workspace/autoskillit-runs/impl-20261011-200500-587134\n\nWorking b...[fixture-truncated 780 chars]"},"uuid":"5067983e-c8ee-44df-8677-6acfb5f2d6d9","timestamp":"2026-10-11T20:05:57.864Z","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} +{"parentUuid":"9ab1fecd-1984-470f-a6a6-e6a60a361312","isSidechain":true,"agentId":"a9ed51b7c02b828dd","message":{"id":"071b1f337786beccec1da5b44669d627","type":"message","role":"assistant","content":[{"type":"text","text":"# Interface Mapping Audit\n\n## SET/READ Trace Table\n\n| Variable | SET Location | SET Value | READ Location | READ Expectation | Match? |\n|----------|-------------|-----------|---------------|------------------|--------|\n| `tests/test_text.py` content | plan step 2 (create) | new file with `normalize_whitespace` tests only | `Taskfile.yml:11-12` (`unittest discover`) | discover runs over existing `t...[fixture-truncated 2117 chars]"}],"model":"MiniMax-M3","stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":918,"cache_creation_input_tokens":0,"cache_read_input_tokens":6939,"output_tokens":1767,"output_tokens_details":{"thinking_tokens":1127},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionAgent":"autoskillit:plan-interface-mapper","attributionSkill":"make-plan","attributionPlugin":"autoskillit","type":"assistant","uuid":"53c38458-ca9f-45dd-a78c-38dcfd68125f","timestamp":"2026-10-11T20:06:23.901Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-aeb96cc0ddbdab294.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-aeb96cc0ddbdab294.jsonl new file mode 100644 index 0000000000..b98512bab6 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-aeb96cc0ddbdab294.jsonl @@ -0,0 +1,2 @@ +{"parentUuid":null,"isSidechain":true,"promptId":"b304e3c8-f9a9-4e0e-8d13-8e7d6617cc4e","agentId":"aeb96cc0ddbdab294","type":"user","message":{"role":"user","content":"You are the plan-foundation-auditor. Perform a step-by-step control-flow audit of the following draft implementation plan.\n\nContrastive frame: \"A junior reviewer found this plan's control flow acceptable — what did they miss?\"\n\nDo NOT suggest scope expansion. Identify only gaps in what the plan already claims to do.\n\nCodebase root: /workspace/autoskillit-runs/impl-20261011-200500-587134\n\nWorking b...[fixture-truncated 1144 chars]"},"uuid":"1178680b-6e3b-4d4b-9ee4-894c30881464","timestamp":"2026-10-11T20:05:57.914Z","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} +{"parentUuid":"b28e2ef4-916c-41b8-a55f-ec64f01eaece","isSidechain":true,"agentId":"aeb96cc0ddbdab294","message":{"id":"071b1f3371a476db86a8860eb750a903","type":"message","role":"assistant","content":[{"type":"text","text":"# Plan Audit — Findings\n\n## Step 1–2 — Enumerate & Draw Control Flow\n\n**Target function:** `normalize_whitespace(text: str) -> str` (new, in `sandbox/text.py`).\n\nPlanned body is a two-call pipeline (`text.strip()` then `re.sub(r'\\s+', ' ', ...)`). No `if`/`match`/`return` branches inside the function — every input traverses the same path. The audit therefore pivots to the **test file and the plan'...[fixture-truncated 2234 chars]"}],"model":"MiniMax-M3","stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":594,"cache_creation_input_tokens":0,"cache_read_input_tokens":7345,"output_tokens":3617,"output_tokens_details":{"thinking_tokens":2955},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionAgent":"autoskillit:plan-foundation-auditor","attributionSkill":"make-plan","attributionPlugin":"autoskillit","type":"assistant","uuid":"f522d9e2-3194-4afc-9b88-8a072e3ff485","timestamp":"2026-10-11T20:06:30.307Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl new file mode 100644 index 0000000000..d92ba7be4c --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl @@ -0,0 +1,23 @@ +{"type":"queue-operation","operation":"enqueue","timestamp":"2026-10-11T20:04:40.218Z","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","content":"You are an L2 food truck orchestrator. Execute the recipe 'implementation' autonomously.\nTimeout: 7200s. Campaign: 48915f19-d220-4069-8d55-b81efff84c05. Dispatch: 4666c3fd-486f-4ce1-b51f-9d9d660a5560.\n\n--- SECTION 1: ADMIRAL DISCIPLINE (L2/FOOD-TRUCK SUBSET) ---\n\n## CONTEXT LIMIT ROUTING — MANDATORY\n\nWhen `run_skill` returns `needs_retry=true` for **any step**:\n\n- **If `retry_reason: resume` AND `...[fixture-truncated 33134 chars]"} +{"type":"queue-operation","operation":"dequeue","timestamp":"2026-10-11T20:04:40.218Z","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b"} +{"parentUuid":null,"isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":"You are an L2 food truck orchestrator. Execute the recipe 'implementation' autonomously.\nTimeout: 7200s. Campaign: 48915f19-d220-4069-8d55-b81efff84c05. Dispatch: 4666c3fd-486f-4ce1-b51f-9d9d660a5560.\n\n--- SECTION 1: ADMIRAL DISCIPLINE (L2/FOOD-TRUCK SUBSET) ---\n\n## CONTEXT LIMIT ROUTING — MANDATORY\n\nWhen `run_skill` returns `needs_retry=true` for **any step**:\n\n- **If `retry_reason: resume` AND `...[fixture-truncated 33134 chars]"},"uuid":"e1be918a-cba5-4f0b-9960-60a08420f2b2","timestamp":"2026-10-11T20:04:40.246Z","permissionMode":"bypassPermissions","promptSource":"sdk","turnOrigin":"sdk","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"9aa8bfde-3315-49d7-b568-c907349a3dbd","isSidechain":false,"message":{"id":"071b1eea9dc5d3c03c383ed25b5ec610","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c9191907360b2a092bf","name":"mcp__plugin_autoskillit_autoskillit__bootstrap_clone","input":{"source_dir":"/workspace/sandbox","run_name":"impl","base_branch":"main","step_name":"clone"}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":1301,"cache_creation_input_tokens":0,"cache_read_input_tokens":42278,"output_tokens":222,"output_tokens_details":{"thinking_tokens":143},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"get_recipe_section","type":"assistant","uuid":"66199695-80b0-448d-9390-b9bcb3712e6e","timestamp":"2026-10-11T20:04:59.937Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"66199695-80b0-448d-9390-b9bcb3712e6e","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c9191907360b2a092bf","type":"tool_result","content":"## bootstrap_clone\n\nwork_dir: /workspace/autoskillit-runs/impl-20261011-200500-587134\nremote_url: https://github.com/TalonT-Org/autoskillit-e2e-sandbox.git\nbase_sha: d6c143d2494488deeb6680d7d5cf2e55e3e01732\nmerge_target: main\ntimings:\n clone_ms: 851\n rev_parse_ms: 3\n\n--- RECIPE SEGMENT ---\n{\"bodies\":[{\"body\":\"{\\\"claim_and_resolve\\\":{\\\"tool\\\":\\\"claim_and_resolve_issue\\\",\\\"with\\\":{\\\"step_name\\\":\\\"...[fixture-truncated 8153 chars]"}]},"uuid":"6cdc0d3f-fc36-426f-97c3-206e93e21f49","timestamp":"2026-10-11T20:05:01.170Z","toolUseResult":"## bootstrap_clone\n\nwork_dir: /workspace/autoskillit-runs/impl-20261011-200500-587134\nremote_url: https://github.com/TalonT-Org/autoskillit-e2e-sandbox.git\nbase_sha: d6c143d2494488deeb6680d7d5cf2e55e3e01732\nmerge_target: main\ntimings:\n clone_ms: 851\n rev_parse_ms: 3\n\n--- RECIPE SEGMENT ---\n{\"bodies\":[{\"body\":\"{\\\"claim_and_resolve\\\":{\\\"tool\\\":\\\"claim_and_resolve_issue\\\",\\\"with\\\":{\\\"step_name\\\":\\\"...[fixture-truncated 8153 chars]","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"work_dir\":\"/workspace/autoskillit-runs/impl-20261011-200500-587134\",\"remote_url\":\"https://github.com/TalonT-Org/autoskillit-e2e-sandbox.git\",\"base_sha\":\"d6c143d2494488deeb6680d7d5cf2e55e3e01732\",\"merge_target\":\"main\",\"timings\":{\"clone_ms\":851,\"rev_parse_ms\":3},\"recipe_segment\":{\"kind\":\"success\",\"source_step\":\"clone\",\"segments\":[{\"index\":1,\"name\":\"I1\"}],\"bodies\":[{\"step\":\"claim_and_resolve\",\"body\":\"{\\\"claim_and_resolve\\\":{\\\"tool\\\":\\\"claim_and_resolve_issue\\\",\\\"with\\\":{\\\"step_name\\\":\\\"claim_and_resolve\\\",\\\"issue_url\\\":\\\"${{ inputs.issue_url }}\\\",\\\"allow_reentry\\\":false},\\\"capture\\\":{\\\"issue_number\\\":\\\"${{ result.issue_number }}\\\",\\\"issue_title\\\":\\\"${{ result.issue_title }}\\\",\\\"issue_slug\\\":\\\"${{ result.issue_slug }}\\\",\\\"claimed\\\":\\\"${{ result.claimed }}\\\"},\\\"on_result\\\":[{\\\"when\\\":\\\"${{ result.claimed }} == true\\\",\\\"route\\\":\\\"create_and_publish\\\"},{\\\"route\\\":\\\"register_clone_failure\\\"}],\\\"on_failure\\\":\\\"register_clone_failure\\\",\\\"note\\\":\\\"Fetches issue title/slug and claims the issue in one turn. Returns issue_number, issue_title, issue_slug, and claimed boolean. When claimed=false (another session owns it), routes to register_clone_failure before escalate_stop so the clone directory is registered for cleanup. issue_slug and issue_number are passed to create_and_publish for branch naming.\\\\n\\\"}}\",\"body_sha256\":\"sha256:2dff84b001815660dc762ea29e4eeeb9eea19b11c77638b46aec51039509a257\"},{\"step\":\"create_and_publish\",\"body\":\"{\\\"create_and_publish\\\":{\\\"tool\\\":\\\"create_and_publish_branch\\\",\\\"with\\\":{\\\"issue_slug\\\":\\\"${{ context.issue_slug }}\\\",\\\"run_name\\\":\\\"${{ inputs.run_name }}\\\",\\\"issue_number\\\":\\\"${{ context.issue_number }}\\\",\\\"work_dir\\\":\\\"${{ context.work_dir }}\\\",\\\"remote_url\\\":\\\"${{ context.remote_url }}\\\",\\\"step_name\\\":\\\"create_and_publish\\\"},\\\"capture\\\":{\\\"merge_target\\\":\\\"${{ result.merge_target }}\\\"},\\\"on_success\\\":\\\"plan\\\",\\\"on_failure\\\":\\\"release_issue_failure\\\",\\\"note\\\":\\\"Computes branch name, creates it (with collision suffix if needed), and pushes to origin in one turn. Overrides merge_target with the feature branch name. merge_worktree requires refs/remotes/origin/{branch} to exist after fetch (step 5.5). When open_pr=false, this step is skipped and merge_target remains as base_branch.\\\\n\\\"}}\",\"body_sha256\":\"sha256:f102a40c0ec99f47038934ff163d56cee9d5df486246d7575221ce2c6a89fcf1\"},{\"step\":\"plan\",\"body\":\"{\\\"plan\\\":{\\\"tool\\\":\\\"run_skill\\\",\\\"model\\\":\\\"\\\",\\\"with\\\":{\\\"skill_command\\\":\\\"/autoskillit:make-plan\\\",\\\"skill_inputs\\\":{\\\"adversarial_review_level\\\":\\\"auto\\\",\\\"task\\\":\\\"${{ inputs.task }}\\\",\\\"issue_url\\\":\\\"${{ inputs.issue_url }}\\\",\\\"audit_cycle_path\\\":\\\"${{ context.audit_cycle_path }}\\\",\\\"plan_set_coverage_gaps\\\":\\\"${{ context.plan_set_coverage_gaps }}\\\"},\\\"cwd\\\":\\\"${{ context.work_dir }}\\\",\\\"step_name\\\":\\\"plan\\\"},\\\"capture\\\":{\\\"plan_path\\\":\\\"${{ result.plan_path }}\\\",\\\"all_plan_paths\\\":\\\"${{ result.plan_path }}\\\",\\\"verdict\\\":\\\"${{ result.verdict }}\\\",\\\"plan_disposition_path\\\":\\\"${{ result.plan_disposition_path }}\\\"},\\\"capture_list\\\":{\\\"plan_parts\\\":\\\"${{ result.plan_parts }}\\\"},\\\"retries\\\":0,\\\"on_result\\\":[{\\\"when\\\":\\\"${{ result.verdict }} == plan\\\",\\\"route\\\":\\\"bind_plan_set\\\"}],\\\"on_failure\\\":\\\"release_issue_failure\\\",\\\"on_rate_limit\\\":\\\"release_issue_failure\\\",\\\"on_context_limit\\\":\\\"salvage_plan\\\",\\\"optional_context_refs\\\":[\\\"audit_cycle_path\\\",\\\"plan_set_coverage_gaps\\\"],\\\"note\\\":\\\"Produces an ordinary plan when no audit_cycle_path is present. On NO GO re-entry, consumes that exact current authority and emits both plan_path and plan_disposition_path. all_plan_paths appends result.plan_path across re-entries; verdict=plan is the only successful result.\\\"}}\",\"body_sha256\":\"sha256:c44e11b44d35bcdf9620f407c17b7936a9b47a1f348e3f377cfd7b3ae9c4765a\",\"invocation_template_digest\":\"sha256:deec5b08a76bdabe0f7f520b9bb66f78789645f615012ac8eaf24806bef8e05e\"},{\"step\":\"salvage_plan\",\"body\":\"{\\\"salvage_plan\\\":{\\\"tool\\\":\\\"run_python\\\",\\\"with\\\":{\\\"step_name\\\":\\\"salvage_plan\\\",\\\"callable\\\":\\\"autoskillit.recipe._cmd_rpc.verify_plan_artifacts\\\",\\\"plan_parts\\\":\\\"${{ context.plan_parts }}\\\",\\\"audit_cycle_path\\\":\\\"${{ context.audit_cycle_path }}\\\"},\\\"capture\\\":{\\\"plan_path\\\":\\\"${{ result.plan_path }}\\\",\\\"plan_disposition_path\\\":\\\"${{ result.plan_disposition_path }}\\\",\\\"all_plan_paths\\\":\\\"${{ result.plan_path }}\\\"},\\\"capture_list\\\":{\\\"plan_parts\\\":\\\"${{ result.plan_parts }}\\\"},\\\"optional_context_refs\\\":[\\\"audit_cycle_path\\\"],\\\"retries\\\":0,\\\"on_result\\\":[{\\\"when\\\":\\\"${{ result.verdict }} == salvaged\\\",\\\"route\\\":\\\"bind_plan_set\\\"},{\\\"route\\\":\\\"release_issue_failure\\\"}],\\\"on_failure\\\":\\\"release_issue_failure\\\",\\\"note\\\":\\\"Deterministic salvage for plan context-limit stumbles: verifies captured plan_parts artifacts exist on disk. salvaged routes onward as if plan succeeded; unsalvageable falls through to plan's original on_failure destination. Substitute the captured ${{ context.plan_parts }} value into the plan_parts argument verbatim, unmodified — do not reformat, quote, or re-derive it; the callable's input gramm...[fixture-truncated 30 chars]\\\"}}\",\"body_sha256\":\"sha256:08f1e03e1927e09d6d304a2925760a2215f0f78b768fb856ab0261f695558d1f\"},{\"step\":\"bind_plan_set\",\"body\":\"{\\\"bind_plan_set\\\":{\\\"tool\\\":\\\"bind_plan_set\\\",\\\"with\\\":{\\\"plan_parts\\\":\\\"${{ context.plan_parts }}\\\",\\\"cwd\\\":\\\"${{ context.work_dir }}\\\",\\\"issue_url\\\":\\\"${{ inputs.issue_url }}\\\",\\\"seal\\\":\\\"true\\\",\\\"step_name\\\":\\\"bind_plan_set\\\"},\\\"capture\\\":{\\\"plan_set_authority_path\\\":\\\"${{ result.plan_set_authority_path }}\\\",\\\"plan_set_authority_digest\\\":\\\"${{ result.plan_set_authority_digest }}\\\",\\\"plan_set_parts\\\":\\\"${{ result.plan_set_parts }}\\\",\\\"plan_set_coverage_gaps\\\":\\\"${{ result.coverage_gaps }}\\\"},\\\"on_result\\\":[{\\\"when\\\":\\\"${{ result.success }} == true\\\",\\\"route\\\":\\\"verify\\\"},{\\\"when\\\":\\\"${{ result.reason }} == coverage_failed\\\",\\\"route\\\":\\\"check_replan_iteration\\\"},{\\\"route\\\":\\\"release_issue_failure\\\"}],\\\"on_failure\\\":\\\"release_issue_failure\\\",\\\"note\\\":\\\"Bind exactly the plan_parts value captured from the producer that just ran. A successful replan replaces the earlier list with its returned list: do not append rejected paths or rebuild the list from a directory listing. When salvage_plan is the producer, bind its returned list unchanged under the existing salvage recovery and strict coverage checks. Aggregate plan-set gate before every walkthroug...[fixture-truncated 277 chars]\\\"}}\",\"body_sha256\":\"sha256:ac7464747141373f7d61c06815e6021b52cda0a41170cde8e83296541e2229e6\"},{\"step\":\"check_replan_iteration\",\"body\":\"{\\\"check_replan_iteration\\\":{\\\"tool\\\":\\\"run_python\\\",\\\"with\\\":{\\\"step_name\\\":\\\"check_replan_iteration\\\",\\\"callable\\\":\\\"autoskillit.smoke_utils.check_loop_iteration\\\",\\\"current_iteration\\\":\\\"${{ context.plan_set_replan_count }}\\\",\\\"max_iterations\\\":\\\"2\\\"},\\\"capture\\\":{\\\"plan_set_replan_count\\\":\\\"${{ result.next_iteration }}\\\"},\\\"optional_context_refs\\\":[\\\"plan_set_replan_count\\\"],\\\"on_result\\\":[{\\\"when\\\":\\\"${{ result.max_exceeded }} == true\\\",\\\"route\\\":\\\"release_issue_failure\\\"},{\\\"route\\\":\\\"plan\\\"}],\\\"on_failure\\\":\\\"release_issue_failure\\\"}}\",\"body_sha256\":\"sha256:6ef878d992680fbac602153f1c0ac829f8a5c0657be7718c4426c2d25812f9c3\"}],\"recipe_execution\":{\"execution_id\":\"1dda3d6d29e545cbae216029ec657356\",\"invocation_template_digests\":{\"plan\":\"sha256:deec5b08a76bdabe0f7f520b9bb66f78789645f615012ac8eaf24806bef8e05e\"},\"skill_input_shapes\":{\"plan\":{\"keys\":[\"task\",\"issue_url\",\"adversarial_review_level\",\"audit_cycle_path\",\"plan_set_coverage_gaps\"],\"absence_values\":{\"audit_cycle_path\":\"\",\"plan_set_coverage_gaps\":\"\"}}},\"snapshot_digest\":\"sha256:1269b37298577647aec24baeb183326c56ad0d0bf2fecbd0231b0ad6e5b81fba\"},\"pull_closures\":[],\"recipe_pull\":{\"producer_tool\":\"open_kitchen\",\"recipe_name\":\"implementation\",\"descriptor_version\":2,\"schema_version\":2,\"payload_sha256\":\"sha256:ea2e62f28435745f27a7a742cbb98d2e34cdb00d82eef53117118c3b574d9497\",\"artifact_blob_sha256\":\"sha256:cd5e7fff1ca8145532aa9ce8ca2a26ddbfe48fb704316310789b8123319f8d63\",\"artifact_blob_size_bytes\":576427,\"body_sha256\":\"sha256:3f14e54f1505f8a5c4d13c2bc5658394855e4c7dc624cc527ddb4bf0711954d5\",\"body_size_bytes\":115869,\"flow_schema_version\":2,\"flow_sha256\":\"sha256:3b9ff899035aad7555d5863de6c70c6ed95e6ffac5f19cea0c46cc17261f4f2a\",\"flow_size_bytes\":101558,\"flow_record_count\":694,\"pull_tool\":\"get_recipe_section\"}}}"}},"sourceToolAssistantUUID":"66199695-80b0-448d-9390-b9bcb3712e6e","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"0b144546-f16f-4dd1-94b8-87b4f092a3e6","isSidechain":false,"message":{"id":"071b1eedfe3e75016da0bd16d2a6ae53","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c919df971e2b8b09c37","name":"mcp__plugin_autoskillit_autoskillit__claim_and_resolve_issue","input":{"issue_url":"https://github.com/TalonT-Org/autoskillit-e2e-sandbox/issues/1","allow_reentry":false,"step_name":"claim_and_resolve"}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":2643,"cache_creation_input_tokens":0,"cache_read_input_tokens":43579,"output_tokens":218,"output_tokens_details":{"thinking_tokens":124},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"bootstrap_clone","type":"assistant","uuid":"e8638011-011a-4b7d-8a7f-a09899915dfb","timestamp":"2026-10-11T20:05:03.120Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"e8638011-011a-4b7d-8a7f-a09899915dfb","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c919df971e2b8b09c37","type":"tool_result","content":"## claim_and_resolve_issue\n\nsuccess: True\nclaimed: True\nlabel: in-progress\nreview_approach_recommended: False\ninvestigation_complete: False\nissue_number: 1\nissue_title: E2E implementation fixture: normalize whitespace\nissue_slug: e2e-implementation-fixture-normalize-whitespace\ntimings:\n fetch_title_ms: 359\n claim_ms: 2475\n\n--- RECIPE SEGMENT ---\n{\"bodies\":[],\"kind\":\"success\",\"pull_closures\":[],\"...[fixture-truncated 943 chars]"}]},"uuid":"b689f776-8c6f-4831-bd03-e33b6ded7492","timestamp":"2026-10-11T20:05:06.239Z","toolUseResult":"## claim_and_resolve_issue\n\nsuccess: True\nclaimed: True\nlabel: in-progress\nreview_approach_recommended: False\ninvestigation_complete: False\nissue_number: 1\nissue_title: E2E implementation fixture: normalize whitespace\nissue_slug: e2e-implementation-fixture-normalize-whitespace\ntimings:\n fetch_title_ms: 359\n claim_ms: 2475\n\n--- RECIPE SEGMENT ---\n{\"bodies\":[],\"kind\":\"success\",\"pull_closures\":[],\"...[fixture-truncated 943 chars]","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"success\":true,\"claimed\":true,\"label\":\"in-progress\",\"review_approach_recommended\":false,\"investigation_complete\":false,\"issue_number\":1,\"issue_title\":\"E2E implementation fixture: normalize whitespace\",\"issue_slug\":\"e2e-implementation-fixture-normalize-whitespace\",\"timings\":{\"fetch_title_ms\":359,\"claim_ms\":2475},\"recipe_segment\":{\"kind\":\"success\",\"source_step\":\"claim_and_resolve\",\"segments\":[],\"bodies\":[],\"recipe_execution\":{\"execution_id\":\"1dda3d6d29e545cbae216029ec657356\",\"invocation_template_digests\":{},\"skill_input_shapes\":{},\"snapshot_digest\":\"sha256:1269b37298577647aec24baeb183326c56ad0d0bf2fecbd0231b0ad6e5b81fba\"},\"pull_closures\":[],\"recipe_pull\":{\"producer_tool\":\"open_kitchen\",\"recipe_name\":\"implementation\",\"descriptor_version\":2,\"schema_version\":2,\"payload_sha256\":\"sha256:ea2e62f28435745f27a7a742cbb98d2e34cdb00d82eef53117118c3b574d9497\",\"artifact_blob_sha256\":\"sha256:cd5e7fff1ca8145532aa9ce8ca2a26ddbfe48fb704316310789b8123319f8d63\",\"artifact_blob_size_bytes\":576427,\"body_sha256\":\"sha256:3f14e54f1505f8a5c4d13c2bc5658394855e4c7dc624cc527ddb4bf0711954d5\",\"body_size_bytes\":115869,\"flow_schema_version\":2,\"flow_sha256\":\"sha256:3b9ff899035aad7555d5863de6c70c6ed95e6ffac5f19cea0c46cc17261f4f2a\",\"flow_size_bytes\":101558,\"flow_record_count\":694,\"pull_tool\":\"get_recipe_section\"}}}"}},"sourceToolAssistantUUID":"e8638011-011a-4b7d-8a7f-a09899915dfb","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"09cc0463-fc85-4985-ae35-09732330b279","isSidechain":false,"message":{"id":"071b1ef27e5cad826dc3ce73f99787ac","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c91b227749fbd03b2b0","name":"mcp__plugin_autoskillit_autoskillit__create_and_publish_branch","input":{"issue_slug":"e2e-implementation-fixture-normalize-whitespace","run_name":"impl","issue_number":"1","work_dir":"/workspace/autoskillit-runs/impl-20261011-200500-587134","remote_url":"https://github.com/TalonT-Org/autoskillit-e2e-sandbox.git","step_name":"create_and_publish"}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":737,"cache_creation_input_tokens":0,"cache_read_input_tokens":46222,"output_tokens":182,"output_tokens_details":{"thinking_tokens":32},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"claim_and_resolve_issue","type":"assistant","uuid":"fd55e15c-8988-49ee-8a19-20eff2eb5919","timestamp":"2026-10-11T20:05:08.278Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"fd55e15c-8988-49ee-8a19-20eff2eb5919","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c91b227749fbd03b2b0","type":"tool_result","content":"## create_and_publish_branch\n\nmerge_target: e2e-implementation-fixture-normalize-whitespace/1-3\nwas_unique: False\ntimings:\n compute_ms: 0\n branch_create_ms: 1131\n push_ms: 1068"}]},"uuid":"d5b2d789-548b-4f26-be64-599540dab17e","timestamp":"2026-10-11T20:05:10.770Z","toolUseResult":"## create_and_publish_branch\n\nmerge_target: e2e-implementation-fixture-normalize-whitespace/1-3\nwas_unique: False\ntimings:\n compute_ms: 0\n branch_create_ms: 1131\n push_ms: 1068","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"merge_target\":\"e2e-implementation-fixture-normalize-whitespace/1-3\",\"was_unique\":false,\"timings\":{\"compute_ms\":0,\"branch_create_ms\":1131,\"push_ms\":1068}}"}},"sourceToolAssistantUUID":"fd55e15c-8988-49ee-8a19-20eff2eb5919","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"914bc97f-052b-46b7-ad36-0fa7d53b614f","isSidechain":false,"message":{"id":"071b1efbdcae033e3bee1954f2c790c3","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c91d9517369b630b48c","name":"mcp__plugin_autoskillit_autoskillit__run_skill","input":{"skill_command":"/autoskillit:make-plan","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","step_name":"plan","skill_inputs":{"adversarial_review_level":"auto","task":"E2E implementation fixture: normalize whitespace\n\n# Reusable implementation E2E fixture\n\nAdd `normalize_whitespace(text: str) -> str` to `sandbox/text.py`.\n\nThe function must remove leading and trailing whitespace and replace each run of\nwhitespace (including spaces, tabs, and newlines) with one ASCII space. Preserve\nthe case and punctuation of non-whitespace characters. Empty and whitespace-only\n...[fixture-truncated 737 chars]","issue_url":"https://github.com/TalonT-Org/autoskillit-e2e-sandbox/issues/1","audit_cycle_path":"","plan_set_coverage_gaps":""}}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":209,"cache_creation_input_tokens":0,"cache_read_input_tokens":47866,"output_tokens":435,"output_tokens_details":{"thinking_tokens":9},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"record_pipeline_step","type":"assistant","uuid":"25cae19a-d6ea-4f11-a8b3-c8937bd543b9","timestamp":"2026-10-11T20:05:18.314Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"25cae19a-d6ea-4f11-a8b3-c8937bd543b9","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c91d9517369b630b48c","type":"tool_result","content":"run_skill: FAIL [FAIL]\nexit_code: \nneeds_retry: False\nerror: RECIPE EXECUTION REJECTED [recipe_execution_attestation_missing]: an active recipe requires recipe_execution_id and invocation_template_digest; take both from the recipe_execution block of the complete_recipe_initialization receipt (bounded delivery) or of the open_kitchen response (inline delivery), using invocation_template_digests[ste...[fixture-truncated 274 chars]"}]},"uuid":"3936d669-1f07-41d4-bc84-3770955f1e04","timestamp":"2026-10-11T20:05:18.870Z","toolUseResult":"run_skill: FAIL [FAIL]\nexit_code: \nneeds_retry: False\nerror: RECIPE EXECUTION REJECTED [recipe_execution_attestation_missing]: an active recipe requires recipe_execution_id and invocation_template_digest; take both from the recipe_execution block of the complete_recipe_initialization receipt (bounded delivery) or of the open_kitchen response (inline delivery), using invocation_template_digests[ste...[fixture-truncated 274 chars]","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"success\":false,\"is_error\":true,\"error\":\"RECIPE EXECUTION REJECTED [recipe_execution_attestation_missing]: an active recipe requires recipe_execution_id and invocation_template_digest; take both from the recipe_execution block of the complete_recipe_initialization receipt (bounded delivery) or of the open_kitchen response (inline delivery), using invocation_template_digests[step_name] for this step; structured calls must initialize skill...[fixture-truncated 213 chars]\",\"stage\":\"preflight:recipe_execution\",\"retriable\":false}"}},"sourceToolAssistantUUID":"25cae19a-d6ea-4f11-a8b3-c8937bd543b9","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"02c03783-dcd5-4563-8311-e06aa7b8c603","isSidechain":false,"message":{"id":"071b1efe8d9be3c62f44b3fd56e52baa","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c91e6c07518b24e8d31","name":"mcp__plugin_autoskillit_autoskillit__run_skill","input":{"skill_command":"/autoskillit:make-plan","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","step_name":"plan","recipe_execution_id":"1dda3d6d29e545cbae216029ec657356","invocation_template_digest":"sha256:deec5b08a76bdabe0f7f520b9bb66f78789645f615012ac8eaf24806bef8e05e","skill_inputs":{"adversarial_review_level":"auto","task":"E2E implementation fixture: normalize whitespace\n\n# Reusable implementation E2E fixture\n\nAdd `normalize_whitespace(text: str) -> str` to `sandbox/text.py`.\n\nThe function must remove leading and trailing whitespace and replace each run of\nwhitespace (including spaces, tabs, and newlines) with one ASCII space. Preserve\nthe case and punctuation of non-whitespace characters. Empty and whitespace-only\n...[fixture-truncated 737 chars]","issue_url":"https://github.com/TalonT-Org/autoskillit-e2e-sandbox/issues/1","audit_cycle_path":"","plan_set_coverage_gaps":""}}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":630,"cache_creation_input_tokens":0,"cache_read_input_tokens":48075,"output_tokens":607,"output_tokens_details":{"thinking_tokens":94},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"run_skill","type":"assistant","uuid":"ef71da75-f7f0-44ec-a940-f1b9c8affbba","timestamp":"2026-10-11T20:05:21.760Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"ef71da75-f7f0-44ec-a940-f1b9c8affbba","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c91e6c07518b24e8d31","type":"tool_result","content":"run_skill: OK [success]\nsession_id: 8ef8f993-8caa-47dd-83b0-26f9e5b51139\nreceipt_id: 0739af7533c34089af07c982a53a62c1\nexit_code: 0 (infra-terminated after completion — grace exceeded)\nneeds_retry: False\noutcome_fields: {'plan_parts': '/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md', 'plan_path': '/workspace/autoski...[fixture-truncated 444 chars]"}]},"uuid":"bd81bead-a8ba-4a75-8388-a924aadc126d","timestamp":"2026-10-11T20:07:17.201Z","toolUseResult":"run_skill: OK [success]\nsession_id: 8ef8f993-8caa-47dd-83b0-26f9e5b51139\nreceipt_id: 0739af7533c34089af07c982a53a62c1\nexit_code: 0 (infra-terminated after completion — grace exceeded)\nneeds_retry: False\noutcome_fields: {'plan_parts': '/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md', 'plan_path': '/workspace/autoski...[fixture-truncated 444 chars]","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"success\":true,\"result\":\"verdict = plan\\nplan_path = /workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md\\nplan_parts = /workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md\",\"session_id\":\"8ef8f993-8caa-47dd-83b0-26f9e5b51139\",\"subtype\":\"success\",\"cli_subtype\":\"success\",\"is_error\":false,\"exit_code\":0,\"kill_reason\":\"kill_after_completion\",\"needs_retry\":false,\"retry_reason\":\"none\",\"stderr\":\"\",\"token_usage\":{\"backend\":\"claude-code\",\"provider_used\":\"anthropic\",\"input_tokens\":{\"state\":\"measured\",\"value\":54534},\"output_tokens\":{\"state\":\"measured\",\"value\":8817},\"cache_write_tokens\":{\"state\":\"measured_zero\",\"value\":0},\"cache_read_tokens\":{\"state\":\"measured\",\"value\":405101},\"model_breakdown\":{\"MiniMax-M3\":{\"input_tokens\":{\"state\":\"measured_zero\",\"value\":0},\"output_tokens\":{\"state\":\"measured_zero\",\"value\":0},\"cache_read_tokens\":{\"state\":\"unknown\",\"value\":null},\"cache_write_tokens\":{\"state\":\"unknown\",\"value\":null}}},\"peak_context\":{\"state\":\"unknown\",\"value\":null},\"turn_count\":10},\"write_path_warnings\":[],\"write_call_count\":1,\"fs_writes_detected\":true,\"git_writes_detected\":false,\"file_changes_count\":0,\"has_progress_evidence\":true,\"has_implementation_progress\":true,\"completion_required\":true,\"outcome_fields\":{\"plan_parts\":\"/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md\",\"plan_path\":\"/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md\",\"verdict\":\"plan\"},\"outcome_invariant_violated\":false,\"outcome_qualifier\":null,\"adjudication_verdict\":null,\"last_stop_reason\":\"\",\"lifespan_started\":true,\"provider_fallback\":false,\"provider_used\":\"anthropic\",\"infra_exit_category\":\"completed\",\"infra_cleanup_incomplete\":false,\"infra_fault_domain\":\"logic\",\"api_retry_count\":0,\"api_retry_last_error\":\"\",\"api_retry_last_status\":null,\"api_retry_exhausted\":false,\"api_error_status\":null,\"api_terminal_reason\":\"completed\",\"api_error_code\":\"\",\"api_error_message_seen\":false,\"candidate_exhausted\":false,\"execution_selection\":{\"selection_id\":\"838046e6cee249598edf9a01cc35eb1e\",\"campaign_id\":\"48915f19-d220-4069-8d55-b81efff84c05\",\"attempts\":[{\"candidate_id\":\"838046e6cee249598edf9a01cc35eb1e:0\",\"ordinal\":0,\"attempt\":1,\"parent_backend\":\"claude-code\",\"parent_provider\":\"anthropic\",\"parent_model\":\"sonnet\",\"requested_backend\":\"\",\"requested_provider\":\"\",\"requested_model\":\"\",\"effective_backend\":\"claude-code\",\"effective_provider\":\"anthropic\",\"effective_model\":\"sonnet\",\"backend_source_path\":\"agent_backend.backend\",\"provider_source_path\":\"backend.native_provider\",\"model_source_path\":\"backend.claude-code.model\",\"admission_status\":\"started\",\"admission_at_epoch\":1791749122,\"admission_cache_status\":\"quota_guard_disabled\",\"admission_cache_age_seconds\":null,\"rate_limit_status\":\"\",\"rate_limit_type\":\"\",\"rate_limit_resets_at_epoch\":null,\"rejection_reason\":null,\"transition\":\"primary\",\"execution_started\":true,\"child_session_id\":\"8ef8f993-8caa-47dd-83b0-26f9e5b51139\"}],\"terminal_candidate_id\":\"838046e6cee249598edf9a01cc35eb1e:0\",\"manifest_ref\":\"execution-candidates/838046e6cee249598edf9a01cc35eb1e.json\",\"invocation_deadline_epoch\":1791756276,\"remaining_retry_budget\":2,\"backend_rerouted\":false,\"candidate_fallback\":false,\"provider_fallback\":false,\"continuation\":null,\"completed\":true},\"rate_limit_status\":\"\",\"rate_limit_type\":\"\",\"rate_limit_resets_at_epoch\":null,\"pre_contamination_retry_reason\":\"none\",\"pre_contamination_subtype\":\"\",\"ndjson_unknown_event_count\":0,\"ndjson_unknown_item_count\":0,\"session_errors\":[],\"audit_status\":null,\"audit_verdict\":null,\"audit_cycle_path\":null,\"audit_attempt_id\":null,\"execution_identity\":{\"requested_parent_backend\":\"claude-code\",\"effective_parent_backend\":\"\",\"requested_parent_model\":\"sonnet\",\"effective_parent_model\":\"\",\"requested_parent_effort\":\"\",\"effective_parent_effort\":\"\",\"cli_version\":\"\",\"override_tier\":\"\",\"override_key_path\":\"\",\"parent_session_id\":\"\",\"children\":[]},\"order_id\":\"4666c3fd-486f-4ce1-b51f-9d9d660a5560\",\"receipt_id\":\"0739af7533c34089af07c982a53a62c1\"}"}},"sourceToolAssistantUUID":"ef71da75-f7f0-44ec-a940-f1b9c8affbba","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"9583a00d-f2bf-4f54-9e39-859d6ac3e38d","isSidechain":false,"message":{"id":"071b1f7556fb87c19d84e09c50262e1d","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c93aed67125a3a7504e","name":"mcp__plugin_autoskillit_autoskillit__complete_run_skill_result","input":{"receipt_id":"0739af7533c34089af07c982a53a62c1"}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":959,"cache_creation_input_tokens":0,"cache_read_input_tokens":48705,"output_tokens":89,"output_tokens_details":{"thinking_tokens":24},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"run_skill","type":"assistant","uuid":"fd7e9b53-087c-4247-9289-e909ff5fd4e5","timestamp":"2026-10-11T20:07:18.500Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"fd7e9b53-087c-4247-9289-e909ff5fd4e5","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c93aed67125a3a7504e","type":"tool_result","content":"## complete_run_skill_result\n\nsuccess: True\nreceipt_id: 0739af7533c34089af07c982a53a62c1\nrun_skill_success: True\nclassification: success\nsession_id: 8ef8f993-8caa-47dd-83b0-26f9e5b51139\ntracker:\n success: True\n incarnation_matches: True\n step: plan\n order_id: 4666c3fd-486f-4ce1-b51f-9d9d660a5560\n status: complete\n pipeline_id: 4666c3fd-486f-4ce1-b51f-9d9d660a5560\n done: 1\n total: 135\ntrack...[fixture-truncated 1025 chars]"}]},"uuid":"ac39e42b-7c63-4205-b7c9-a61025b1b4d6","timestamp":"2026-10-11T20:07:18.890Z","toolUseResult":"## complete_run_skill_result\n\nsuccess: True\nreceipt_id: 0739af7533c34089af07c982a53a62c1\nrun_skill_success: True\nclassification: success\nsession_id: 8ef8f993-8caa-47dd-83b0-26f9e5b51139\ntracker:\n success: True\n incarnation_matches: True\n step: plan\n order_id: 4666c3fd-486f-4ce1-b51f-9d9d660a5560\n status: complete\n pipeline_id: 4666c3fd-486f-4ce1-b51f-9d9d660a5560\n done: 1\n total: 135\ntrack...[fixture-truncated 1025 chars]","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"success\":true,\"receipt_id\":\"0739af7533c34089af07c982a53a62c1\",\"run_skill_success\":true,\"classification\":\"success\",\"session_id\":\"8ef8f993-8caa-47dd-83b0-26f9e5b51139\",\"tracker\":{\"success\":true,\"incarnation_matches\":true,\"step\":\"plan\",\"order_id\":\"4666c3fd-486f-4ce1-b51f-9d9d660a5560\",\"status\":\"complete\",\"pipeline_id\":\"4666c3fd-486f-4ce1-b51f-9d9d660a5560\",\"done\":1,\"total\":135},\"tracker_repairable\":false,\"recipe_segment\":{\"kind\":\"success\",\"source_step\":\"plan\",\"segments\":[],\"bodies\":[],\"recipe_execution\":{\"execution_id\":\"1dda3d6d29e545cbae216029ec657356\",\"invocation_template_digests\":{},\"skill_input_shapes\":{},\"snapshot_digest\":\"sha256:1269b37298577647aec24baeb183326c56ad0d0bf2fecbd0231b0ad6e5b81fba\"},\"pull_closures\":[],\"recipe_pull\":{\"producer_tool\":\"open_kitchen\",\"recipe_name\":\"implementation\",\"descriptor_version\":2,\"schema_version\":2,\"payload_sha256\":\"sha256:ea2e62f28435745f27a7a742cbb98d2e34cdb00d82eef53117118c3b574d9497\",\"artifact_blob_sha256\":\"sha256:cd5e7fff1ca8145532aa9ce8ca2a26ddbfe48fb704316310789b8123319f8d63\",\"artifact_blob_size_bytes\":576427,\"body_sha256\":\"sha256:3f14e54f1505f8a5c4d13c2bc5658394855e4c7dc624cc527ddb4bf0711954d5\",\"body_size_bytes\":115869,\"flow_schema_version\":2,\"flow_sha256\":\"sha256:3b9ff899035aad7555d5863de6c70c6ed95e6ffac5f19cea0c46cc17261f4f2a\",\"flow_size_bytes\":101558,\"flow_record_count\":694,\"pull_tool\":\"get_recipe_section\"}}}"}},"sourceToolAssistantUUID":"fd7e9b53-087c-4247-9289-e909ff5fd4e5","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"ce90af08-dd2d-4343-a661-8c8b229e79c6","isSidechain":false,"message":{"id":"071b20897697dd2a864edf9dfcc2d180","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c97eda97598bed1c282","name":"mcp__plugin_autoskillit_autoskillit__run_skill","input":{"skill_command":"/autoskillit:dry-walkthrough","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","step_name":"verify","recipe_execution_id":"1dda3d6d29e545cbae216029ec657356","invocation_template_digest":"sha256:7447506c5306026f763953953af00ee3cf4a723f01b786bf9aa989751756c5cf","output_dir":".autoskillit/temp","skill_inputs":{"plan_path":"/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md","issue_url":"https://github.com/TalonT-Org/autoskillit-e2e-sandbox/issues/1","review_path":"","audit_cycle_path":"","plan_disposition_path":"","plan_set_authority_path":"/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/plan-set-authority/planset-407e119e8ff1c06eeaf93992/0c6efb4faa404ec4fb97f4c0.json"}}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":3378,"cache_creation_input_tokens":0,"cache_read_input_tokens":160144,"output_tokens":534,"output_tokens_details":{"thinking_tokens":131},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"run_python","type":"assistant","uuid":"b02275c1-d87d-4a7b-a28e-c87b020aac8d","timestamp":"2026-10-11T20:11:56.730Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"b02275c1-d87d-4a7b-a28e-c87b020aac8d","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c97eda97598bed1c282","type":"tool_result","content":"run_skill: FAIL [zero_writes]\nsession_id: 54f18144-1188-45d7-a392-17b4529f721e\nreceipt_id: 18af52568f97412b9eecc304d8a75e7c\nexit_code: 0\nneeds_retry: True\nretry_reason: zero_writes\nhas_progress_evidence: True\nadjudication_verdict: {'reason_kind': 'zero_writes', 'subtype': 'zero_writes', 'detail': 'expected implementation writes were not observed', 'outcome_fields': None, 'defects': []}\n\nresult:\nex...[fixture-truncated 46 chars]"}]},"uuid":"e945da13-1cc3-48a0-9ce5-06578b883602","timestamp":"2026-10-11T20:14:47.218Z","toolUseResult":"run_skill: FAIL [zero_writes]\nsession_id: 54f18144-1188-45d7-a392-17b4529f721e\nreceipt_id: 18af52568f97412b9eecc304d8a75e7c\nexit_code: 0\nneeds_retry: True\nretry_reason: zero_writes\nhas_progress_evidence: True\nadjudication_verdict: {'reason_kind': 'zero_writes', 'subtype': 'zero_writes', 'detail': 'expected implementation writes were not observed', 'outcome_fields': None, 'defects': []}\n\nresult:\nex...[fixture-truncated 46 chars]","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"success\":false,\"result\":\"expected implementation writes were not observed\",\"session_id\":\"54f18144-1188-45d7-a392-17b4529f721e\",\"subtype\":\"zero_writes\",\"cli_subtype\":\"success\",\"is_error\":false,\"exit_code\":0,\"kill_reason\":\"natural_exit\",\"needs_retry\":true,\"retry_reason\":\"zero_writes\",\"stderr\":\"\",\"token_usage\":{\"backend\":\"claude-code\",\"provider_used\":\"anthropic\",\"input_tokens\":{\"state\":\"measured\",\"value\":48030},\"output_tokens\":{\"state\":\"measured\",\"value\":4349},\"cache_write_tokens\":{\"state\":\"measured_zero\",\"value\":0},\"cache_read_tokens\":{\"state\":\"measured\",\"value\":166797},\"model_breakdown\":{\"MiniMax-M3\":{\"input_tokens\":{\"state\":\"measured_zero\",\"value\":0},\"output_tokens\":{\"state\":\"measured_zero\",\"value\":0},\"cache_read_tokens\":{\"state\":\"unknown\",\"value\":null},\"cache_write_tokens\":{\"state\":\"unknown\",\"value\":null}}},\"peak_context\":{\"state\":\"unknown\",\"value\":null},\"turn_count\":5},\"write_path_warnings\":[],\"write_call_count\":0,\"fs_writes_detected\":true,\"git_writes_detected\":false,\"file_changes_count\":0,\"has_progress_evidence\":true,\"has_implementation_progress\":false,\"completion_required\":false,\"outcome_fields\":null,\"outcome_invariant_violated\":false,\"outcome_qualifier\":null,\"adjudication_verdict\":{\"reason_kind\":\"zero_writes\",\"subtype\":\"zero_writes\",\"detail\":\"expected implementation writes were not observed\",\"outcome_fields\":null,\"defects\":[]},\"last_stop_reason\":\"\",\"lifespan_started\":true,\"provider_fallback\":false,\"provider_used\":\"anthropic\",\"infra_exit_category\":\"completed\",\"infra_cleanup_incomplete\":false,\"infra_fault_domain\":\"logic\",\"api_retry_count\":0,\"api_retry_last_error\":\"\",\"api_retry_last_status\":null,\"api_retry_exhausted\":false,\"api_error_status\":null,\"api_terminal_reason\":\"completed\",\"api_error_code\":\"\",\"api_error_message_seen\":false,\"candidate_exhausted\":false,\"execution_selection\":{\"selection_id\":\"77495ee8add84a1b8334155a172ef192\",\"campaign_id\":\"48915f19-d220-4069-8d55-b81efff84c05\",\"attempts\":[{\"candidate_id\":\"77495ee8add84a1b8334155a172ef192:0\",\"ordinal\":0,\"attempt\":1,\"parent_backend\":\"claude-code\",\"parent_provider\":\"anthropic\",\"parent_model\":\"sonnet\",\"requested_backend\":\"\",\"requested_provider\":\"\",\"requested_model\":\"\",\"effective_backend\":\"claude-code\",\"effective_provider\":\"anthropic\",\"effective_model\":\"sonnet\",\"backend_source_path\":\"agent_backend.backend\",\"provider_source_path\":\"backend.native_provider\",\"model_source_path\":\"backend.claude-code.model\",\"admission_status\":\"started\",\"admission_at_epoch\":1791749517,\"admission_cache_status\":\"quota_guard_disabled\",\"admission_cache_age_seconds\":null,\"rate_limit_status\":\"\",\"rate_limit_type\":\"\",\"rate_limit_resets_at_epoch\":null,\"rejection_reason\":null,\"transition\":\"primary\",\"execution_started\":true,\"child_session_id\":\"54f18144-1188-45d7-a392-17b4529f721e\"}],\"terminal_candidate_id\":\"77495ee8add84a1b8334155a172ef192:0\",\"manifest_ref\":\"execution-candidates/77495ee8add84a1b8334155a172ef192.json\",\"invocation_deadline_epoch\":1791756276,\"remaining_retry_budget\":2,\"backend_rerouted\":false,\"candidate_fallback\":false,\"provider_fallback\":false,\"continuation\":null,\"completed\":true},\"rate_limit_status\":\"\",\"rate_limit_type\":\"\",\"rate_limit_resets_at_epoch\":null,\"pre_contamination_retry_reason\":\"none\",\"pre_contamination_subtype\":\"\",\"ndjson_unknown_event_count\":0,\"ndjson_unknown_item_count\":0,\"session_errors\":[],\"audit_status\":null,\"audit_verdict\":null,\"audit_cycle_path\":null,\"audit_attempt_id\":null,\"execution_identity\":{\"requested_parent_backend\":\"claude-code\",\"effective_parent_backend\":\"\",\"requested_parent_model\":\"sonnet\",\"effective_parent_model\":\"\",\"requested_parent_effort\":\"\",\"effective_parent_effort\":\"\",\"cli_version\":\"\",\"override_tier\":\"\",\"override_key_path\":\"\",\"parent_session_id\":\"\",\"children\":[]},\"order_id\":\"4666c3fd-486f-4ce1-b51f-9d9d660a5560\",\"receipt_id\":\"18af52568f97412b9eecc304d8a75e7c\"}"}},"sourceToolAssistantUUID":"b02275c1-d87d-4a7b-a28e-c87b020aac8d","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"3a518c19-addd-4ba2-a391-10c4149a6cd4","isSidechain":false,"message":{"id":"071b21375dfedbd6689fd160582e771b","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c9a935971faa349a923","name":"mcp__plugin_autoskillit_autoskillit__complete_run_skill_result","input":{"receipt_id":"18af52568f97412b9eecc304d8a75e7c"}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":727,"cache_creation_input_tokens":0,"cache_read_input_tokens":163522,"output_tokens":385,"output_tokens_details":{"thinking_tokens":319},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"run_skill","type":"assistant","uuid":"d5d574c9-e7be-4957-b3c9-4f1786d9de2a","timestamp":"2026-10-11T20:14:50.215Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"d5d574c9-e7be-4957-b3c9-4f1786d9de2a","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c9a935971faa349a923","type":"tool_result","content":"## complete_run_skill_result\n\nsuccess: True\nreceipt_id: 18af52568f97412b9eecc304d8a75e7c\nrun_skill_success: False\nclassification: zero_writes\nsession_id: 54f18144-1188-45d7-a392-17b4529f721e\ntracker:\n success: True\n status: not_applied\ntracker_repairable: False\n\n--- RECIPE SEGMENT ---\n{\"kind\":\"recovery\",\"pull_closure\":[\"renew_plan_set\",\"create_impl_worktree\",\"implement\",\"release_issue_failure\",\"...[fixture-truncated 1059 chars]"}]},"uuid":"fc21e657-3713-460e-b508-f14b28a65117","timestamp":"2026-10-11T20:14:50.513Z","toolUseResult":"## complete_run_skill_result\n\nsuccess: True\nreceipt_id: 18af52568f97412b9eecc304d8a75e7c\nrun_skill_success: False\nclassification: zero_writes\nsession_id: 54f18144-1188-45d7-a392-17b4529f721e\ntracker:\n success: True\n status: not_applied\ntracker_repairable: False\n\n--- RECIPE SEGMENT ---\n{\"kind\":\"recovery\",\"pull_closure\":[\"renew_plan_set\",\"create_impl_worktree\",\"implement\",\"release_issue_failure\",\"...[fixture-truncated 1059 chars]","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"success\":true,\"receipt_id\":\"18af52568f97412b9eecc304d8a75e7c\",\"run_skill_success\":false,\"classification\":\"zero_writes\",\"session_id\":\"54f18144-1188-45d7-a392-17b4529f721e\",\"tracker\":{\"success\":true,\"status\":\"not_applied\"},\"tracker_repairable\":false,\"recipe_segment\":{\"kind\":\"recovery\",\"source_step\":\"verify\",\"target_steps\":[\"renew_plan_set\",\"release_issue_failure\"],\"pull_closure\":[\"renew_plan_set\",\"create_impl_worktree\",\"implement\",\"release_issue_failure\",\"register_clone_failure\",\"escalate_stop\"],\"pull_requests\":[{\"section\":\"renew_plan_set\",\"part\":0},{\"section\":\"create_impl_worktree\",\"part\":0},{\"section\":\"implement\",\"part\":0},{\"section\":\"release_issue_failure\",\"part\":0},{\"section\":\"register_clone_failure\",\"part\":0},{\"section\":\"escalate_stop\",\"part\":0}],\"recipe_pull\":{\"producer_tool\":\"open_kitchen\",\"recipe_name\":\"implementation\",\"descriptor_version\":2,\"schema_version\":2,\"payload_sha256\":\"sha256:ea2e62f28435745f27a7a742cbb98d2e34cdb00d82eef53117118c3b574d9497\",\"artifact_blob_sha256\":\"sha256:cd5e7fff1ca8145532aa9ce8ca2a26ddbfe48fb704316310789b8123319f8d63\",\"artifact_blob_size_bytes\":576427,\"body_sha256\":\"sha256:3f14e54f1505f8a5c4d13c2bc5658394855e4c7dc624cc527ddb4bf0711954d5\",\"body_size_bytes\":115869,\"flow_schema_version\":2,\"flow_sha256\":\"sha256:3b9ff899035aad7555d5863de6c70c6ed95e6ffac5f19cea0c46cc17261f4f2a\",\"flow_size_bytes\":101558,\"flow_record_count\":694,\"pull_tool\":\"get_recipe_section\"}}}"}},"sourceToolAssistantUUID":"d5d574c9-e7be-4957-b3c9-4f1786d9de2a","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":null,"logicalParentUuid":"282278b8-9dd2-4a6f-bbaa-1105acc27bfb","isSidechain":false,"type":"system","subtype":"compact_boundary","content":"Conversation compacted","level":"info","compactMetadata":{"trigger":"auto","preTokens":166958,"postTokens":3121,"cumulativeDroppedTokens":163837,"durationMs":21279,"preservedSegment":{"headUuid":"b00cd523-124f-4830-9333-a77e74e0f7f5","anchorUuid":"c73a3283-5a21-4482-8124-50d7d3251b94","tailUuid":"282278b8-9dd2-4a6f-bbaa-1105acc27bfb"},"preservedMessages":{"anchorUuid":"c73a3283-5a21-4482-8124-50d7d3251b94","uuids":["b00cd523-124f-4830-9333-a77e74e0f7f5","5c92e852-aedc-4502-aabd-4ecc5cb13843","f2cc56c1-0cfa-4ba2-bf38-64115b02adea","282278b8-9dd2-4a6f-bbaa-1105acc27bfb"],"allUuids":["b00cd523-124f-4830-9333-a77e74e0f7f5","2aac9b5f-a477-43f9-bb2f-bf74d1f49f8c","05edcb98-33b3-4462-b670-bca0163361fe","c9579bc1-e850-46cf-8f41-586f87a894ae","8afdb5bf-7d28-49c8-af1a-08f5a3c343fe","62b75b38-cc5e-4922-b5a5-398281472724","52c3981f-6147-4b1f-8256-531acee3828c","4f22a38f-6841-42e3-ba27-40a45b71387f","d09734c1-860a-4015-8590-1c8491e75c14","18b90c12-6bfb-4bd0-8c79-f7d3c7602c87","0287af2b-1d5a-4266-8e75-29c3e96ec868","0a3f287e-7607-4ba9-b042-dd1701ebea9d","0e407ba0-f60d-4f94-9ba8-4543674e2e3f","cb4e9a54-e7ed-433e-950c-e93211025deb","6aa3d78b-4514-456f-a42a-a76e136692d6","04dd0b8b-f81f-4fdd-a777-59a57d69eeb5","5c92e852-aedc-4502-aabd-4ecc5cb13843","fb3efad1-a773-4c1d-9d47-b6760694a153","f2cc56c1-0cfa-4ba2-bf38-64115b02adea","282278b8-9dd2-4a6f-bbaa-1105acc27bfb"]}},"uuid":"9a99fc5b-0ab0-4f91-a84b-4b2196d3b702","timestamp":"2026-10-11T20:15:22.874Z","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main","slug":"shimmying-tinkering-dream"} +{"parentUuid":"a286e33c-9ae5-4118-90ea-99d928d70a15","isSidechain":false,"message":{"id":"071b2190376d0b42f31507604c7b85ae","type":"message","role":"assistant","content":[{"type":"text","text":"The analyze_pipeline_health_error is `optional: true` with `skip_when_false: inputs.pipeline_health`. Since pipeline_health wasn't set, this will skip to escalate_stop, which is the stop action.\n\nThe verify step's zero_writes failure has been correctly routed to release_issue_failure → register_clone_failure → escalate_stop. The pipeline is now at the terminal stop point. Per the message:\n\n> \"Pipe...[fixture-truncated 1476 chars]"}],"model":"MiniMax-M3","stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":455,"cache_creation_input_tokens":0,"cache_read_input_tokens":41427,"output_tokens":472,"output_tokens_details":{"thinking_tokens":0},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":0,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"run_cmd","type":"assistant","uuid":"8136c073-129f-45bf-a17c-71a2d795dd38","timestamp":"2026-10-11T20:16:20.582Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main","slug":"shimmying-tinkering-dream"} +{"parentUuid":"8136c073-129f-45bf-a17c-71a2d795dd38","isSidechain":false,"type":"system","subtype":"stop_hook_summary","hookCount":2,"hookInfos":[{"command":"python3 -B \"${CLAUDE_PLUGIN_ROOT}/hooks/_dispatch.py\" guards/join_stop_guard","durationMs":117},{"command":"python3 -B \"${CLAUDE_PLUGIN_ROOT}/hooks/_dispatch.py\" session_lifetime_notice_hook","durationMs":99}],"hookErrors":[],"hookAdditionalContext":[],"preventedContinuation":false,"stopReason":"","hasOutput":false,"level":"suggestion","timestamp":"2026-10-11T20:16:20.704Z","uuid":"28868d68-9669-4c8c-874c-7b76c09003b0","toolUseID":"fdb4c1c9-f5b5-4999-8095-64dd918d9388","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main","slug":"shimmying-tinkering-dream"} +{"type":"cost-state","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","totalCostUSD":3.1986767999999994,"totalAPIDuration":314689,"totalAPIDurationWithoutRetries":314423,"totalToolDuration":300139,"totalLinesAdded":0,"totalLinesRemoved":0,"totalDuration":704309,"startTime":1791749076557,"modelUsage":{"claude-sonnet-5":{"inputTokens":206383,"outputTokens":43841,"thinkingTokens":17593,"cacheReadInputTokens":11737504,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":3.1986767999999994}},"hasUnknownModelCost":false} diff --git a/tests/infra/test_canonical_json_producer_convention.py b/tests/infra/test_canonical_json_producer_convention.py index 0038c812a7..5d9ed71ada 100644 --- a/tests/infra/test_canonical_json_producer_convention.py +++ b/tests/infra/test_canonical_json_producer_convention.py @@ -132,7 +132,7 @@ def _scan_child_tool_authority_producer_sites() -> set[tuple[str, int, str]]: producer_symbol="write_audit_disposition_bundle", producer_path=_TYPED_PRODUCER_MODULE, producer_function="_write_disposition_report", - skill_md_refs=(("src/autoskillit/skills_extended/make-plan/SKILL.md", 383, 395),), + skill_md_refs=(("src/autoskillit/skills_extended/make-plan/SKILL.md", 386, 398),), ), "inventory": CanonicalArtifactDef( consumer_site=("src/autoskillit/core/audit/audit_cycle_verifier.py", (660,)), @@ -146,7 +146,7 @@ def _scan_child_tool_authority_producer_sites() -> set[tuple[str, int, str]]: producer_symbol="write_audit_disposition_bundle", producer_path=_TYPED_PRODUCER_MODULE, producer_function="_write_plan_association", - skill_md_refs=(("src/autoskillit/skills_extended/make-plan/SKILL.md", 383, 395),), + skill_md_refs=(("src/autoskillit/skills_extended/make-plan/SKILL.md", 386, 398),), ), "audit_semantic_result": CanonicalArtifactDef( consumer_site=("src/autoskillit/core/audit/audit_semantic_codec.py", 423), diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index cc2ea845d6..b3d0143897 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -13,10 +13,12 @@ from pathlib import Path from types import SimpleNamespace from typing import Any +from urllib.parse import quote import pytest from tests.infra._complexity_helpers import load_check_script +from tests.infra._e2e_evidence import load_provenance, materialize_run pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] @@ -26,6 +28,206 @@ harness = load_check_script("_autoskillit_e2e_harness", _HARNESS_SCRIPT) e2e_catalog = harness.e2e_catalog + +@pytest.fixture(autouse=True) +def _isolated_sandbox(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(harness, "SANDBOX_CLONE", tmp_path / "workspace" / "sandbox") + + +@pytest.fixture +def failure_sources(tmp_path: Path) -> dict[Path, Path]: + runs = harness.SANDBOX_CLONE.parent / harness.PIPELINE_RUNS_DIR + project = tmp_path / "home" / ".claude" / "projects" / "project" + sources = { + runs / "clone-a/.autoskillit/temp/make-plan/a_plan.md": Path( + "pipeline-artifacts/clone-a/make-plan/a_plan.md" + ), + runs / "clone-b/.autoskillit/temp/rectify/b_plan.md": Path( + "pipeline-artifacts/clone-b/rectify/b_plan.md" + ), + runs / "clone-a/.autoskillit/temp/make-plan/nested/metadata.json": Path( + "pipeline-artifacts/clone-a/make-plan/nested/metadata.json" + ), + project / "session.jsonl": Path("transcripts/project/session.jsonl"), + project / "session/subagents/agent-one.jsonl": Path( + "transcripts/project/session/subagents/agent-one.jsonl" + ), + } + for source, relative in sources.items(): + source.parent.mkdir(parents=True, exist_ok=True) + source.write_bytes(f"evidence for {relative}\n".encode()) + (project / "ignored.txt").write_text("not a transcript") + return sources + + +@pytest.mark.parametrize( + ("case", "outcome", "passed"), + ( + ("failed", "failed", False), + ("exception", "failed", False), + ("expected_failure", "expected_failure", False), + ("passed", "passed", True), + ), +) +def test_run_test_retains_evidence_only_when_it_does_not_pass( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure_sources, case, outcome, passed +) -> None: + def run(*_args, **_kwargs): + if case == "exception": + raise RuntimeError("harness crashed") + return ( + ["test failed"] if case == "failed" else [], + [_SMOKE_FAILURE] if case == "expected_failure" else [], + ) + + monkeypatch.setattr(harness, "_run_test", run) + failures, result = _run_recipe(tmp_path, FakeRunner(lambda _argv: None)) + out = tmp_path / "out" + assert result["passed"] is passed + assert result["outcome"] == outcome + assert result["failures"] == failures + if passed: + assert "evidence" not in result + assert not (out / "pipeline-artifacts").exists() + assert not (out / "transcripts").exists() + return + assert result["evidence"] == {"plan_files": 3, "transcripts": 2, "errors": []} + for source, relative in failure_sources.items(): + assert (out / relative).read_bytes() == source.read_bytes() + assert not (out / "transcripts/project/ignored.txt").exists() + + +@pytest.mark.parametrize("failure_site", ("copy", "walk")) +def test_evidence_errors_preserve_the_verdict_and_continue_other_sources( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure_sources, failure_site +) -> None: + source = next(path for path in failure_sources if path.name == "a_plan.md") + target = source if failure_site == "copy" else source.parent + owner = harness.shutil if failure_site == "copy" else harness.os + name = "copyfile" if failure_site == "copy" else "scandir" + original = getattr(owner, name) + + def unreadable(path, *args, **kwargs): + if path in (target, str(target)): + raise OSError("cannot read evidence") + return original(path, *args, **kwargs) + + monkeypatch.setattr(owner, name, unreadable) + monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: (["test failed"], [])) + failures, result = _run_recipe(tmp_path, FakeRunner(lambda _argv: None)) + assert failures == ["test failed"] + assert result["outcome"] == "failed" + assert result["passed"] is False + assert result["evidence"]["transcripts"] == 2 + assert any(str(target) in error for error in result["evidence"]["errors"]) + readable = next(path for path in failure_sources if path.name == "b_plan.md") + assert (tmp_path / "out" / failure_sources[readable]).read_bytes() == readable.read_bytes() + + +def test_unexpected_collection_error_still_writes_the_result( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def crash(*_args, **_kwargs): + try: + raise ValueError("collection root cause") + except ValueError as exc: + raise RuntimeError("collection crashed") from exc + + monkeypatch.setattr(harness, "collect_failure_evidence", crash) + monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: (["test failed"], [])) + failures, result = _run_recipe(tmp_path, FakeRunner(lambda _argv: None)) + assert failures == ["test failed"] + assert result["outcome"] == "failed" + assert result["evidence"]["plan_files"] == result["evidence"]["transcripts"] == 0 + assert "collection crashed" in result["evidence"]["errors"][0] + assert "ValueError: collection root cause" in result["evidence"]["exception"] + assert "RuntimeError: collection crashed" in result["evidence"]["exception"] + assert 'raise RuntimeError("collection crashed") from exc' in result["evidence"]["exception"] + + +def test_nothing_to_collect_adds_no_evidence_key( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: (["test failed"], [])) + _, result = _run_recipe(tmp_path, FakeRunner(lambda _argv: None)) + assert "evidence" not in result + + +def test_failure_evidence_skips_symlinks_and_linked_scan_ancestors(tmp_path: Path) -> None: + runs = harness.SANDBOX_CLONE.parent / harness.PIPELINE_RUNS_DIR + plans = runs / "clone-a/.autoskillit/temp/make-plan" + plans.mkdir(parents=True) + (plans / "kept.md").write_text("kept") + external = tmp_path / "external" + (external / "make-plan").mkdir(parents=True) + (external / "make-plan" / "outside.md").write_text("outside") + (plans / "file-link.md").symlink_to(external / "make-plan" / "outside.md") + (plans / "dir-link").symlink_to(external, target_is_directory=True) + ancestor = runs / "clone-b/.autoskillit" + ancestor.mkdir(parents=True) + (ancestor / "temp").symlink_to(external, target_is_directory=True) + (runs / "linked-clone").symlink_to(runs / "clone-a", target_is_directory=True) + home = tmp_path / "home" + home.mkdir() + (external / "projects").mkdir() + (external / "projects" / "outside.jsonl").write_text("outside transcript") + (home / ".claude").symlink_to(external, target_is_directory=True) + out = tmp_path / "out" + + result = harness.collect_failure_evidence(out, home) + + assert result == {"plan_files": 1, "transcripts": 0, "errors": []} + assert [path.relative_to(out) for path in out.rglob("*") if path.is_file()] == [ + Path("pipeline-artifacts/clone-a/make-plan/kept.md") + ] + + +def test_collected_plans_and_transcripts_are_redacted( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure_sources +) -> None: + plan = next(path for path in failure_sources if path.name == "a_plan.md") + transcript = next(path for path in failure_sources if path.name == "session.jsonl") + plan.write_bytes(MINIMAX_KEY.encode()) + transcript.write_bytes(SANDBOX_TOKEN.encode()) + monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: (["test failed"], [])) + _run_recipe(tmp_path, FakeRunner(lambda _argv: None)) + upload = tmp_path / "upload" + + harness.redact_tree([tmp_path / "out"], upload, [MINIMAX_KEY, SANDBOX_TOKEN]) + + for source, secret in ((plan, MINIMAX_KEY), (transcript, SANDBOX_TOKEN)): + copied = (upload / "out" / failure_sources[source]).read_bytes() + assert harness.REDACTED in copied + assert secret.encode() not in copied + + +def test_runs_scan_error_keeps_readable_transcripts( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure_sources +) -> None: + runs = harness.SANDBOX_CLONE.parent / harness.PIPELINE_RUNS_DIR + original = Path.iterdir + + def unreadable(path): + if path == runs: + raise OSError("cannot enumerate runs") + return original(path) + + monkeypatch.setattr(Path, "iterdir", unreadable) + monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: (["test failed"], [])) + failures, result = _run_recipe(tmp_path, FakeRunner(lambda _argv: None)) + assert failures == ["test failed"] + assert result["outcome"] == "failed" + assert result["evidence"]["plan_files"] == 0 + assert result["evidence"]["transcripts"] == 2 + assert str(runs) in result["evidence"]["errors"][0] + + +def test_pipeline_runs_directory_matches_clone_source_of_truth() -> None: + from autoskillit.workspace.clone._detect import RUNS_DIR + + assert harness.PIPELINE_RUNS_DIR == RUNS_DIR + + MINIMAX_KEY = "minimax-test-key" SANDBOX_TOKEN = "sandbox-test-token" SANDBOX = "owner/sandbox" @@ -182,7 +384,9 @@ def _env(**overrides: str) -> dict[str, str]: return {name: value for name, value in env.items() if value} -def _pipeline_catalog(expected_failures: list[dict[str, str]] | None = None): +def _pipeline_catalog( + expected_failures: list[dict[str, str]] | None = None, *, repository=SANDBOX +): entry = { "name": "implementation", "kind": "recipe", @@ -200,7 +404,7 @@ def _pipeline_catalog(expected_failures: list[dict[str, str]] | None = None): } if expected_failures is not None: entry["expected_failures"] = expected_failures - return e2e_catalog.parse_catalog({"sandbox_repository": SANDBOX, "tests": [entry]}) + return e2e_catalog.parse_catalog({"sandbox_repository": repository, "tests": [entry]}) def _pipeline_pr(number: int = 8, *, branch: str = _PIPELINE_BRANCH) -> dict[str, Any]: @@ -243,6 +447,7 @@ def _pipeline_execution_response(argv: list[str], state: dict[str, Any]): if argv[:3] == ["autoskillit", "fleet", "run"]: if state["fleet_exception"] is not None: raise state["fleet_exception"] + state["head_refs"].extend(state["published_refs"]) return _completed(argv, json.dumps(state["fleet_envelope"]) + "\n") if argv[:2] == ["task", "test-check"]: state["test_check_count"] += 1 @@ -287,11 +492,29 @@ def _pipeline_pull_request_api_response(argv: list[str], endpoint: str, state: d return _completed(argv, json.dumps(state["pr_details"][number])) +def _pipeline_refs_response(argv: list[str], endpoint: str, state: dict[str, Any]): + if "/git/matching-refs/heads/" not in endpoint: + return None + state["refs_list_calls"] += 1 + error = state["refs_list_faults"].get(state["refs_list_calls"]) + if error is not None: + raise error + response = state["refs_list_response"] + if response is None: + response = [[{"ref": f"refs/heads/{name}"} for name in state["head_refs"]]] + return _completed(argv, json.dumps(response)) + + def _pipeline_cleanup_api_response(argv: list[str], endpoint: str, state: dict[str, Any]): if argv[2] == "--method": error = state["cleanup_errors"].get("branch_delete") if error is not None: raise error + if argv[3] == "DELETE" and "/git/refs/heads/" in endpoint: + ref_name = endpoint.split("/git/refs/heads/", 1)[1] + state["head_refs"] = [ + name for name in state["head_refs"] if quote(name, safe="") != ref_name + ] return None if "/git/refs/heads/" in endpoint: error = state["cleanup_errors"].get("branch_get") @@ -308,6 +531,7 @@ def _pipeline_api_response(argv: list[str], state: dict[str, Any]): for route in ( _pipeline_ci_api_response, _pipeline_pull_request_api_response, + _pipeline_refs_response, _pipeline_cleanup_api_response, ): response = route(argv, endpoint, state) @@ -391,6 +615,11 @@ def _pipeline_runner() -> tuple[FakeRunner, dict[str, Any]]: "l3_payload": {"reason": "implementation_complete"}, }, "fleet_exception": None, + "head_refs": ["main"], + "published_refs": [], + "refs_list_calls": 0, + "refs_list_faults": {}, + "refs_list_response": None, "cleanup_errors": {}, "proof": {"branches": [_PIPELINE_BRANCH], "claimed": True, "violations": []}, "summary": {"coverage": {"complete": True}, "peak_sessions": 4, "violations": []}, @@ -426,11 +655,17 @@ def _run_pipeline( *, env=None, expected_failures=None, + real_measurement=False, + repository=SANDBOX, ): - catalog = _pipeline_catalog(expected_failures) + catalog = _pipeline_catalog(expected_failures, repository=repository) out = tmp_path / "out" home = tmp_path / "home" - observed = _patch_pipeline_measurement(monkeypatch, state) + if real_measurement: + monkeypatch.setattr(harness.time, "sleep", lambda _seconds: None) + observed = None + else: + observed = _patch_pipeline_measurement(monkeypatch, state) failures = harness.run_test( catalog.get("implementation"), catalog, @@ -1721,7 +1956,7 @@ def test_steps_run_in_order_with_the_token_only_on_stdin(self, tmp_path: Path) - ], ["gh", "auth", "login", "--hostname", "github.com", "--with-token"], ["gh", "auth", "setup-git"], - ["gh", "repo", "clone", SANDBOX, "/workspace/sandbox"], + ["gh", "repo", "clone", SANDBOX, str(harness.SANDBOX_CLONE)], ["gh", "pr", "list", "--repo", SANDBOX, "--state", "all"] + ["--limit", "1", "--json", "number"], ["autoskillit", "fleet", "run", "implementation", "-i", "task=Add a greeting"] @@ -2262,6 +2497,236 @@ def test_fleet_exception_and_cleanup_exception_are_both_preserved( assert any("cleanup branch: branch delete failed" in failure for failure in failures) assert any(call.argv[:3] == ["gh", "issue", "edit"] for call in runner.calls) + def test_branch_created_without_ownership_proof_fails_cleanup_and_is_preserved( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["fleet_envelope"] = { + "success": False, + "dispatch_status": "retry_exhausted", + "dispatch_id": "dispatch-5233", + "dispatched_session_id": "session-5233", + "reason": "known orchestration failure", + } + state["listed_prs"] = [] + state["pr_details"] = {} + state["final_issue"] = {"state": "OPEN", "labels": []} + state["proof"] = {"branches": [], "claimed": False, "violations": []} + state["published_refs"] = ["impl/leaked"] + + failures, _result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert ( + "cleanup: branch impl/leaked was created during this run without ownership proof" + in failures + ) + assert not any( + call.argv[:4] == ["gh", "api", "--method", "DELETE"] for call in runner.calls + ) + assert "impl/leaked" in state["head_refs"] + + def test_proven_branch_is_deleted_and_leaves_nothing_unexplained( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["published_refs"] = [_PIPELINE_BRANCH] + original_list_head_refs = harness.list_head_refs + snapshots = {} + + def record_head_refs(repository, out, env, runner, stage): + refs = original_list_head_refs(repository, out, env, runner, stage) + snapshots[stage] = refs + return refs + + monkeypatch.setattr(harness, "list_head_refs", record_head_refs) + failures, _result, out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + delete = [ + "gh", + "api", + "--method", + "DELETE", + f"repos/{SANDBOX}/git/refs/heads/{quote(_PIPELINE_BRANCH, safe='')}", + ] + assert not any(failure.startswith("cleanup") for failure in failures) + assert any(call.argv == delete for call in runner.calls) + assert state["head_refs"] == ["main"] + assert snapshots["before"] == snapshots["after"] == frozenset({"main"}) + for stage in ("before", "after"): + assert json.loads( + (out / f"head-refs-{stage}.stdout.txt").read_text(encoding="utf-8") + ) == [[{"ref": "refs/heads/main"}]] + + def test_branch_present_before_the_run_is_never_reported(self, tmp_path, monkeypatch) -> None: + runner, state = _pipeline_runner() + state["fleet_envelope"] = { + "success": False, + "dispatch_status": "retry_exhausted", + "dispatch_id": "dispatch-5233", + "dispatched_session_id": "session-5233", + "reason": "known orchestration failure", + } + state["listed_prs"] = [] + state["pr_details"] = {} + state["final_issue"] = {"state": "OPEN", "labels": []} + state["head_refs"] = ["main", "impl/old-leftover"] + state["proof"] = {"branches": [], "claimed": False, "violations": []} + + failures, _result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert not any("impl/old-leftover" in failure for failure in failures) + assert not any( + call.argv[:4] == ["gh", "api", "--method", "DELETE"] for call in runner.calls + ) + + @pytest.mark.parametrize( + ("call_number", "expected_prefix"), [(1, "preflight:"), (2, "cleanup refs:")] + ) + def test_ref_listing_failure_is_reported( + self, tmp_path: Path, monkeypatch, call_number: int, expected_prefix: str + ) -> None: + runner, state = _pipeline_runner() + state["published_refs"] = [_PIPELINE_BRANCH] + state["refs_list_faults"] = {call_number: RuntimeError("refs unavailable")} + + failures, _result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + listing_failures = [failure for failure in failures if failure.startswith(expected_prefix)] + assert len(listing_failures) == 1 + assert "refs unavailable" in listing_failures[0] + if call_number == 1: + assert not any( + call.argv[:3] == ["autoskillit", "fleet", "run"] for call in runner.calls + ) + else: + delete = [ + "gh", + "api", + "--method", + "DELETE", + f"repos/{SANDBOX}/git/refs/heads/{quote(_PIPELINE_BRANCH, safe='')}", + ] + assert any(call.argv == delete for call in runner.calls) + assert any( + call.argv[:3] == ["gh", "issue", "edit"] + and call.argv[-2:] == ["--remove-label", "in-progress"] + for call in runner.calls + ) + + @pytest.mark.parametrize( + "response", + [[], [[{}]], [[{"ref": "refs/tags/v1"}]]], + ) + def test_malformed_ref_listing_fails_preflight( + self, tmp_path: Path, monkeypatch, response + ) -> None: + runner, state = _pipeline_runner() + state["refs_list_response"] = response + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert sum(failure.startswith("preflight:") for failure in failures) == 1 + assert not any(call.argv[:3] == ["autoskillit", "fleet", "run"] for call in runner.calls) + + def test_proven_branch_whose_delete_fails_is_not_reported_as_unexplained( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["published_refs"] = [_PIPELINE_BRANCH] + state["cleanup_errors"] = {"branch_delete": RuntimeError("branch delete failed")} + + failures, _result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert sum("cleanup branch: branch delete failed" in failure for failure in failures) == 1 + assert not any("without ownership proof" in failure for failure in failures) + + def test_failed_run_teardown_deletes_the_branch_proven_by_captured_evidence( + self, tmp_path: Path, monkeypatch + ) -> None: + provenance = load_provenance("run_38170521660_attempt_1") + repository = provenance["repository"] + issue_url = provenance["issue_url"] + claim_label = provenance["claim_label"] + branch = "e2e-implementation-fixture-normalize-whitespace/1-3" + runner, state = _pipeline_runner() + state["seed"] = { + **state["seed"], + "number": int(issue_url.rsplit("/", 1)[1]), + "url": issue_url, + } + state["config"]["github"]["in_progress_label"] = claim_label + state["listed_prs"] = [] + state["pr_details"] = {} + state["final_issue"] = {"state": "OPEN", "labels": []} + state["published_refs"] = [branch] + out = tmp_path / "out" + home = tmp_path / "home" + original_handler = runner.handler + + def materialize_before_fleet(argv): + if argv[:3] == ["autoskillit", "fleet", "run"]: + state["fleet_envelope"] = materialize_run( + "run_38170521660_attempt_1", + trace=out / "session-events.jsonl", + log_root=home / ".local" / "share" / "autoskillit" / "logs", + projects=home / ".claude" / "projects", + ) + return original_handler(argv) + + runner.handler = materialize_before_fleet + failures, result, out, _home, _catalog, _observed = _run_pipeline( + tmp_path, + runner, + state, + monkeypatch, + env=_env(E2E_IMPLEMENTATION_ISSUE_URL=issue_url), + real_measurement=True, + repository=repository, + ) + + delete = [ + "gh", + "api", + "--method", + "DELETE", + f"repos/{repository}/git/refs/heads/{quote(branch, safe='')}", + ] + assert any(call.argv == delete for call in runner.calls) + assert state["head_refs"] == ["main"] + assert not any(failure.startswith(("ownership:", "cleanup")) for failure in failures) + assert result["outcome"] == "failed" + assert ( + sum( + failure.startswith("fleet_envelope:") + and "resumable: fleet_l3_no_result_block" in failure + for failure in failures + ) + == 1 + ) + assert ( + sum( + failure.startswith("session measurement: incomplete lifecycle coverage") + for failure in failures + ) + == 1 + ) + assert json.loads((out / "ownership.json").read_text(encoding="utf-8"))["branches"] == [ + branch + ] + class TestCanaryFlow: def test_canary_runs_one_claude_prompt(self, tmp_path: Path, monkeypatch) -> None: @@ -2529,6 +2994,7 @@ def test_stale_expectation_fails_and_unexpected_diagnostic_is_unlisted( def test_cli_names_expected_failure_and_run_test_handles_exceptions( self, tmp_path, monkeypatch, capsys ): + monkeypatch.setattr(harness.Path, "home", classmethod(lambda _cls: tmp_path / "home")) diagnostic = _diagnostic("error", "runtime", "known clean-install failure") expected = {**diagnostic, "issue": _CLEAN_INSTALL_ISSUE} catalog_path = tmp_path / "catalog.json" diff --git a/tests/infra/test_e2e_sessions.py b/tests/infra/test_e2e_sessions.py index b2bce94d4e..bd86616213 100644 --- a/tests/infra/test_e2e_sessions.py +++ b/tests/infra/test_e2e_sessions.py @@ -9,6 +9,7 @@ import pytest +from autoskillit.core import DIRECT_PREFIX, PLUGIN_PREFIX from tests.infra._complexity_helpers import load_check_script pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] @@ -138,7 +139,40 @@ def _base_sessions( return log_root, trace, child_transcript -def _tool_use(tool_id: str, name: str, tool_input: dict[str, Any]) -> dict[str, Any]: +def _worker(log_root: Path, session_id: str) -> Path: + transcript = _session(log_root, session_id) + index_path = log_root / "sessions.jsonl" + entries = [json.loads(line) for line in index_path.read_text(encoding="utf-8").splitlines()] + entries.append(_entry(session_id, transcript)) + _index(log_root, entries) + return transcript + + +def _trace_root_and_worker( + trace: Path, + log_root: Path, + worker_id: str, + worker_transcript: Path, +) -> None: + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionStart", worker_id, 2, transcript=worker_transcript), + _event("SessionEnd", worker_id, 3, transcript=worker_transcript), + _event("SessionEnd", ROOT_ID, 4, transcript=root_transcript), + ], + ) + + +def _tool_use( + tool_id: str, + name: str, + tool_input: dict[str, Any], + *, + prefix: str = PLUGIN_PREFIX, +) -> dict[str, Any]: return { "type": "assistant", "message": { @@ -146,7 +180,7 @@ def _tool_use(tool_id: str, name: str, tool_input: dict[str, Any]) -> dict[str, { "type": "tool_use", "id": tool_id, - "name": f"mcp__autoskillit__{name}", + "name": f"{prefix}{name}", "input": tool_input, } ] @@ -154,19 +188,70 @@ def _tool_use(tool_id: str, name: str, tool_input: dict[str, Any]) -> dict[str, } -def _tool_result(tool_id: str, result: dict[str, Any]) -> dict[str, Any]: - return { +def _tool_result( + tool_id: str, + result: dict[str, Any], + *, + carrier: str = "mcp_meta", + is_error: bool = False, +) -> dict[str, Any]: + display = "tool reply\n\nsuccess: True" + if carrier == "mcp_meta" or carrier == "display_only": + content = display + elif carrier == "block_json": + content = json.dumps(result) + else: + raise ValueError(f"unsupported tool result carrier: {carrier}") + + row: dict[str, Any] = { "type": "user", "message": { "content": [ { "type": "tool_result", "tool_use_id": tool_id, - "content": json.dumps(result), + "content": content, + "is_error": is_error, } ] }, } + if carrier == "mcp_meta": + row["mcpMeta"] = { + "_meta": {"fastmcp": {"wrap_result": True}}, + "structuredContent": {"result": json.dumps(result)}, + } + return row + + +def _launch( + tool_id: str, + reply: dict[str, Any], + *, + tool: str = "run_skill", + is_error: bool = False, +) -> list[dict[str, Any]]: + return [ + _tool_use(tool_id, tool, {}), + _tool_result(tool_id, reply, is_error=is_error), + ] + + +def _root_ownership_proof( + tmp_path: Path, + records: list[dict[str, Any]], +) -> dict[str, Any]: + log_root, trace, _ = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl(root_transcript, records) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=root_transcript), + ], + ) + return sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) def test_hook_capture_appends_concurrent_allow_listed_records(tmp_path: Path) -> None: @@ -261,6 +346,7 @@ def test_summary_collapses_repeated_events_excludes_unrelated_and_sweeps_half_op assert summary["peak_sessions"] == 1 assert summary["coverage"]["complete"] is True + assert summary["violations"] == [] assert summary["coverage"]["managed_session_ids"] == ["session-child"] assert [identity["session_id"] for identity in summary["identities"]] == [ ROOT_ID, @@ -336,6 +422,29 @@ def test_summary_enumerates_native_agent_transcripts_and_requires_endpoints( ] +def test_native_transcript_symlink_loop_is_reported_missing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + transcript = tmp_path / "agent-agent-a.jsonl" + transcript.symlink_to(transcript.name) + + def resolve_loop(_path: Path) -> Path: + raise RuntimeError("Symlink loop") + + monkeypatch.setattr(Path, "resolve", resolve_loop) + violations: list[str] = [] + proof = sessions._verify_native_transcript( + ROOT_ID, + "agent-a", + transcript, + [{"event": "SubagentStart"}, {"event": "SubagentStop"}], + violations, + ) + + assert proof is None + assert violations == [f"native:{ROOT_ID}:agent-a:transcript_missing"] + + def test_indexed_owned_child_without_hooks_is_incomplete(tmp_path: Path) -> None: log_root, trace, _ = _base_sessions(tmp_path, child_id="missing-hooks") transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" @@ -613,3 +722,506 @@ def test_ownership_isolates_branch_and_claim_scenarios( ] else: assert proof["violations"] == [] + + +@pytest.mark.parametrize("prefix", [DIRECT_PREFIX, PLUGIN_PREFIX]) +def test_ownership_accepts_both_canonical_namespaces(tmp_path: Path, prefix: str) -> None: + proof = _root_ownership_proof( + tmp_path, + [ + _tool_use( + "claim", + "claim_and_resolve_issue", + {"issue_url": ISSUE_URL, "allow_reentry": False}, + prefix=prefix, + ), + _tool_result("claim", {"success": True, "claimed": True, "issue_number": 5233}), + _tool_use( + "branch", + "create_and_publish_branch", + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + prefix=prefix, + ), + _tool_result("branch", {"merge_target": "impl/5233-fix", "was_unique": True}), + ], + ) + + assert proof["branches"] == ["impl/5233-fix"] + assert proof["claimed"] is True + assert proof["violations"] == [] + + +@pytest.mark.parametrize( + "name", + [ + "create_and_publish_branch", + "mcp__other__create_and_publish_branch", + f"{PLUGIN_PREFIX}extra__create_and_publish_branch", + ], +) +def test_tracked_tool_under_another_namespace_is_reported(tmp_path: Path, name: str) -> None: + proof = _root_ownership_proof( + tmp_path, + [ + _tool_use( + "branch", + name, + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + prefix="", + ), + _tool_result("branch", {"merge_target": "impl/5233-fix"}), + ], + ) + + assert proof["branches"] == [] + assert f"transcript:{ROOT_ID}:unrecognised_tool_namespace:1" in proof["violations"] + + +@pytest.mark.parametrize("carrier", ["mcp_meta", "block_json"]) +def test_reply_is_read_from_each_supported_carrier(tmp_path: Path, carrier: str) -> None: + proof = _root_ownership_proof( + tmp_path, + [ + _tool_use( + "branch", + "create_and_publish_branch", + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + ), + _tool_result("branch", {"merge_target": "impl/5233-fix"}, carrier=carrier), + ], + ) + + assert proof["branches"] == ["impl/5233-fix"] + assert proof["violations"] == [] + + +def test_non_json_display_text_proves_no_reply(tmp_path: Path) -> None: + result_row = _tool_result( + "branch", + {"merge_target": "impl/evil"}, + carrier="display_only", + ) + result_row["message"]["content"][0]["content"] += "\nmerge_target: impl/evil" + proof = _root_ownership_proof( + tmp_path, + [ + _tool_use( + "branch", + "create_and_publish_branch", + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + ), + result_row, + ], + ) + + assert proof["branches"] == [] + assert "ownership:branch_result_failed" in proof["violations"] + + +def test_invalid_row_metadata_does_not_fall_back_to_block_json(tmp_path: Path) -> None: + result_row = _tool_result( + "branch", + {"merge_target": "impl/5233-fix"}, + carrier="block_json", + ) + result_row["mcpMeta"] = { + "_meta": {"fastmcp": {"wrap_result": True}}, + "structuredContent": {"result": "not json"}, + } + proof = _root_ownership_proof( + tmp_path, + [ + _tool_use( + "branch", + "create_and_publish_branch", + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + ), + result_row, + ], + ) + + assert proof["branches"] == [] + assert "ownership:branch_result_failed" in proof["violations"] + + +def test_row_level_reply_is_ignored_when_a_row_holds_several_results(tmp_path: Path) -> None: + result = {"merge_target": "impl/5233-fix"} + proof = _root_ownership_proof( + tmp_path, + [ + _tool_use( + "branch", + "create_and_publish_branch", + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + ), + _tool_use("claim", "claim_and_resolve_issue", {"issue_url": "https://example.com"}), + { + "type": "user", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": "branch", + "content": "tool reply\n\nsuccess: True", + }, + { + "type": "tool_result", + "tool_use_id": "claim", + "content": "tool reply\n\nsuccess: True", + }, + ] + }, + "mcpMeta": { + "_meta": {"fastmcp": {"wrap_result": True}}, + "structuredContent": {"result": json.dumps(result)}, + }, + }, + ], + ) + + assert proof["branches"] == [] + assert "ownership:branch_result_failed" in proof["violations"] + + +def test_untracked_merge_target_reply_is_not_a_branch(tmp_path: Path) -> None: + proof = _root_ownership_proof( + tmp_path, + [ + _tool_use("clone", "bootstrap_clone", {}), + _tool_result("clone", {"merge_target": "main"}), + ], + ) + + assert proof["branches"] == [] + assert proof["violations"] == [] + + +@pytest.mark.parametrize( + ("tool", "success"), + [ + ("run_skill", True), + ("complete_run_skill_result", True), + ("recover_run_skill_result", True), + ("run_skill", False), + ], +) +def test_launch_receipt_adopts_a_worker_with_empty_parent_fields( + tmp_path: Path, + tool: str, + success: bool, +) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + root_transcript, + _launch( + "launch", + {"success": success, "receipt_id": "r1", "session_id": worker_id}, + tool=tool, + ), + ) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["managed_session_ids"] == [worker_id] + assert summary["coverage"]["complete"] is True + + +def test_acknowledgement_alone_adopts_a_worker(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + root_transcript, + [ + *_launch( + "launch", + {"success": True, "receipt_id": "r1", "audit_status": "published"}, + ), + *_launch( + "ack", + {"success": True, "receipt_id": "r1", "session_id": worker_id}, + tool="complete_run_skill_result", + ), + ], + ) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["managed_session_ids"] == [worker_id] + assert summary["violations"] == [] + + +@pytest.mark.parametrize("worker_dispatch", [DISPATCH_ID, "another-dispatch"]) +def test_overlapping_session_without_receipt_or_parent_is_reported_not_adopted( + tmp_path: Path, + worker_dispatch: str, +) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + index_path = log_root / "sessions.jsonl" + entries = [json.loads(line) for line in index_path.read_text(encoding="utf-8").splitlines()] + next(row for row in entries if row["session_id"] == worker_id)["dispatch_id"] = worker_dispatch + _index(log_root, entries) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + proof = sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) + + violation = f"lineage:{worker_id}:unattributed" + assert worker_id not in summary["coverage"]["managed_session_ids"] + assert violation in summary["violations"] + assert summary["coverage"]["complete"] is False + assert violation in proof["violations"] + + +def test_receipt_in_an_unowned_transcript_does_not_adopt(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + foreign_id = "session-foreign" + foreign_transcript = _session( + log_root, + foreign_id, + parent_session_id="other-parent", + records=_launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": worker_id}, + ), + ) + index_path = log_root / "sessions.jsonl" + entries = [json.loads(line) for line in index_path.read_text(encoding="utf-8").splitlines()] + entries.append( + _entry( + foreign_id, + foreign_transcript, + parent_session_id="other-parent", + caller_session_id="other-parent", + ) + ) + _index(log_root, entries) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert worker_id not in summary["coverage"]["managed_session_ids"] + assert f"lineage:{worker_id}:unattributed" in summary["violations"] + + +@pytest.mark.parametrize( + "reply", + [ + { + "success": False, + "is_error": True, + "error": "permission denied", + "stage": "preflight", + "retriable": False, + }, + {"success": False, "session_id": ""}, + {"success": False, "session_id": "headless-0123456789ab"}, + ], +) +def test_reply_without_receipt_id_proves_no_launch( + tmp_path: Path, + reply: dict[str, Any], +) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl(root_transcript, _launch("launch", reply)) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["managed_session_ids"] == [] + assert summary["violations"] == [] + + +def test_launch_reply_marked_is_error_is_not_evidence(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + root_transcript, + _launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": worker_id}, + is_error=True, + ), + ) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert worker_id not in summary["coverage"]["managed_session_ids"] + assert f"lineage:{worker_id}:unattributed" in summary["violations"] + + +def test_launched_session_absent_from_trace_and_index_is_reported(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + root_transcript, + _launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": "session-ghost"}, + ), + ) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert "lineage:session-ghost:launched_session_unobserved" in summary["violations"] + + +@pytest.mark.parametrize( + ("index_count", "receipt_names_worker"), + [(0, False), (0, True), (2, False), (2, True)], +) +def test_candidate_without_exactly_one_index_row_is_reported( + tmp_path: Path, + index_count: int, + receipt_names_worker: bool, +) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + index_path = log_root / "sessions.jsonl" + entries = [json.loads(line) for line in index_path.read_text(encoding="utf-8").splitlines()] + worker_entry = next(row for row in entries if row["session_id"] == worker_id) + entries = [row for row in entries if row["session_id"] != worker_id] + if index_count == 2: + entries.extend([worker_entry, dict(worker_entry)]) + _index(log_root, entries) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + root_transcript, + _launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": worker_id}, + ) + if receipt_names_worker + else [], + ) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert worker_id not in summary["coverage"]["managed_session_ids"] + assert f"sessions_index:{worker_id}:missing_or_ambiguous" in summary["violations"] + + +def test_receipt_named_worker_from_another_dispatch_is_not_adopted(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + index_path = log_root / "sessions.jsonl" + entries = [json.loads(line) for line in index_path.read_text(encoding="utf-8").splitlines()] + next(row for row in entries if row["session_id"] == worker_id)["dispatch_id"] = ( + "another-dispatch" + ) + _index(log_root, entries) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + root_transcript, + _launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": worker_id}, + ), + ) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert worker_id not in summary["coverage"]["managed_session_ids"] + assert f"sessions_index:{worker_id}:dispatch_mismatch" in summary["violations"] + + +def test_receipt_adopted_worker_with_a_contradicting_declared_parent_is_reported( + tmp_path: Path, +) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + summary_path = log_root / "sessions" / worker_id / "summary.json" + worker_summary = json.loads(summary_path.read_text(encoding="utf-8")) + worker_summary["execution_identity"]["parent_session_id"] = "other-parent" + summary_path.write_text(json.dumps(worker_summary), encoding="utf-8") + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + root_transcript, + _launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": worker_id}, + ), + ) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["managed_session_ids"] == [worker_id] + assert f"summary:{worker_id}:parent_mismatch" in summary["violations"] + + +def test_receipt_adopted_worker_without_lifecycle_endpoints_is_incomplete(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + worker_id = "session-worker" + _worker(log_root, worker_id) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + root_transcript, + _launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": worker_id}, + ), + ) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["managed_session_ids"] == [worker_id] + assert summary["coverage"]["complete"] is False + assert f"lifecycle:{worker_id}:missing_endpoints" in summary["violations"] diff --git a/tests/infra/test_e2e_sessions_captured.py b/tests/infra/test_e2e_sessions_captured.py new file mode 100644 index 0000000000..00fb9d2126 --- /dev/null +++ b/tests/infra/test_e2e_sessions_captured.py @@ -0,0 +1,248 @@ +"""Tests for sanitized, captured E2E session evidence.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any + +import pytest + +from autoskillit.core import PLUGIN_PREFIX +from tests.infra._complexity_helpers import load_check_script +from tests.infra._e2e_evidence import load_provenance, materialize_run + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] + +REPO_ROOT = Path(__file__).resolve().parents[2] +FIXTURE_NAME = "run_38170521660_attempt_1" +FIXTURE_DIR = Path(__file__).with_name("fixtures") / "e2e_sessions" / FIXTURE_NAME +_SCRIPT = REPO_ROOT / "scripts" / "e2e" / "e2e_sessions.py" +sessions = load_check_script("_autoskillit_e2e_sessions_captured", _SCRIPT) +ROOT_SESSION_ID = "2292658a-34cf-44a0-a0b1-f7a20218874b" +WORKER_SESSION_IDS = ( + "8ef8f993-8caa-47dd-83b0-26f9e5b51139", + "54f18144-1188-45d7-a392-17b4529f721e", +) +UNMATCHED_AGENT_ID = "a610a4bda1f28ca35" +NATIVE_AGENT_IDS = { + "a516cbae85151dc7c", + "a9ed51b7c02b828dd", + "aeb96cc0ddbdab294", + "a7e613554b849d6b2", +} + +_INPUTS_TO_PRESERVE = { + PLUGIN_PREFIX + "claim_and_resolve_issue", + PLUGIN_PREFIX + "create_and_publish_branch", +} +_TRUNCATION = re.compile(r"\.\.\.\[fixture-truncated (\d+) chars\]$") + + +def _rows(path: Path) -> list[dict[str, Any]]: + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + + +def _tool_reply(rows: list[dict[str, Any]], suffix: str) -> tuple[dict[str, Any], dict[str, Any]]: + matches = [ + (row, block) + for row in rows + for block in row.get("message", {}).get("content", []) + if isinstance(block, dict) + and block.get("type") == "tool_use" + and isinstance(block.get("name"), str) + and block["name"].endswith(suffix) + ] + assert len(matches) == 1 + use_row, use_block = matches[0] + reply = next(row for row in rows if row.get("sourceToolAssistantUUID") == use_row["uuid"]) + result = next( + block + for block in reply.get("message", {}).get("content", []) + if isinstance(block, dict) + and block.get("type") == "tool_result" + and block.get("tool_use_id") == use_block.get("id") + ) + return reply, result + + +def _assert_truncated_prose(value: Any, *, preserve_input: bool = False) -> None: + if isinstance(value, str): + stripped = value.lstrip() + if not preserve_input and stripped.startswith(("{", "[")): + try: + parsed = json.loads(value) + except json.JSONDecodeError: + parsed = None + if isinstance(parsed, (dict, list)): + _assert_truncated_prose(parsed) + return + if not preserve_input and len(value) > 400: + marker = _TRUNCATION.search(value) + assert marker is not None + assert marker.start() == 400 + assert int(marker.group(1)) > 0 + return + if isinstance(value, list): + for item in value: + _assert_truncated_prose(item, preserve_input=preserve_input) + return + if isinstance(value, dict): + keep_full_input = ( + value.get("type") == "tool_use" and value.get("name") in _INPUTS_TO_PRESERVE + ) + for key, item in value.items(): + _assert_truncated_prose( + item, + preserve_input=preserve_input or (key == "input" and keep_full_input), + ) + + +def test_captured_fixture_is_sanitized_and_keeps_the_real_wire_shape() -> None: + provenance = load_provenance(FIXTURE_NAME) + for path in FIXTURE_DIR.rglob("*"): + if path.is_file() and path.name != "PROVENANCE.json": + assert b"/home/" not in path.read_bytes(), path.relative_to(FIXTURE_DIR) + + envelope = json.loads((FIXTURE_DIR / "out/envelope.json").read_text(encoding="utf-8")) + root_transcript = ( + FIXTURE_DIR + / "projects" + / "-workspace-sandbox" + / f"{envelope['dispatched_session_id']}.jsonl" + ) + rows = _rows(root_transcript) + version_rows = [row["version"] for row in rows if "version" in row] + assert version_rows + assert all(version == provenance["cli_version"] for version in version_rows) + + tool_uses = [ + block + for row in rows + for block in row.get("message", {}).get("content", []) + if isinstance(block, dict) and block.get("type") == "tool_use" + ] + autoskillit_names = [ + block["name"] + for block in tool_uses + if isinstance(block.get("name"), str) and "autoskillit" in block["name"] + ] + assert autoskillit_names + assert all(name.startswith(PLUGIN_PREFIX) for name in autoskillit_names) + + for suffix in ("claim_and_resolve_issue", "create_and_publish_branch"): + reply, result_block = _tool_reply(rows, suffix) + structured = reply["mcpMeta"]["structuredContent"]["result"] + assert isinstance(json.loads(structured), dict) + with pytest.raises(json.JSONDecodeError): + json.loads(result_block["content"]) + + bootstrap_reply, _ = _tool_reply(rows, "bootstrap_clone") + bootstrap_result = json.loads(bootstrap_reply["mcpMeta"]["structuredContent"]["result"]) + assert bootstrap_result["merge_target"] == "main" + _assert_truncated_prose(rows) + + +def test_captured_run_proves_claim_and_published_branch(tmp_path: Path) -> None: + provenance = load_provenance(FIXTURE_NAME) + trace = tmp_path / "trace" / "nested" / "session-events.jsonl" + log_root = tmp_path / "log-root" + projects = tmp_path / "projects" + envelope = materialize_run( + FIXTURE_NAME, + trace=trace, + log_root=log_root, + projects=projects, + ) + + proof = sessions.collect_ownership( + trace, + envelope, + log_root, + provenance["issue_url"], + provenance["repository"], + claim_label=provenance["claim_label"], + ) + assert proof["branches"] == ["e2e-implementation-fixture-normalize-whitespace/1-3"] + assert proof["claimed"] is True + assert proof["violations"] == [] + assert envelope["l3_payload"] is None + + +def test_captured_run_accounts_for_successful_and_failed_workers( + tmp_path: Path, +) -> None: + provenance = load_provenance(FIXTURE_NAME) + trace = tmp_path / "trace" / "session-events.jsonl" + log_root = tmp_path / "log-root" + envelope = materialize_run( + FIXTURE_NAME, + trace=trace, + log_root=log_root, + projects=tmp_path / "projects", + ) + + summary = sessions.summarize(trace, envelope, log_root) + coverage = summary["coverage"] + assert coverage["managed_session_ids"] == sorted(WORKER_SESSION_IDS) + assert coverage["owned_session_ids"] == sorted((ROOT_SESSION_ID, *WORKER_SESSION_IDS)) + assert summary["peak_sessions"] == 5 + assert {agent["agent_id"] for agent in coverage["native_agents"]} == NATIVE_AGENT_IDS + assert coverage["complete"] is False + assert summary["violations"] == sorted( + [ + f"lifecycle:agent:{ROOT_SESSION_ID}:{UNMATCHED_AGENT_ID}:orphan_end", + f"native:{ROOT_SESSION_ID}:{UNMATCHED_AGENT_ID}:lifecycle_incomplete", + f"native:{ROOT_SESSION_ID}:{UNMATCHED_AGENT_ID}:transcript_unenumerated", + ] + ) + assert envelope["l3_payload"] is None + + ownership = sessions.collect_ownership( + trace, + envelope, + log_root, + provenance["issue_url"], + provenance["repository"], + claim_label=provenance["claim_label"], + ) + assert ownership["branches"] == ["e2e-implementation-fixture-normalize-whitespace/1-3"] + assert ownership["claimed"] is True + assert ownership["violations"] == [] + assert [item["session_id"] for item in ownership["transcripts"]] == sorted( + (ROOT_SESSION_ID, *WORKER_SESSION_IDS) + ) + + +def test_captured_run_is_complete_once_the_unmatched_stop_is_removed( + tmp_path: Path, +) -> None: + trace = tmp_path / "trace" / "session-events.jsonl" + log_root = tmp_path / "log-root" + envelope = materialize_run( + FIXTURE_NAME, + trace=trace, + log_root=log_root, + projects=tmp_path / "projects", + ) + events = _rows(trace) + unmatched_stops = [ + event + for event in events + if event.get("event") == "SubagentStop" and event.get("agent_id") == UNMATCHED_AGENT_ID + ] + assert len(unmatched_stops) == 1 + trace.write_text( + "".join( + json.dumps(event, separators=(",", ":"), ensure_ascii=False) + "\n" + for event in events + if event not in unmatched_stops + ), + encoding="utf-8", + ) + + summary = sessions.summarize(trace, envelope, log_root) + assert summary["coverage"]["complete"] is True + assert summary["violations"] == [] + assert summary["peak_sessions"] == 5 diff --git a/tests/infra/test_e2e_workflow.py b/tests/infra/test_e2e_workflow.py index 16c7f1a384..942f39f6de 100644 --- a/tests/infra/test_e2e_workflow.py +++ b/tests/infra/test_e2e_workflow.py @@ -11,6 +11,7 @@ import pytest +from autoskillit.core import MIN_FREE_BYTES_THRESHOLD from autoskillit.core.io import load_yaml pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] @@ -289,6 +290,17 @@ def test_headless_smoke_exit_trap_removes_exact_container_then_runs_cleanup( assert statuses[key] == value +def test_e2e_test_container_exceeds_pytest_capacity_floor(workflow: dict, tmp_path: Path) -> None: + result, events, _, _ = _run_smoke_workflow_shell(workflow, tmp_path) + + assert result.returncode == 0, result.stderr + launch_args = events[0]["args"] + shm_arg = next(arg for arg in launch_args if arg.startswith("--shm-size=")) + size = re.fullmatch(r"--shm-size=(\d+)g", shm_arg) + assert size is not None + assert int(size[1]) * 1024**3 > MIN_FREE_BYTES_THRESHOLD + + @pytest.mark.parametrize("redact_status", [0, 1]) def test_local_lifecycle_records_redaction_and_fails_when_redaction_fails( tmp_path: Path, redact_status: int @@ -416,7 +428,7 @@ def test_clean_install_uses_the_common_user_image_and_artifacts( run = _inline_smoke_cleanup(_step(workflow, "run", step_id=step_id)["run"]) branch, outside = _model_branch(run) - assert run.count("docker_args=()") == 1 + assert run.count("docker_args=(") == 1 assert '"${docker_args[@]}"' in outside assert "docker run --rm" in outside assert '"$IMAGE"' in outside diff --git a/tests/infra/test_pretty_output_recipe.py b/tests/infra/test_pretty_output_recipe.py index 02fe9a80bc..4411d10b42 100644 --- a/tests/infra/test_pretty_output_recipe.py +++ b/tests/infra/test_pretty_output_recipe.py @@ -1058,7 +1058,7 @@ def _assert_compaction_semantics(recipe_name: str, mode_name: str, loaded_result if recipe_name != "remediation": return False rectify_note = original_parsed["steps"]["rectify"]["note"] - assert "Glob plan_dir for *_part_*.md" in rectify_note, ( + assert "plan_parts is exactly the plan_parts value" in rectify_note, ( f"property=remediation rectify note source; recipe={recipe_name}; mode={mode_name}" ) assert compacted_parsed["steps"]["rectify"]["note"] == rectify_note, ( @@ -1350,9 +1350,9 @@ def test_pretty_output_recipe_grid_preserves_semantics_and_budgets(tmp_path, mon f"ingredients_only=both; actual={measured_modes!r}; limit=all registry modes" ) assert maxima == { - "get_recipe_section": (186_838, "remediation", "all_truthy"), - "load_recipe": (186_838, "remediation", "all_truthy"), - "open_kitchen": (186_892, "remediation", "all_truthy"), + "get_recipe_section": (188_124, "remediation", "all_truthy"), + "load_recipe": (188_124, "remediation", "all_truthy"), + "open_kitchen": (188_178, "remediation", "all_truthy"), }, ( "property=canonical baseline maxima; recipe=all; mode=all; tool=all; " f"ingredients_only=both; actual={maxima!r}; limit=measured current baseline" diff --git a/tests/infra/test_release_sanity.py b/tests/infra/test_release_sanity.py index a010cec094..72ad6a68a3 100644 --- a/tests/infra/test_release_sanity.py +++ b/tests/infra/test_release_sanity.py @@ -35,15 +35,20 @@ def test_sync_manifest_in_gitignore(): assert ".autoskillit/sync_manifest.json" in gitignore -def test_autoskillit_root_files_are_all_registered(): - """.autoskillit/ root must only contain files registered in _COMMITTED_BY_DESIGN - or _AUTOSKILLIT_GITIGNORE_ENTRIES (or .gitignore itself). +def test_autoskillit_root_entries_are_all_registered(): + """Every .autoskillit/ root entry must be registered or self-ignored. - Catches any session-scoped file mistakenly placed outside temp/ at CI time. - If .hook_config.json (or any future runtime artifact) is present in the root - rather than temp/, this test fails immediately. + Files may be registered in _COMMITTED_BY_DESIGN or + _AUTOSKILLIT_GITIGNORE_ENTRIES; .gitignore itself is always accepted. A + runtime directory must have a valid self-ignore marker. A directory without + one is a namespace left by an earlier version: delete it or run a dispatch + to repair its marker. """ - from autoskillit.core.io import _AUTOSKILLIT_GITIGNORE_ENTRIES, _COMMITTED_BY_DESIGN + from autoskillit.core.io import ( + _AUTOSKILLIT_GITIGNORE_ENTRIES, + _COMMITTED_BY_DESIGN, + is_self_ignored, + ) autoskillit_dir = REPO_ROOT / ".autoskillit" if not autoskillit_dir.is_dir(): @@ -58,13 +63,15 @@ def test_autoskillit_root_files_are_all_registered(): unregistered = [ p.name for p in autoskillit_dir.iterdir() - if not p.is_dir() and p.name not in registered_names + if p.name not in registered_names and not (p.is_dir() and is_self_ignored(p)) ] assert not unregistered, ( - f"Unregistered files found in .autoskillit/: {unregistered}. " - f"Session-scoped runtime files belong in .autoskillit/temp/ where they " - f"are auto-gitignored by temp/.gitignore ('*'). Persistent files that " - f"belong in .autoskillit/ root need an entry added to " + f"Unregistered entries found in .autoskillit/: {unregistered}. " + f"A runtime directory without a valid self-ignore marker is a namespace " + f"left by an earlier version; delete it or run a dispatch to repair the " + f"marker. Runtime files belong in .autoskillit/temp/ where they are " + f"auto-gitignored by temp/.gitignore ('*'). Persistent files that belong " + f"in .autoskillit/ root need an entry added to " f"_AUTOSKILLIT_GITIGNORE_ENTRIES in src/autoskillit/core/io/io.py." ) diff --git a/tests/recipe/test_bundled_recipes_pipeline_structure.py b/tests/recipe/test_bundled_recipes_pipeline_structure.py index 2ab38aff1b..8a34c2d94f 100644 --- a/tests/recipe/test_bundled_recipes_pipeline_structure.py +++ b/tests/recipe/test_bundled_recipes_pipeline_structure.py @@ -7,6 +7,8 @@ from __future__ import annotations +from pathlib import Path + import pytest from autoskillit.core.types import Severity @@ -16,6 +18,91 @@ pytestmark = [pytest.mark.layer("recipe"), pytest.mark.small] +_TERMINAL_COVERAGE_BINDS = { + ("implementation-groups", "bind_group_parts"), + ("implementation-groups", "renew_plan_set"), + ("implementation", "renew_plan_set"), + ("remediation", "renew_plan_set"), + ("remediation", "renew_before_retry"), +} + + +def _assert_bind_coverage_disposition(recipe, step_name: str) -> None: + bind = recipe.steps[step_name] + assert bind.on_result is not None + coverage_routes = [ + condition + for condition in bind.on_result.conditions + if "result.reason" in (condition.when or "") + and "coverage_failed" in (condition.when or "") + ] + if (recipe.name, step_name) in _TERMINAL_COVERAGE_BINDS: + assert not coverage_routes + return + assert len(coverage_routes) == 1, f"undecided bind site: {recipe.name}/{step_name}" + assert bind.capture["plan_set_coverage_gaps"].from_ == "${{ result.coverage_gaps }}" + guard = recipe.steps[coverage_routes[0].route] + assert guard.tool == "run_python" + assert guard.with_args["callable"] == "autoskillit.smoke_utils.check_loop_iteration" + assert guard.with_args["max_iterations"] == "2" + assert guard.on_result is not None + routes = [condition.route for condition in guard.on_result.conditions if not condition.when] + assert len(routes) == 1 + replan_name = routes[0] + replan = recipe.steps[replan_name] + assert replan.tool == "run_skill" + inputs = replan.with_args["skill_inputs"] + assert isinstance(inputs, dict) + assert "${{ context.plan_set_coverage_gaps }}" in inputs.values() + assert any( + recipe.steps[target].tool == "bind_plan_set" + for target in _build_success_step_graph(recipe)[replan_name] + ) + + +def test_all_bind_sites_have_decided_coverage_failure_disposition() -> None: + bind_sites: set[tuple[str, str]] = set() + for path in sorted(builtin_recipes_dir().glob("*.yaml")): + recipe = load_recipe(path) + for step_name, step in recipe.steps.items(): + if step.tool == "bind_plan_set": + bind_sites.add((recipe.name, step_name)) + _assert_bind_coverage_disposition(recipe, step_name) + assert _TERMINAL_COVERAGE_BINDS <= bind_sites + + +def test_bind_disposition_guard_names_an_undecided_bind(tmp_path: Path) -> None: + source = (builtin_recipes_dir() / "implementation.yaml").read_text() + scratch = tmp_path / "implementation.yaml" + scratch.write_text( + source.replace( + "\nsteps:\n", + "\nsteps:\n undecided_bind:\n tool: bind_plan_set\n" + " on_result:\n - route: release_issue_failure\n", + 1, + ) + ) + recipe = load_recipe(scratch) + with pytest.raises(AssertionError, match="undecided bind site: implementation/undecided_bind"): + _assert_bind_coverage_disposition(recipe, "undecided_bind") + + +def test_binds_use_the_producer_part_list_without_directory_discovery() -> None: + for path in sorted(builtin_recipes_dir().glob("*.yaml")): + recipe = load_recipe(path) + binds = {name for name, step in recipe.steps.items() if step.tool == "bind_plan_set"} + if binds: + assert all("*_part_*.md" not in (step.note or "") for step in recipe.steps.values()) + graph = _build_success_step_graph(recipe) + for bind_name in binds: + if recipe.steps[bind_name].with_args.get("plan_parts") != "${{ context.plan_parts }}": + continue + for source, targets in graph.items(): + if bind_name in targets: + assert recipe.steps[source].capture_list["plan_parts"].from_ == ( + "${{ result.plan_parts }}" + ), f"{recipe.name}/{source} must capture the replacement list for {bind_name}" + def _skill_input(step, name: str) -> object: values = step.with_args.get("skill_inputs", {}) @@ -326,14 +413,6 @@ def test_ip5_audit_impl_has_on_failure(self, recipe) -> None: assert step.on_success is None # on_result is used; on_success remains absent assert step.on_failure == "check_audit_integrity_retry" - def test_ip6_plan_step_note_contains_glob_pattern(self, recipe) -> None: - """T_IP6: plan step note must contain *_part_*.md glob pattern for multi-part discovery.""" - note = recipe.steps["plan"].note or "" - assert "*_part_*.md" in note, ( - "plan step note must contain glob pattern for multi-part discovery; " - "if removed, agents will not discover part files" - ) - def test_ip7_verify_step_note_sequential_constraint(self, recipe) -> None: """T_IP7: verify step note must contain sequential execution constraint.""" note = recipe.steps["verify"].note or "" @@ -647,7 +726,13 @@ def test_ip_next_or_done_no_more_groups_route(self, recipe) -> None: def test_ip_coverage_replan_is_bounded(self, recipe) -> None: bind = recipe.steps["bind_plan_set"] assert bind.on_result is not None - assert any(c.route == "check_replan_iteration" for c in bind.on_result.conditions) + assert any( + c.route == "check_replan_iteration" + and c.when + and "result.reason" in c.when + and "coverage_failed" in c.when + for c in bind.on_result.conditions + ) guard = recipe.steps["check_replan_iteration"] assert guard.with_args["max_iterations"] == "2" assert guard.on_result is not None @@ -712,7 +797,13 @@ def test_ig6_next_or_done_routes_more_parts_to_review(self, recipe) -> None: def test_ig_gap_replan_is_bounded(self, recipe) -> None: seal = recipe.steps["seal_plan_set"] assert seal.on_result is not None - assert any(c.route == "check_replan_iteration" for c in seal.on_result.conditions) + assert any( + c.route == "check_replan_iteration" + and c.when + and "result.reason" in c.when + and "coverage_failed" in c.when + for c in seal.on_result.conditions + ) guard = recipe.steps["check_replan_iteration"] assert guard.with_args["max_iterations"] == "2" assert guard.on_result is not None diff --git a/tests/recipe/test_implementation.py b/tests/recipe/test_implementation.py index 3e8f4c2f01..1b2aeb3d6f 100644 --- a/tests/recipe/test_implementation.py +++ b/tests/recipe/test_implementation.py @@ -21,7 +21,7 @@ RECIPE_PATH = builtin_recipes_dir() / "implementation.yaml" _PRE_DELIVERY_STRUCTURE_SHA256 = ( - "sha256:03057194113eef6e9902d5b26ed974c8da839a66f17aa8b20f70b8f8ebdbb375" + "sha256:73734dbf63321cb0ee4d72e28101fa29f1f51dbbb8a8064df96552958b08a9f6" ) _CHECKPOINT_HANDLER_TABLE = { diff --git a/tests/recipe/test_io_json_precompile.py b/tests/recipe/test_io_json_precompile.py index 374f95b293..1f9c514681 100644 --- a/tests/recipe/test_io_json_precompile.py +++ b/tests/recipe/test_io_json_precompile.py @@ -56,6 +56,53 @@ def _compile_json(yaml_path: Path) -> None: json_path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") +def test_load_recipe_dict_expands_shared_notes_in_surrounding_text(tmp_path: Path) -> None: + yaml_path = tmp_path / "recipe.yaml" + _write_yaml( + yaml_path, + { + **_MINIMAL_RECIPE, + "steps": { + "done": { + "action": "stop", + "note": "Before. __RECIPE_NOTE_PLAN_SET_RENEWAL_FAILURE__ After.", + } + }, + }, + ) + _compile_json(yaml_path) + note = load_recipe_dict(yaml_path)["steps"]["done"]["note"] + assert note.startswith("Before. coverage_failed here is terminal by decision.") + assert note.endswith("it names the rows the edit broke. After.") + assert "__RECIPE_NOTE_" not in note + + +def test_raw_recipe_delivery_expands_shared_notes_as_valid_yaml(tmp_path: Path) -> None: + from autoskillit.recipe import load_and_validate + + yaml_path = tmp_path / ".autoskillit" / "recipes" / "test-recipe.yaml" + yaml_path.parent.mkdir(parents=True) + _write_yaml( + yaml_path, + { + **_MINIMAL_RECIPE, + "steps": { + "done": { + "action": "stop", + "message": "Done.", + "note": "Before. __RECIPE_NOTE_PLAN_SET_RENEWAL_FAILURE__ After.", + } + }, + }, + ) + delivered = load_and_validate("test-recipe", project_dir=tmp_path, temp_dir=tmp_path) + assert "__RECIPE_NOTE_" not in delivered["content"] + assert ( + load_yaml(delivered["content"])["steps"]["done"]["note"] + == (load_recipe_dict(yaml_path)["steps"]["done"]["note"]) + ) + + def test_load_recipe_dict_prefers_json_when_fresh(tmp_path, monkeypatch): yaml_path = tmp_path / "recipe.yaml" _write_yaml(yaml_path, _MINIMAL_RECIPE) diff --git a/tests/recipe/test_remediation_recipe.py b/tests/recipe/test_remediation_recipe.py index cfd31877e6..05fdce9211 100644 --- a/tests/recipe/test_remediation_recipe.py +++ b/tests/recipe/test_remediation_recipe.py @@ -22,7 +22,7 @@ Path(__file__).parent.parent.parent / "src" / "autoskillit" / "recipes" / "remediation.yaml" ) _PRE_DELIVERY_STRUCTURE_SHA256 = ( - "sha256:ecd6705d9e08441738c914d0853a975c292bdd341e1bf7957e35256d4ca11777" + "sha256:64e553d7e7669f32968ac401161139869fce71c023dbb129ba977d0dd2e3d831" ) @@ -48,7 +48,13 @@ def test_context_limited_walkthrough_renews_before_retry(recipe) -> None: def test_coverage_replan_is_bounded(recipe) -> None: bind = recipe.steps["bind_plan_set"] assert bind.on_result is not None - assert any(c.route == "check_replan_iteration" for c in bind.on_result.conditions) + assert any( + c.route == "check_replan_iteration" + and c.when + and "result.reason" in c.when + and "coverage_failed" in c.when + for c in bind.on_result.conditions + ) guard = recipe.steps["check_replan_iteration"] assert guard.with_args["max_iterations"] == "2" assert guard.on_result is not None diff --git a/tests/recipe/test_rules_multipart_iteration.py b/tests/recipe/test_rules_multipart_iteration.py index 08e325887d..bd2970132e 100644 --- a/tests/recipe/test_rules_multipart_iteration.py +++ b/tests/recipe/test_rules_multipart_iteration.py @@ -9,32 +9,6 @@ class TestMultipartIterationRule: - def test_mi1_multipart_rule_warns_on_missing_glob_note(self) -> None: - """T_MI1: multipart-glob-note fires when make-plan step has no *_part_*.md in note.""" - recipe = Recipe( - name="test-recipe", - description="test", - ingredients={}, - steps={ - "plan": RecipeStep( - tool="run_skill", - with_args={"skill_command": "/autoskillit:make-plan inputs.task"}, - on_success="verify", - note="Produces a plan file.", - ), - "verify": RecipeStep( - tool="run_skill", - with_args={"skill_command": "/autoskillit:dry-walkthrough context.plan_path"}, - on_success="done", - ), - "done": RecipeStep(action="stop", message="Done"), - }, - kitchen_rules=[], - ) - warnings = run_semantic_rules(recipe) - rule_names = [w.rule for w in warnings] - assert "multipart-iteration-notes" in rule_names - def test_mi2_multipart_rule_passes_compliant_recipe(self) -> None: """T_MI2: Validator emits no multipart warnings when all conventions are present.""" recipe = Recipe( @@ -46,7 +20,7 @@ def test_mi2_multipart_rule_passes_compliant_recipe(self) -> None: tool="run_skill", with_args={"skill_command": "/autoskillit:make-plan inputs.task"}, on_success="verify", - note="Glob plan_dir for *_part_*.md or single plan file.", + note="Produces the plan_parts result.", ), "verify": RecipeStep( tool="run_skill", @@ -78,7 +52,7 @@ def test_mi3_multipart_rule_passes_non_verify_backroute(self) -> None: tool="run_skill", with_args={"skill_command": "/autoskillit:rectify context.investigation_path"}, on_success="dry_walkthrough", - note="Glob plan_dir for *_part_*.md or single plan file.", + note="Produces the plan_parts result.", ), "dry_walkthrough": RecipeStep( tool="run_skill", @@ -111,7 +85,7 @@ def test_mi4_multipart_rule_fires_when_no_more_parts_route(self) -> None: tool="run_skill", with_args={"skill_command": "/autoskillit:make-plan inputs.task"}, on_success="next_or_done", - note="Glob plan_dir for *_part_*.md or single plan file.", + note="Produces the plan_parts result.", ), "next_or_done": RecipeStep( action="route", @@ -138,7 +112,7 @@ def compliant_multipart_recipe_no_list() -> Recipe: tool="run_skill", with_args={"skill_command": "/autoskillit:make-plan inputs.task"}, capture={"plan_path": "${{ result.plan_path }}"}, - note="Glob plan_dir for *_part_*.md or single plan file. Sort into plan_parts[].", + note="Produces the plan_parts result.", on_success="done", ), "done": RecipeStep(action="stop", message="Done"), @@ -161,7 +135,7 @@ def compliant_multipart_recipe_with_list() -> Recipe: capture={"plan_path": "${{ result.plan_path }}"}, capture_list={"plan_parts": "${{ result.plan_parts }}"}, retries=0, - note="Glob plan_dir for *_part_*.md or single plan file. Sort into plan_parts[].", + note="Produces the plan_parts result.", on_success="done", ), "done": RecipeStep(action="stop", message="Done"), diff --git a/tests/server/test_delivery_e2e_verification.py b/tests/server/test_delivery_e2e_verification.py index e346a0380e..0211538777 100644 --- a/tests/server/test_delivery_e2e_verification.py +++ b/tests/server/test_delivery_e2e_verification.py @@ -251,9 +251,9 @@ async def test_implementation_bounded_path_counts_automatic_and_advertised_deliv # Pin normalized delivery costs to detect drift in the bundled recipe. # These measured totals are not production-safe thresholds. assert totals == { - "raw_chars": 18_551, - "utf8_bytes": 18_553, - "client_serialized_chars": 20_634, - "estimated_tokens": 4_636, + "raw_chars": 18_800, + "utf8_bytes": 18_802, + "client_serialized_chars": 20_871, + "estimated_tokens": 4_698, "responses": 5, } diff --git a/tests/server/test_plan_set_materializer.py b/tests/server/test_plan_set_materializer.py index a6e76181fe..a6224ff7b0 100644 --- a/tests/server/test_plan_set_materializer.py +++ b/tests/server/test_plan_set_materializer.py @@ -30,6 +30,23 @@ async def fetch_issue(self, *_args: object, **_kwargs: object) -> dict[str, obje } +def _plan_part( + allocation_rows: str, + *, + before_table: str = "", + steps: str = "### Step 1.1: R1\nImplement R1.", +) -> str: + return f"""# Plan part +{steps} + +## Issue Requirement Allocation +{before_table} +| Requirement ID | Allocation | Implementation Step | +| --- | --- | --- | +{allocation_rows} +""" + + @pytest.mark.asyncio async def test_sealed_binding_writes_content_addressed_authority(tmp_path: Path) -> None: fixture = _FIXTURES / "split_r1_r2" @@ -55,6 +72,120 @@ async def test_sealed_binding_writes_content_addressed_authority(tmp_path: Path) assert Path(result.plan_set_authority_path).is_file() +def _coverage_failure_case( + case: str, +) -> tuple[tuple[str, ...], str, tuple[str, ...], tuple[str, ...], tuple[str, ...]]: + expected_gaps: tuple[str, ...] + expected_errors: tuple[str, ...] = () + expected_forms: tuple[str, ...] = () + + if case == "evidence": + part_texts = ( + _plan_part( + "| R1 | owned | Steps 1, 2 |\n| R2 (Add fn) | owned | Step 1.2 |", + steps=("### Step 1.1: R1\nImplement R1.\n\n### Step 1.2: R2\nImplement R2."), + ), + ) + issue_url = "owner/repo#1" + expected_gaps = ( + "R1", + "R2 (Add fn)", + "Steps 1, 2", + "Step 1.1", + "Step 1.2", + ) + expected_errors = ("allocation_step_missing", "allocation_step_unreferenced") + expected_forms = ("Expected exactly one key such as Step 2.1",) + elif case == "table": + sentence = "The plan explanation appears before the allocation table." + part_texts = (_plan_part("| R1 | owned | Step 1.1 |", before_table=sentence),) + issue_url = "owner/repo#1" + expected_gaps = (sentence,) + expected_forms = ("table rows must be pipe-delimited",) + elif case == "row": + part_texts = (_plan_part("| R1 | partial | Step 1.1 |"),) + issue_url = "owner/repo#1" + expected_gaps = ("R1", "partial") + expected_forms = ("use owned or shared",) + elif case == "two_parts": + part_texts = ( + _plan_part( + "| R1 | owned | Step 1.1 |", + before_table="The first part has prose before its table.", + ), + _plan_part("| R2 | partial | Step 1.1 |"), + ) + issue_url = "owner/repo#1" + expected_gaps = ( + "The first part has prose before its table.", + "R2", + "partial", + ) + expected_forms = ("table rows must be pipe-delimited", "use owned or shared") + elif case == "part_without_obligations": + part_texts = (_plan_part(""),) + issue_url = "" + expected_gaps = ("part_without_obligations",) + else: + fixture = _FIXTURES / "missing_r2" + part_texts = tuple( + (fixture / name).read_text(encoding="utf-8") for name in ("part_a.md", "part_b.md") + ) + issue_url = "owner/repo#1" + expected_gaps = ("R2 [unassigned]",) + + return part_texts, issue_url, expected_gaps, expected_errors, expected_forms + + +@pytest.mark.parametrize( + "case", + ("evidence", "table", "row", "two_parts", "part_without_obligations", "unassigned"), +) +@pytest.mark.asyncio +async def test_coverage_failures_return_feedback(tmp_path: Path, case: str) -> None: + part_texts, issue_url, expected_gaps, expected_errors, expected_forms = _coverage_failure_case( + case + ) + + part_names = ( + ("part_a.md", "part_b.md") + if case == "unassigned" + else tuple(f"{case}_part_{index}.md" for index in range(1, len(part_texts) + 1)) + ) + part_paths = tuple(tmp_path / name for name in part_names) + for path, text in zip(part_paths, part_texts, strict=True): + path.write_text(text, encoding="utf-8") + request = PlanSetBindRequest( + plan_parts_raw="\n".join(str(path) for path in part_paths), + allowed_root=tmp_path, + issue_url=issue_url, + parent_authority_path="", + seal=True, + step_name="bind", + execution_generation="generation", + kitchen_id="kitchen", + dispatch_id="", + binding_mode=PlanSetBindingMode.RECIPE, + ) + + result = await DefaultPlanSetMaterializer(_GitHub()).bind(request) + + assert result.reason is PlanSetRejectReason.COVERAGE_FAILED + assert result.coverage_gaps.strip() + if case != "unassigned": + for path in part_paths: + assert str(path) in result.coverage_gaps + for fragment in expected_gaps: + assert fragment in result.coverage_gaps + for fragment in expected_errors: + assert fragment in result.error + for fragment in expected_forms: + assert fragment in result.coverage_gaps + if expected_forms: + assert "Write a corrected plan part to a new file" in result.coverage_gaps + assert "keep the same requirements allocated" in result.coverage_gaps + + @pytest.mark.asyncio async def test_coverage_failure_writes_no_authority(tmp_path: Path) -> None: fixture = _FIXTURES / "missing_r2" diff --git a/tests/server/test_tools_plan_set.py b/tests/server/test_tools_plan_set.py index 827538d118..89365d8929 100644 --- a/tests/server/test_tools_plan_set.py +++ b/tests/server/test_tools_plan_set.py @@ -116,7 +116,9 @@ def now(_zone): assert repeated["plan_set_authority_path"] == initial["plan_set_authority_path"] -@pytest.mark.parametrize("case", ("empty", "outside", "fetch_failure", "no_issue")) +@pytest.mark.parametrize( + "case", ("empty", "outside", "fetch_failure", "no_issue", "coverage_failed") +) async def test_wrapper_classifies_inputs_and_failures( tmp_path: Path, tool_ctx, monkeypatch: pytest.MonkeyPatch, case: str ) -> None: @@ -139,14 +141,19 @@ async def test_wrapper_classifies_inputs_and_failures( assert result["reason"] == PlanSetRejectReason.ISSUE_FETCH_FAILED.value assert not list((root / "plan-set-authority").rglob("*.json")) else: + step_heading = "#### Step 1.1: P-1" if case == "coverage_failed" else "### Step 1.1: P-1" first.write_text( - "# Plan\n\n### Step 1.1: P-1\nImplement P-1.\n\n" + f"# Plan\n\n{step_heading}\nImplement P-1.\n\n" "## Issue Requirement Allocation\n" "| Requirement ID | Allocation | Implementation Step |\n" "|---|---|---|\n| P-1 | owned | Step 1.1 |\n", encoding="utf-8", ) result = json.loads(await bind_plan_set(plan_parts=str(first), cwd=str(tmp_path))) + if case == "coverage_failed": + assert result["reason"] == PlanSetRejectReason.COVERAGE_FAILED.value + assert result["coverage_gaps"].strip() + return assert result["success"] is True authority = json.loads(Path(result["plan_set_authority_path"]).read_text()) assert authority["inventory_mode"] == "no_issue" diff --git a/tests/test_test_filter_core_cascade.py b/tests/test_test_filter_core_cascade.py index 0ccdcb42f1..febbd4f0bc 100644 --- a/tests/test_test_filter_core_cascade.py +++ b/tests/test_test_filter_core_cascade.py @@ -302,7 +302,9 @@ def test_type_results_cascade(self) -> None: "smoke_utils", } ) - assert MODULE_CASCADE_CORE["_type_results_records"] == MODULE_CASCADE_CORE["_type_results"] + assert MODULE_CASCADE_CORE["_type_results_records"] == ( + MODULE_CASCADE_CORE["_type_results"] | {"config"} + ) def test_type_backend_cascade(self) -> None: assert MODULE_CASCADE_CORE["_type_backend"] == frozenset( @@ -794,7 +796,8 @@ def test_type_results_narrow_routing(self, tmp_path: Path, source_path: str) -> dir_names = {p.name for p in result} for pkg in ["core", "execution", "server", "fleet", "migration"]: assert pkg in dir_names, f"_type_results cascade should include {pkg}" - for excluded in ["config", "hooks", "skills", "planner"]: + assert ("config" in dir_names) == source_path.endswith("_type_results_records.py") + for excluded in ["hooks", "skills", "planner"]: assert excluded not in dir_names, ( f"_type_results cascade should not include {excluded}" ) diff --git a/tests/workspace/test_clone_gate_runtime_state.py b/tests/workspace/test_clone_gate_runtime_state.py new file mode 100644 index 0000000000..2f304579ab --- /dev/null +++ b/tests/workspace/test_clone_gate_runtime_state.py @@ -0,0 +1,128 @@ +"""Git visibility of AutoSkillit-owned state during clone checks.""" + +from __future__ import annotations + +import uuid +from pathlib import Path + +import pytest + +from autoskillit.core import ( + ManagedHeadlessSessionKind, + NativeShellCaptureMode, + resolve_native_shell_capture_decision, +) +from autoskillit.core.io import _AUTOSKILLIT_GITIGNORE_ENTRIES, ensure_project_temp +from autoskillit.execution.session import DefaultManagedHeadlessSessionLineageStore +from autoskillit.workspace.clone import detect_uncommitted_changes +from tests._git_topology import commit_file + +pytestmark = [ + pytest.mark.layer("workspace"), + pytest.mark.medium, + pytest.mark.usefixtures("isolated_git_config"), +] + + +_CANONICAL_PARENT_GITIGNORE = "\n".join(_AUTOSKILLIT_GITIGNORE_ENTRIES) + "\n" + + +def _initialized( + repo: Path, + *, + parent_gitignore: str | None = _CANONICAL_PARENT_GITIGNORE, +) -> Path: + if parent_gitignore is not None: + commit_file( + repo, + ".autoskillit/.gitignore", + parent_gitignore, + "add AutoSkillit ignore entries", + ) + commit_file(repo, ".autoskillit/config.yaml", "project: probe\n", "add project config") + return repo + + +def _create_record(anchor: Path): + return DefaultManagedHeadlessSessionLineageStore().create( + lineage_anchor=anchor, + launch_id=uuid.uuid4().hex, + decision=resolve_native_shell_capture_decision(NativeShellCaptureMode.CAPTURE), + backend="claude-code", + session_kind=ManagedHeadlessSessionKind.FOOD_TRUCK, + dispatch_id=str(uuid.uuid4()), + ) + + +def test_launch_sequence_leaves_checkout_clean(git_repo: Path) -> None: + repo = _initialized(git_repo) + gitignore_path = repo / ".autoskillit" / ".gitignore" + committed_gitignore = gitignore_path.read_bytes() + + ensure_project_temp(repo) + _create_record(repo) + + records_dir = repo / ".autoskillit" / "managed-headless-session-lineage" / "records" + assert records_dir.is_dir() and any(records_dir.iterdir()) + assert detect_uncommitted_changes(str(repo)) == [] + assert gitignore_path.read_bytes() == committed_gitignore + + +@pytest.mark.parametrize( + "parent_gitignore", + [None, "# user rule retained\nprivate-artifacts/\n"], + ids=["absent", "tracked-stale"], +) +def test_launch_sequence_preserves_parent_gitignore_state( + git_repo: Path, parent_gitignore: str | None +) -> None: + repo = _initialized(git_repo, parent_gitignore=parent_gitignore) + gitignore_path = repo / ".autoskillit" / ".gitignore" + + ensure_project_temp(repo) + _create_record(repo) + + assert detect_uncommitted_changes(str(repo)) == [] + if parent_gitignore is None: + assert not gitignore_path.exists() + else: + assert gitignore_path.read_bytes() == parent_gitignore.encode() + + +def test_user_edits_still_block_after_launch(git_repo: Path) -> None: + repo = _initialized(git_repo) + gitignore_path = repo / ".autoskillit" / ".gitignore" + committed_gitignore = gitignore_path.read_bytes() + + ensure_project_temp(repo) + _create_record(repo) + + (repo / "notes.txt").write_text("keep this note\n", encoding="utf-8") + config_path = repo / ".autoskillit" / "config.yaml" + config_path.write_text( + config_path.read_text(encoding="utf-8") + "user_setting: retained\n", + encoding="utf-8", + ) + recipe_path = repo / ".autoskillit" / "recipes" / "mine.yaml" + recipe_path.parent.mkdir(parents=True, exist_ok=True) + recipe_path.write_text("name: mine\n", encoding="utf-8") + + changed = detect_uncommitted_changes(str(repo)) + assert "?? notes.txt" in changed + assert " M .autoskillit/config.yaml" in changed + assert "?? .autoskillit/recipes/" in changed + assert gitignore_path.read_bytes() == committed_gitignore + + +def test_legacy_namespace_is_repaired_by_a_store_read(git_repo: Path) -> None: + repo = _initialized(git_repo) + lineage = _create_record(repo) + namespace = repo / ".autoskillit" / "managed-headless-session-lineage" + (namespace / ".gitignore").unlink(missing_ok=True) + + assert detect_uncommitted_changes(str(repo)) + DefaultManagedHeadlessSessionLineageStore().load( + lineage_anchor=repo, + launch_id=lineage.launch_id, + ) + assert detect_uncommitted_changes(str(repo)) == [] diff --git a/tests/workspace/test_project_local_overrides_resolution.py b/tests/workspace/test_project_local_overrides_resolution.py index 5b838b7724..d514a7a3fd 100644 --- a/tests/workspace/test_project_local_overrides_resolution.py +++ b/tests/workspace/test_project_local_overrides_resolution.py @@ -171,8 +171,8 @@ def test_matching_or_stricter_override_is_effective(tmp_path: Path) -> None: assert not resolver.scan_effective(tmp_path)[1] extended = bundled.replace( - "activate_deps:\n- write-recipe\n", - "activate_deps:\n- write-recipe\nrequires_resources:\n- arch-constraint-catalog\n", + "requires_resources:\n", + "requires_resources:\n- arch-constraint-catalog\n", 1, ) _write_make_plan_override(tmp_path, extended) diff --git a/uv.lock b/uv.lock index 0f1b7cf310..6b3c7ef26f 100644 --- a/uv.lock +++ b/uv.lock @@ -133,7 +133,7 @@ wheels = [ [[package]] name = "autoskillit" -version = "0.10.1266" +version = "0.10.1271" source = { editable = "." } dependencies = [ { name = "anyio" }, From d1eaf42992fb4acbeb0ba5061612d19f8d82d1e9 Mon Sep 17 00:00:00 2001 From: Talon Trecek <36818290+Trecek@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:27:09 -0700 Subject: [PATCH 20/20] fix(e2e): scope native lifecycle capture to named agents (#5277) Unnamed internal-agent `SubagentStop` callbacks made E2E native lifecycle coverage fail. Install `.+` for agent hooks from the collector's shared lifecycle declaration, keep session hooks unfiltered, and reject a main-agent key in user settings before installation. The collector retains delivered callbacks and adds a `capture:...:agent_type_missing` violation for unnamed owned agent records. Regression tests cover settings preservation, main-agent preflight rejection, resolvable agent names, fleet launch configuration, named orphan stops, and the historical trace under the declared capture policy. The README documents the capture precondition and the unchecked external settings channels, whose named orphan stops still fail coverage. Validation: `pre-commit run --all-files` passed. `task test-local-gate`, invoked through `test_check`, first produced exactly the 10 expected regression failures before implementation, then passed with 50,926 tests passing after implementation. Fixes #5271 --- scripts/e2e/README.md | 23 ++++- scripts/e2e/e2e_harness.py | 12 ++- scripts/e2e/e2e_sessions.py | 11 +- tests/_hook_protocol_oracle.py | 19 ++++ tests/arch/test_e2e_sessions_contract.py | 34 +++++- tests/execution/test_flag_contracts.py | 29 ++++++ tests/hooks/test_hook_protocol_oracle.py | 35 +++++++ tests/infra/test_e2e_harness.py | 90 +++++++++++++++- tests/infra/test_e2e_sessions.py | 120 +++++++++++++++++++++- tests/infra/test_e2e_sessions_captured.py | 26 +++-- 10 files changed, 377 insertions(+), 22 deletions(-) diff --git a/scripts/e2e/README.md b/scripts/e2e/README.md index a93f10dad4..2183065579 100644 --- a/scripts/e2e/README.md +++ b/scripts/e2e/README.md @@ -200,6 +200,19 @@ pipeline this declares capacity, not a measured fan-out. The child limit stays s The implementation pipeline installs synchronous command hooks for `SessionStart`, `SessionEnd`, `SubagentStart` and `SubagentStop`, preserving existing hooks and MiniMax settings. These hooks follow the [Claude lifecycle schema](https://code.claude.com/docs/en/hooks). +The agent hooks use `.+` to require a non-empty agent type: under the default main +agent, Claude Code also fires `SubagentStop` for internal agents with an empty type. +An omitted or empty matcher admits those callbacks; `.+` accepts every non-empty +built-in, custom or namespaced plugin agent name without an allowlist or product-registry +imports in these stdlib-only scripts. Session hooks have no matcher. +Sessions must run with the default main agent. The harness fails preflight if the +user-level settings file has an `agent` key; the fleet command builders pass neither +`--agent` nor `--settings`, and the bundled plugin sets no main agent. Project/local +settings in the sandbox, managed settings and other enabled plugins' main-agent defaults +are not checked. A main agent from those sources makes internal stops carry its name: +they fail coverage as a named stop without a start, without the blank-type `capture:` code. +Claude Code 2.1.275 fixed specific matchers admitting blank-type stops; the image's +2.1.280 pin includes that fix. Each identity spans its first start through its final end, so compaction and resume yield a conservative upper bound. Half-open intervals end before simultaneous starts. The summary reconciles the dispatched root, managed lineage and enumerable native transcripts. @@ -212,9 +225,13 @@ Each must have exactly one index row with the run's `dispatch_id`. Parentless se without receipts, missing or ambiguous index rows, and receipt-named sessions absent from both trace and index fail coverage. Shared dispatch IDs or overlapping lifetimes never adopt a session. Missing endpoints, identities or owned transcripts fail coverage; -transcript activity never infers a lifetime. An auto-compacted root can leave an unmatched -`SubagentStop`, which still fails coverage and is tracked separately. A peak exceeding -capacity fails validation. +transcript activity never infers a lifetime. The captured pre-fix trace contains a blank-type +stop immediately before a compact-source session start; its producing feature is unproven. +An owned agent record without a named type fails coverage with a `capture:` violation. +A named stop without its start or transcript still fails coverage, as do traces captured +before the matcher was installed. The collector retains delivered callbacks and adds +violations; it does not filter them out. Individual codes are in `out/session-concurrency.json` +under `violations`. A peak exceeding capacity fails validation. Commission with a successful MiniMax CI run and repeat against the same seed. Inspect planning, dry-walkthrough, implementation, test, integration, audit GO and open-PR evidence, diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index 36aee365fa..f28e9b1bc2 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -938,12 +938,16 @@ def close_open_pull_requests( def install_session_hooks(home: Path, trace: Path) -> None: path = home / ".claude" / "settings.json" settings = json.loads(path.read_text(encoding="utf-8")) + if "agent" in settings: + # Internal-agent callbacks carry the main agent's name, which the matcher admits. + raise ValueError("session hooks: settings.json sets a main agent") command = f"python3 /opt/e2e/e2e_sessions.py {shlex.quote(str(trace))}" hooks = settings.setdefault("hooks", {}) - for event in ("SessionStart", "SessionEnd", "SubagentStart", "SubagentStop"): - hooks.setdefault(event, []).append( - {"hooks": [{"type": "command", "command": command, "timeout": 10}]} - ) + for event, matcher in e2e_sessions.LIFECYCLE_HOOK_MATCHERS.items(): + entry: dict[str, Any] = {"hooks": [{"type": "command", "command": command, "timeout": 10}]} + if matcher is not None: + entry["matcher"] = matcher + hooks.setdefault(event, []).append(entry) path.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8") diff --git a/scripts/e2e/e2e_sessions.py b/scripts/e2e/e2e_sessions.py index 85c8552dd9..94c12b7d49 100644 --- a/scripts/e2e/e2e_sessions.py +++ b/scripts/e2e/e2e_sessions.py @@ -17,6 +17,13 @@ _SESSION_EVENTS = frozenset({"SessionStart", "SessionEnd"}) _AGENT_EVENTS = frozenset({"SubagentStart", "SubagentStop"}) _LIFECYCLE_EVENTS = _SESSION_EVENTS | _AGENT_EVENTS +# With the default main agent, internal agents have an empty agent_type. +# Session matchers filter source/reason, so keep session endpoints unfiltered. +_NATIVE_AGENT_MATCHER = ".+" +LIFECYCLE_HOOK_MATCHERS: Mapping[str, str | None] = { + **{event: None for event in sorted(_SESSION_EVENTS)}, + **{event: _NATIVE_AGENT_MATCHER for event in sorted(_AGENT_EVENTS)}, +} # Pinned to product prefixes by tests/arch/test_e2e_sessions_contract.py. _AUTOSKILLIT_SERVERS = frozenset( {("mcp", "autoskillit"), ("mcp", "plugin_autoskillit_autoskillit")} @@ -494,6 +501,8 @@ def _check_transcript_coverage( elif event["event"] in _AGENT_EVENTS: key = (session_id, event["agent_id"]) observed.setdefault(key, []).append(event) + if not event.get("agent_type"): + violations.append(f"capture:{session_id}:{event['agent_id']}:agent_type_missing") for session_id, paths in session_paths.items(): if len(paths) > 1: violations.append(f"lifecycle:{session_id}:transcript_mismatch") @@ -551,7 +560,7 @@ def _verify_native_transcript( ) -> dict[str, str] | None: label = f"native:{session_id}:{agent_id}" names = {event["event"] for event in events} - if not {"SubagentStart", "SubagentStop"} <= names: + if not _AGENT_EVENTS <= names: violations.append(f"{label}:lifecycle_incomplete") if transcript is None: violations.append(f"{label}:transcript_unenumerated") diff --git a/tests/_hook_protocol_oracle.py b/tests/_hook_protocol_oracle.py index 3a1328c4ac..2c03697604 100644 --- a/tests/_hook_protocol_oracle.py +++ b/tests/_hook_protocol_oracle.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import re from collections.abc import Callable, Mapping from pathlib import Path from typing import Any, Literal, NamedTuple @@ -445,6 +446,24 @@ def codex_verdict( # These keys and event exceptions come from Claude's JSON output and # per-event decision-control tables in hooks.md. +_EXACT_MATCHER = re.compile(r"[A-Za-z0-9_\- ,|]*") + + +def claude_matcher_admits(matcher: str | None, value: str) -> bool: + """Apply Claude Code's documented hook-matcher rule to one candidate value. + + Source: the "Matcher patterns" section of the page named by + ``CLAUDE_PROTOCOL_SOURCE``, re-read 2026-10-13. Python's ``re.search`` stands in + for ``RegExp.prototype.test``. ``FileChanged`` and ``StopFailure`` use a narrower + exact-match set and are not modelled. + """ + if matcher is None or matcher in ("", "*"): + return True + if _EXACT_MATCHER.fullmatch(matcher): + return value in {item.strip() for item in re.split(r"[|,]", matcher)} + return re.search(matcher, value) is not None + + CLAUDE_HOOK_SPECIFIC_OUTPUT_FIELDS: Mapping[str, frozenset[str]] = { "PreToolUse": frozenset( { diff --git a/tests/arch/test_e2e_sessions_contract.py b/tests/arch/test_e2e_sessions_contract.py index ff8eb67a1d..35f6e3820c 100644 --- a/tests/arch/test_e2e_sessions_contract.py +++ b/tests/arch/test_e2e_sessions_contract.py @@ -1,11 +1,14 @@ -"""Pin the standalone collector's identities and tools to product definitions.""" +"""Pin the standalone collector's identities, tools and hook policy to the product.""" +import json from pathlib import Path import pytest -from autoskillit.core import DIRECT_PREFIX, PLUGIN_PREFIX +from autoskillit.core import DIRECT_PREFIX, PLUGIN_PREFIX, ClaudeFlags, pkg_root from autoskillit.core.types import GATED_TOOLS, UNGATED_TOOLS +from autoskillit.execution.backends import claude_resolvable_agent_names +from tests._hook_protocol_oracle import claude_matcher_admits from tests.infra._complexity_helpers import load_check_script pytestmark = pytest.mark.medium @@ -25,3 +28,30 @@ def test_server_identities_match_canonical_prefixes() -> None: def test_tracked_tools_are_registered_product_tools() -> None: assert sessions._TRACKED_TOOLS assert sessions._TRACKED_TOOLS <= GATED_TOOLS | UNGATED_TOOLS + + +def test_agent_hooks_admit_every_resolvable_agent_and_reject_an_unnamed_one() -> None: + names = claude_resolvable_agent_names() + for event in sessions._AGENT_EVENTS: + matcher = sessions.LIFECYCLE_HOOK_MATCHERS[event] + assert matcher is not None + assert not claude_matcher_admits(matcher, "") + assert all(claude_matcher_admits(matcher, name) for name in names) + + +def test_session_hooks_stay_unfiltered() -> None: + matchers = sessions.LIFECYCLE_HOOK_MATCHERS + assert set(matchers) == sessions._LIFECYCLE_EVENTS + assert all(matchers[event] is None for event in sessions._SESSION_EVENTS) + + +def test_product_selects_no_main_agent() -> None: + # Internal-agent callbacks are unnamed only while no main agent is set. + assert {"--agent", "--settings"}.isdisjoint(flag.value for flag in ClaudeFlags) + root = pkg_root() + manifest = json.loads((root / ".claude-plugin" / "plugin.json").read_text(encoding="utf-8")) + assert "agent" not in manifest.get("settings", {}) + plugin_settings = root / "settings.json" + assert not plugin_settings.exists() or "agent" not in json.loads( + plugin_settings.read_text(encoding="utf-8") + ) diff --git a/tests/execution/test_flag_contracts.py b/tests/execution/test_flag_contracts.py index a027254c7d..d6d70fb412 100644 --- a/tests/execution/test_flag_contracts.py +++ b/tests/execution/test_flag_contracts.py @@ -7,10 +7,14 @@ one place, and all tests catch the ripple. """ +from pathlib import Path + import pytest from autoskillit.core import ClaudeFlags +from autoskillit.core.types import OutputFormat from autoskillit.execution.backends.claude import ClaudeCodeBackend +from tests.execution.backends._plugin_binding import plugin_binding pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] @@ -69,6 +73,31 @@ def _extract_flags(cmd: tuple[str, ...] | list[str]) -> set[str]: class TestFlagRegistryAudit: + @pytest.mark.parametrize("fmt", list(OutputFormat)) + def test_fleet_cmd_flags_are_all_registered(self, fmt: OutputFormat) -> None: + backend = ClaudeCodeBackend() + skill = backend.build_skill_session_cmd( + "/investigate foo", + cwd="/tmp", + completion_marker="%%DONE%%", + model=None, + plugin_binding=None, + output_format=fmt, + ) + with plugin_binding(Path("/p")) as binding: + root = backend.build_food_truck_cmd( + orchestrator_prompt="Orchestrator", + plugin_binding=binding, + cwd="/tmp", + completion_marker="%%DONE%%", + output_format=fmt, + ) + resume = backend.build_resume_cmd( + resume_session_id="abc", prompt="Emit", output_format=fmt + ) + for spec in (skill, root, resume): + assert not _extract_flags(spec.cmd) - KNOWN_CLAUDE_FLAGS + def test_interactive_cmd_flags_are_all_registered(self): result = ClaudeCodeBackend().build_interactive_cmd(model="claude-sonnet-4-6") unknown = _extract_flags(result.cmd) - KNOWN_CLAUDE_FLAGS diff --git a/tests/hooks/test_hook_protocol_oracle.py b/tests/hooks/test_hook_protocol_oracle.py index eb52ff0e88..12287963c4 100644 --- a/tests/hooks/test_hook_protocol_oracle.py +++ b/tests/hooks/test_hook_protocol_oracle.py @@ -18,6 +18,7 @@ STATUS_COMPLETED, STATUS_FAILED, STATUS_STOPPED, + claude_matcher_admits, claude_verdict, codex_verdict, ) @@ -25,6 +26,40 @@ pytestmark = [pytest.mark.layer("hooks"), pytest.mark.small] +@pytest.mark.parametrize( + ("matcher", "value", "admitted"), + [ + (None, "Bash", True), + (None, "", True), + ("", "Bash", True), + ("", "", True), + ("*", "Bash", True), + ("*", "", True), + ("Bash", "Bash", True), + ("Bash", "BashOutput", False), + ("Edit|Write", "Edit", True), + ("Edit|Write", "Write", True), + ("Edit|Write", "NotebookEdit", False), + ("Edit, Write", "Edit", True), + ("Edit, Write", "Write", True), + ("Edit, Write", "NotebookEdit", False), + ("code-reviewer", "code-reviewer", True), + ("code-reviewer", "other-code-reviewer", False), + ("^Notebook", "NotebookEdit", True), + ("Edit.*", "NotebookEdit", True), + ("^Edit$", "NotebookEdit", False), + ("^my-plugin:reviewer$", "my-plugin:reviewer", True), + ("general-purpose", "", False), + (".+", "", False), + (".*", "", True), + ], +) +def test_claude_matcher_follows_the_documented_rule( + matcher: str | None, value: str, admitted: bool +) -> None: + assert claude_matcher_admits(matcher, value) is admitted + + @pytest.mark.parametrize("event", ["PreCompact", "PostCompact"]) def test_codex_compact_halt_preserves_system_message(event: str) -> None: # codex-rs/hooks/src/events/compact.rs:260-278 records warnings before stopping. diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index b3d0143897..c659ae2644 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -5,6 +5,7 @@ import json import math import os +import shlex import stat import subprocess import tempfile @@ -17,6 +18,7 @@ import pytest +from tests._hook_protocol_oracle import claude_matcher_admits from tests.infra._complexity_helpers import load_check_script from tests.infra._e2e_evidence import load_provenance, materialize_run @@ -2095,6 +2097,92 @@ def test_missing_sandbox_token_fails_before_any_launch(self, tmp_path: Path) -> class TestPipelineRecipeFlow: + def test_session_hooks_install_the_declared_policy_and_keep_existing_settings( + self, tmp_path: Path + ) -> None: + home = tmp_path / "home" + settings_path = home / ".claude" / "settings.json" + settings_path.parent.mkdir(parents=True, exist_ok=True) + foreign_subagent_stop = { + "matcher": "ForeignAgent", + "hooks": [{"type": "command", "command": "foreign"}], + } + other_pre_tool_use = { + "matcher": "Bash", + "hooks": [{"type": "command", "command": "other"}], + } + settings_path.write_text( + json.dumps( + { + "theme": "dark", + "env": {"KEEP1": "1"}, + "hooks": { + "SubagentStop": [foreign_subagent_stop], + "PreToolUse": [other_pre_tool_use], + }, + } + ), + encoding="utf-8", + ) + trace = tmp_path / "trace with space" / "events.jsonl" + + harness.install_session_hooks(home, trace) + + settings = json.loads(settings_path.read_text(encoding="utf-8")) + lifecycle_events = {"SessionStart", "SessionEnd", "SubagentStart", "SubagentStop"} + declarations = harness.e2e_sessions.LIFECYCLE_HOOK_MATCHERS + assert set(declarations) == lifecycle_events + assert set(settings["hooks"]) == lifecycle_events | {"PreToolUse"} + assert settings["theme"] == "dark" + assert settings["env"] == {"KEEP1": "1"} + assert settings["hooks"]["PreToolUse"] == [other_pre_tool_use] + assert settings["hooks"]["SubagentStop"][0] == foreign_subagent_stop + + expected_command_hook = { + "type": "command", + "command": f"python3 /opt/e2e/e2e_sessions.py {shlex.quote(str(trace))}", + "timeout": 10, + } + for event in lifecycle_events: + installed = settings["hooks"][event] + if event == "SubagentStop": + installed = installed[1:] + assert len(installed) == 1 + entry = installed[0] + assert entry["hooks"] == [expected_command_hook] + declared_matcher = declarations[event] + if declared_matcher is None: + assert "matcher" not in entry + else: + assert entry["matcher"] == declared_matcher + assert not claude_matcher_admits(entry["matcher"], "") + + def test_session_hooks_refuse_a_custom_main_agent(self, tmp_path: Path) -> None: + home = tmp_path / "home" + settings_path = home / ".claude" / "settings.json" + settings_path.parent.mkdir(parents=True, exist_ok=True) + settings_path.write_bytes(b'{"agent":"custom-main"}') + original = settings_path.read_bytes() + + with pytest.raises(ValueError, match="main agent"): + harness.install_session_hooks(home, tmp_path / "events.jsonl") + + assert settings_path.read_bytes() == original + + def test_custom_main_agent_setting_fails_preflight(self, tmp_path: Path, monkeypatch) -> None: + settings_path = tmp_path / "home" / ".claude" / "settings.json" + settings_path.parent.mkdir(parents=True, exist_ok=True) + settings_path.write_text('{"agent":"custom-main"}', encoding="utf-8") + runner, state = _pipeline_runner() + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert failures == ["preflight: session hooks: settings.json sets a main agent"] + assert result["outcome"] == "failed" + assert not any(call.argv[:3] == ["autoskillit", "fleet", "run"] for call in runner.calls) + def test_pipeline_uses_the_seed_issue_and_cleans_only_its_owned_pr( self, tmp_path: Path, monkeypatch ) -> None: @@ -2214,7 +2302,7 @@ def test_pipeline_uses_the_seed_issue_and_cleans_only_its_owned_pr( assert settings["env"] == _EXPECTED_SETTINGS_ENV assert all( len(settings["hooks"][event]) == 1 - for event in ("SessionStart", "SessionEnd", "SubagentStart", "SubagentStop") + for event in harness.e2e_sessions.LIFECYCLE_HOOK_MATCHERS ) close = next(call for call in calls if call.argv[:3] == ["gh", "pr", "close"]) diff --git a/tests/infra/test_e2e_sessions.py b/tests/infra/test_e2e_sessions.py index bd86616213..465de639c6 100644 --- a/tests/infra/test_e2e_sessions.py +++ b/tests/infra/test_e2e_sessions.py @@ -85,6 +85,7 @@ def _event( *, transcript: Path | None = None, agent_id: str | None = None, + agent_type: str | None = None, agent_transcript: Path | None = None, ) -> dict[str, Any]: row: dict[str, Any] = { @@ -96,6 +97,8 @@ def _event( row["transcript_path"] = str(transcript) if agent_id is not None: row["agent_id"] = agent_id + if agent_type is not None: + row["agent_type"] = agent_type if agent_transcript is not None: row["agent_transcript_path"] = str(agent_transcript) return row @@ -292,6 +295,38 @@ def test_hook_capture_appends_concurrent_allow_listed_records(tmp_path: Path) -> assert len(trace.read_text(encoding="utf-8").splitlines()) == 32 +@pytest.mark.parametrize("agent_type", ["general-purpose", ""]) +def test_hook_capture_preserves_agent_type_and_strips_stop_message( + tmp_path: Path, agent_type: str +) -> None: + trace = tmp_path / "events.jsonl" + captured = sessions.capture_hook_event( + trace, + { + "hook_event_name": "SubagentStop", + "session_id": ROOT_ID, + "agent_id": "agent-x", + "agent_type": agent_type, + "transcript_path": "/tmp/root.jsonl", + "agent_transcript_path": "/tmp/agent-x.jsonl", + "last_assistant_message": "must not be persisted", + }, + ) + + assert captured is True + [record] = [json.loads(line) for line in trace.read_text(encoding="utf-8").splitlines()] + assert record["agent_type"] == agent_type + assert set(record) == { + "event", + "timestamp", + "session_id", + "agent_id", + "agent_type", + "transcript_path", + "agent_transcript_path", + } + + def test_summary_collapses_repeated_events_excludes_unrelated_and_sweeps_half_open( tmp_path: Path, ) -> None: @@ -401,9 +436,20 @@ def test_summary_enumerates_native_agent_transcripts_and_requires_endpoints( trace, [ _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), - _event("SubagentStart", ROOT_ID, 2, agent_id="agent-a"), _event( - "SubagentStop", ROOT_ID, 4, agent_id="agent-a", agent_transcript=agent_transcript + "SubagentStart", + ROOT_ID, + 2, + agent_id="agent-a", + agent_type="general-purpose", + ), + _event( + "SubagentStop", + ROOT_ID, + 4, + agent_id="agent-a", + agent_type="general-purpose", + agent_transcript=agent_transcript, ), _event("SessionEnd", ROOT_ID, 5, transcript=root_transcript), ], @@ -422,6 +468,76 @@ def test_summary_enumerates_native_agent_transcripts_and_requires_endpoints( ] +@pytest.mark.parametrize( + ("agent_type", "extra_violation"), + [ + pytest.param("custom-main", None, id="named-custom-main"), + pytest.param("", f"capture:{ROOT_ID}:agent-x:agent_type_missing", id="blank-type"), + pytest.param(None, f"capture:{ROOT_ID}:agent-x:agent_type_missing", id="missing-type"), + ], +) +def test_orphan_agent_stop_is_reported_even_without_an_admitted_type( + tmp_path: Path, + agent_type: str | None, + extra_violation: str | None, +) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + missing_agent_transcript = ( + log_root / "transcripts" / ROOT_ID / "subagents" / "agent-agent-x.jsonl" + ) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event( + "SubagentStop", + ROOT_ID, + 2, + agent_id="agent-x", + agent_type=agent_type, + agent_transcript=missing_agent_transcript, + ), + _event("SessionEnd", ROOT_ID, 3, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + expected_violations = [ + f"lifecycle:agent:{ROOT_ID}:agent-x:orphan_end", + f"native:{ROOT_ID}:agent-x:lifecycle_incomplete", + f"native:{ROOT_ID}:agent-x:transcript_unenumerated", + ] + if extra_violation is not None: + expected_violations.append(extra_violation) + assert summary["coverage"]["complete"] is False + assert summary["violations"] == sorted(expected_violations) + assert summary["peak_sessions"] == 1 + + +def test_enumerated_native_transcript_without_agent_hooks_is_incomplete( + tmp_path: Path, +) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + agent_transcript = log_root / "transcripts" / ROOT_ID / "subagents" / "agent-agent-a.jsonl" + agent_transcript.parent.mkdir(parents=True) + agent_transcript.write_text("{}\n", encoding="utf-8") + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 3, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["complete"] is False + assert f"native:{ROOT_ID}:agent-a:lifecycle_incomplete" in summary["violations"] + + def test_native_transcript_symlink_loop_is_reported_missing( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/infra/test_e2e_sessions_captured.py b/tests/infra/test_e2e_sessions_captured.py index 00fb9d2126..03b4d4b52d 100644 --- a/tests/infra/test_e2e_sessions_captured.py +++ b/tests/infra/test_e2e_sessions_captured.py @@ -10,6 +10,7 @@ import pytest from autoskillit.core import PLUGIN_PREFIX +from tests._hook_protocol_oracle import claude_matcher_admits from tests.infra._complexity_helpers import load_check_script from tests.infra._e2e_evidence import load_provenance, materialize_run @@ -195,6 +196,7 @@ def test_captured_run_accounts_for_successful_and_failed_workers( f"lifecycle:agent:{ROOT_SESSION_ID}:{UNMATCHED_AGENT_ID}:orphan_end", f"native:{ROOT_SESSION_ID}:{UNMATCHED_AGENT_ID}:lifecycle_incomplete", f"native:{ROOT_SESSION_ID}:{UNMATCHED_AGENT_ID}:transcript_unenumerated", + f"capture:{ROOT_SESSION_ID}:{UNMATCHED_AGENT_ID}:agent_type_missing", ] ) assert envelope["l3_payload"] is None @@ -215,7 +217,7 @@ def test_captured_run_accounts_for_successful_and_failed_workers( ) -def test_captured_run_is_complete_once_the_unmatched_stop_is_removed( +def test_captured_run_is_complete_under_the_declared_capture_policy( tmp_path: Path, ) -> None: trace = tmp_path / "trace" / "session-events.jsonl" @@ -227,17 +229,20 @@ def test_captured_run_is_complete_once_the_unmatched_stop_is_removed( projects=tmp_path / "projects", ) events = _rows(trace) - unmatched_stops = [ - event - for event in events - if event.get("event") == "SubagentStop" and event.get("agent_id") == UNMATCHED_AGENT_ID - ] - assert len(unmatched_stops) == 1 + admitted = [] + excluded: list[tuple[str, str, str]] = [] + for event in events: + matcher = sessions.LIFECYCLE_HOOK_MATCHERS[event["event"]] + agent_type = event.get("agent_type", "") + if claude_matcher_admits(matcher, agent_type): + admitted.append(event) + else: + excluded.append((event["event"], event.get("agent_id", ""), agent_type)) + assert excluded == [("SubagentStop", UNMATCHED_AGENT_ID, "")] trace.write_text( "".join( json.dumps(event, separators=(",", ":"), ensure_ascii=False) + "\n" - for event in events - if event not in unmatched_stops + for event in admitted ), encoding="utf-8", ) @@ -246,3 +251,6 @@ def test_captured_run_is_complete_once_the_unmatched_stop_is_removed( assert summary["coverage"]["complete"] is True assert summary["violations"] == [] assert summary["peak_sessions"] == 5 + assert { + agent["agent_id"] for agent in summary["coverage"]["native_agents"] + } == NATIVE_AGENT_IDS