diff --git a/.autoskillit/test-filter-manifest.yaml b/.autoskillit/test-filter-manifest.yaml index 5f4ee22d77..2d2f4b78e1 100644 --- a/.autoskillit/test-filter-manifest.yaml +++ b/.autoskillit/test-filter-manifest.yaml @@ -243,6 +243,9 @@ tests/execution/backends/fixtures/codex_ndjson/*.json: tests/execution/fixtures/*.json: - execution/ +tests/infra/fixtures/e2e_sessions/**: + - infra/ + tests/execution/fixtures/*.jsonl: - execution/ diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a35b998ce8..aedb28d507 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -104,12 +104,13 @@ jobs: env: MINIMAX_API_KEY: ${{ matrix.kind != 'clean-install' && secrets.MINIMAX_API_KEY || '' }} E2E_SANDBOX_TOKEN: ${{ matrix.kind != 'clean-install' && secrets.E2E_SANDBOX_TOKEN || '' }} + E2E_IMPLEMENTATION_ISSUE_URL: ${{ vars.E2E_IMPLEMENTATION_ISSUE_URL }} TEST: ${{ matrix.test }} run: | set -euo pipefail docker_args=(--shm-size=3g) if [[ '${{ matrix.kind }}' != 'clean-install' ]]; then - docker_args+=(--env MINIMAX_API_KEY --env E2E_SANDBOX_TOKEN) + docker_args+=(--env MINIMAX_API_KEY --env E2E_SANDBOX_TOKEN --env E2E_IMPLEMENTATION_ISSUE_URL) docker_args+=(--volume "$RUNNER_TEMP/e2e/data:/home/autoskillit/.local/share/autoskillit") fi if [[ "$TEST" == 'headless-smoke' ]]; then diff --git a/scripts/e2e/README.md b/scripts/e2e/README.md index da923f52bd..14cf4554a3 100644 --- a/scripts/e2e/README.md +++ b/scripts/e2e/README.md @@ -11,6 +11,7 @@ This directory holds everything the workflow runs: | `e2e_catalog.py` | Stdlib loader and validator for the catalog | | `e2e_select.py` | Picks the tests an event runs (`select`) and judges the required check (`gate`) | | `e2e_harness.py` | Runs one test (`run`), performs exact sandbox teardown (`cleanup`) and redacts artifacts (`redact`) | +| `e2e_sessions.py` | Captures synchronous lifecycle events, measures concurrency and pairs ownership evidence | | `autoskillit-config.yaml` | AutoSkillit user-layer config the harness installs | | `post-e2e-failure.sh` | Appends a failure to the test's tracking issue | @@ -40,6 +41,11 @@ model tests: because MiniMax serves the Anthropic Messages API. 3. Create the `e2e` label in this repository. 4. In branch protection, mark only `e2e-gate` as required — never `select` or `run (…)`. +5. Before publishing a branch with the selectable `implementation` entry, obtain a passing + #5232 sandbox smoke CI run. A maintainer must seed a fixed OPEN sandbox issue describing + a small feature and its test, then set repository variable `E2E_IMPLEMENTATION_ISSUE_URL` + to its canonical `https://github.com/OWNER/REPO/issues/NUMBER` URL. Reuse that issue on + every run. Missing configuration fails preflight; it is not an expected product defect. ## Sandbox repository requirements @@ -192,7 +198,11 @@ matrix is always parseable (`{"include":[]}` when nothing is selected). bypasses sampling because a labelled pull request whose sample fell out would otherwise never run for that commit. - **Picking:** tests whose `trigger_paths` match a changed path, up to `MAX_TESTS` (2); when - none match, exactly one test. Ties are ordered by the SHA-256 hex digest of + none match, one eligible ordinary test. A pipeline requires actual size at or above a + threshold and a matching trigger path, even with the `e2e` label. This filter also applies + to fallback; a pipeline-only ineligible pool selects nothing. At most one pipeline is + selected within the total cap. Named workflow dispatch bypasses PR eligibility. + Ties are ordered by the SHA-256 hex digest of `"{seed}:{name}"`, so the decision is a pure function of the payload and the changed paths. Patterns use `fnmatch.fnmatchcase`, where `*` also matches `/` (`scripts/e2e/*` matches every file below `scripts/e2e/`). @@ -208,11 +218,52 @@ succeeded or nothing was selected and `run` was skipped. Every other combination The `run` job uses one repository-wide concurrency group (`group: e2e`, `queue: max`, `cancel-in-progress: false`) and `max-parallel: 1`, so every selected test of every run waits its turn and never cancels another. `select` and `e2e-gate` are not queued. A -catalog entry's `peak_sessions` is capped at `MAX_PEAK_SESSIONS` (8): one step's fan-out of -up to six subagents plus the orchestrator and step session. With one test running at a -time, CI stays at or below eight of the roughly twelve concurrent MiniMax sessions. +catalog entry's `peak_sessions` is capped at `MAX_PEAK_SESSIONS` (8). For an uncommissioned +pipeline this declares capacity, not a measured fan-out. The child limit stays six. `clean-install` requires exactly zero `peak_sessions`. +The implementation pipeline installs synchronous command hooks for `SessionStart`, +`SessionEnd`, `SubagentStart` and `SubagentStop`, preserving existing hooks and MiniMax +settings. These hooks follow the [Claude lifecycle schema](https://code.claude.com/docs/en/hooks). +The agent hooks use `.+` to require a non-empty agent type: under the default main +agent, Claude Code also fires `SubagentStop` for internal agents with an empty type. +An omitted or empty matcher admits those callbacks; `.+` accepts every non-empty +built-in, custom or namespaced plugin agent name without an allowlist or product-registry +imports in these stdlib-only scripts. Session hooks have no matcher. +Sessions must run with the default main agent. The harness fails preflight if the +user-level settings file has an `agent` key; the fleet command builders pass neither +`--agent` nor `--settings`, and the bundled plugin sets no main agent. Project/local +settings in the sandbox, managed settings and other enabled plugins' main-agent defaults +are not checked. A main agent from those sources makes internal stops carry its name: +they fail coverage as a named stop without a start, without the blank-type `capture:` code. +Claude Code 2.1.275 fixed specific matchers admitting blank-type stops; the image's +2.1.280 pin includes that fix. +Each identity spans its first start through its final end, so compaction and resume yield +a conservative upper bound. Half-open intervals end before simultaneous starts. The +summary reconciles the dispatched root, managed lineage and enumerable native transcripts. +AutoSkillit writes `logs/sessions.jsonl` and each session's `summary.json`; their dispatch +and parent/caller fields are AutoSkillit provenance, separate from Claude Code transcript +and lifecycle hook fields. Empty links do not imply a Claude Code schema guarantee. +Managed sessions are adopted through declared parent fields or receipted `run_skill`, +`complete_run_skill_result` or `recover_run_skill_result` replies in an owned transcript. +Each must have exactly one index row with the run's `dispatch_id`. Parentless sessions +without receipts, missing or ambiguous index rows, and receipt-named sessions absent +from both trace and index fail coverage. Shared dispatch IDs or overlapping lifetimes +never adopt a session. Missing endpoints, identities or owned transcripts fail coverage; +transcript activity never infers a lifetime. The captured pre-fix trace contains a blank-type +stop immediately before a compact-source session start; its producing feature is unproven. +An owned agent record without a named type fails coverage with a `capture:` violation. +A named stop without its start or transcript still fails coverage, as do traces captured +before the matcher was installed. The collector retains delivered callbacks and adds +violations; it does not filter them out. Individual codes are in `out/session-concurrency.json` +under `violations`. A peak exceeding capacity fails validation. + +Commission with a successful MiniMax CI run and repeat against the same seed. Inspect +planning, dry-walkthrough, implementation, test, integration, audit GO and open-PR evidence, +plus cleanup and the sandbox CI association. Replace provisional `peak_sessions` with +the observed conservative peak and set `peak_sessions_evidence` to the CI run URL, tested +commit and first-start/final-end method. Expected failures do not satisfy commissioning. + Every harness subprocess has a timeout that fires before the job's: `fleet run` gets the test's `timeout_sec` plus `HARNESS_GRACE_SEC`, and the job gets that budget in minutes plus `JOB_OVERHEAD_MINUTES` for building the image, redaction and upload. So `result.json`, PR @@ -237,6 +288,13 @@ redaction and upload steps as the other output artifacts. Collection also runs f expected failure or a harness exception. A container killed before or during collection may leave no evidence or only partial evidence. +Pipeline evidence includes `session-events.jsonl`, `session-concurrency.json`, compact +`ownership.json`, the seed, effective configuration, fleet stdout/stderr/envelope, captured +PR metadata/files, detached-head test commands, associated CI runs/jobs, and sandbox head-ref +listings before the run and after cleanup (`head-refs-before.*`, `head-refs-after.*`). Collection +occurs inside the container before transcript paths disappear. Incomplete evidence is +retained and fails the scenario; redaction and upload still run after failure. + `clean-install` mounts no home directories and retains evidence under `out/`. Its redactor receives the same required `--secret-env` names with variables unset and skips the absent home-log tree. Redaction and upload still run after test failures. @@ -301,6 +359,57 @@ row and close the bug. Fleet translates the L3 result's `reason` into the failed command envelope's `error` and `user_visible_message` fields; the harness matches that envelope rather than the recipe payload's `failed_step`/`reason`. +For the implementation pipeline, the project-owned +[`DispatchCompleted.to_envelope`](../../src/autoskillit/fleet/campaign_state/state_outcomes.py) +must report success, `dispatch_status="success"` and root/dispatch identities. The nested +[`implementation` terminal](../../src/autoskillit/recipes/implementation.yaml) must have +`reason="implementation_complete"`. Successful no-change and already-done terminals remain +valid recipe behavior but fail this scenario. No PR URL is required in the envelope. + +The harness resolves the configured seed once and dispatches its exact URL and title/body +task once. Effective project-aware config comes from `autoskillit config show`, whose +implementation calls `load_config(Path.cwd())`; both configured and sandbox default branches +must be `main`. No caller `base_branch` ingredient is sent. Sandbox setup and baseline tests +must succeed before launch. + +Paired root transcript tool calls/results prove the created branch; paired root/managed +claim results prove any added claim label. The collector accepts the direct +`mcp__autoskillit__` and plugin `mcp__plugin_autoskillit_autoskillit__` namespaces; +a tracked tool under any other namespace fails the proof. Claude Code persisted +transcripts carry replies in row-level `mcpMeta.structuredContent` when `mcpMeta` +is present, otherwise in JSON block content. Malformed metadata or metadata on a +row with several results fails the paired resource call. Hook display text never +replaces unusable metadata. The pretty-output hook emits `updatedMCPToolOutput`; +`_result_object` remains the shared envelope decoder, and metadata presence alone +does not identify whether the hook ran. These shapes are pinned by +`tests/infra/fixtures/e2e_sessions/` from Claude Code 2.1.280; re-capture them when +an otherwise successful call reports a failed branch or claim result. + +Discovery paginates through the PR baseline +and leaves unrelated PRs untouched. Require exactly one owned open, unmerged sandbox PR +targeting `main`, without auto-merge, with the canonical seed URL and non-removed changes +under both `sandbox/` and `tests/`. Fetch and detach at the captured head before setup and +tests. Require a successful associated sandbox `CI` run and `test` job whose nested PR +association has the exact branch revision; an outer merge SHA is not the branch SHA. +Finally re-read the head and state to reject movement during validation. + +Teardown runs after validation, failure or exception. It independently collects metrics, +closes owned open PRs, deletes evidenced branches (including branches without a PR), and +removes only the effective claim label proven added by this run. The seed remains OPEN +and other labels are preserved. Missing ownership proof preserves uncertain resources and +fails cleanup. Cleanup errors remain fatal alongside the original failure. + +Branch reconciliation lists the sandbox's branches before the run and after cleanup. +A branch created during the run that `ownership.json` does not explain is preserved +and fails cleanup; baseline branches are excluded from unexplained-branch reports. +Branches named by the ownership proof retain the existing cleanup rule. The original +API stdout saved as `head-refs-before.stdout.txt` and `head-refs-after.stdout.txt` supplies +the in-memory snapshots; accompanying stderr and command records diagnose failed requests. +This check only detects unexplained branches: the creating run records the failure once, +and the branch and any PR on it must be removed by hand. The next run includes it in +its baseline. A manual run overlapping CI can trigger the check because CI serialization +does not cover local runs. + ## Catalog schema ```json @@ -329,15 +438,38 @@ the harness matches that envelope rather than the recipe payload's `failed_step` ``` `recipe`, `ingredients` and `expected_pull_request_state` are required for `recipe` tests -and forbidden for `canary` and `clean-install` tests. `expected_failures` is optional for -`clean-install` and recipe tests. Clean-install rows use the exact doctor `severity`, `check` -and `message`; recipe rows require `severity: error`, `check` and `message` matching the failed -envelope. Every row includes an open bug issue URL -(`https://github.com/TalonT-Org/AutoSkillit/issues/`). Duplicate diagnostic identities -and unknown keys are rejected. `peak_sessions` must be +and forbidden for `canary` and `clean-install` tests. Optional recipe-only fields are +`pipeline` (boolean, default false), `issue_url_env` (seed variable name), +`required_changed_paths` (nonempty relative prefixes), `test_command` (nonempty argv), and +`peak_sessions_evidence` (nonempty provenance string). Environment-owned seeds cannot also +specify literal `task` or `issue_url` ingredients. Path prefixes cannot be absolute or +traverse parents. For example, the implementation entry uses: + +```json +{"pipeline": true, "issue_url_env": "E2E_IMPLEMENTATION_ISSUE_URL", + "required_changed_paths": ["sandbox/", "tests/"], "test_command": ["task", "test-check"]} +``` + +The recipe_fixture field names a staged YAML fixture for the headless smoke scenario. + +The expected_failures field is optional for recipes and clean-install. Pipeline rows +contain exactly check, message, issue; ordinary recipe and clean-install rows also +contain severity. Ordinary recipes require severity: error and match the failed +envelope's error and user_visible_message; clean-install allows error or warning +and matches doctor diagnostics. Pipeline rows match structured implementation findings. +Every row links to an open AutoSkillit bug issue. Duplicate diagnostic identities and +unknown keys are rejected. The peak_sessions value must be 1–8 for model tests and exactly zero for clean-install. `timeout_sec` must keep the derived job timeout within 360 minutes. +Recipe expectations match exact stable `(check, message)` product findings linked to real +bug issues. Setup, discovery, malformed evidence and cleanup failures cannot be admitted. +Unknown findings and absent expected findings fail. Every recipe records `outcome`: +`passed`, `expected_failure` (with `passed=false`), or `failed`. A linked known defect may +use the accepted CLI exit convention, but only `passed` with no expected findings counts +as successful commissioning. File or deduplicate observed product bugs separately; do not +predeclare a provider defect or replace the MiniMax settings-file route. + ### Adding a test 1. Add the entry to `catalog.json` with a realistic `peak_sessions` and `timeout_sec`. diff --git a/scripts/e2e/autoskillit-config.yaml b/scripts/e2e/autoskillit-config.yaml index 511b455990..331721be11 100644 --- a/scripts/e2e/autoskillit-config.yaml +++ b/scripts/e2e/autoskillit-config.yaml @@ -3,6 +3,9 @@ quota_guard: enabled: false +branching: + default_base_branch: main + model: default: sonnet[1m] diff --git a/scripts/e2e/catalog.json b/scripts/e2e/catalog.json index 00bf957ce1..ba100c3b0b 100644 --- a/scripts/e2e/catalog.json +++ b/scripts/e2e/catalog.json @@ -70,6 +70,35 @@ "recipe_fixture": "sandbox-smoke.yaml", "ingredients": {}, "expected_pull_request_state": "closed" + }, + { + "name": "implementation", + "kind": "recipe", + "pipeline": true, + "peak_sessions": 8, + "timeout_sec": 7200, + "trigger_paths": [ + "src/autoskillit/recipes/*", + "src/autoskillit/skills/*", + "src/autoskillit/skills_extended/*", + "src/autoskillit/server/*", + "src/autoskillit/execution/*" + ], + "recipe": "implementation", + "ingredients": { + "source_dir": "/workspace/sandbox", + "open_pr": "true", + "auto_merge": "false", + "arch_lenses": "false", + "audit_impl": "true" + }, + "expected_pull_request_state": "open", + "issue_url_env": "E2E_IMPLEMENTATION_ISSUE_URL", + "required_changed_paths": [ + "sandbox/", + "tests/" + ], + "test_command": ["task", "test-check"] } ] } diff --git a/scripts/e2e/e2e_catalog.py b/scripts/e2e/e2e_catalog.py index 46354d43f4..2940e8e472 100644 --- a/scripts/e2e/e2e_catalog.py +++ b/scripts/e2e/e2e_catalog.py @@ -7,7 +7,7 @@ import re from collections.abc import Mapping from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath from types import MappingProxyType CATALOG_PATH = Path(__file__).with_name("catalog.json") @@ -28,7 +28,21 @@ _CATALOG_KEYS = frozenset({"sandbox_repository", "tests"}) _COMMON_KEYS = frozenset({"name", "kind", "peak_sessions", "timeout_sec", "trigger_paths"}) _RECIPE_KEYS = frozenset({"recipe", "ingredients", "expected_pull_request_state"}) +_PIPELINE_METADATA_KEYS = frozenset( + { + "issue_url_env", + "required_changed_paths", + "test_command", + "peak_sessions_evidence", + } +) +_RECIPE_OPTIONAL_KEYS = _PIPELINE_METADATA_KEYS | { + "pipeline", + "expected_failures", + "recipe_fixture", +} _EXPECTED_FAILURE_KEYS = frozenset({"severity", "check", "message", "issue"}) +_PIPELINE_EXPECTED_FAILURE_KEYS = frozenset({"check", "message", "issue"}) _BUG_URL_PATTERN = re.compile(r"https://github\.com/TalonT-Org/AutoSkillit/issues/[1-9][0-9]*") @@ -50,6 +64,11 @@ class CatalogTest: expected_pull_request_state: str | None expected_failures: tuple[Mapping[str, str], ...] = () recipe_fixture: str | None = None + pipeline: bool = False + issue_url_env: str | None = None + required_changed_paths: tuple[str, ...] = () + test_command: tuple[str, ...] = () + peak_sessions_evidence: str | None = None @dataclass(frozen=True, slots=True) @@ -83,6 +102,13 @@ def _require_str(value: object, where: str) -> str: return value +def _require_nonblank_str(value: object, where: str) -> str: + result = _require_str(value, where) + if not result.strip(): + raise CatalogError(f"{where} must be non-empty") + return result + + def _require_int(value: object, where: str, *, low: int, high: int | None = None) -> int: if isinstance(value, bool) or not isinstance(value, int): raise CatalogError(f"{where} must be an integer") @@ -111,7 +137,7 @@ def _parse_ingredients(value: object, where: str) -> tuple[tuple[str, str], ...] def _parse_recipe_fields( raw: Mapping[str, object], where: str -) -> tuple[str, tuple[tuple[str, str], ...], str]: +) -> tuple[str, tuple[tuple[str, str], ...], str, str | None]: recipe = _require_str(raw["recipe"], f"{where}.recipe") ingredients = _parse_ingredients(raw["ingredients"], f"{where}.ingredients") state = _require_str( @@ -121,28 +147,40 @@ def _parse_recipe_fields( raise CatalogError( f"{where}.expected_pull_request_state must be one of {PULL_REQUEST_STATES}" ) - return recipe, ingredients, state + fixture = None + if "recipe_fixture" in raw: + fixture = _require_str(raw["recipe_fixture"], f"{where}.recipe_fixture") + if not _RECIPE_FIXTURE_PATTERN.fullmatch(fixture): + raise CatalogError(f"{where}.recipe_fixture must be a .yaml basename") + return recipe, ingredients, state, fixture -def _parse_expected_failures(value: object, where: str) -> tuple[Mapping[str, str], ...]: +def _parse_expected_failures( + value: object, where: str, *, pipeline: bool = False +) -> tuple[Mapping[str, str], ...]: where = f"{where}.expected_failures" if not isinstance(value, list): raise CatalogError(f"{where} must be a list") + keys = _PIPELINE_EXPECTED_FAILURE_KEYS if pipeline else _EXPECTED_FAILURE_KEYS rows: list[Mapping[str, str]] = [] - seen: set[tuple[str, str, str]] = set() + seen: set[tuple[str, ...]] = set() for index, raw in enumerate(value): location = f"{where}[{index}]" if not isinstance(raw, dict): raise CatalogError(f"{location} must be an object") - _require_keys(raw, _EXPECTED_FAILURE_KEYS, location) - row = {key: _require_str(raw[key], f"{location}.{key}") for key in _EXPECTED_FAILURE_KEYS} + _require_keys(raw, keys, location) + row = {key: _require_str(raw[key], f"{location}.{key}") for key in keys} if any(not item.strip() for item in row.values()): raise CatalogError(f"{location} fields must be non-empty") - if row["severity"] not in ("error", "warning"): + if not pipeline and row["severity"] not in ("error", "warning"): raise CatalogError(f"{location}.severity must be error or warning") if not _BUG_URL_PATTERN.fullmatch(row["issue"]): raise CatalogError(f"{location}.issue must link to an AutoSkillit bug issue") - identity = (row["severity"], row["check"], row["message"]) + identity = ( + (row["check"], row["message"]) + if pipeline + else (row["severity"], row["check"], row["message"]) + ) if identity in seen: raise CatalogError(f"{location}: duplicate diagnostic {identity}") seen.add(identity) @@ -150,6 +188,85 @@ def _parse_expected_failures(value: object, where: str) -> tuple[Mapping[str, st return tuple(rows) +def _parse_recipe_options( + raw: Mapping[str, object], where: str, ingredients: tuple[tuple[str, str], ...] +) -> tuple[str | None, tuple[str, ...], tuple[str, ...], str | None]: + issue_url_env = ( + _require_nonblank_str(raw["issue_url_env"], f"{where}.issue_url_env") + if "issue_url_env" in raw + else None + ) + if issue_url_env and {"issue_url", "task"}.intersection(key for key, _ in ingredients): + raise CatalogError( + f"{where}.ingredients cannot contain issue_url or task when issue_url_env is set" + ) + changed_paths = ( + _parse_required_changed_paths( + raw["required_changed_paths"], f"{where}.required_changed_paths" + ) + if "required_changed_paths" in raw + else () + ) + test_command = ( + _parse_test_command(raw["test_command"], f"{where}.test_command") + if "test_command" in raw + else () + ) + evidence = ( + _require_nonblank_str(raw["peak_sessions_evidence"], f"{where}.peak_sessions_evidence") + if "peak_sessions_evidence" in raw + else None + ) + return issue_url_env, changed_paths, test_command, evidence + + +def _parse_required_changed_paths(value: object, where: str) -> tuple[str, ...]: + if not isinstance(value, list) or not value: + raise CatalogError(f"{where} must be a non-empty list") + changed_paths: list[str] = [] + for item in value: + prefix = _require_nonblank_str(item, f"{where}[]") + posix_path = PurePosixPath(prefix) + windows_path = PureWindowsPath(prefix) + invalid_path = any( + ( + posix_path.is_absolute(), + windows_path.is_absolute(), + bool(windows_path.drive), + ".." in posix_path.parts, + ".." in windows_path.parts, + ) + ) + if invalid_path: + raise CatalogError(f"{where}[] must be a relative path prefix without '..'") + changed_paths.append(prefix) + return tuple(changed_paths) + + +def _parse_test_command(command: object, where: str) -> tuple[str, ...]: + if not isinstance(command, list) or not command: + raise CatalogError(f"{where} must be a non-empty argv list") + return tuple(_require_nonblank_str(item, f"{where}[]") for item in command) + + +def _validate_test_keys(raw: Mapping[str, object], kind: str, where: str) -> bool: + keys = _COMMON_KEYS | _RECIPE_KEYS if kind == "recipe" else _COMMON_KEYS + if kind == "clean-install" and "expected_failures" in raw: + keys |= {"expected_failures"} + if kind != "recipe": + _require_keys(raw, keys, where) + return False + + missing = sorted(keys - raw.keys()) + unknown = sorted(raw.keys() - keys - _RECIPE_OPTIONAL_KEYS) + if missing or unknown: + raise CatalogError(f"{where}: missing keys {missing}, unknown keys {unknown}") + pipeline = raw.get("pipeline", False) + if not isinstance(pipeline, bool): + raise CatalogError(f"{where}.pipeline must be a boolean") + return pipeline + + def _parse_test(raw: object, index: int) -> CatalogTest: where = f"tests[{index}]" if not isinstance(raw, dict): @@ -157,26 +274,38 @@ def _parse_test(raw: object, index: int) -> CatalogTest: kind = raw.get("kind") if kind not in KINDS: raise CatalogError(f"{where}.kind must be one of {KINDS}") - keys = _COMMON_KEYS | _RECIPE_KEYS if kind == "recipe" else _COMMON_KEYS - if kind == "recipe" and "recipe_fixture" in raw: - keys |= {"recipe_fixture"} - if kind in ("recipe", "clean-install") and "expected_failures" in raw: - keys |= {"expected_failures"} - _require_keys(raw, keys, where) + pipeline = _validate_test_keys(raw, kind, where) name = _require_str(raw["name"], f"{where}.name") if not _NAME_PATTERN.fullmatch(name): raise CatalogError(f"{where}.name must match {_NAME_PATTERN.pattern}") - recipe, ingredients, state = ( - _parse_recipe_fields(raw, where) if kind == "recipe" else (None, (), None) + recipe, ingredients, state, recipe_fixture = ( + _parse_recipe_fields(raw, where) if kind == "recipe" else (None, (), None, None) ) - recipe_fixture = None - if kind == "recipe" and "recipe_fixture" in raw: - recipe_fixture = _require_str(raw["recipe_fixture"], f"{where}.recipe_fixture") - if not _RECIPE_FIXTURE_PATTERN.fullmatch(recipe_fixture): - raise CatalogError(f"{where}.recipe_fixture must be a .yaml basename") - expected_failures = _parse_expected_failures(raw.get("expected_failures", []), where) - if kind == "recipe" and any(row["severity"] != "error" for row in expected_failures): + issue_url_env: str | None = None + required_changed_paths: tuple[str, ...] = () + test_command: tuple[str, ...] = () + peak_sessions_evidence: str | None = None + if kind == "recipe": + expected_failures = _parse_expected_failures( + raw.get("expected_failures", []), where, pipeline=pipeline + ) + elif kind == "clean-install": + expected_failures = _parse_expected_failures(raw.get("expected_failures", []), where) + else: + expected_failures = () + if ( + kind == "recipe" + and not pipeline + and any(row["severity"] != "error" for row in expected_failures) + ): raise CatalogError(f"{where}.expected_failures severity must be error for recipe tests") + if kind == "recipe": + ( + issue_url_env, + required_changed_paths, + test_command, + peak_sessions_evidence, + ) = _parse_recipe_options(raw, where, ingredients) test = CatalogTest( name=name, kind=kind, @@ -193,6 +322,11 @@ def _parse_test(raw: object, index: int) -> CatalogTest: expected_pull_request_state=state, expected_failures=expected_failures, recipe_fixture=recipe_fixture, + pipeline=pipeline, + issue_url_env=issue_url_env, + required_changed_paths=required_changed_paths, + test_command=test_command, + peak_sessions_evidence=peak_sessions_evidence, ) if job_timeout_minutes(test) > MAX_JOB_MINUTES: raise CatalogError(f"{where}.timeout_sec exceeds the {MAX_JOB_MINUTES}-minute job limit") diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index 15e810040d..6e696b7fc4 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -9,6 +9,7 @@ import json import os import re +import shlex import shutil import subprocess import sys @@ -19,8 +20,10 @@ from collections.abc import Callable, Mapping, Sequence from pathlib import Path from typing import Any +from urllib.parse import quote import e2e_catalog +import e2e_sessions MINIMAX_BASE_URL = "https://api.minimax.io/anthropic" MINIMAX_MODEL = "MiniMax-M3[1m]" @@ -43,6 +46,9 @@ _SCRUBBED_ENV = frozenset({"CLAUDE_CODE_OAUTH_TOKEN", *SECRET_ENV}) _STDERR_TAIL_CHARS = 2000 +_SESSION_TRACE_FILENAME = "session-events.jsonl" +_CI_POLL_ATTEMPTS = 6 +_CI_POLL_INTERVAL_SEC = 10 CLEAN_INSTALL_ALLOWED_WARNINGS: dict[tuple[str, str], str] = { ( "pytest_temp_capacity", @@ -278,28 +284,40 @@ def latest_pull_request_number( def new_pull_requests( - repository: str, baseline: int, env: Mapping[str, str], runner: Runner + repository: str, + baseline: int, + env: Mapping[str, str], + runner: Runner, + *, + pipeline: bool = False, ) -> tuple[list[dict[str, Any]] | None, str | None]: - rows, failure = _list_pull_requests( - repository, 20, "number,state,additions,deletions", env, runner - ) - if rows is None: - return None, failure - return [row for row in rows if int(row["number"]) > baseline], None + limit = 20 + fields = "number,state,additions,deletions" + if pipeline: + fields += ( + ",body,headRefOid,headRefName,headRepository,baseRefName,autoMergeRequest,mergedAt" + ) + while True: + rows, failure = _list_pull_requests(repository, limit, fields, env, runner) + if rows is None: + return None, failure + if not pipeline or len(rows) < limit or any(row["number"] <= baseline for row in rows): + return [row for row in rows if row["number"] > baseline], None + limit *= 2 def fleet_run_argv( - test: e2e_catalog.CatalogTest, runtime_ingredients: Mapping[str, str] | None = None + test: e2e_catalog.CatalogTest, *, ingredients: Mapping[str, str] | None = None ) -> list[str]: - effective = dict(test.ingredients) - effective.update(runtime_ingredients or {}) - ingredients = [arg for key, value in effective.items() for arg in ("-i", f"{key}={value}")] + values = dict(test.ingredients) + values.update(ingredients or {}) + arguments = [arg for key, value in values.items() for arg in ("-i", f"{key}={value}")] return [ "autoskillit", "fleet", "run", str(test.recipe), - *ingredients, + *arguments, "--disable-quota-guard", "--timeout-sec", str(test.timeout_sec), @@ -326,6 +344,33 @@ def check_envelope(returncode: int, stdout: str) -> tuple[list[str], dict[str, A return failures, envelope +def check_implementation_envelope( + envelope: Mapping[str, Any], +) -> tuple[list[str], list[dict[str, str]]]: + failures: list[str] = [] + findings: list[dict[str, str]] = [] + payload = envelope.get("l3_payload") + reason = payload.get("reason") if isinstance(payload, dict) else None + status = envelope.get("dispatch_status") + if envelope.get("success") is not True or status != "success": + stable_reason = envelope.get("reason") or reason + if isinstance(stable_reason, str) and stable_reason: + findings.append({"check": "fleet_envelope", "message": f"{status}: {stable_reason}"}) + else: + failures.append("fleet run: unsuccessful envelope has no specific failure reason") + elif reason != "implementation_complete": + if isinstance(reason, str) and reason: + findings.append({"check": "implementation_terminal", "message": reason}) + else: + failures.append("fleet run: implementation terminal reason is missing") + if not all( + isinstance(envelope.get(key), str) and envelope[key] + for key in ("dispatch_id", "dispatched_session_id") + ): + failures.append("fleet run: dispatch/session identity is missing") + return failures, findings + + def session_index_dir_names(log_dir: Path) -> frozenset[str]: index = log_dir / "sessions.jsonl" if not index.exists(): @@ -384,29 +429,51 @@ def run_fleet( out: Path, env: Mapping[str, str], runner: Runner, - runtime_ingredients: Mapping[str, str] | None = None, -) -> tuple[list[str], dict[str, Any] | None]: + *, + ingredients: Mapping[str, str] | None = None, +) -> tuple[dict[str, Any] | None, list[str], list[dict[str, str]]]: log_dir = Path(env["AUTOSKILLIT_LOG_DIR"]) baseline = session_index_dir_names(log_dir) failures: list[str] = [] + findings: list[dict[str, str]] = [] envelope: dict[str, Any] | None = None try: result, failure = _call( runner, - fleet_run_argv(test, runtime_ingredients), + fleet_run_argv(test, ingredients=ingredients), what="fleet run", + evidence=out / "fleet-run", cwd=SANDBOX_CLONE, env=env, timeout=test.timeout_sec + e2e_catalog.HARNESS_GRACE_SEC, - evidence=out / "fleet-run", ) + stdout = (out / "fleet-run.stdout.txt").read_text(encoding="utf-8", errors="replace") + shutil.copyfile(out / "fleet-run.stdout.txt", out / "fleet-run.stdout.log") + shutil.copyfile(out / "fleet-run.stderr.txt", out / "fleet-run.stderr.log") + line = last_line(stdout) + (out / "envelope.json").write_text(line + "\n", encoding="utf-8") + if not test.pipeline: + if result is None: + failures.append(str(failure)) + else: + envelope_failures, envelope = check_envelope(result.returncode, stdout) + failures.extend(envelope_failures) + return envelope, failures, findings + try: + envelope = json.loads(line) + except json.JSONDecodeError: + envelope = None + if not isinstance(envelope, dict): + envelope = None + failures.append("fleet run: the last stdout line is not a JSON object envelope") if result is None: failures.append(str(failure)) - else: - (out / "fleet-run.stdout.log").write_text(result.stdout, encoding="utf-8") - (out / "fleet-run.stderr.log").write_text(result.stderr, encoding="utf-8") - (out / "envelope.json").write_text(last_line(result.stdout) + "\n", encoding="utf-8") - failures, envelope = check_envelope(result.returncode, result.stdout) + elif envelope is not None: + envelope_failures, findings = check_implementation_envelope(envelope) + failures.extend(envelope_failures) + if failure and not findings: + failures.append(failure) + return envelope, failures, findings finally: failures.extend( check_session_windows( @@ -415,7 +482,6 @@ def run_fleet( sessions_required=envelope is not None and envelope.get("success") is True, ) ) - return failures, envelope def match_recipe_failure( @@ -545,7 +611,7 @@ def run_smoke_recipe( if runtime is None: return failures, [] (out / "smoke-launched").write_text(test.name + "\n", encoding="utf-8") - fleet_failures, envelope = run_fleet(test, out, env, runner, runtime) + envelope, fleet_failures, _ = run_fleet(test, out, env, runner, ingredients=runtime) fleet_failures, matched = match_recipe_failure(test, fleet_failures, envelope) failures += fleet_failures if ( @@ -955,6 +1021,506 @@ def close_open_pull_requests( return failures +def install_session_hooks(home: Path, trace: Path) -> None: + path = home / ".claude" / "settings.json" + settings = json.loads(path.read_text(encoding="utf-8")) + if "agent" in settings: + # Internal-agent callbacks carry the main agent's name, which the matcher admits. + raise ValueError("session hooks: settings.json sets a main agent") + command = f"python3 /opt/e2e/e2e_sessions.py {shlex.quote(str(trace))}" + hooks = settings.setdefault("hooks", {}) + for event, matcher in e2e_sessions.LIFECYCLE_HOOK_MATCHERS.items(): + entry: dict[str, Any] = {"hooks": [{"type": "command", "command": command, "timeout": 10}]} + if matcher is not None: + entry["matcher"] = matcher + hooks.setdefault(event, []).append(entry) + path.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8") + + +def _json_call( + argv: Sequence[str], + env: Mapping[str, str], + runner: Runner, + *, + evidence: Path | None = None, + cwd: Path | None = None, +) -> Any: + result, failure = _setup_call(runner, argv, env, evidence=evidence, cwd=cwd) + if failure or result is None: + raise ValueError(failure or "missing command result") + try: + return json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise ValueError(f"{' '.join(argv[:3])}: invalid JSON") from exc + + +def _api_pages(endpoint: str, env: Mapping[str, str], runner: Runner, evidence: Path) -> list[Any]: + pages = _json_call( + ["gh", "api", endpoint, "--paginate", "--slurp"], env, runner, evidence=evidence + ) + if not isinstance(pages, list): + raise ValueError("GitHub API: expected paginated JSON array") + return pages + + +def list_head_refs( + repository: str, out: Path, env: Mapping[str, str], runner: Runner, stage: str +) -> frozenset[str]: + """Return the branch names currently present in the sandbox repository.""" + pages = _api_pages( + f"repos/{repository}/git/matching-refs/heads/", env, runner, out / f"head-refs-{stage}" + ) + if not pages: + raise ValueError("head-ref listing must contain a page") + refs: set[str] = set() + for page in pages: + if not isinstance(page, list): + raise ValueError("head-ref listing page must be an array") + for row in page: + ref = row.get("ref") if isinstance(row, dict) else None + if ( + not isinstance(ref, str) + or not ref.startswith("refs/heads/") + or ref == "refs/heads/" + ): + raise ValueError("head-ref listing must contain nonempty refs/heads/ objects") + refs.add(ref.removeprefix("refs/heads/")) + return frozenset(refs) + + +def prepare_seed( + test: e2e_catalog.CatalogTest, + repository: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> tuple[dict[str, Any], dict[str, str], str]: + url = env.get(test.issue_url_env or "", "") + match = re.fullmatch( + r"https://github\.com/" + re.escape(repository) + r"/issues/([1-9][0-9]*)", url + ) + if not match: + raise ValueError("seed: missing or noncanonical sandbox issue URL") + config = _json_call( + ["autoskillit", "config", "show"], + env, + runner, + evidence=out / "effective-config", + cwd=SANDBOX_CLONE, + ) + label = config["github"]["in_progress_label"] + seed = _json_call( + ["gh", "issue", "view", url, "--json", "number,url,state,title,body,labels"], + env, + runner, + evidence=out / "seed", + ) + if ( + not isinstance(seed, dict) + or seed.get("url") != url + or type(seed.get("number")) is not int + or seed["number"] != int(match[1]) + or seed.get("state") != "OPEN" + or not isinstance(seed.get("title"), str) + or not seed["title"].strip() + or not isinstance(seed.get("body"), str) + or not isinstance(seed.get("labels"), list) + or any( + not isinstance(row, dict) or not isinstance(row.get("name"), str) + for row in seed["labels"] + ) + ): + raise ValueError("seed: expected the configured OPEN sandbox issue with valid metadata") + if label in {row["name"] for row in seed["labels"]}: + raise ValueError("seed: issue is already claimed") + repo = _json_call( + ["gh", "repo", "view", repository, "--json", "defaultBranchRef"], env, runner + ) + default_branch = repo.get("defaultBranchRef") if isinstance(repo, dict) else None + if ( + not isinstance(default_branch, dict) + or default_branch.get("name") != "main" + or config["branching"]["default_base_branch"] != "main" + ): + raise ValueError("seed: sandbox and effective configured base branch must both be main") + ingredients = dict(test.ingredients) + ingredients.update(issue_url=url, task=f"{seed['title']}\n\n{seed['body']}") + for stage, argv in ( + ("baseline-install", ["task", "install-worktree"]), + ("baseline-test", ["task", "test-check"]), + ): + _, failure = _call( + runner, + argv, + what=stage, + evidence=out / stage, + cwd=SANDBOX_CLONE, + env=env, + timeout=SETUP_COMMAND_TIMEOUT_SEC, + ) + if failure: + raise ValueError(failure) + return seed, ingredients, label + + +def _repo_matches(value: Any, repository: str) -> bool: + return isinstance(value, dict) and ( + value.get("full_name") == repository + or value.get("url") == f"https://api.github.com/repos/{repository}" + ) + + +def owned_pull_requests( + rows: Sequence[Mapping[str, Any]], + proof: Mapping[str, Any], + repository: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[dict[str, Any]]: + owned = [] + for row in rows: + pr = _json_call( + ["gh", "api", f"repos/{repository}/pulls/{row['number']}"], + env, + runner, + evidence=out / f"pr-{row['number']}", + ) + if not isinstance(pr, dict): + raise ValueError("PR discovery: expected an object") + head = pr.get("head") + if ( + isinstance(head, dict) + and head.get("ref") in proof["branches"] + and _repo_matches(head.get("repo"), repository) + ): + pr["_listed_head"] = row.get("headRefOid") + owned.append(pr) + return owned + + +def validate_sandbox_ci( + repository: str, + number: int, + head: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> None: + workflows = _api_pages( + f"repos/{repository}/actions/workflows?per_page=100", env, runner, out / "ci-workflows" + ) + candidates = [ + workflow + for page in workflows + for workflow in page.get("workflows", []) + if workflow.get("name") == "CI" and workflow.get("state") == "active" + ] + if len(candidates) != 1: + raise ValueError("sandbox CI: expected one active CI workflow") + for attempt in range(_CI_POLL_ATTEMPTS): + pages = _api_pages( + f"repos/{repository}/actions/workflows/{candidates[0]['id']}/runs?event=pull_request&per_page=100", + env, + runner, + out / "ci-runs", + ) + runs = [ + run + for page in pages + for run in page.get("workflow_runs", []) + if any( + pr.get("number") == number + and pr.get("head", {}).get("sha") == head + and _repo_matches(pr.get("head", {}).get("repo"), repository) + and _repo_matches(pr.get("base", {}).get("repo"), repository) + for pr in run.get("pull_requests", []) + ) + ] + if runs: + run = max(runs, key=lambda value: (value["id"], value.get("run_attempt", 1))) + if run.get("status") == "completed": + jobs = _api_pages( + f"repos/{repository}/actions/runs/{run['id']}/jobs?per_page=100", + env, + runner, + out / "ci-jobs", + ) + tests = [ + job + for page in jobs + for job in page.get("jobs", []) + if job.get("name") == "test" + ] + if ( + run.get("conclusion") != "success" + or not tests + or any( + job.get("status") != "completed" or job.get("conclusion") != "success" + for job in tests + ) + ): + raise ValueError("sandbox CI: CI run and test job must succeed") + return + if attempt + 1 < _CI_POLL_ATTEMPTS: + time.sleep(_CI_POLL_INTERVAL_SEC) + raise ValueError("sandbox CI: no completed CI run associated with the exact PR head") + + +def validate_implementation_pr( + test: e2e_catalog.CatalogTest, + prs: Sequence[Mapping[str, Any]], + seed: Mapping[str, Any], + repository: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + if len(prs) != 1: + return [f"expected exactly one new owned sandbox pull request, found {len(prs)}"] + pr = prs[0] + number = pr["number"] + head = pr.get("head", {}) + base = pr.get("base", {}) + oid = head.get("sha") + if ( + pr.get("state") != "open" + or pr.get("merged") is not False + or pr.get("merged_at") + or pr.get("auto_merge") is not None + or base.get("ref") != "main" + or head.get("ref") == "main" + or not isinstance(oid, str) + or not oid + or not _repo_matches(base.get("repo"), repository) + or not _repo_matches(head.get("repo"), repository) + ): + return [ + "implementation PR: expected an open, unmerged sandbox PR targeting main " + "without auto-merge" + ] + if oid != pr.get("_listed_head"): + return ["implementation PR: REST head differs from captured headRefOid"] + if not re.search(re.escape(seed["url"]) + r"(?![A-Za-z0-9_/#?=-])", str(pr.get("body", ""))): + return ["implementation PR: body does not reference the canonical seed URL"] + if ( + type(pr.get("additions")) is not int + or type(pr.get("deletions")) is not int + or (pr["additions"] + pr["deletions"] <= 0) + ): + return ["implementation PR: empty or malformed diff"] + pages = _api_pages( + f"repos/{repository}/pulls/{number}/files?per_page=100", env, runner, out / "pr-files" + ) + files = [row for page in pages for row in page] + for prefix in test.required_changed_paths: + if not any( + row.get("status") != "removed" + and isinstance(row.get("filename"), str) + and row["filename"].startswith(prefix) + and row.get("additions", 0) + row.get("deletions", 0) > 0 + for row in files + ): + return [f"implementation PR: missing non-removed changes under {prefix}"] + failures = test_pr_head(test, number, oid, out, env, runner) + if failures: + return failures + validate_sandbox_ci(repository, number, oid, out, env, runner) + final = _json_call( + ["gh", "api", f"repos/{repository}/pulls/{number}"], env, runner, evidence=out / "pr-final" + ) + if ( + final.get("head", {}).get("sha") != oid + or final.get("state") != "open" + or final.get("merged") is not False + or final.get("merged_at") + ): + return ["implementation PR: head or open/unmerged state changed during validation"] + return [] + + +def test_pr_head( + test: e2e_catalog.CatalogTest, + number: int, + oid: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + commands = ( + ("pr-fetch", ["git", "fetch", "origin", f"refs/pull/{number}/head"]), + ("pr-oid", ["git", "rev-parse", "FETCH_HEAD"]), + ("pr-checkout", ["git", "checkout", "--detach", oid]), + ("pr-install", ["task", "install-worktree"]), + ("pr-test", list(test.test_command)), + ) + for stage, argv in commands: + result, failure = _call( + runner, + argv, + what=stage, + evidence=out / stage, + cwd=SANDBOX_CLONE, + env=env, + timeout=SETUP_COMMAND_TIMEOUT_SEC, + ) + if failure: + return [failure] + if stage == "pr-oid" and (result is None or result.stdout.strip() != oid): + return ["implementation PR: fetched head differs from captured head"] + return [] + + +def discover_cleanup_prs( + repository: str, + baseline: int, + proof: Mapping[str, Any], + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> tuple[list[dict[str, Any]], list[str]]: + failures: list[str] = [] + owned: list[dict[str, Any]] = [] + for attempt in range(2): + try: + rows, failure = new_pull_requests(repository, baseline, env, runner, pipeline=True) + if rows is None: + raise ValueError(failure) + owned = owned_pull_requests(rows, proof, repository, out, env, runner) + if owned or attempt == 1: + if rows and not proof["branches"]: + failures.append("cleanup: PR ownership could not be established") + break + except Exception as exc: + if attempt == 1: + failures.append(f"cleanup discovery: {exc}") + if attempt == 0: + time.sleep(1) + return owned, failures + + +def delete_owned_branch( + repository: str, branch: str, env: Mapping[str, str], runner: Runner +) -> None: + if branch == "main" or not branch or branch.startswith("refs/"): + raise ValueError("refusing to delete an unsafe/default branch") + endpoint = f"repos/{repository}/git/refs/heads/{quote(branch, safe='')}" + result, failure = _setup_call(runner, ["gh", "api", endpoint], env) + if failure and result is not None and "404" in result.stderr: + return + if failure: + raise ValueError(failure) + time.sleep(1) + _, failure = _setup_call(runner, ["gh", "api", "--method", "DELETE", endpoint], env) + if failure: + raise ValueError(failure) + + +def release_owned_claim( + proof: Mapping[str, Any], + seed: Mapping[str, Any], + label: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + current = _json_call( + ["gh", "issue", "view", seed["url"], "--json", "state,labels"], + env, + runner, + evidence=out / "seed-final", + ) + failures = [] if current.get("state") == "OPEN" else ["cleanup: seed issue is no longer OPEN"] + if label not in {row["name"] for row in current["labels"]}: + return failures + if not proof["claimed"]: + return [*failures, "cleanup: claim label is present without this run's claim proof"] + time.sleep(1) + _, failure = _setup_call( + runner, ["gh", "issue", "edit", seed["url"], "--remove-label", label], env + ) + if failure: + failures.append(f"cleanup: {failure}") + return failures + + +def unexplained_branch_failures( + repository: str, + head_refs: frozenset[str], + proof: Mapping[str, Any], + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + """Report branches created during the run that the ownership proof does not explain.""" + try: + current = list_head_refs(repository, out, env, runner, "after") + except Exception as exc: + return [f"cleanup refs: {exc}"] + return [ + f"cleanup: branch {branch} was created during this run without ownership proof" + for branch in sorted(current - head_refs - set(proof["branches"])) + ] + + +def cleanup_implementation( + repository: str, + baseline: int, + head_refs: frozenset[str], + proof: Mapping[str, Any], + seed: Mapping[str, Any], + label: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + owned, failures = discover_cleanup_prs(repository, baseline, proof, out, env, runner) + for pr in owned: + try: + if pr.get("state") == "open": + time.sleep(1) + _, failure = _setup_call( + runner, ["gh", "pr", "close", str(pr["number"]), "--repo", repository], env + ) + if failure: + failures.append(f"cleanup: {failure}") + except Exception as exc: + failures.append(f"cleanup PR: {exc}") + for branch in proof["branches"]: + try: + delete_owned_branch(repository, branch, env, runner) + except Exception as exc: + failures.append(f"cleanup branch: {exc}") + failures += unexplained_branch_failures(repository, head_refs, proof, out, env, runner) + try: + failures += release_owned_claim(proof, seed, label, out, env, runner) + except Exception as exc: + failures.append(f"cleanup claim: {exc}") + return failures + + +def match_recipe_findings( + failures: list[str], + findings: Sequence[Mapping[str, str]], + expected: Sequence[Mapping[str, str]], +) -> tuple[list[str], list[dict[str, str]]]: + annotations = {(row["check"], row["message"]): row for row in expected} + matched: dict[tuple[str, str], dict[str, str]] = {} + for row in findings: + identity = (row["check"], row["message"]) + if identity in annotations: + matched[identity] = dict(annotations[identity]) + else: + failures.append(f"{row['check']}: {row['message']}") + for identity, annotation in annotations.items(): + if identity not in matched: + failures.append( + f"expected recipe finding absent {identity}; remove stale expected failure " + f"for {annotation['issue']}" + ) + return failures, list(matched.values()) + + def run_recipe( test: e2e_catalog.CatalogTest, repository: str, @@ -962,6 +1528,8 @@ def run_recipe( out: Path, env: Mapping[str, str], runner: Runner, + *, + home: Path | None = None, ) -> tuple[list[str], list[dict[str, str]]]: failure = ( configure_git(env, runner) @@ -972,15 +1540,46 @@ def run_recipe( return [failure], [] if test.recipe_fixture is not None: return run_smoke_recipe(test, repository, out, env, runner) + seed: dict[str, Any] = {} + head_refs: frozenset[str] = frozenset() + ingredients = dict(test.ingredients) + label = "" + home = home or Path.home() + trace = out / _SESSION_TRACE_FILENAME + if test.pipeline or test.issue_url_env: + try: + seed, ingredients, label = prepare_seed(test, repository, out, env, runner) + if test.pipeline: + head_refs = list_head_refs(repository, out, env, runner, "before") + trace.write_text("", encoding="utf-8") + install_session_hooks(home, trace) + except Exception as exc: + return [f"preflight: {exc}"], [] baseline, failure = latest_pull_request_number(repository, env, runner) if baseline is None: return [str(failure)], [] + if not test.pipeline: + return launch_regular_recipe(test, repository, baseline, ingredients, out, env, runner) + return launch_recipe( + test, repository, baseline, head_refs, seed, ingredients, label, out, home, env, runner + ) + + +def launch_regular_recipe( + test: e2e_catalog.CatalogTest, + repository: str, + baseline: int, + ingredients: Mapping[str, str], + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> tuple[list[str], list[dict[str, str]]]: failures: list[str] = [] + matched: list[dict[str, str]] = [] listed: list[dict[str, Any]] | None = None try: - fleet_failures, envelope = run_fleet(test, out, env, runner) - fleet_failures, matched = match_recipe_failure(test, fleet_failures, envelope) - failures += fleet_failures + envelope, failures, _ = run_fleet(test, out, env, runner, ingredients=ingredients) + failures, matched = match_recipe_failure(test, failures, envelope) if not matched: listed, failure = new_pull_requests(repository, baseline, env, runner) if listed is None: @@ -992,6 +1591,121 @@ def run_recipe( return failures, matched +def save_ownership( + trace: Path, + envelope: dict[str, Any] | None, + log_root: Path, + seed: Mapping[str, Any], + repository: str, + out: Path, + label: str, +) -> dict[str, Any]: + proof = e2e_sessions.collect_ownership( + trace, envelope, log_root, seed["url"], repository, claim_label=label + ) + (out / "ownership.json").write_text(json.dumps(proof, indent=2) + "\n", encoding="utf-8") + return proof + + +def finish_pipeline( + test: e2e_catalog.CatalogTest, + repository: str, + baseline: int, + head_refs: frozenset[str], + seed: Mapping[str, Any], + label: str, + out: Path, + home: Path, + env: Mapping[str, str], + runner: Runner, + envelope: dict[str, Any] | None, + proof: dict[str, Any] | None, +) -> list[str]: + failures: list[str] = [] + trace = out / _SESSION_TRACE_FILENAME + log_root = e2e_sessions.default_log_root(home) + try: + summary = e2e_sessions.summarize(trace, envelope, log_root) + if not summary["coverage"]["complete"]: + failures.append("session measurement: incomplete lifecycle coverage") + if summary["peak_sessions"] > test.peak_sessions: + failures.append("session measurement: observed peak exceeds catalog capacity") + except Exception as exc: + failures.append(f"session measurement: {exc}") + if proof is None: + try: + proof = save_ownership(trace, envelope, log_root, seed, repository, out, label) + failures += [f"ownership: {value}" for value in proof["violations"]] + except Exception as exc: + failures.append(f"cleanup ownership: {exc}") + proof = proof or {"branches": [], "claimed": False} + failures += cleanup_implementation( + repository, baseline, head_refs, proof, seed, label, out, env, runner + ) + return failures + + +def launch_recipe( + test: e2e_catalog.CatalogTest, + repository: str, + baseline: int, + head_refs: frozenset[str], + seed: Mapping[str, Any], + ingredients: Mapping[str, str], + label: str, + out: Path, + home: Path, + env: Mapping[str, str], + runner: Runner, +) -> tuple[list[str], list[dict[str, str]]]: + failures: list[str] = [] + findings: list[dict[str, str]] = [] + envelope: dict[str, Any] | None = None + proof: dict[str, Any] | None = None + try: + envelope, fleet_failures, findings = run_fleet( + test, out, env, runner, ingredients=ingredients + ) + failures += fleet_failures + proof = save_ownership( + out / _SESSION_TRACE_FILENAME, + envelope, + e2e_sessions.default_log_root(home), + seed, + repository, + out, + label, + ) + failures += [f"ownership: {value}" for value in proof["violations"]] + listed, failure = new_pull_requests(repository, baseline, env, runner, pipeline=True) + if listed is None: + failures.append(str(failure)) + else: + owned = owned_pull_requests(listed, proof, repository, out, env, runner) + if not fleet_failures and not findings: + failures += validate_implementation_pr( + test, owned, seed, repository, out, env, runner + ) + except Exception as exc: + failures.append(f"harness error: {exc}") + finally: + failures += finish_pipeline( + test, + repository, + baseline, + head_refs, + seed, + label, + out, + home, + env, + runner, + envelope, + proof, + ) + return match_recipe_findings(failures, findings, test.expected_failures) + + def check_clean_install_doctor( stdout: str, expected_failures: Sequence[Mapping[str, str]] ) -> tuple[list[str], list[dict[str, str]]]: @@ -1107,7 +1821,7 @@ def _run_test( **scrubbed, "AUTOSKILLIT_LOG_DIR": str(home / ".local" / "share" / "autoskillit" / "logs"), } - return run_recipe(test, catalog.sandbox_repository, token, out, recipe_env, runner) + return run_recipe(test, catalog.sandbox_repository, token, out, recipe_env, runner, home=home) def _copy_failure_files( diff --git a/scripts/e2e/e2e_select.py b/scripts/e2e/e2e_select.py index 906e66a69b..56a8dbd246 100644 --- a/scripts/e2e/e2e_select.py +++ b/scripts/e2e/e2e_select.py @@ -21,6 +21,7 @@ MIN_CHANGED_LINES = 300 SAMPLE_PROBABILITY = 0.5 MAX_TESTS = 2 +MAX_PIPELINE_TESTS = 1 @dataclass(frozen=True, slots=True) @@ -68,10 +69,12 @@ def selected(self) -> bool: def is_eligible(pr: PullRequestFacts) -> bool: + return E2E_LABEL in pr.labels or _is_large_change(pr) + + +def _is_large_change(pr: PullRequestFacts) -> bool: return ( - E2E_LABEL in pr.labels - or pr.changed_files >= MIN_CHANGED_FILES - or pr.additions + pr.deletions >= MIN_CHANGED_LINES + pr.changed_files >= MIN_CHANGED_FILES or pr.additions + pr.deletions >= MIN_CHANGED_LINES ) @@ -97,13 +100,32 @@ def _touches(test: e2e_catalog.CatalogTest, changed_paths: Sequence[str]) -> boo def pick_tests( - catalog: e2e_catalog.Catalog, changed_paths: Sequence[str], seed: str + catalog: e2e_catalog.Catalog, + changed_paths: Sequence[str], + seed: str, + *, + large_change: bool = False, ) -> tuple[str, ...]: - ranked = sorted(catalog.tests, key=lambda test: _rank(seed, test.name)) - touched = [test.name for test in ranked if _touches(test, changed_paths)] + eligible = [ + test + for test in catalog.tests + if not test.pipeline or (large_change and _touches(test, changed_paths)) + ] + ranked = sorted(eligible, key=lambda test: _rank(seed, test.name)) + touched = [test for test in ranked if _touches(test, changed_paths)] if touched: - return tuple(touched[:MAX_TESTS]) - return (ranked[0].name,) + picked: list[str] = [] + pipeline_count = 0 + for test in touched: + if test.pipeline: + if pipeline_count >= MAX_PIPELINE_TESTS: + continue + pipeline_count += 1 + picked.append(test.name) + if len(picked) == MAX_TESTS: + break + return tuple(picked) + return (ranked[0].name,) if ranked else () def select_for_pull_request( @@ -122,10 +144,16 @@ def select_for_pull_request( ) seed = f"{pr.number}:{pr.head_sha}" if E2E_LABEL in pr.labels: - return Selection(pick_tests(catalog, changed_paths, seed), f"{E2E_LABEL} label") + return Selection( + pick_tests(catalog, changed_paths, seed, large_change=_is_large_change(pr)), + f"{E2E_LABEL} label", + ) if not is_sampled(pr, SAMPLE_PROBABILITY): return Selection((), "eligible but not sampled for this head commit") - return Selection(pick_tests(catalog, changed_paths, seed), "eligible and sampled") + return Selection( + pick_tests(catalog, changed_paths, seed, large_change=_is_large_change(pr)), + "eligible and sampled", + ) def select_for_dispatch(requested: str, catalog: e2e_catalog.Catalog) -> Selection: diff --git a/scripts/e2e/e2e_sessions.py b/scripts/e2e/e2e_sessions.py new file mode 100644 index 0000000000..94c12b7d49 --- /dev/null +++ b/scripts/e2e/e2e_sessions.py @@ -0,0 +1,959 @@ +#!/usr/bin/env python3 +"""Capture and summarize Claude session lifecycle evidence for E2E runs.""" + +from __future__ import annotations + +import json +import math +import os +import re +import sys +import time +from collections.abc import Mapping +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit + +_SESSION_EVENTS = frozenset({"SessionStart", "SessionEnd"}) +_AGENT_EVENTS = frozenset({"SubagentStart", "SubagentStop"}) +_LIFECYCLE_EVENTS = _SESSION_EVENTS | _AGENT_EVENTS +# With the default main agent, internal agents have an empty agent_type. +# Session matchers filter source/reason, so keep session endpoints unfiltered. +_NATIVE_AGENT_MATCHER = ".+" +LIFECYCLE_HOOK_MATCHERS: Mapping[str, str | None] = { + **{event: None for event in sorted(_SESSION_EVENTS)}, + **{event: _NATIVE_AGENT_MATCHER for event in sorted(_AGENT_EVENTS)}, +} +# Pinned to product prefixes by tests/arch/test_e2e_sessions_contract.py. +_AUTOSKILLIT_SERVERS = frozenset( + {("mcp", "autoskillit"), ("mcp", "plugin_autoskillit_autoskillit")} +) +_RESOURCE_TOOLS = frozenset({"claim_and_resolve_issue", "create_and_publish_branch"}) +_LAUNCH_TOOLS = frozenset({"run_skill", "complete_run_skill_result", "recover_run_skill_result"}) +_TRACKED_TOOLS = _RESOURCE_TOOLS | _LAUNCH_TOOLS +_MAX_HOOK_INPUT_BYTES = 65_536 +_MAX_TRACE_RECORD_BYTES = 8_192 + + +def capture_hook_event(trace_path: Path, payload: Mapping[str, Any]) -> bool: + """Append one allow-listed lifecycle record; return False for other hooks.""" + record = _hook_record(payload) + if record is None: + return False + encoded = (json.dumps(record, separators=(",", ":")) + "\n").encode("utf-8") + if len(encoded) > _MAX_TRACE_RECORD_BYTES: + raise ValueError("lifecycle hook record exceeds the size limit") + trace_path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(trace_path, os.O_APPEND | os.O_CREAT | os.O_WRONLY, 0o600) + try: + written = os.write(fd, encoded) + if written != len(encoded): + raise OSError("short lifecycle trace append") + finally: + os.close(fd) + return True + + +def _hook_record(payload: Mapping[str, Any]) -> dict[str, Any] | None: + event = payload.get("hook_event_name", payload.get("event")) + if event is None: + return None + if not isinstance(event, str): + raise ValueError("lifecycle hook event must be a string") + if event not in _LIFECYCLE_EVENTS: + return None + + session_id = payload.get("session_id") + if not isinstance(session_id, str) or not session_id: + raise ValueError("lifecycle hook has no session_id") + + record: dict[str, Any] = { + "event": event, + "timestamp": time.time(), + "session_id": session_id, + } + for key in ( + "agent_id", + "agent_type", + "source", + "transcript_path", + "agent_transcript_path", + ): + value = payload.get(key) + if value is None: + continue + if not isinstance(value, str): + raise ValueError(f"lifecycle hook {key} must be a string") + record[key] = value + if event in _AGENT_EVENTS and not record.get("agent_id"): + raise ValueError("subagent lifecycle hook has no agent_id") + return record + + +def _read_jsonl( + path: Path, label: str, violations: list[str], *, collect: bool = True +) -> list[dict[str, Any]]: + try: + handle = path.open(encoding="utf-8", errors="replace") + except OSError: + violations.append(f"{label}:unavailable") + return [] + + records: list[dict[str, Any]] = [] + with handle: + for line_number, line in enumerate(handle, 1): + if not line.strip(): + violations.append(f"{label}:empty_line:{line_number}") + continue + try: + item = json.loads(line) + except (json.JSONDecodeError, UnicodeDecodeError): + violations.append(f"{label}:invalid_json:{line_number}") + continue + if not isinstance(item, dict): + violations.append(f"{label}:invalid_record:{line_number}") + continue + if collect: + records.append(item) + return records + + +def _load_trace(trace_path: Path, violations: list[str]) -> list[dict[str, Any]]: + records = _read_jsonl(trace_path, "trace", violations) + valid: list[dict[str, Any]] = [] + for index, record in enumerate(records, 1): + event = record.get("event") + timestamp = record.get("timestamp") + session_id = record.get("session_id") + if not isinstance(event, str) or event not in _LIFECYCLE_EVENTS: + violations.append(f"trace:unknown_event:{index}") + continue + finite_timestamp = False + if isinstance(timestamp, (int, float)) and not isinstance(timestamp, bool): + try: + finite_timestamp = math.isfinite(timestamp) + except OverflowError: + pass + if not finite_timestamp: + violations.append(f"trace:invalid_timestamp:{index}") + continue + if not isinstance(session_id, str) or not session_id: + violations.append(f"trace:invalid_session_id:{index}") + continue + optional_invalid = any( + key in record and record[key] is not None and not isinstance(record[key], str) + for key in ( + "agent_id", + "agent_type", + "source", + "transcript_path", + "agent_transcript_path", + ) + ) + if optional_invalid: + violations.append(f"trace:invalid_optional_field:{index}") + continue + if event in _AGENT_EVENTS and not record.get("agent_id"): + violations.append(f"trace:missing_agent_id:{index}") + continue + valid.append(record) + return valid + + +def _read_envelope(envelope: dict[str, Any] | None, violations: list[str]) -> tuple[str, str]: + if not isinstance(envelope, dict): + violations.append("envelope:missing") + return "", "" + session_id = envelope.get("dispatched_session_id") + dispatch_id = envelope.get("dispatch_id") + if not isinstance(session_id, str) or not session_id: + violations.append("envelope:missing_dispatched_session_id") + session_id = "" + if not isinstance(dispatch_id, str) or not dispatch_id: + violations.append("envelope:missing_dispatch_id") + dispatch_id = "" + return session_id, dispatch_id + + +def _load_index(log_root: Path, violations: list[str]) -> list[dict[str, Any]]: + index_path = log_root / "sessions.jsonl" + records = _read_jsonl(index_path, "sessions_index", violations) + for line_number, row in enumerate(records, 1): + if not isinstance(row.get("session_id"), str) or not row.get("session_id"): + violations.append(f"sessions_index:missing_session_id:{line_number}") + return records + + +def _safe_session_dir(log_root: Path, row: Mapping[str, Any]) -> Path | None: + name = row.get("dir_name") + if not isinstance(name, str) or not name or name in {".", ".."} or Path(name).name != name: + return None + return log_root / "sessions" / name + + +def _read_summary(session_dir: Path, expected_session_id: str) -> dict[str, Any] | None: + try: + summary = json.loads((session_dir / "summary.json").read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError, UnicodeDecodeError): + return None + if not isinstance(summary, dict) or summary.get("session_id") != expected_session_id: + return None + identity = summary.get("execution_identity") + if not isinstance(identity, dict) or not isinstance(identity.get("parent_session_id"), str): + return None + return summary + + +def default_log_root(home: Path) -> Path: + return home / ".local" / "share" / "autoskillit" / "logs" + + +def _discover_owned( + trace_path: Path, + envelope: dict[str, Any] | None, + log_root: Path | None, +) -> dict[str, Any]: + """Resolve only the dispatched session and fresh, linked managed descendants.""" + root = log_root or default_log_root(Path.home()) + violations: list[str] = [] + events = _load_trace(trace_path, violations) + root_id, dispatch_id = _read_envelope(envelope, violations) + trace_session_ids = { + record["session_id"] for record in events if record["event"] in _SESSION_EVENTS + } + rows = _load_index(root, violations) + rows_by_id: dict[str, list[dict[str, Any]]] = {} + for row in rows: + session_id = row.get("session_id") + if isinstance(session_id, str) and session_id: + rows_by_id.setdefault(session_id, []).append(row) + owned: dict[str, dict[str, Any]] = {} + root_rows = rows_by_id.get(root_id, []) + if not root_id or len(root_rows) != 1: + violations.append("sessions_index:root_missing_or_ambiguous") + else: + if root_id not in trace_session_ids: + violations.append("trace:root_session_missing") + root_row = root_rows[0] + if not dispatch_id or root_row.get("dispatch_id") != dispatch_id: + violations.append("sessions_index:root_dispatch_mismatch") + _add_owned_session(root, root_row, root_id, violations, owned) + candidates = _fresh_candidates( + root, root_id, dispatch_id, trace_session_ids, rows_by_id, violations + ) + _add_linked_descendants(root, dispatch_id, candidates, owned, violations) + observed = trace_session_ids | set(rows_by_id) + for session_id in sorted(_launched_session_ids(owned) - observed): + violations.append(f"lineage:{session_id}:launched_session_unobserved") + + return { + "root_id": root_id, + "dispatch_id": dispatch_id, + "log_root": root, + "events": events, + "owned": owned, + "violations": violations, + "trace_session_ids": trace_session_ids, + } + + +def _fresh_candidates( + log_root: Path, + root_id: str, + dispatch_id: str, + trace_session_ids: set[str], + rows_by_id: Mapping[str, list[dict[str, Any]]], + violations: list[str], +) -> dict[str, tuple[dict[str, Any], dict[str, Any] | None]]: + candidates: dict[str, tuple[dict[str, Any], dict[str, Any] | None]] = {} + candidate_ids = trace_session_ids | { + session_id + for session_id, rows in rows_by_id.items() + if dispatch_id and any(row.get("dispatch_id") == dispatch_id for row in rows) + } + for session_id in candidate_ids: + if session_id == root_id: + continue + matching_rows = rows_by_id.get(session_id, []) + if len(matching_rows) != 1: + violations.append(f"sessions_index:{session_id}:missing_or_ambiguous") + continue + row = matching_rows[0] + session_dir = _safe_session_dir(log_root, row) + summary = _read_summary(session_dir, session_id) if session_dir else None + candidates[session_id] = row, summary + return candidates + + +def _launched_session_ids(owned: Mapping[str, Mapping[str, Any]]) -> set[str]: + """Return the session ids named by receipted launch replies in owned transcripts.""" + launched: set[str] = set() + for item in owned.values(): + for tool, _tool_input, reply, is_error in item["calls"]: + if tool not in _LAUNCH_TOOLS or is_error or reply is None: + continue + session_id = reply.get("session_id") + if reply.get("receipt_id") and isinstance(session_id, str) and session_id: + launched.add(session_id) + return launched + + +def _report_unattributed( + remaining: Mapping[str, tuple[dict[str, Any], dict[str, Any] | None]], violations: list[str] +) -> None: + """Report sessions of the run that declare no parent and were named by no launch receipt.""" + for session_id, (row, summary) in sorted(remaining.items()): + declared = [row.get("parent_session_id"), row.get("caller_session_id")] + if summary: + declared.append(summary["execution_identity"]["parent_session_id"]) + if not any(isinstance(value, str) and value for value in declared): + violations.append(f"lineage:{session_id}:unattributed") + + +def _add_linked_descendants( + log_root: Path, + dispatch_id: str, + candidates: dict[str, tuple[dict[str, Any], dict[str, Any] | None]], + owned: dict[str, dict[str, Any]], + violations: list[str], +) -> None: + remaining = dict(candidates) + while remaining: + changed = False + owned_ids = set(owned) + launched = _launched_session_ids(owned) + for session_id, (row, summary) in list(remaining.items()): + parents = { + value + for key in ("parent_session_id", "caller_session_id") + if isinstance((value := row.get(key)), str) and value + } + summary_parent = summary["execution_identity"]["parent_session_id"] if summary else "" + by_receipt = session_id in launched + linked = by_receipt or parents.intersection(owned_ids) or summary_parent in owned_ids + if not linked: + continue + if not dispatch_id or row.get("dispatch_id") != dispatch_id: + violations.append(f"sessions_index:{session_id}:dispatch_mismatch") + del remaining[session_id] + continue + if summary is None: + violations.append(f"summary:{session_id}:missing_or_malformed") + elif summary_parent not in owned_ids and (summary_parent or not by_receipt): + violations.append(f"summary:{session_id}:parent_mismatch") + if parents and not parents.intersection(owned_ids): + violations.append(f"sessions_index:{session_id}:parent_mismatch") + _add_owned_session(log_root, row, session_id, violations, owned, summary=summary) + del remaining[session_id] + changed = True + if not changed: + break + _report_unattributed(remaining, violations) + + +def _add_owned_session( + log_root: Path, + row: Mapping[str, Any], + session_id: str, + violations: list[str], + owned: dict[str, dict[str, Any]], + *, + summary: dict[str, Any] | None = None, +) -> None: + session_dir = _safe_session_dir(log_root, row) + if session_dir is None: + violations.append(f"sessions_index:{session_id}:invalid_dir_name") + return + required_index_fields = ( + "session_id", + "dir_name", + "dispatch_id", + "parent_session_id", + "caller_session_id", + "claude_code_log", + ) + if any(key not in row for key in required_index_fields): + violations.append(f"sessions_index:{session_id}:missing_fields") + if summary is None: + summary = _read_summary(session_dir, session_id) + if summary is None: + violations.append(f"summary:{session_id}:missing_or_malformed") + elif summary.get("dispatch_id", "") != row.get("dispatch_id", ""): + violations.append(f"summary:{session_id}:dispatch_mismatch") + transcript_value = row.get("claude_code_log") + transcript: Path | None = None + if isinstance(transcript_value, str) and transcript_value: + transcript = Path(transcript_value) + if not transcript.is_absolute(): + transcript = log_root / transcript + else: + violations.append(f"sessions_index:{session_id}:missing_transcript") + calls = [] + if transcript is None or not transcript.is_file(): + violations.append(f"transcript:{session_id}:missing") + else: + _read_jsonl(transcript, f"transcript:{session_id}", violations, collect=False) + calls = _transcript_tool_pairs(transcript, session_id, violations) + owned[session_id] = { + "row": dict(row), + "summary": summary, + "session_dir": session_dir, + "transcript": transcript, + "calls": calls, + } + + +def _lifecycle_intervals( + events: list[dict[str, Any]], + session_ids: set[str], + violations: list[str], +) -> tuple[list[dict[str, Any]], set[str]]: + grouped = _group_lifecycle_events(events, session_ids) + intervals: list[dict[str, Any]] = [] + seen_sessions: set[str] = set() + for identity, endpoints in grouped.items(): + item = _make_interval(identity, endpoints, violations) + if item is None: + continue + if identity[0] == "session": + seen_sessions.add(identity[1]) + intervals.append(item) + return intervals, seen_sessions + + +def _group_lifecycle_events( + events: list[dict[str, Any]], session_ids: set[str] +) -> dict[tuple[str, ...], dict[str, list[dict[str, Any]]]]: + grouped: dict[tuple[str, ...], dict[str, list[dict[str, Any]]]] = {} + for event in events: + name = event["event"] + session_id = event["session_id"] + if session_id not in session_ids: + continue + identity: tuple[str, ...] + if name in _SESSION_EVENTS: + identity = ("session", session_id) + role = "start" if name == "SessionStart" else "end" + else: + identity = ("agent", session_id, event["agent_id"]) + role = "start" if name == "SubagentStart" else "end" + grouped.setdefault(identity, {"start": [], "end": []})[role].append(event) + return grouped + + +def _make_interval( + identity: tuple[str, ...], + endpoints: Mapping[str, list[dict[str, Any]]], + violations: list[str], +) -> dict[str, Any] | None: + starts, stops = endpoints["start"], endpoints["end"] + label = ":".join(identity) + if not starts: + violations.append(f"lifecycle:{label}:orphan_end") + return None + if not stops: + violations.append(f"lifecycle:{label}:missing_end") + return None + start = min(float(item["timestamp"]) for item in starts) + end = max(float(item["timestamp"]) for item in stops) + if any(float(item["timestamp"]) < start for item in stops): + violations.append(f"lifecycle:{label}:orphan_end_before_start") + if end < start or any(float(event["timestamp"]) > end for event in starts): + reason = "end_before_start" if end < start else "missing_final_end" + violations.append(f"lifecycle:{label}:{reason}") + return None + item = {"kind": identity[0], "session_id": identity[1], "start": start, "end": end} + if identity[0] == "agent": + item["agent_id"] = identity[2] + path_field = "agent_transcript_path" + else: + path_field = "transcript_path" + paths = { + str(Path(event[path_field]).resolve()) + for event in starts + stops + if isinstance(event.get(path_field), str) and event[path_field] + } + if len(paths) > 1: + violations.append(f"lifecycle:{label}:transcript_mismatch") + if paths: + item["transcript_path"] = sorted(paths)[0] + for key in ("agent_type", "source"): + values = {event[key] for event in starts + stops if isinstance(event.get(key), str)} + if len(values) == 1: + item[key] = next(iter(values)) + return item + + +def _check_transcript_coverage( + events: list[dict[str, Any]], + owned: Mapping[str, Mapping[str, Any]], + violations: list[str], +) -> list[dict[str, str]]: + session_ids = set(owned) + session_paths: dict[str, set[str]] = {session_id: set() for session_id in session_ids} + observed: dict[tuple[str, str], list[dict[str, Any]]] = {} + for event in events: + session_id = event["session_id"] + if session_id not in session_ids: + continue + if event["event"] in _SESSION_EVENTS and event.get("transcript_path"): + session_paths[session_id].add(str(Path(event["transcript_path"]).resolve())) + elif event["event"] in _AGENT_EVENTS: + key = (session_id, event["agent_id"]) + observed.setdefault(key, []).append(event) + if not event.get("agent_type"): + violations.append(f"capture:{session_id}:{event['agent_id']}:agent_type_missing") + for session_id, paths in session_paths.items(): + if len(paths) > 1: + violations.append(f"lifecycle:{session_id}:transcript_mismatch") + enumerated = _enumerate_native_transcripts(owned, session_paths, violations) + + native_agents: list[dict[str, str]] = [] + agent_keys = set(observed) + agent_keys.update( + (session_id, agent_id) for session_id, files in enumerated.items() for agent_id in files + ) + for session_id, agent_id in sorted(agent_keys): + transcript = enumerated.get(session_id, {}).get(agent_id) + proof = _verify_native_transcript( + session_id, agent_id, transcript, observed.get((session_id, agent_id), []), violations + ) + if proof is not None: + native_agents.append(proof) + return native_agents + + +def _enumerate_native_transcripts( + owned: Mapping[str, Mapping[str, Any]], + session_paths: Mapping[str, set[str]], + violations: list[str], +) -> dict[str, dict[str, Path]]: + enumerated: dict[str, dict[str, Path]] = {} + for session_id, item in owned.items(): + transcript = item.get("transcript") + if not isinstance(transcript, Path) or not transcript.is_file(): + continue + paths = session_paths.get(session_id, set()) + if not paths: + violations.append(f"lifecycle:{session_id}:transcript_path_missing") + elif paths != {str(transcript.resolve())}: + violations.append(f"lifecycle:{session_id}:transcript_path_mismatch") + subagents_dir = transcript.parent / session_id / "subagents" + files: dict[str, Path] = {} + for path in sorted(subagents_dir.glob("agent-*.jsonl")): + agent_id = path.name.removeprefix("agent-").removesuffix(".jsonl") + if not agent_id: + continue + files[agent_id] = path + if not path.is_file(): + violations.append(f"native:{session_id}:{agent_id}:transcript_missing") + enumerated[session_id] = files + return enumerated + + +def _verify_native_transcript( + session_id: str, + agent_id: str, + transcript: Path | None, + events: list[dict[str, Any]], + violations: list[str], +) -> dict[str, str] | None: + label = f"native:{session_id}:{agent_id}" + names = {event["event"] for event in events} + if not _AGENT_EVENTS <= names: + violations.append(f"{label}:lifecycle_incomplete") + if transcript is None: + violations.append(f"{label}:transcript_unenumerated") + return None + if not transcript.is_file(): + violations.append(f"{label}:transcript_missing") + return None + expected_path = str(transcript.resolve()) + paths = { + str(Path(event["agent_transcript_path"]).resolve()) + for event in events + if isinstance(event.get("agent_transcript_path"), str) + } + if paths != {expected_path}: + violations.append(f"{label}:transcript_mismatch") + if not paths: + violations.append(f"{label}:transcript_path_missing") + _read_jsonl(transcript, f"transcript:{session_id}:{agent_id}", violations, collect=False) + return {"session_id": session_id, "agent_id": agent_id, "transcript_path": expected_path} + + +def _peak_sessions(intervals: list[dict[str, Any]]) -> int: + points: list[tuple[float, int]] = [] + for item in intervals: + if item["start"] == item["end"]: + continue + points.append((item["end"], -1)) + points.append((item["start"], 1)) + active = peak = 0 + for _timestamp, delta in sorted(points, key=lambda point: (point[0], point[1])): + active += delta + peak = max(peak, active) + return peak + + +def summarize( + trace_path: Path, + envelope: dict[str, Any] | None, + log_root: Path | None = None, +) -> dict[str, Any]: + """Write a bounded concurrency summary beside the raw lifecycle trace.""" + context = _discover_owned(trace_path, envelope, log_root) + violations = list(context["violations"]) + owned: dict[str, dict[str, Any]] = context["owned"] + owned_ids = set(owned) + intervals, seen_sessions = _lifecycle_intervals(context["events"], owned_ids, violations) + for session_id in sorted(owned_ids - seen_sessions): + violations.append(f"lifecycle:{session_id}:missing_endpoints") + native_agents = _check_transcript_coverage(context["events"], owned, violations) + root_id = context["root_id"] + root_seen = root_id in seen_sessions + summary = { + "method": "hook_lifecycle_intervals", + "trace_path": str(trace_path), + "peak_sessions": _peak_sessions(intervals), + "identities": sorted( + intervals, + key=lambda item: ( + item["start"], + item["kind"], + item["session_id"], + item.get("agent_id", ""), + ), + ), + "coverage": { + "complete": not violations, + "root_session_id": root_id or None, + "root_session_seen": root_seen, + "managed_session_ids": sorted(owned_ids - ({root_id} if root_id else set())), + "owned_session_ids": sorted(owned_ids), + "native_agents": native_agents, + }, + "violations": sorted(set(violations)), + } + output_path = trace_path.with_name("session-concurrency.json") + output_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return summary + + +def _issue_number(issue_url: str) -> str: + match = re.search(r"/issues/(\d+)/?$", urlsplit(issue_url).path) + if match: + return match.group(1) + match = re.search(r"#(\d+)$", issue_url) + return match.group(1) if match else "" + + +def _issue_matches_repository(issue_url: str, repository: str) -> bool: + parsed = urlsplit(issue_url) + return ( + parsed.scheme == "https" + and parsed.netloc.lower() == "github.com" + and (parsed.path.strip("/").split("/issues/", 1)[0].lower() == repository.lower()) + ) + + +def _remote_matches_repository(remote_url: Any, repository: str) -> bool: + if not isinstance(remote_url, str): + return False + https = re.fullmatch(r"https://github\.com/([^/]+/[^/]+?)(?:\.git)?/?", remote_url) + ssh = re.fullmatch(r"git@github\.com:([^/]+/[^/]+)\.git", remote_url) + match = https or ssh + return bool(match and match.group(1).lower() == repository.lower()) + + +def _result_object(value: Any) -> dict[str, Any] | None: + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError: + return None + if isinstance(value, dict): + if any( + key in value for key in ("success", "claimed", "merge_target", "error", "error_type") + ): + return value + nested = value.get("structuredContent", value.get("result", value.get("content"))) + if isinstance(nested, list): + return _result_text_blocks(nested) + elif isinstance(nested, str): + return _result_object(nested) + if isinstance(value, list): + return _result_text_blocks(value) + return None + + +def _result_text_blocks(blocks: list[Any]) -> dict[str, Any] | None: + for block in blocks: + if not isinstance(block, dict) or not isinstance(block.get("text"), str): + continue + try: + decoded = json.loads(block["text"]) + except json.JSONDecodeError: + continue + if isinstance(decoded, dict): + return decoded + return None + + +def _transcript_tool_pairs( + transcript: Path, + violation_prefix: str, + violations: list[str], +) -> list[tuple[str, dict[str, Any], dict[str, Any] | None, bool]]: + """Return paired target calls as name, input, result, and result-error flag.""" + try: + handle = transcript.open(encoding="utf-8", errors="replace") + except OSError: + violations.append(f"transcript:{violation_prefix}:unavailable") + return [] + pending: dict[str, tuple[str, dict[str, Any]]] = {} + paired: list[tuple[str, dict[str, Any], dict[str, Any] | None, bool]] = [] + with handle: + for line_number, line in enumerate(handle, 1): + try: + row = json.loads(line) + except (json.JSONDecodeError, UnicodeDecodeError): + violations.append(f"transcript:{violation_prefix}:invalid_json:{line_number}") + continue + if isinstance(row, dict): + _pair_record(row, line_number, violation_prefix, pending, paired, violations) + for tool_id, (name, tool_input) in pending.items(): + violations.append(f"transcript:{violation_prefix}:unpaired_tool_use:{tool_id}") + paired.append((name, tool_input, None, False)) + return paired + + +def _receipt( + row: Mapping[str, Any], content: list[Any], block: Mapping[str, Any] +) -> dict[str, Any] | None: + """Decode row metadata when present, otherwise decode the JSON block carrier.""" + meta = row.get("mcpMeta") + if "mcpMeta" in row: + results = [ + item + for item in content + if isinstance(item, dict) and item.get("type") == "tool_result" + ] + if len(results) != 1 or not isinstance(meta, dict): + return None + return _result_object(meta.get("structuredContent")) + return _result_object(block.get("content")) + + +def _pair_record( + row: dict[str, Any], + line_number: int, + violation_prefix: str, + pending: dict[str, tuple[str, dict[str, Any]]], + paired: list[tuple[str, dict[str, Any], dict[str, Any] | None, bool]], + violations: list[str], +) -> None: + message = row.get("message") + if not isinstance(message, dict): + message = row + content = message.get("content", row.get("content", [])) + if not isinstance(content, list): + return + role = row.get("type", message.get("role")) + for block in content: + if not isinstance(block, dict): + continue + if role == "assistant" and block.get("type") == "tool_use": + _remember_tool_use(block, line_number, violation_prefix, pending, violations) + elif role == "user" and block.get("type") == "tool_result": + tool_id = block.get("tool_use_id") + if isinstance(tool_id, str) and tool_id in pending: + name, tool_input = pending.pop(tool_id) + result = _receipt(row, content, block) + paired.append((name, tool_input, result, block.get("is_error") is True)) + + +def _remember_tool_use( + block: dict[str, Any], + line_number: int, + prefix: str, + pending: dict[str, tuple[str, dict[str, Any]]], + violations: list[str], +) -> None: + name = block.get("name") + if not isinstance(name, str): + return + parts = tuple(name.split("__")) + tool = parts[-1] + if tool not in _TRACKED_TOOLS: + return + if len(parts) != 3 or parts[:2] not in _AUTOSKILLIT_SERVERS: + violations.append(f"transcript:{prefix}:unrecognised_tool_namespace:{line_number}") + return + tool_id, tool_input = block.get("id"), block.get("input") + if not isinstance(tool_id, str) or not tool_id or not isinstance(tool_input, dict): + violations.append(f"transcript:{prefix}:malformed_tool_use:{line_number}") + return + if tool_id in pending: + violations.append(f"transcript:{prefix}:duplicate_tool_use:{line_number}") + pending[tool_id] = (tool, tool_input) + + +def collect_ownership( + trace_path: Path, + envelope: dict[str, Any] | None, + log_root: Path | None = None, + issue_url: str = "", + repository: str = "", + claim_label: str | None = None, +) -> dict[str, Any]: + """Collect compact branch and claim proofs from the owned transcript lineage.""" + context = _discover_owned(trace_path, envelope, log_root) + violations = list(context["violations"]) + root_id = context["root_id"] + expected_number = _issue_number(issue_url) + valid_issue_reference = bool( + expected_number and repository and _issue_matches_repository(issue_url, repository) + ) + if not valid_issue_reference: + violations.append("ownership:invalid_issue_reference") + + branches, claim_calls, transcript_evidence = _owned_tool_proofs( + context["owned"], + root_id, + valid_issue_reference, + expected_number, + issue_url, + repository, + violations, + ) + claimed = _claim_succeeded(claim_calls, expected_number, claim_label, violations) + return { + "branches": sorted(set(branches)), + "claimed": claimed, + "transcripts": sorted(transcript_evidence, key=lambda item: item["session_id"]), + "violations": sorted(set(violations)), + } + + +def _owned_tool_proofs( + owned: Mapping[str, Mapping[str, Any]], + root_id: str, + valid_issue_reference: bool, + expected_number: str, + issue_url: str, + repository: str, + violations: list[str], +) -> tuple[ + list[str], list[tuple[dict[str, Any], dict[str, Any] | None, bool]], list[dict[str, str]] +]: + branches: list[str] = [] + claims: list[tuple[dict[str, Any], dict[str, Any] | None, bool]] = [] + transcripts: list[dict[str, str]] = [] + for session_id, item in owned.items(): + transcript = item.get("transcript") + if not isinstance(transcript, Path) or not transcript.is_file(): + continue + transcripts.append({"session_id": session_id, "path": str(transcript)}) + for name, tool_input, result, is_error in item["calls"]: + target = _branch_target( + session_id, + root_id, + name, + tool_input, + result, + is_error, + valid_issue_reference, + expected_number, + repository, + violations, + ) + if target is not None: + branches.append(target) + if name == "claim_and_resolve_issue" and valid_issue_reference: + if tool_input.get("issue_url") == issue_url: + claims.append((tool_input, result, is_error)) + return branches, claims, transcripts + + +def _branch_target( + session_id: str, + root_id: str, + name: str, + tool_input: dict[str, Any], + result: dict[str, Any] | None, + is_error: bool, + valid_issue_reference: bool, + expected_number: str, + repository: str, + violations: list[str], +) -> str | None: + if name != "create_and_publish_branch" or session_id != root_id or not valid_issue_reference: + return None + if tool_input.get("issue_number") != expected_number: + return None + if not _remote_matches_repository(tool_input.get("remote_url"), repository): + return None + if ( + is_error + or result is None + or "error" in result + or "error_type" in result + or not isinstance(result.get("merge_target"), str) + or not result["merge_target"].strip() + ): + violations.append("ownership:branch_result_failed") + return None + return result["merge_target"].strip() + + +def _claim_succeeded( + claims: list[tuple[dict[str, Any], dict[str, Any] | None, bool]], + expected_number: str, + claim_label: str | None, + violations: list[str], +) -> bool: + if len(claims) > 1: + violations.append("ownership:claim_reentry") + return False + if not claims: + return False + tool_input, result, is_error = claims[0] + requested_label = tool_input.get("label") + label_matches_request = claim_label is None or requested_label in (None, "", claim_label) + claimed = bool( + not is_error + and label_matches_request + and tool_input.get("allow_reentry") is not True + and result is not None + and result.get("success") is True + and result.get("claimed") is True + and str(result.get("issue_number", "")) == expected_number + and result.get("reentry") is not True + ) + if not claimed: + violations.append("ownership:claim_result_failed") + return claimed + + +def main(argv: list[str] | None = None) -> int: + args = sys.argv[1:] if argv is None else argv + if len(args) != 1: + print("usage: e2e_sessions.py ", file=sys.stderr) + return 2 + try: + raw = sys.stdin.buffer.read(_MAX_HOOK_INPUT_BYTES + 1) + if len(raw) > _MAX_HOOK_INPUT_BYTES: + raise ValueError("lifecycle hook input exceeds the size limit") + payload = json.loads(raw) + if not isinstance(payload, dict): + raise ValueError("lifecycle hook input must be an object") + capture_hook_event(Path(args[0]), payload) + except (OSError, ValueError, json.JSONDecodeError, UnicodeDecodeError) as exc: + print(f"e2e lifecycle capture failed: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/autoskillit/core/types/audit/_type_plan_set_authority.py b/src/autoskillit/core/types/audit/_type_plan_set_authority.py index cbf64aa99a..6755167244 100644 --- a/src/autoskillit/core/types/audit/_type_plan_set_authority.py +++ b/src/autoskillit/core/types/audit/_type_plan_set_authority.py @@ -501,6 +501,10 @@ def _validate_collections(self) -> None: raise ValueError( "coverage.parts_without_obligations must match actual empty parts" ) + elif self.coverage.parts_without_obligations: + raise ValueError( + "coverage.parts_without_obligations is invalid for enumerated inventory" + ) @classmethod def create( diff --git a/src/autoskillit/recipe/helpers/_io_loading.py b/src/autoskillit/recipe/helpers/_io_loading.py index d799ad8099..9b6f6c72d7 100644 --- a/src/autoskillit/recipe/helpers/_io_loading.py +++ b/src/autoskillit/recipe/helpers/_io_loading.py @@ -58,7 +58,7 @@ def _substitute_recipe_note_placeholders(value: Any) -> Any: - """Substitute recipe note placeholders. Only strings reach this layer.""" + """Expand note placeholders in strings, leaving non-string values unchanged.""" if isinstance(value, str): for placeholder, note in _RECIPE_NOTE_TABLE.items(): value = value.replace(placeholder, note) diff --git a/tests/_hook_protocol_oracle.py b/tests/_hook_protocol_oracle.py index f3fd523aa0..385f9b213a 100644 --- a/tests/_hook_protocol_oracle.py +++ b/tests/_hook_protocol_oracle.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import re from collections.abc import Callable, Mapping from pathlib import Path from typing import Any, Literal, NamedTuple @@ -446,6 +447,24 @@ def codex_verdict( # These keys and event exceptions come from Claude's JSON output and # per-event decision-control tables in hooks.md. +_EXACT_MATCHER = re.compile(r"[A-Za-z0-9_\- ,|]*") + + +def claude_matcher_admits(matcher: str | None, value: str) -> bool: + """Apply Claude Code's documented hook-matcher rule to one candidate value. + + Source: the "Matcher patterns" section of the page named by + ``CLAUDE_PROTOCOL_SOURCE``, re-read 2026-10-13. Python's ``re.search`` stands in + for ``RegExp.prototype.test``. ``FileChanged`` and ``StopFailure`` use a narrower + exact-match set and are not modelled. + """ + if matcher is None or matcher in ("", "*"): + return True + if _EXACT_MATCHER.fullmatch(matcher): + return value in {item.strip() for item in re.split(r"[|,]", matcher)} + return re.search(matcher, value) is not None + + CLAUDE_HOOK_SPECIFIC_OUTPUT_FIELDS: Mapping[str, frozenset[str]] = { "PreToolUse": frozenset( { diff --git a/tests/arch/test_e2e_sessions_contract.py b/tests/arch/test_e2e_sessions_contract.py new file mode 100644 index 0000000000..35f6e3820c --- /dev/null +++ b/tests/arch/test_e2e_sessions_contract.py @@ -0,0 +1,57 @@ +"""Pin the standalone collector's identities, tools and hook policy to the product.""" + +import json +from pathlib import Path + +import pytest + +from autoskillit.core import DIRECT_PREFIX, PLUGIN_PREFIX, ClaudeFlags, pkg_root +from autoskillit.core.types import GATED_TOOLS, UNGATED_TOOLS +from autoskillit.execution.backends import claude_resolvable_agent_names +from tests._hook_protocol_oracle import claude_matcher_admits +from tests.infra._complexity_helpers import load_check_script + +pytestmark = pytest.mark.medium + +sessions = load_check_script( + "_autoskillit_e2e_sessions_contract", + Path(__file__).resolve().parents[2] / "scripts" / "e2e" / "e2e_sessions.py", +) + + +def test_server_identities_match_canonical_prefixes() -> None: + assert sessions._AUTOSKILLIT_SERVERS == frozenset( + tuple(prefix.split("__")[:-1]) for prefix in (DIRECT_PREFIX, PLUGIN_PREFIX) + ) + + +def test_tracked_tools_are_registered_product_tools() -> None: + assert sessions._TRACKED_TOOLS + assert sessions._TRACKED_TOOLS <= GATED_TOOLS | UNGATED_TOOLS + + +def test_agent_hooks_admit_every_resolvable_agent_and_reject_an_unnamed_one() -> None: + names = claude_resolvable_agent_names() + for event in sessions._AGENT_EVENTS: + matcher = sessions.LIFECYCLE_HOOK_MATCHERS[event] + assert matcher is not None + assert not claude_matcher_admits(matcher, "") + assert all(claude_matcher_admits(matcher, name) for name in names) + + +def test_session_hooks_stay_unfiltered() -> None: + matchers = sessions.LIFECYCLE_HOOK_MATCHERS + assert set(matchers) == sessions._LIFECYCLE_EVENTS + assert all(matchers[event] is None for event in sessions._SESSION_EVENTS) + + +def test_product_selects_no_main_agent() -> None: + # Internal-agent callbacks are unnamed only while no main agent is set. + assert {"--agent", "--settings"}.isdisjoint(flag.value for flag in ClaudeFlags) + root = pkg_root() + manifest = json.loads((root / ".claude-plugin" / "plugin.json").read_text(encoding="utf-8")) + assert "agent" not in manifest.get("settings", {}) + plugin_settings = root / "settings.json" + assert not plugin_settings.exists() or "agent" not in json.loads( + plugin_settings.read_text(encoding="utf-8") + ) diff --git a/tests/arch/test_mcp_prefix_literal_containment.py b/tests/arch/test_mcp_prefix_literal_containment.py index aba6020538..01b6ff01b7 100644 --- a/tests/arch/test_mcp_prefix_literal_containment.py +++ b/tests/arch/test_mcp_prefix_literal_containment.py @@ -8,6 +8,7 @@ import pytest from autoskillit.core import find_qualified_autoskillit_tool_names, paths +from tests.arch._helpers import SRC_ROOT pytestmark = [pytest.mark.layer("arch"), pytest.mark.small] @@ -42,14 +43,17 @@ def test_mcp_prefix_literals_confined_to_canonical_module() -> None: """The string literals for both MCP prefix forms appear only in _plugin_ids.py.""" src_root = paths.pkg_root() violations: list[str] = [] - for py_file in sorted(src_root.rglob("*.py")): - rel = py_file.relative_to(src_root).as_posix() - if rel == _CANONICAL_MODULE: - continue - for lineno, value in _string_literals_in_module(py_file): - for literal in _CONFINED_LITERALS: - if literal in value: - violations.append(f"{rel}:{lineno}: contains {literal!r} in {value!r}") + for root in (src_root, SRC_ROOT.parents[1] / "scripts"): + files = sorted(path for path in root.rglob("*.py") if "__pycache__" not in path.parts) + assert files, f"{root} scan universe must not be empty" + for py_file in files: + rel = py_file.relative_to(root).as_posix() + if root == src_root and rel == _CANONICAL_MODULE: + continue + for lineno, value in _string_literals_in_module(py_file): + for literal in _CONFINED_LITERALS: + if literal in value: + violations.append(f"{rel}:{lineno}: contains {literal!r} in {value!r}") assert not violations, ( "MCP prefix literals must appear only in " f"{_CANONICAL_MODULE}. Violations:\n" + "\n".join(violations) diff --git a/tests/core/types/test_plan_set_authority.py b/tests/core/types/test_plan_set_authority.py index 19d236496f..708ee18b9f 100644 --- a/tests/core/types/test_plan_set_authority.py +++ b/tests/core/types/test_plan_set_authority.py @@ -85,6 +85,15 @@ def test_create_digest_and_canonical_round_trip(tmp_path: Path) -> None: }, "unknown part key", ), + ( + { + "inventory_mode": InventoryMode.ENUMERATED, + "coverage": CoverageResultDef( + CoverageStatus.PASS, parts_without_obligations=("P1",) + ), + }, + "invalid for enumerated inventory", + ), ], ) def test_invalid_authority_states_fail_closed( diff --git a/tests/execution/test_flag_contracts.py b/tests/execution/test_flag_contracts.py index a027254c7d..d6d70fb412 100644 --- a/tests/execution/test_flag_contracts.py +++ b/tests/execution/test_flag_contracts.py @@ -7,10 +7,14 @@ one place, and all tests catch the ripple. """ +from pathlib import Path + import pytest from autoskillit.core import ClaudeFlags +from autoskillit.core.types import OutputFormat from autoskillit.execution.backends.claude import ClaudeCodeBackend +from tests.execution.backends._plugin_binding import plugin_binding pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] @@ -69,6 +73,31 @@ def _extract_flags(cmd: tuple[str, ...] | list[str]) -> set[str]: class TestFlagRegistryAudit: + @pytest.mark.parametrize("fmt", list(OutputFormat)) + def test_fleet_cmd_flags_are_all_registered(self, fmt: OutputFormat) -> None: + backend = ClaudeCodeBackend() + skill = backend.build_skill_session_cmd( + "/investigate foo", + cwd="/tmp", + completion_marker="%%DONE%%", + model=None, + plugin_binding=None, + output_format=fmt, + ) + with plugin_binding(Path("/p")) as binding: + root = backend.build_food_truck_cmd( + orchestrator_prompt="Orchestrator", + plugin_binding=binding, + cwd="/tmp", + completion_marker="%%DONE%%", + output_format=fmt, + ) + resume = backend.build_resume_cmd( + resume_session_id="abc", prompt="Emit", output_format=fmt + ) + for spec in (skill, root, resume): + assert not _extract_flags(spec.cmd) - KNOWN_CLAUDE_FLAGS + def test_interactive_cmd_flags_are_all_registered(self): result = ClaudeCodeBackend().build_interactive_cmd(model="claude-sonnet-4-6") unknown = _extract_flags(result.cmd) - KNOWN_CLAUDE_FLAGS diff --git a/tests/execution/test_headless_enum_recovery.py b/tests/execution/test_headless_enum_recovery.py index 57dfcf5efc..96c243507d 100644 --- a/tests/execution/test_headless_enum_recovery.py +++ b/tests/execution/test_headless_enum_recovery.py @@ -23,15 +23,16 @@ from autoskillit.recipe._contracts_types import SkillContract, SkillOutput from autoskillit.recipe.contracts import get_skill_contract, load_bundled_manifest from tests.conftest import _make_result -from tests.execution.conftest import _launch_inputs, _mock_backend, _success_session_json +from tests.execution.conftest import ( + _launch_inputs, + _mock_backend, + _success_session_json, + _unmarked_rollout_snapshot, +) pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] -def _unmarked_rollout_snapshot() -> tuple[None, str]: - return None, "2000-01-01T00:00:00+00:00" - - def _make_plan_contract( *, write_behavior: str | None = "conditional", diff --git a/tests/execution/test_model_backend_launch_contract.py b/tests/execution/test_model_backend_launch_contract.py index 76d3e4a061..173b2f1843 100644 --- a/tests/execution/test_model_backend_launch_contract.py +++ b/tests/execution/test_model_backend_launch_contract.py @@ -17,7 +17,7 @@ from autoskillit.core import LaunchValueSourceKind from autoskillit.core.types import LaunchContractError, RetryReason, SkillResult -from .conftest import _backend_authority, _launch_inputs, _mock_backend +from .conftest import _backend_authority, _launch_inputs, _mock_backend, _unmarked_rollout_snapshot pytestmark = [pytest.mark.layer("execution"), pytest.mark.small, pytest.mark.model_contract] @@ -180,7 +180,7 @@ async def test_rejected_admission_does_not_capture_rollout_or_invoke_runner(tmp_ from autoskillit.execution.headless._headless_launch import _run_headless_attempt kwargs, runner = _build_codex_attempt_kwargs("gpt-6-sol", tmp_path) - capture = Mock(return_value=(None, "2000-01-01T00:00:00+00:00")) + capture = Mock(side_effect=_unmarked_rollout_snapshot) kwargs["capture_rollout_on_launch"] = capture async def reject(contract): @@ -226,7 +226,7 @@ def build_spec(_binding, _extras, _managed_attempt_id=None): "launch_resolver": resolver, "launch_preparation": preparation, "expected_launch_contract": None, - "capture_rollout_on_launch": lambda: (None, "2000-01-01T00:00:00+00:00"), + "capture_rollout_on_launch": _unmarked_rollout_snapshot, "plugin_authority": None, "plugin_load_mode": PluginLoadMode.NONE, "provider_extras": None, diff --git a/tests/hooks/test_hook_protocol_oracle.py b/tests/hooks/test_hook_protocol_oracle.py index a66d8c8e79..42f3aa34cf 100644 --- a/tests/hooks/test_hook_protocol_oracle.py +++ b/tests/hooks/test_hook_protocol_oracle.py @@ -18,6 +18,7 @@ STATUS_COMPLETED, STATUS_FAILED, STATUS_STOPPED, + claude_matcher_admits, claude_verdict, codex_verdict, ) @@ -25,6 +26,40 @@ pytestmark = [pytest.mark.layer("hooks"), pytest.mark.small] +@pytest.mark.parametrize( + ("matcher", "value", "admitted"), + [ + (None, "Bash", True), + (None, "", True), + ("", "Bash", True), + ("", "", True), + ("*", "Bash", True), + ("*", "", True), + ("Bash", "Bash", True), + ("Bash", "BashOutput", False), + ("Edit|Write", "Edit", True), + ("Edit|Write", "Write", True), + ("Edit|Write", "NotebookEdit", False), + ("Edit, Write", "Edit", True), + ("Edit, Write", "Write", True), + ("Edit, Write", "NotebookEdit", False), + ("code-reviewer", "code-reviewer", True), + ("code-reviewer", "other-code-reviewer", False), + ("^Notebook", "NotebookEdit", True), + ("Edit.*", "NotebookEdit", True), + ("^Edit$", "NotebookEdit", False), + ("^my-plugin:reviewer$", "my-plugin:reviewer", True), + ("general-purpose", "", False), + (".+", "", False), + (".*", "", True), + ], +) +def test_claude_matcher_follows_the_documented_rule( + matcher: str | None, value: str, admitted: bool +) -> None: + assert claude_matcher_admits(matcher, value) is admitted + + @pytest.mark.parametrize("event", ["PreCompact", "PostCompact"]) def test_codex_compact_halt_preserves_system_message(event: str) -> None: # codex-rs/hooks/src/events/compact.rs:260-278 records warnings before stopping. diff --git a/tests/infra/_e2e_evidence.py b/tests/infra/_e2e_evidence.py new file mode 100644 index 0000000000..b847eee377 --- /dev/null +++ b/tests/infra/_e2e_evidence.py @@ -0,0 +1,49 @@ +"""Load and materialize captured E2E session evidence fixtures.""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import Any + +_PROJECTS_TOKEN = "@CLAUDE_PROJECTS@" + + +def _fixture_dir(name: str) -> Path: + return Path(__file__).with_name("fixtures") / "e2e_sessions" / name + + +def load_provenance(name: str) -> dict[str, Any]: + """Read the provenance record for a captured E2E run.""" + return json.loads((_fixture_dir(name) / "PROVENANCE.json").read_text(encoding="utf-8")) + + +def _replace_projects_token(path: Path, projects: Path) -> None: + text = path.read_text(encoding="utf-8") + path.write_text(text.replace(_PROJECTS_TOKEN, str(projects)), encoding="utf-8") + + +def materialize_run( + name: str, + *, + trace: Path, + log_root: Path, + projects: Path, +) -> dict[str, Any]: + """Copy a captured run to the given locations and return its envelope.""" + fixture = _fixture_dir(name) + trace.parent.mkdir(parents=True, exist_ok=True) + log_root.mkdir(parents=True, exist_ok=True) + projects.mkdir(parents=True, exist_ok=True) + + shutil.copyfile(fixture / "out/session-events.jsonl", trace) + shutil.copytree(fixture / "logs", log_root, dirs_exist_ok=True) + shutil.copytree(fixture / "projects", projects, dirs_exist_ok=True) + + _replace_projects_token(trace, projects) + _replace_projects_token(log_root / "sessions.jsonl", projects) + for summary in (log_root / "sessions").glob("*/summary.json"): + _replace_projects_token(summary, projects) + + return json.loads((fixture / "out/envelope.json").read_text(encoding="utf-8")) diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/PROVENANCE.json b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/PROVENANCE.json new file mode 100644 index 0000000000..77b3700720 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/PROVENANCE.json @@ -0,0 +1,9 @@ +{ + "producer": "claude-code", + "cli_version": "2.1.280", + "source_run": "38170521660", + "issue_url": "https://github.com/TalonT-Org/autoskillit-e2e-sandbox/issues/1", + "repository": "TalonT-Org/autoskillit-e2e-sandbox", + "claim_label": "in-progress", + "sanitization": "Derived from the retained run artifact. The root transcript retains rows 1-3, 54-55, 61-62, 66-67, 81, 84, 88-89, 93-94, 547-548, 552-553, 569, 679, 680, and 682; each retained row keeps every source key. Worker transcripts retain their first three and final two rows; native subagent transcripts retain their first and final rows. Prose strings longer than 400 characters are truncated with a removed-character marker, while the claim and branch tool inputs remain complete. Serialized JSON carriers, including mcpMeta.structuredContent.result, are parsed, recursively sanitized, and reserialized so structural and ownership fields remain intact. The container prefix /home/autoskillit/.claude/projects is replaced with @CLAUDE_PROJECTS@ in the event trace, session index, and summaries; other /home/ paths are tokenized. Index and summary data retain identity fields needed to resolve the captured lineage. logs/skill-session-contracts/ and logs/otlp.jsonl are excluded." +} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions.jsonl new file mode 100644 index 0000000000..52f67da240 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions.jsonl @@ -0,0 +1,3 @@ +{"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","dir_name":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","dispatch_id":"4666c3fd-486f-4ce1-b51f-9d9d660a5560","parent_session_id":"","caller_session_id":"","claude_code_log":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl","session_type":"skill","step_name":"plan","order_id":"4666c3fd-486f-4ce1-b51f-9d9d660a5560","campaign_id":"48915f19-d220-4069-8d55-b81efff84c05","kitchen_id":"48915f19-d220-4069-8d55-b81efff84c05","timestamp":"2026-10-11T20:05:22.655547+00:00","claude_code_version":"2.1.280 (Claude Code)","autoskillit_version":"0.10.1271","backend":"claude-code","provider_used":"anthropic","model_identifier":"MiniMax-M3","configured_model":"sonnet","success":true,"schema_version":17,"recipe_name":"implementation","recipe_version":"1.0.0","requested_parent_backend":"claude-code","requested_parent_model":"sonnet"} +{"session_id":"54f18144-1188-45d7-a392-17b4529f721e","dir_name":"54f18144-1188-45d7-a392-17b4529f721e","dispatch_id":"4666c3fd-486f-4ce1-b51f-9d9d660a5560","parent_session_id":"","caller_session_id":"","claude_code_log":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl","session_type":"skill","step_name":"verify","order_id":"4666c3fd-486f-4ce1-b51f-9d9d660a5560","campaign_id":"48915f19-d220-4069-8d55-b81efff84c05","kitchen_id":"48915f19-d220-4069-8d55-b81efff84c05","timestamp":"2026-10-11T20:11:57.256584+00:00","claude_code_version":"2.1.280 (Claude Code)","autoskillit_version":"0.10.1271","backend":"claude-code","provider_used":"anthropic","model_identifier":"MiniMax-M3","configured_model":"sonnet","success":false,"schema_version":17,"recipe_name":"implementation","recipe_version":"1.0.0","requested_parent_backend":"claude-code","requested_parent_model":"sonnet"} +{"session_id":"2292658a-34cf-44a0-a0b1-f7a20218874b","dir_name":"2292658a-34cf-44a0-a0b1-f7a20218874b","dispatch_id":"4666c3fd-486f-4ce1-b51f-9d9d660a5560","parent_session_id":"","caller_session_id":"","claude_code_log":"@CLAUDE_PROJECTS@/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl","session_type":"orchestrator","step_name":"","order_id":"4666c3fd-486f-4ce1-b51f-9d9d660a5560","campaign_id":"48915f19-d220-4069-8d55-b81efff84c05","kitchen_id":"48915f19-d220-4069-8d55-b81efff84c05","timestamp":"2026-10-11T20:04:36.539164+00:00","claude_code_version":"2.1.280 (Claude Code)","autoskillit_version":"0.10.1271","backend":"claude-code","provider_used":"anthropic","model_identifier":"MiniMax-M3","configured_model":"sonnet","success":true,"schema_version":17,"recipe_name":"","recipe_version":"","requested_parent_backend":"","requested_parent_model":""} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/2292658a-34cf-44a0-a0b1-f7a20218874b/summary.json b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/2292658a-34cf-44a0-a0b1-f7a20218874b/summary.json new file mode 100644 index 0000000000..d13696603e --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/2292658a-34cf-44a0-a0b1-f7a20218874b/summary.json @@ -0,0 +1,23 @@ +{ + "backend": "claude-code", + "campaign_id": "48915f19-d220-4069-8d55-b81efff84c05", + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "end_ts": "2026-10-11T20:16:23.869672+00:00", + "execution_identity": { + "children": [], + "cli_version": "", + "effective_parent_backend": "", + "effective_parent_effort": "", + "effective_parent_model": "", + "override_key_path": "", + "override_tier": "", + "parent_session_id": "", + "requested_parent_backend": "", + "requested_parent_effort": "", + "requested_parent_model": "" + }, + "provider_used": "anthropic", + "session_id": "2292658a-34cf-44a0-a0b1-f7a20218874b", + "session_type": "orchestrator", + "start_ts": "2026-10-11T20:04:36.539164+00:00" +} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/54f18144-1188-45d7-a392-17b4529f721e/summary.json b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/54f18144-1188-45d7-a392-17b4529f721e/summary.json new file mode 100644 index 0000000000..38000961d4 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/54f18144-1188-45d7-a392-17b4529f721e/summary.json @@ -0,0 +1,23 @@ +{ + "backend": "claude-code", + "campaign_id": "48915f19-d220-4069-8d55-b81efff84c05", + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "end_ts": "2026-10-11T20:14:46.893573+00:00", + "execution_identity": { + "children": [], + "cli_version": "", + "effective_parent_backend": "", + "effective_parent_effort": "", + "effective_parent_model": "", + "override_key_path": "", + "override_tier": "", + "parent_session_id": "", + "requested_parent_backend": "claude-code", + "requested_parent_effort": "", + "requested_parent_model": "sonnet" + }, + "provider_used": "anthropic", + "session_id": "54f18144-1188-45d7-a392-17b4529f721e", + "session_type": "skill", + "start_ts": "2026-10-11T20:11:57.256584+00:00" +} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/8ef8f993-8caa-47dd-83b0-26f9e5b51139/summary.json b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/8ef8f993-8caa-47dd-83b0-26f9e5b51139/summary.json new file mode 100644 index 0000000000..e2c1ce5cc3 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/logs/sessions/8ef8f993-8caa-47dd-83b0-26f9e5b51139/summary.json @@ -0,0 +1,23 @@ +{ + "backend": "claude-code", + "campaign_id": "48915f19-d220-4069-8d55-b81efff84c05", + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "end_ts": "2026-10-11T20:07:16.866414+00:00", + "execution_identity": { + "children": [], + "cli_version": "", + "effective_parent_backend": "", + "effective_parent_effort": "", + "effective_parent_model": "", + "override_key_path": "", + "override_tier": "", + "parent_session_id": "", + "requested_parent_backend": "claude-code", + "requested_parent_effort": "", + "requested_parent_model": "sonnet" + }, + "provider_used": "anthropic", + "session_id": "8ef8f993-8caa-47dd-83b0-26f9e5b51139", + "session_type": "skill", + "start_ts": "2026-10-11T20:05:22.655547+00:00" +} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/out/envelope.json b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/out/envelope.json new file mode 100644 index 0000000000..ed1e998ea1 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/out/envelope.json @@ -0,0 +1,114 @@ +{ + "kind": "completed", + "success": false, + "dispatch_status": "resumable", + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "dispatched_session_id": "2292658a-34cf-44a0-a0b1-f7a20218874b", + "reason": "fleet_l3_no_result_block", + "effect_provenance": { + "operation_id": "b6908330d53b40b98656d73e3a8fdb05", + "aggregate_phase": "started", + "retry_disposition": "resume_by_identity", + "effects": [ + { + "name": "campaign_state_write", + "phase": "confirmed", + "effect_id": "b6908330d53b40b98656d73e3a8fdb05:campaign_state_write", + "retry_relevant": true, + "confirmation_receipt": "per-dispatch state and captures persisted", + "known_downstream_identities": { + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "state_path": "/workspace/sandbox/.autoskillit/temp/dispatches/4666c3fd-486f-4ce1-b51f-9d9d660a5560.json" + }, + "ambiguity": "" + }, + { + "name": "child_discovery", + "phase": "confirmed", + "effect_id": "b6908330d53b40b98656d73e3a8fdb05:child_discovery", + "retry_relevant": true, + "confirmation_receipt": "captured one process identity tuple", + "known_downstream_identities": { + "boot_id": "95f60de8-f823-4b7a-abc9-9e3f15310bc6", + "create_time": "1791749076.16", + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "identity_degraded": "False", + "pid": "106", + "starttime_ticks": "15116" + }, + "ambiguity": "" + }, + { + "name": "dispatch_allocation", + "phase": "confirmed", + "effect_id": "b6908330d53b40b98656d73e3a8fdb05:dispatch_allocation", + "retry_relevant": true, + "confirmation_receipt": "per-dispatch state identity persisted", + "known_downstream_identities": { + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "state_path": "/workspace/sandbox/.autoskillit/temp/dispatches/4666c3fd-486f-4ce1-b51f-9d9d660a5560.json" + }, + "ambiguity": "" + }, + { + "name": "process_spawn", + "phase": "confirmed", + "effect_id": "b6908330d53b40b98656d73e3a8fdb05:process_spawn", + "retry_relevant": true, + "confirmation_receipt": "executor reported spawned process and authoritative session identity", + "known_downstream_identities": { + "dispatch_id": "4666c3fd-486f-4ce1-b51f-9d9d660a5560", + "dispatched_session_id": "2292658a-34cf-44a0-a0b1-f7a20218874b", + "pid": "106", + "starttime_ticks": "15116" + }, + "ambiguity": "" + } + ], + "cancel_requested": false, + "local_cleanup": null, + "state_cleanup_confirmed": false, + "labels_cleanup_confirmed": false + }, + "token_usage": { + "backend": "claude-code", + "provider_used": "anthropic", + "input_tokens": { + "state": "measured", + "value": 205063 + }, + "output_tokens": { + "state": "measured", + "value": 39864 + }, + "cache_read_tokens": { + "state": "measured", + "value": 11571251 + }, + "cache_write_tokens": { + "state": "measured_zero", + "value": 0 + }, + "peak_context": { + "state": "unknown", + "value": null + } + }, + "l3_payload": null, + "l3_parse_source": "stdout", + "lifespan_started": true, + "stderr": "", + "elapsed_seconds": 708.0250108242035, + "error": "fleet_l3_no_result_block", + "user_visible_message": "fleet_l3_no_result_block", + "resume_checkpoint": { + "completed_items": [ + "plan" + ], + "step_name": "plan", + "progress_pct": 0.007407407407407408, + "ts": "2026-10-11T20:07:18.776245+00:00", + "backend_name": "claude-code", + "skill_name": "implementation" + } +} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/out/session-events.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/out/session-events.jsonl new file mode 100644 index 0000000000..54806953b0 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/out/session-events.jsonl @@ -0,0 +1,16 @@ +{"event":"SessionStart","timestamp":1791749076.9067402,"session_id":"2292658a-34cf-44a0-a0b1-f7a20218874b","source":"startup","transcript_path":"@CLAUDE_PROJECTS@/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl"} +{"event":"SessionStart","timestamp":1791749123.0673993,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","source":"startup","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl"} +{"event":"SubagentStart","timestamp":1791749157.9630604,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","agent_id":"a9ed51b7c02b828dd","agent_type":"autoskillit:plan-interface-mapper","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl"} +{"event":"SubagentStart","timestamp":1791749158.0156765,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","agent_id":"aeb96cc0ddbdab294","agent_type":"autoskillit:plan-foundation-auditor","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl"} +{"event":"SubagentStart","timestamp":1791749158.1193779,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","agent_id":"a516cbae85151dc7c","agent_type":"autoskillit:plan-registry-tracer","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl"} +{"event":"SubagentStop","timestamp":1791749183.3175266,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","agent_id":"a516cbae85151dc7c","agent_type":"autoskillit:plan-registry-tracer","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl","agent_transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a516cbae85151dc7c.jsonl"} +{"event":"SubagentStop","timestamp":1791749183.933718,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","agent_id":"a9ed51b7c02b828dd","agent_type":"autoskillit:plan-interface-mapper","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl","agent_transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a9ed51b7c02b828dd.jsonl"} +{"event":"SubagentStop","timestamp":1791749190.3404713,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","agent_id":"aeb96cc0ddbdab294","agent_type":"autoskillit:plan-foundation-auditor","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl","agent_transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-aeb96cc0ddbdab294.jsonl"} +{"event":"SessionEnd","timestamp":1791749233.8509,"session_id":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl"} +{"event":"SessionStart","timestamp":1791749517.653047,"session_id":"54f18144-1188-45d7-a392-17b4529f721e","source":"startup","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl"} +{"event":"SubagentStart","timestamp":1791749551.111115,"session_id":"54f18144-1188-45d7-a392-17b4529f721e","agent_id":"a7e613554b849d6b2","agent_type":"general-purpose","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl"} +{"event":"SubagentStop","timestamp":1791749679.8737457,"session_id":"54f18144-1188-45d7-a392-17b4529f721e","agent_id":"a7e613554b849d6b2","agent_type":"general-purpose","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl","agent_transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e/subagents/agent-a7e613554b849d6b2.jsonl"} +{"event":"SessionEnd","timestamp":1791749686.7281744,"session_id":"54f18144-1188-45d7-a392-17b4529f721e","transcript_path":"@CLAUDE_PROJECTS@/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl"} +{"event":"SubagentStop","timestamp":1791749722.6609342,"session_id":"2292658a-34cf-44a0-a0b1-f7a20218874b","agent_id":"a610a4bda1f28ca35","agent_type":"","transcript_path":"@CLAUDE_PROJECTS@/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl","agent_transcript_path":"@CLAUDE_PROJECTS@/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b/subagents/agent-a610a4bda1f28ca35.jsonl"} +{"event":"SessionStart","timestamp":1791749722.700151,"session_id":"2292658a-34cf-44a0-a0b1-f7a20218874b","source":"compact","transcript_path":"@CLAUDE_PROJECTS@/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl"} +{"event":"SessionEnd","timestamp":1791749780.901994,"session_id":"2292658a-34cf-44a0-a0b1-f7a20218874b","transcript_path":"@CLAUDE_PROJECTS@/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl"} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl new file mode 100644 index 0000000000..5fc748643b --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e.jsonl @@ -0,0 +1,5 @@ +{"type":"queue-operation","operation":"enqueue","timestamp":"2026-10-11T20:12:00.782Z","sessionId":"54f18144-1188-45d7-a392-17b4529f721e","content":"Use the /dry-walkthrough skill AUTOSKILLIT_BOUND_INVOCATION_V1\n{\"skill_inputs\":[{\"name\":\"plan_path\",\"value\":\"/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md\"},{\"name\":\"issue_url\",\"value\":\"https://github.com/TalonT-Org/autoskillit-e2e-sandbox/issues/1\"},{\"name\":\"review_path\",\"value\":\"\"},{\"name\":\"audit_cycle_path\",\"va...[fixture-truncated 2001 chars]"} +{"type":"queue-operation","operation":"dequeue","timestamp":"2026-10-11T20:12:00.783Z","sessionId":"54f18144-1188-45d7-a392-17b4529f721e"} +{"parentUuid":null,"isSidechain":false,"promptId":"6ec22124-b069-42f2-a3d7-361204649ceb","type":"user","message":{"role":"user","content":"Use the /dry-walkthrough skill AUTOSKILLIT_BOUND_INVOCATION_V1\n{\"skill_inputs\":[{\"name\":\"plan_path\",\"value\":\"/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md\"},{\"name\":\"issue_url\",\"value\":\"https://github.com/TalonT-Org/autoskillit-e2e-sandbox/issues/1\"},{\"name\":\"review_path\",\"value\":\"\"},{\"name\":\"audit_cycle_path\",\"va...[fixture-truncated 2001 chars]"},"uuid":"1fa895db-0f63-4e4d-9319-5e0a1ea3bd2e","timestamp":"2026-10-11T20:12:00.823Z","permissionMode":"bypassPermissions","promptSource":"sdk","turnOrigin":"sdk","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"54f18144-1188-45d7-a392-17b4529f721e","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} +{"type":"last-prompt","lastPrompt":"Use the /dry-walkthrough skill AUTOSKILLIT_BOUND_INVOCATION_V1 {\"skill_inputs\":[{\"name\":\"plan_path\",\"value\":\"/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normali…","leafUuid":"9a7737fa-95fb-4950-9f81-a0f69f85140a","sessionId":"54f18144-1188-45d7-a392-17b4529f721e"} +{"type":"cost-state","sessionId":"54f18144-1188-45d7-a392-17b4529f721e","totalCostUSD":0.4940016000000001,"totalAPIDuration":149446,"totalAPIDurationWithoutRetries":149406,"totalToolDuration":130687,"totalLinesAdded":3,"totalLinesRemoved":1,"totalDuration":169414,"startTime":1791749517277,"modelUsage":{"claude-sonnet-5":{"inputTokens":93799,"outputTokens":19776,"thinkingTokens":13280,"cacheReadInputTokens":543218,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.4940016000000001}},"hasUnknownModelCost":false} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e/subagents/agent-a7e613554b849d6b2.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e/subagents/agent-a7e613554b849d6b2.jsonl new file mode 100644 index 0000000000..9b09bf65bd --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/54f18144-1188-45d7-a392-17b4529f721e/subagents/agent-a7e613554b849d6b2.jsonl @@ -0,0 +1,2 @@ +{"parentUuid":null,"isSidechain":true,"promptId":"6ec22124-b069-42f2-a3d7-361204649ceb","agentId":"a7e613554b849d6b2","type":"user","message":{"role":"user","content":"You are validating a proposed implementation plan by performing a dry walkthrough of each change without implementing. Your role is technical validation, not strategic decision-making. Fix factual inaccuracies (wrong file paths, nonexistent functions, incorrect line numbers). Preserve all goals and scope.\n\n## Plan to Validate\nRead the exact plan file at:\n`/workspace/autoskillit-runs/impl-20261011-...[fixture-truncated 7212 chars]"},"uuid":"bd262550-a49c-499e-8443-0dad17d21c1a","timestamp":"2026-10-11T20:12:31.074Z","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"54f18144-1188-45d7-a392-17b4529f721e","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} +{"parentUuid":"1bee3141-ddeb-4c5a-8e05-5c397c01fc75","isSidechain":true,"agentId":"a7e613554b849d6b2","message":{"id":"071b212510333ffa9b724744f66211b1","type":"message","role":"assistant","content":[{"type":"text","text":"# Dry Walkthrough Report\n\n## Plan Validation Summary\n\n**File**: `/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md`\n**Stamp**: `Dry-walkthrough verified = TRUE` (verified present on line 1)\n\n---\n\n## Step 1: Multi-Part Detection\n- Filename has no `_part_` suffix. PASS\n- Plan contains no PART scope warning block. PASS\n\n...[fixture-truncated 4672 chars]"}],"model":"MiniMax-M3","stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":188,"cache_creation_input_tokens":0,"cache_read_input_tokens":45709,"output_tokens":1882,"output_tokens_details":{"thinking_tokens":532},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionAgent":"general-purpose","attributionSkill":"dry-walkthrough","type":"assistant","uuid":"3819a31b-bbba-46b9-83ca-61def8b3f88f","timestamp":"2026-10-11T20:14:39.838Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"54f18144-1188-45d7-a392-17b4529f721e","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl new file mode 100644 index 0000000000..3f16a49374 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139.jsonl @@ -0,0 +1,5 @@ +{"type":"queue-operation","operation":"enqueue","timestamp":"2026-10-11T20:05:26.169Z","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","content":"Use the /make-plan skill AUTOSKILLIT_BOUND_INVOCATION_V1\n{\"skill_inputs\":[{\"name\":\"task\",\"value\":\"E2E implementation fixture: normalize whitespace\\n\\n# Reusable implementation E2E fixture\\n\\nAdd `normalize_whitespace(text: str) -> str` to `sandbox/text.py`.\\n\\nThe function must remove leading and trailing whitespace and replace each run of\\nwhitespace (including spaces, tabs, and newlines) with on...[fixture-truncated 2035 chars]"} +{"type":"queue-operation","operation":"dequeue","timestamp":"2026-10-11T20:05:26.169Z","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139"} +{"parentUuid":null,"isSidechain":false,"promptId":"b304e3c8-f9a9-4e0e-8d13-8e7d6617cc4e","type":"user","message":{"role":"user","content":"Use the /make-plan skill AUTOSKILLIT_BOUND_INVOCATION_V1\n{\"skill_inputs\":[{\"name\":\"task\",\"value\":\"E2E implementation fixture: normalize whitespace\\n\\n# Reusable implementation E2E fixture\\n\\nAdd `normalize_whitespace(text: str) -> str` to `sandbox/text.py`.\\n\\nThe function must remove leading and trailing whitespace and replace each run of\\nwhitespace (including spaces, tabs, and newlines) with on...[fixture-truncated 2035 chars]"},"uuid":"0b38c9ec-2c12-4619-8935-2121f8c249ad","timestamp":"2026-10-11T20:05:26.210Z","permissionMode":"bypassPermissions","promptSource":"sdk","turnOrigin":"sdk","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} +{"type":"last-prompt","lastPrompt":"Use the /make-plan skill AUTOSKILLIT_BOUND_INVOCATION_V1 {\"skill_inputs\":[{\"name\":\"task\",\"value\":\"E2E implementation fixture: normalize whitespace\\n\\n# Reusable implementation E2E fixture\\n\\nAdd `norm…","leafUuid":"f50148a8-7b86-4926-a4d2-961755148f52","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139"} +{"type":"cost-state","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","totalCostUSD":0.41991880000000004,"totalAPIDuration":123555,"totalAPIDurationWithoutRetries":123501,"totalToolDuration":91414,"totalLinesAdded":199,"totalLinesRemoved":0,"totalDuration":111137,"startTime":1791749122675,"modelUsage":{"claude-sonnet-5":{"inputTokens":79993,"outputTokens":16833,"thinkingTokens":8494,"cacheReadInputTokens":458014,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.41991880000000004}},"hasUnknownModelCost":false} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a516cbae85151dc7c.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a516cbae85151dc7c.jsonl new file mode 100644 index 0000000000..e37cb2a70e --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a516cbae85151dc7c.jsonl @@ -0,0 +1,2 @@ +{"parentUuid":null,"isSidechain":true,"promptId":"b304e3c8-f9a9-4e0e-8d13-8e7d6617cc4e","agentId":"a516cbae85151dc7c","type":"user","message":{"role":"user","content":"You are the plan-registry-tracer. Perform a registry and artifact trace audit of the following draft implementation plan.\n\nContrastive frame: \"A junior reviewer found this plan's registry coverage complete — what did they miss?\"\n\nDo NOT suggest scope expansion. Identify only gaps in what the plan already claims to do.\n\nCodebase root: /workspace/autoskillit-runs/impl-20261011-200500-587134\n\nWorking...[fixture-truncated 715 chars]"},"uuid":"0a0a8820-6e0f-43bd-9b90-5a9dadd38fb9","timestamp":"2026-10-11T20:05:58.067Z","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} +{"parentUuid":"8349df76-a174-48ed-87fa-55fcd3bb3647","isSidechain":true,"agentId":"a516cbae85151dc7c","message":{"id":"071b1f367fc2617d89e17e0512b28e11","type":"message","role":"assistant","content":[{"type":"text","text":"# Registry Trace Findings\n\n## Touched Symbols\n- `normalize_whitespace` — new function in `sandbox/text.py` (no rename, no removal)\n\n## Grep/LSP Results\n\n| Symbol | File Found | Reference Type | Updated by Plan? |\n|--------|-----------|----------------|------------------|\n| normalize_whitespace | (none) | N/A — net-new symbol, no prior references anywhere in repo | N/A |\n\nA recursive grep across `/...[fixture-truncated 1847 chars]"}],"model":"MiniMax-M3","stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":1894,"cache_creation_input_tokens":0,"cache_read_input_tokens":9755,"output_tokens":1025,"output_tokens_details":{"thinking_tokens":474},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionAgent":"autoskillit:plan-registry-tracer","attributionSkill":"make-plan","attributionPlugin":"autoskillit","type":"assistant","uuid":"27b5b29b-5719-4134-b996-7009d7bee590","timestamp":"2026-10-11T20:06:23.283Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a9ed51b7c02b828dd.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a9ed51b7c02b828dd.jsonl new file mode 100644 index 0000000000..b2c3fede89 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-a9ed51b7c02b828dd.jsonl @@ -0,0 +1,2 @@ +{"parentUuid":null,"isSidechain":true,"promptId":"b304e3c8-f9a9-4e0e-8d13-8e7d6617cc4e","agentId":"a9ed51b7c02b828dd","type":"user","message":{"role":"user","content":"You are the plan-interface-mapper. Perform a variable and data-flow trace audit of the following draft implementation plan.\n\nContrastive frame: \"A junior reviewer found this plan's variable usage correct — what did they miss?\"\n\nDo NOT suggest scope expansion. Identify only gaps in what the plan already claims to do.\n\nCodebase root: /workspace/autoskillit-runs/impl-20261011-200500-587134\n\nWorking b...[fixture-truncated 780 chars]"},"uuid":"5067983e-c8ee-44df-8677-6acfb5f2d6d9","timestamp":"2026-10-11T20:05:57.864Z","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} +{"parentUuid":"9ab1fecd-1984-470f-a6a6-e6a60a361312","isSidechain":true,"agentId":"a9ed51b7c02b828dd","message":{"id":"071b1f337786beccec1da5b44669d627","type":"message","role":"assistant","content":[{"type":"text","text":"# Interface Mapping Audit\n\n## SET/READ Trace Table\n\n| Variable | SET Location | SET Value | READ Location | READ Expectation | Match? |\n|----------|-------------|-----------|---------------|------------------|--------|\n| `tests/test_text.py` content | plan step 2 (create) | new file with `normalize_whitespace` tests only | `Taskfile.yml:11-12` (`unittest discover`) | discover runs over existing `t...[fixture-truncated 2117 chars]"}],"model":"MiniMax-M3","stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":918,"cache_creation_input_tokens":0,"cache_read_input_tokens":6939,"output_tokens":1767,"output_tokens_details":{"thinking_tokens":1127},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionAgent":"autoskillit:plan-interface-mapper","attributionSkill":"make-plan","attributionPlugin":"autoskillit","type":"assistant","uuid":"53c38458-ca9f-45dd-a78c-38dcfd68125f","timestamp":"2026-10-11T20:06:23.901Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-aeb96cc0ddbdab294.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-aeb96cc0ddbdab294.jsonl new file mode 100644 index 0000000000..b98512bab6 --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-autoskillit-runs-impl-20261011-200500-587134/8ef8f993-8caa-47dd-83b0-26f9e5b51139/subagents/agent-aeb96cc0ddbdab294.jsonl @@ -0,0 +1,2 @@ +{"parentUuid":null,"isSidechain":true,"promptId":"b304e3c8-f9a9-4e0e-8d13-8e7d6617cc4e","agentId":"aeb96cc0ddbdab294","type":"user","message":{"role":"user","content":"You are the plan-foundation-auditor. Perform a step-by-step control-flow audit of the following draft implementation plan.\n\nContrastive frame: \"A junior reviewer found this plan's control flow acceptable — what did they miss?\"\n\nDo NOT suggest scope expansion. Identify only gaps in what the plan already claims to do.\n\nCodebase root: /workspace/autoskillit-runs/impl-20261011-200500-587134\n\nWorking b...[fixture-truncated 1144 chars]"},"uuid":"1178680b-6e3b-4d4b-9ee4-894c30881464","timestamp":"2026-10-11T20:05:57.914Z","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} +{"parentUuid":"b28e2ef4-916c-41b8-a55f-ec64f01eaece","isSidechain":true,"agentId":"aeb96cc0ddbdab294","message":{"id":"071b1f3371a476db86a8860eb750a903","type":"message","role":"assistant","content":[{"type":"text","text":"# Plan Audit — Findings\n\n## Step 1–2 — Enumerate & Draw Control Flow\n\n**Target function:** `normalize_whitespace(text: str) -> str` (new, in `sandbox/text.py`).\n\nPlanned body is a two-call pipeline (`text.strip()` then `re.sub(r'\\s+', ' ', ...)`). No `if`/`match`/`return` branches inside the function — every input traverses the same path. The audit therefore pivots to the **test file and the plan'...[fixture-truncated 2234 chars]"}],"model":"MiniMax-M3","stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":594,"cache_creation_input_tokens":0,"cache_read_input_tokens":7345,"output_tokens":3617,"output_tokens_details":{"thinking_tokens":2955},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionAgent":"autoskillit:plan-foundation-auditor","attributionSkill":"make-plan","attributionPlugin":"autoskillit","type":"assistant","uuid":"f522d9e2-3194-4afc-9b88-8a072e3ff485","timestamp":"2026-10-11T20:06:30.307Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","sessionId":"8ef8f993-8caa-47dd-83b0-26f9e5b51139","version":"2.1.280","gitBranch":"e2e-implementation-fixture-normalize-whitespace/1-3"} diff --git a/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl new file mode 100644 index 0000000000..d92ba7be4c --- /dev/null +++ b/tests/infra/fixtures/e2e_sessions/run_38170521660_attempt_1/projects/-workspace-sandbox/2292658a-34cf-44a0-a0b1-f7a20218874b.jsonl @@ -0,0 +1,23 @@ +{"type":"queue-operation","operation":"enqueue","timestamp":"2026-10-11T20:04:40.218Z","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","content":"You are an L2 food truck orchestrator. Execute the recipe 'implementation' autonomously.\nTimeout: 7200s. Campaign: 48915f19-d220-4069-8d55-b81efff84c05. Dispatch: 4666c3fd-486f-4ce1-b51f-9d9d660a5560.\n\n--- SECTION 1: ADMIRAL DISCIPLINE (L2/FOOD-TRUCK SUBSET) ---\n\n## CONTEXT LIMIT ROUTING — MANDATORY\n\nWhen `run_skill` returns `needs_retry=true` for **any step**:\n\n- **If `retry_reason: resume` AND `...[fixture-truncated 33134 chars]"} +{"type":"queue-operation","operation":"dequeue","timestamp":"2026-10-11T20:04:40.218Z","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b"} +{"parentUuid":null,"isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":"You are an L2 food truck orchestrator. Execute the recipe 'implementation' autonomously.\nTimeout: 7200s. Campaign: 48915f19-d220-4069-8d55-b81efff84c05. Dispatch: 4666c3fd-486f-4ce1-b51f-9d9d660a5560.\n\n--- SECTION 1: ADMIRAL DISCIPLINE (L2/FOOD-TRUCK SUBSET) ---\n\n## CONTEXT LIMIT ROUTING — MANDATORY\n\nWhen `run_skill` returns `needs_retry=true` for **any step**:\n\n- **If `retry_reason: resume` AND `...[fixture-truncated 33134 chars]"},"uuid":"e1be918a-cba5-4f0b-9960-60a08420f2b2","timestamp":"2026-10-11T20:04:40.246Z","permissionMode":"bypassPermissions","promptSource":"sdk","turnOrigin":"sdk","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"9aa8bfde-3315-49d7-b568-c907349a3dbd","isSidechain":false,"message":{"id":"071b1eea9dc5d3c03c383ed25b5ec610","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c9191907360b2a092bf","name":"mcp__plugin_autoskillit_autoskillit__bootstrap_clone","input":{"source_dir":"/workspace/sandbox","run_name":"impl","base_branch":"main","step_name":"clone"}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":1301,"cache_creation_input_tokens":0,"cache_read_input_tokens":42278,"output_tokens":222,"output_tokens_details":{"thinking_tokens":143},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"get_recipe_section","type":"assistant","uuid":"66199695-80b0-448d-9390-b9bcb3712e6e","timestamp":"2026-10-11T20:04:59.937Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"66199695-80b0-448d-9390-b9bcb3712e6e","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c9191907360b2a092bf","type":"tool_result","content":"## bootstrap_clone\n\nwork_dir: /workspace/autoskillit-runs/impl-20261011-200500-587134\nremote_url: https://github.com/TalonT-Org/autoskillit-e2e-sandbox.git\nbase_sha: d6c143d2494488deeb6680d7d5cf2e55e3e01732\nmerge_target: main\ntimings:\n clone_ms: 851\n rev_parse_ms: 3\n\n--- RECIPE SEGMENT ---\n{\"bodies\":[{\"body\":\"{\\\"claim_and_resolve\\\":{\\\"tool\\\":\\\"claim_and_resolve_issue\\\",\\\"with\\\":{\\\"step_name\\\":\\\"...[fixture-truncated 8153 chars]"}]},"uuid":"6cdc0d3f-fc36-426f-97c3-206e93e21f49","timestamp":"2026-10-11T20:05:01.170Z","toolUseResult":"## bootstrap_clone\n\nwork_dir: /workspace/autoskillit-runs/impl-20261011-200500-587134\nremote_url: https://github.com/TalonT-Org/autoskillit-e2e-sandbox.git\nbase_sha: d6c143d2494488deeb6680d7d5cf2e55e3e01732\nmerge_target: main\ntimings:\n clone_ms: 851\n rev_parse_ms: 3\n\n--- RECIPE SEGMENT ---\n{\"bodies\":[{\"body\":\"{\\\"claim_and_resolve\\\":{\\\"tool\\\":\\\"claim_and_resolve_issue\\\",\\\"with\\\":{\\\"step_name\\\":\\\"...[fixture-truncated 8153 chars]","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"work_dir\":\"/workspace/autoskillit-runs/impl-20261011-200500-587134\",\"remote_url\":\"https://github.com/TalonT-Org/autoskillit-e2e-sandbox.git\",\"base_sha\":\"d6c143d2494488deeb6680d7d5cf2e55e3e01732\",\"merge_target\":\"main\",\"timings\":{\"clone_ms\":851,\"rev_parse_ms\":3},\"recipe_segment\":{\"kind\":\"success\",\"source_step\":\"clone\",\"segments\":[{\"index\":1,\"name\":\"I1\"}],\"bodies\":[{\"step\":\"claim_and_resolve\",\"body\":\"{\\\"claim_and_resolve\\\":{\\\"tool\\\":\\\"claim_and_resolve_issue\\\",\\\"with\\\":{\\\"step_name\\\":\\\"claim_and_resolve\\\",\\\"issue_url\\\":\\\"${{ inputs.issue_url }}\\\",\\\"allow_reentry\\\":false},\\\"capture\\\":{\\\"issue_number\\\":\\\"${{ result.issue_number }}\\\",\\\"issue_title\\\":\\\"${{ result.issue_title }}\\\",\\\"issue_slug\\\":\\\"${{ result.issue_slug }}\\\",\\\"claimed\\\":\\\"${{ result.claimed }}\\\"},\\\"on_result\\\":[{\\\"when\\\":\\\"${{ result.claimed }} == true\\\",\\\"route\\\":\\\"create_and_publish\\\"},{\\\"route\\\":\\\"register_clone_failure\\\"}],\\\"on_failure\\\":\\\"register_clone_failure\\\",\\\"note\\\":\\\"Fetches issue title/slug and claims the issue in one turn. Returns issue_number, issue_title, issue_slug, and claimed boolean. When claimed=false (another session owns it), routes to register_clone_failure before escalate_stop so the clone directory is registered for cleanup. issue_slug and issue_number are passed to create_and_publish for branch naming.\\\\n\\\"}}\",\"body_sha256\":\"sha256:2dff84b001815660dc762ea29e4eeeb9eea19b11c77638b46aec51039509a257\"},{\"step\":\"create_and_publish\",\"body\":\"{\\\"create_and_publish\\\":{\\\"tool\\\":\\\"create_and_publish_branch\\\",\\\"with\\\":{\\\"issue_slug\\\":\\\"${{ context.issue_slug }}\\\",\\\"run_name\\\":\\\"${{ inputs.run_name }}\\\",\\\"issue_number\\\":\\\"${{ context.issue_number }}\\\",\\\"work_dir\\\":\\\"${{ context.work_dir }}\\\",\\\"remote_url\\\":\\\"${{ context.remote_url }}\\\",\\\"step_name\\\":\\\"create_and_publish\\\"},\\\"capture\\\":{\\\"merge_target\\\":\\\"${{ result.merge_target }}\\\"},\\\"on_success\\\":\\\"plan\\\",\\\"on_failure\\\":\\\"release_issue_failure\\\",\\\"note\\\":\\\"Computes branch name, creates it (with collision suffix if needed), and pushes to origin in one turn. Overrides merge_target with the feature branch name. merge_worktree requires refs/remotes/origin/{branch} to exist after fetch (step 5.5). When open_pr=false, this step is skipped and merge_target remains as base_branch.\\\\n\\\"}}\",\"body_sha256\":\"sha256:f102a40c0ec99f47038934ff163d56cee9d5df486246d7575221ce2c6a89fcf1\"},{\"step\":\"plan\",\"body\":\"{\\\"plan\\\":{\\\"tool\\\":\\\"run_skill\\\",\\\"model\\\":\\\"\\\",\\\"with\\\":{\\\"skill_command\\\":\\\"/autoskillit:make-plan\\\",\\\"skill_inputs\\\":{\\\"adversarial_review_level\\\":\\\"auto\\\",\\\"task\\\":\\\"${{ inputs.task }}\\\",\\\"issue_url\\\":\\\"${{ inputs.issue_url }}\\\",\\\"audit_cycle_path\\\":\\\"${{ context.audit_cycle_path }}\\\",\\\"plan_set_coverage_gaps\\\":\\\"${{ context.plan_set_coverage_gaps }}\\\"},\\\"cwd\\\":\\\"${{ context.work_dir }}\\\",\\\"step_name\\\":\\\"plan\\\"},\\\"capture\\\":{\\\"plan_path\\\":\\\"${{ result.plan_path }}\\\",\\\"all_plan_paths\\\":\\\"${{ result.plan_path }}\\\",\\\"verdict\\\":\\\"${{ result.verdict }}\\\",\\\"plan_disposition_path\\\":\\\"${{ result.plan_disposition_path }}\\\"},\\\"capture_list\\\":{\\\"plan_parts\\\":\\\"${{ result.plan_parts }}\\\"},\\\"retries\\\":0,\\\"on_result\\\":[{\\\"when\\\":\\\"${{ result.verdict }} == plan\\\",\\\"route\\\":\\\"bind_plan_set\\\"}],\\\"on_failure\\\":\\\"release_issue_failure\\\",\\\"on_rate_limit\\\":\\\"release_issue_failure\\\",\\\"on_context_limit\\\":\\\"salvage_plan\\\",\\\"optional_context_refs\\\":[\\\"audit_cycle_path\\\",\\\"plan_set_coverage_gaps\\\"],\\\"note\\\":\\\"Produces an ordinary plan when no audit_cycle_path is present. On NO GO re-entry, consumes that exact current authority and emits both plan_path and plan_disposition_path. all_plan_paths appends result.plan_path across re-entries; verdict=plan is the only successful result.\\\"}}\",\"body_sha256\":\"sha256:c44e11b44d35bcdf9620f407c17b7936a9b47a1f348e3f377cfd7b3ae9c4765a\",\"invocation_template_digest\":\"sha256:deec5b08a76bdabe0f7f520b9bb66f78789645f615012ac8eaf24806bef8e05e\"},{\"step\":\"salvage_plan\",\"body\":\"{\\\"salvage_plan\\\":{\\\"tool\\\":\\\"run_python\\\",\\\"with\\\":{\\\"step_name\\\":\\\"salvage_plan\\\",\\\"callable\\\":\\\"autoskillit.recipe._cmd_rpc.verify_plan_artifacts\\\",\\\"plan_parts\\\":\\\"${{ context.plan_parts }}\\\",\\\"audit_cycle_path\\\":\\\"${{ context.audit_cycle_path }}\\\"},\\\"capture\\\":{\\\"plan_path\\\":\\\"${{ result.plan_path }}\\\",\\\"plan_disposition_path\\\":\\\"${{ result.plan_disposition_path }}\\\",\\\"all_plan_paths\\\":\\\"${{ result.plan_path }}\\\"},\\\"capture_list\\\":{\\\"plan_parts\\\":\\\"${{ result.plan_parts }}\\\"},\\\"optional_context_refs\\\":[\\\"audit_cycle_path\\\"],\\\"retries\\\":0,\\\"on_result\\\":[{\\\"when\\\":\\\"${{ result.verdict }} == salvaged\\\",\\\"route\\\":\\\"bind_plan_set\\\"},{\\\"route\\\":\\\"release_issue_failure\\\"}],\\\"on_failure\\\":\\\"release_issue_failure\\\",\\\"note\\\":\\\"Deterministic salvage for plan context-limit stumbles: verifies captured plan_parts artifacts exist on disk. salvaged routes onward as if plan succeeded; unsalvageable falls through to plan's original on_failure destination. Substitute the captured ${{ context.plan_parts }} value into the plan_parts argument verbatim, unmodified — do not reformat, quote, or re-derive it; the callable's input gramm...[fixture-truncated 30 chars]\\\"}}\",\"body_sha256\":\"sha256:08f1e03e1927e09d6d304a2925760a2215f0f78b768fb856ab0261f695558d1f\"},{\"step\":\"bind_plan_set\",\"body\":\"{\\\"bind_plan_set\\\":{\\\"tool\\\":\\\"bind_plan_set\\\",\\\"with\\\":{\\\"plan_parts\\\":\\\"${{ context.plan_parts }}\\\",\\\"cwd\\\":\\\"${{ context.work_dir }}\\\",\\\"issue_url\\\":\\\"${{ inputs.issue_url }}\\\",\\\"seal\\\":\\\"true\\\",\\\"step_name\\\":\\\"bind_plan_set\\\"},\\\"capture\\\":{\\\"plan_set_authority_path\\\":\\\"${{ result.plan_set_authority_path }}\\\",\\\"plan_set_authority_digest\\\":\\\"${{ result.plan_set_authority_digest }}\\\",\\\"plan_set_parts\\\":\\\"${{ result.plan_set_parts }}\\\",\\\"plan_set_coverage_gaps\\\":\\\"${{ result.coverage_gaps }}\\\"},\\\"on_result\\\":[{\\\"when\\\":\\\"${{ result.success }} == true\\\",\\\"route\\\":\\\"verify\\\"},{\\\"when\\\":\\\"${{ result.reason }} == coverage_failed\\\",\\\"route\\\":\\\"check_replan_iteration\\\"},{\\\"route\\\":\\\"release_issue_failure\\\"}],\\\"on_failure\\\":\\\"release_issue_failure\\\",\\\"note\\\":\\\"Bind exactly the plan_parts value captured from the producer that just ran. A successful replan replaces the earlier list with its returned list: do not append rejected paths or rebuild the list from a directory listing. When salvage_plan is the producer, bind its returned list unchanged under the existing salvage recovery and strict coverage checks. Aggregate plan-set gate before every walkthroug...[fixture-truncated 277 chars]\\\"}}\",\"body_sha256\":\"sha256:ac7464747141373f7d61c06815e6021b52cda0a41170cde8e83296541e2229e6\"},{\"step\":\"check_replan_iteration\",\"body\":\"{\\\"check_replan_iteration\\\":{\\\"tool\\\":\\\"run_python\\\",\\\"with\\\":{\\\"step_name\\\":\\\"check_replan_iteration\\\",\\\"callable\\\":\\\"autoskillit.smoke_utils.check_loop_iteration\\\",\\\"current_iteration\\\":\\\"${{ context.plan_set_replan_count }}\\\",\\\"max_iterations\\\":\\\"2\\\"},\\\"capture\\\":{\\\"plan_set_replan_count\\\":\\\"${{ result.next_iteration }}\\\"},\\\"optional_context_refs\\\":[\\\"plan_set_replan_count\\\"],\\\"on_result\\\":[{\\\"when\\\":\\\"${{ result.max_exceeded }} == true\\\",\\\"route\\\":\\\"release_issue_failure\\\"},{\\\"route\\\":\\\"plan\\\"}],\\\"on_failure\\\":\\\"release_issue_failure\\\"}}\",\"body_sha256\":\"sha256:6ef878d992680fbac602153f1c0ac829f8a5c0657be7718c4426c2d25812f9c3\"}],\"recipe_execution\":{\"execution_id\":\"1dda3d6d29e545cbae216029ec657356\",\"invocation_template_digests\":{\"plan\":\"sha256:deec5b08a76bdabe0f7f520b9bb66f78789645f615012ac8eaf24806bef8e05e\"},\"skill_input_shapes\":{\"plan\":{\"keys\":[\"task\",\"issue_url\",\"adversarial_review_level\",\"audit_cycle_path\",\"plan_set_coverage_gaps\"],\"absence_values\":{\"audit_cycle_path\":\"\",\"plan_set_coverage_gaps\":\"\"}}},\"snapshot_digest\":\"sha256:1269b37298577647aec24baeb183326c56ad0d0bf2fecbd0231b0ad6e5b81fba\"},\"pull_closures\":[],\"recipe_pull\":{\"producer_tool\":\"open_kitchen\",\"recipe_name\":\"implementation\",\"descriptor_version\":2,\"schema_version\":2,\"payload_sha256\":\"sha256:ea2e62f28435745f27a7a742cbb98d2e34cdb00d82eef53117118c3b574d9497\",\"artifact_blob_sha256\":\"sha256:cd5e7fff1ca8145532aa9ce8ca2a26ddbfe48fb704316310789b8123319f8d63\",\"artifact_blob_size_bytes\":576427,\"body_sha256\":\"sha256:3f14e54f1505f8a5c4d13c2bc5658394855e4c7dc624cc527ddb4bf0711954d5\",\"body_size_bytes\":115869,\"flow_schema_version\":2,\"flow_sha256\":\"sha256:3b9ff899035aad7555d5863de6c70c6ed95e6ffac5f19cea0c46cc17261f4f2a\",\"flow_size_bytes\":101558,\"flow_record_count\":694,\"pull_tool\":\"get_recipe_section\"}}}"}},"sourceToolAssistantUUID":"66199695-80b0-448d-9390-b9bcb3712e6e","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"0b144546-f16f-4dd1-94b8-87b4f092a3e6","isSidechain":false,"message":{"id":"071b1eedfe3e75016da0bd16d2a6ae53","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c919df971e2b8b09c37","name":"mcp__plugin_autoskillit_autoskillit__claim_and_resolve_issue","input":{"issue_url":"https://github.com/TalonT-Org/autoskillit-e2e-sandbox/issues/1","allow_reentry":false,"step_name":"claim_and_resolve"}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":2643,"cache_creation_input_tokens":0,"cache_read_input_tokens":43579,"output_tokens":218,"output_tokens_details":{"thinking_tokens":124},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"bootstrap_clone","type":"assistant","uuid":"e8638011-011a-4b7d-8a7f-a09899915dfb","timestamp":"2026-10-11T20:05:03.120Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"e8638011-011a-4b7d-8a7f-a09899915dfb","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c919df971e2b8b09c37","type":"tool_result","content":"## claim_and_resolve_issue\n\nsuccess: True\nclaimed: True\nlabel: in-progress\nreview_approach_recommended: False\ninvestigation_complete: False\nissue_number: 1\nissue_title: E2E implementation fixture: normalize whitespace\nissue_slug: e2e-implementation-fixture-normalize-whitespace\ntimings:\n fetch_title_ms: 359\n claim_ms: 2475\n\n--- RECIPE SEGMENT ---\n{\"bodies\":[],\"kind\":\"success\",\"pull_closures\":[],\"...[fixture-truncated 943 chars]"}]},"uuid":"b689f776-8c6f-4831-bd03-e33b6ded7492","timestamp":"2026-10-11T20:05:06.239Z","toolUseResult":"## claim_and_resolve_issue\n\nsuccess: True\nclaimed: True\nlabel: in-progress\nreview_approach_recommended: False\ninvestigation_complete: False\nissue_number: 1\nissue_title: E2E implementation fixture: normalize whitespace\nissue_slug: e2e-implementation-fixture-normalize-whitespace\ntimings:\n fetch_title_ms: 359\n claim_ms: 2475\n\n--- RECIPE SEGMENT ---\n{\"bodies\":[],\"kind\":\"success\",\"pull_closures\":[],\"...[fixture-truncated 943 chars]","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"success\":true,\"claimed\":true,\"label\":\"in-progress\",\"review_approach_recommended\":false,\"investigation_complete\":false,\"issue_number\":1,\"issue_title\":\"E2E implementation fixture: normalize whitespace\",\"issue_slug\":\"e2e-implementation-fixture-normalize-whitespace\",\"timings\":{\"fetch_title_ms\":359,\"claim_ms\":2475},\"recipe_segment\":{\"kind\":\"success\",\"source_step\":\"claim_and_resolve\",\"segments\":[],\"bodies\":[],\"recipe_execution\":{\"execution_id\":\"1dda3d6d29e545cbae216029ec657356\",\"invocation_template_digests\":{},\"skill_input_shapes\":{},\"snapshot_digest\":\"sha256:1269b37298577647aec24baeb183326c56ad0d0bf2fecbd0231b0ad6e5b81fba\"},\"pull_closures\":[],\"recipe_pull\":{\"producer_tool\":\"open_kitchen\",\"recipe_name\":\"implementation\",\"descriptor_version\":2,\"schema_version\":2,\"payload_sha256\":\"sha256:ea2e62f28435745f27a7a742cbb98d2e34cdb00d82eef53117118c3b574d9497\",\"artifact_blob_sha256\":\"sha256:cd5e7fff1ca8145532aa9ce8ca2a26ddbfe48fb704316310789b8123319f8d63\",\"artifact_blob_size_bytes\":576427,\"body_sha256\":\"sha256:3f14e54f1505f8a5c4d13c2bc5658394855e4c7dc624cc527ddb4bf0711954d5\",\"body_size_bytes\":115869,\"flow_schema_version\":2,\"flow_sha256\":\"sha256:3b9ff899035aad7555d5863de6c70c6ed95e6ffac5f19cea0c46cc17261f4f2a\",\"flow_size_bytes\":101558,\"flow_record_count\":694,\"pull_tool\":\"get_recipe_section\"}}}"}},"sourceToolAssistantUUID":"e8638011-011a-4b7d-8a7f-a09899915dfb","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"09cc0463-fc85-4985-ae35-09732330b279","isSidechain":false,"message":{"id":"071b1ef27e5cad826dc3ce73f99787ac","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c91b227749fbd03b2b0","name":"mcp__plugin_autoskillit_autoskillit__create_and_publish_branch","input":{"issue_slug":"e2e-implementation-fixture-normalize-whitespace","run_name":"impl","issue_number":"1","work_dir":"/workspace/autoskillit-runs/impl-20261011-200500-587134","remote_url":"https://github.com/TalonT-Org/autoskillit-e2e-sandbox.git","step_name":"create_and_publish"}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":737,"cache_creation_input_tokens":0,"cache_read_input_tokens":46222,"output_tokens":182,"output_tokens_details":{"thinking_tokens":32},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"claim_and_resolve_issue","type":"assistant","uuid":"fd55e15c-8988-49ee-8a19-20eff2eb5919","timestamp":"2026-10-11T20:05:08.278Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"fd55e15c-8988-49ee-8a19-20eff2eb5919","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c91b227749fbd03b2b0","type":"tool_result","content":"## create_and_publish_branch\n\nmerge_target: e2e-implementation-fixture-normalize-whitespace/1-3\nwas_unique: False\ntimings:\n compute_ms: 0\n branch_create_ms: 1131\n push_ms: 1068"}]},"uuid":"d5b2d789-548b-4f26-be64-599540dab17e","timestamp":"2026-10-11T20:05:10.770Z","toolUseResult":"## create_and_publish_branch\n\nmerge_target: e2e-implementation-fixture-normalize-whitespace/1-3\nwas_unique: False\ntimings:\n compute_ms: 0\n branch_create_ms: 1131\n push_ms: 1068","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"merge_target\":\"e2e-implementation-fixture-normalize-whitespace/1-3\",\"was_unique\":false,\"timings\":{\"compute_ms\":0,\"branch_create_ms\":1131,\"push_ms\":1068}}"}},"sourceToolAssistantUUID":"fd55e15c-8988-49ee-8a19-20eff2eb5919","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"914bc97f-052b-46b7-ad36-0fa7d53b614f","isSidechain":false,"message":{"id":"071b1efbdcae033e3bee1954f2c790c3","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c91d9517369b630b48c","name":"mcp__plugin_autoskillit_autoskillit__run_skill","input":{"skill_command":"/autoskillit:make-plan","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","step_name":"plan","skill_inputs":{"adversarial_review_level":"auto","task":"E2E implementation fixture: normalize whitespace\n\n# Reusable implementation E2E fixture\n\nAdd `normalize_whitespace(text: str) -> str` to `sandbox/text.py`.\n\nThe function must remove leading and trailing whitespace and replace each run of\nwhitespace (including spaces, tabs, and newlines) with one ASCII space. Preserve\nthe case and punctuation of non-whitespace characters. Empty and whitespace-only\n...[fixture-truncated 737 chars]","issue_url":"https://github.com/TalonT-Org/autoskillit-e2e-sandbox/issues/1","audit_cycle_path":"","plan_set_coverage_gaps":""}}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":209,"cache_creation_input_tokens":0,"cache_read_input_tokens":47866,"output_tokens":435,"output_tokens_details":{"thinking_tokens":9},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"record_pipeline_step","type":"assistant","uuid":"25cae19a-d6ea-4f11-a8b3-c8937bd543b9","timestamp":"2026-10-11T20:05:18.314Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"25cae19a-d6ea-4f11-a8b3-c8937bd543b9","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c91d9517369b630b48c","type":"tool_result","content":"run_skill: FAIL [FAIL]\nexit_code: \nneeds_retry: False\nerror: RECIPE EXECUTION REJECTED [recipe_execution_attestation_missing]: an active recipe requires recipe_execution_id and invocation_template_digest; take both from the recipe_execution block of the complete_recipe_initialization receipt (bounded delivery) or of the open_kitchen response (inline delivery), using invocation_template_digests[ste...[fixture-truncated 274 chars]"}]},"uuid":"3936d669-1f07-41d4-bc84-3770955f1e04","timestamp":"2026-10-11T20:05:18.870Z","toolUseResult":"run_skill: FAIL [FAIL]\nexit_code: \nneeds_retry: False\nerror: RECIPE EXECUTION REJECTED [recipe_execution_attestation_missing]: an active recipe requires recipe_execution_id and invocation_template_digest; take both from the recipe_execution block of the complete_recipe_initialization receipt (bounded delivery) or of the open_kitchen response (inline delivery), using invocation_template_digests[ste...[fixture-truncated 274 chars]","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"success\":false,\"is_error\":true,\"error\":\"RECIPE EXECUTION REJECTED [recipe_execution_attestation_missing]: an active recipe requires recipe_execution_id and invocation_template_digest; take both from the recipe_execution block of the complete_recipe_initialization receipt (bounded delivery) or of the open_kitchen response (inline delivery), using invocation_template_digests[step_name] for this step; structured calls must initialize skill...[fixture-truncated 213 chars]\",\"stage\":\"preflight:recipe_execution\",\"retriable\":false}"}},"sourceToolAssistantUUID":"25cae19a-d6ea-4f11-a8b3-c8937bd543b9","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"02c03783-dcd5-4563-8311-e06aa7b8c603","isSidechain":false,"message":{"id":"071b1efe8d9be3c62f44b3fd56e52baa","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c91e6c07518b24e8d31","name":"mcp__plugin_autoskillit_autoskillit__run_skill","input":{"skill_command":"/autoskillit:make-plan","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","step_name":"plan","recipe_execution_id":"1dda3d6d29e545cbae216029ec657356","invocation_template_digest":"sha256:deec5b08a76bdabe0f7f520b9bb66f78789645f615012ac8eaf24806bef8e05e","skill_inputs":{"adversarial_review_level":"auto","task":"E2E implementation fixture: normalize whitespace\n\n# Reusable implementation E2E fixture\n\nAdd `normalize_whitespace(text: str) -> str` to `sandbox/text.py`.\n\nThe function must remove leading and trailing whitespace and replace each run of\nwhitespace (including spaces, tabs, and newlines) with one ASCII space. Preserve\nthe case and punctuation of non-whitespace characters. Empty and whitespace-only\n...[fixture-truncated 737 chars]","issue_url":"https://github.com/TalonT-Org/autoskillit-e2e-sandbox/issues/1","audit_cycle_path":"","plan_set_coverage_gaps":""}}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":630,"cache_creation_input_tokens":0,"cache_read_input_tokens":48075,"output_tokens":607,"output_tokens_details":{"thinking_tokens":94},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"run_skill","type":"assistant","uuid":"ef71da75-f7f0-44ec-a940-f1b9c8affbba","timestamp":"2026-10-11T20:05:21.760Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"ef71da75-f7f0-44ec-a940-f1b9c8affbba","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c91e6c07518b24e8d31","type":"tool_result","content":"run_skill: OK [success]\nsession_id: 8ef8f993-8caa-47dd-83b0-26f9e5b51139\nreceipt_id: 0739af7533c34089af07c982a53a62c1\nexit_code: 0 (infra-terminated after completion — grace exceeded)\nneeds_retry: False\noutcome_fields: {'plan_parts': '/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md', 'plan_path': '/workspace/autoski...[fixture-truncated 444 chars]"}]},"uuid":"bd81bead-a8ba-4a75-8388-a924aadc126d","timestamp":"2026-10-11T20:07:17.201Z","toolUseResult":"run_skill: OK [success]\nsession_id: 8ef8f993-8caa-47dd-83b0-26f9e5b51139\nreceipt_id: 0739af7533c34089af07c982a53a62c1\nexit_code: 0 (infra-terminated after completion — grace exceeded)\nneeds_retry: False\noutcome_fields: {'plan_parts': '/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md', 'plan_path': '/workspace/autoski...[fixture-truncated 444 chars]","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"success\":true,\"result\":\"verdict = plan\\nplan_path = /workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md\\nplan_parts = /workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md\",\"session_id\":\"8ef8f993-8caa-47dd-83b0-26f9e5b51139\",\"subtype\":\"success\",\"cli_subtype\":\"success\",\"is_error\":false,\"exit_code\":0,\"kill_reason\":\"kill_after_completion\",\"needs_retry\":false,\"retry_reason\":\"none\",\"stderr\":\"\",\"token_usage\":{\"backend\":\"claude-code\",\"provider_used\":\"anthropic\",\"input_tokens\":{\"state\":\"measured\",\"value\":54534},\"output_tokens\":{\"state\":\"measured\",\"value\":8817},\"cache_write_tokens\":{\"state\":\"measured_zero\",\"value\":0},\"cache_read_tokens\":{\"state\":\"measured\",\"value\":405101},\"model_breakdown\":{\"MiniMax-M3\":{\"input_tokens\":{\"state\":\"measured_zero\",\"value\":0},\"output_tokens\":{\"state\":\"measured_zero\",\"value\":0},\"cache_read_tokens\":{\"state\":\"unknown\",\"value\":null},\"cache_write_tokens\":{\"state\":\"unknown\",\"value\":null}}},\"peak_context\":{\"state\":\"unknown\",\"value\":null},\"turn_count\":10},\"write_path_warnings\":[],\"write_call_count\":1,\"fs_writes_detected\":true,\"git_writes_detected\":false,\"file_changes_count\":0,\"has_progress_evidence\":true,\"has_implementation_progress\":true,\"completion_required\":true,\"outcome_fields\":{\"plan_parts\":\"/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md\",\"plan_path\":\"/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md\",\"verdict\":\"plan\"},\"outcome_invariant_violated\":false,\"outcome_qualifier\":null,\"adjudication_verdict\":null,\"last_stop_reason\":\"\",\"lifespan_started\":true,\"provider_fallback\":false,\"provider_used\":\"anthropic\",\"infra_exit_category\":\"completed\",\"infra_cleanup_incomplete\":false,\"infra_fault_domain\":\"logic\",\"api_retry_count\":0,\"api_retry_last_error\":\"\",\"api_retry_last_status\":null,\"api_retry_exhausted\":false,\"api_error_status\":null,\"api_terminal_reason\":\"completed\",\"api_error_code\":\"\",\"api_error_message_seen\":false,\"candidate_exhausted\":false,\"execution_selection\":{\"selection_id\":\"838046e6cee249598edf9a01cc35eb1e\",\"campaign_id\":\"48915f19-d220-4069-8d55-b81efff84c05\",\"attempts\":[{\"candidate_id\":\"838046e6cee249598edf9a01cc35eb1e:0\",\"ordinal\":0,\"attempt\":1,\"parent_backend\":\"claude-code\",\"parent_provider\":\"anthropic\",\"parent_model\":\"sonnet\",\"requested_backend\":\"\",\"requested_provider\":\"\",\"requested_model\":\"\",\"effective_backend\":\"claude-code\",\"effective_provider\":\"anthropic\",\"effective_model\":\"sonnet\",\"backend_source_path\":\"agent_backend.backend\",\"provider_source_path\":\"backend.native_provider\",\"model_source_path\":\"backend.claude-code.model\",\"admission_status\":\"started\",\"admission_at_epoch\":1791749122,\"admission_cache_status\":\"quota_guard_disabled\",\"admission_cache_age_seconds\":null,\"rate_limit_status\":\"\",\"rate_limit_type\":\"\",\"rate_limit_resets_at_epoch\":null,\"rejection_reason\":null,\"transition\":\"primary\",\"execution_started\":true,\"child_session_id\":\"8ef8f993-8caa-47dd-83b0-26f9e5b51139\"}],\"terminal_candidate_id\":\"838046e6cee249598edf9a01cc35eb1e:0\",\"manifest_ref\":\"execution-candidates/838046e6cee249598edf9a01cc35eb1e.json\",\"invocation_deadline_epoch\":1791756276,\"remaining_retry_budget\":2,\"backend_rerouted\":false,\"candidate_fallback\":false,\"provider_fallback\":false,\"continuation\":null,\"completed\":true},\"rate_limit_status\":\"\",\"rate_limit_type\":\"\",\"rate_limit_resets_at_epoch\":null,\"pre_contamination_retry_reason\":\"none\",\"pre_contamination_subtype\":\"\",\"ndjson_unknown_event_count\":0,\"ndjson_unknown_item_count\":0,\"session_errors\":[],\"audit_status\":null,\"audit_verdict\":null,\"audit_cycle_path\":null,\"audit_attempt_id\":null,\"execution_identity\":{\"requested_parent_backend\":\"claude-code\",\"effective_parent_backend\":\"\",\"requested_parent_model\":\"sonnet\",\"effective_parent_model\":\"\",\"requested_parent_effort\":\"\",\"effective_parent_effort\":\"\",\"cli_version\":\"\",\"override_tier\":\"\",\"override_key_path\":\"\",\"parent_session_id\":\"\",\"children\":[]},\"order_id\":\"4666c3fd-486f-4ce1-b51f-9d9d660a5560\",\"receipt_id\":\"0739af7533c34089af07c982a53a62c1\"}"}},"sourceToolAssistantUUID":"ef71da75-f7f0-44ec-a940-f1b9c8affbba","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"9583a00d-f2bf-4f54-9e39-859d6ac3e38d","isSidechain":false,"message":{"id":"071b1f7556fb87c19d84e09c50262e1d","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c93aed67125a3a7504e","name":"mcp__plugin_autoskillit_autoskillit__complete_run_skill_result","input":{"receipt_id":"0739af7533c34089af07c982a53a62c1"}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":959,"cache_creation_input_tokens":0,"cache_read_input_tokens":48705,"output_tokens":89,"output_tokens_details":{"thinking_tokens":24},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"run_skill","type":"assistant","uuid":"fd7e9b53-087c-4247-9289-e909ff5fd4e5","timestamp":"2026-10-11T20:07:18.500Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"fd7e9b53-087c-4247-9289-e909ff5fd4e5","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c93aed67125a3a7504e","type":"tool_result","content":"## complete_run_skill_result\n\nsuccess: True\nreceipt_id: 0739af7533c34089af07c982a53a62c1\nrun_skill_success: True\nclassification: success\nsession_id: 8ef8f993-8caa-47dd-83b0-26f9e5b51139\ntracker:\n success: True\n incarnation_matches: True\n step: plan\n order_id: 4666c3fd-486f-4ce1-b51f-9d9d660a5560\n status: complete\n pipeline_id: 4666c3fd-486f-4ce1-b51f-9d9d660a5560\n done: 1\n total: 135\ntrack...[fixture-truncated 1025 chars]"}]},"uuid":"ac39e42b-7c63-4205-b7c9-a61025b1b4d6","timestamp":"2026-10-11T20:07:18.890Z","toolUseResult":"## complete_run_skill_result\n\nsuccess: True\nreceipt_id: 0739af7533c34089af07c982a53a62c1\nrun_skill_success: True\nclassification: success\nsession_id: 8ef8f993-8caa-47dd-83b0-26f9e5b51139\ntracker:\n success: True\n incarnation_matches: True\n step: plan\n order_id: 4666c3fd-486f-4ce1-b51f-9d9d660a5560\n status: complete\n pipeline_id: 4666c3fd-486f-4ce1-b51f-9d9d660a5560\n done: 1\n total: 135\ntrack...[fixture-truncated 1025 chars]","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"success\":true,\"receipt_id\":\"0739af7533c34089af07c982a53a62c1\",\"run_skill_success\":true,\"classification\":\"success\",\"session_id\":\"8ef8f993-8caa-47dd-83b0-26f9e5b51139\",\"tracker\":{\"success\":true,\"incarnation_matches\":true,\"step\":\"plan\",\"order_id\":\"4666c3fd-486f-4ce1-b51f-9d9d660a5560\",\"status\":\"complete\",\"pipeline_id\":\"4666c3fd-486f-4ce1-b51f-9d9d660a5560\",\"done\":1,\"total\":135},\"tracker_repairable\":false,\"recipe_segment\":{\"kind\":\"success\",\"source_step\":\"plan\",\"segments\":[],\"bodies\":[],\"recipe_execution\":{\"execution_id\":\"1dda3d6d29e545cbae216029ec657356\",\"invocation_template_digests\":{},\"skill_input_shapes\":{},\"snapshot_digest\":\"sha256:1269b37298577647aec24baeb183326c56ad0d0bf2fecbd0231b0ad6e5b81fba\"},\"pull_closures\":[],\"recipe_pull\":{\"producer_tool\":\"open_kitchen\",\"recipe_name\":\"implementation\",\"descriptor_version\":2,\"schema_version\":2,\"payload_sha256\":\"sha256:ea2e62f28435745f27a7a742cbb98d2e34cdb00d82eef53117118c3b574d9497\",\"artifact_blob_sha256\":\"sha256:cd5e7fff1ca8145532aa9ce8ca2a26ddbfe48fb704316310789b8123319f8d63\",\"artifact_blob_size_bytes\":576427,\"body_sha256\":\"sha256:3f14e54f1505f8a5c4d13c2bc5658394855e4c7dc624cc527ddb4bf0711954d5\",\"body_size_bytes\":115869,\"flow_schema_version\":2,\"flow_sha256\":\"sha256:3b9ff899035aad7555d5863de6c70c6ed95e6ffac5f19cea0c46cc17261f4f2a\",\"flow_size_bytes\":101558,\"flow_record_count\":694,\"pull_tool\":\"get_recipe_section\"}}}"}},"sourceToolAssistantUUID":"fd7e9b53-087c-4247-9289-e909ff5fd4e5","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"ce90af08-dd2d-4343-a661-8c8b229e79c6","isSidechain":false,"message":{"id":"071b20897697dd2a864edf9dfcc2d180","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c97eda97598bed1c282","name":"mcp__plugin_autoskillit_autoskillit__run_skill","input":{"skill_command":"/autoskillit:dry-walkthrough","cwd":"/workspace/autoskillit-runs/impl-20261011-200500-587134","step_name":"verify","recipe_execution_id":"1dda3d6d29e545cbae216029ec657356","invocation_template_digest":"sha256:7447506c5306026f763953953af00ee3cf4a723f01b786bf9aa989751756c5cf","output_dir":".autoskillit/temp","skill_inputs":{"plan_path":"/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/make-plan/normalize_whitespace_plan_2026-10-11_200500.md","issue_url":"https://github.com/TalonT-Org/autoskillit-e2e-sandbox/issues/1","review_path":"","audit_cycle_path":"","plan_disposition_path":"","plan_set_authority_path":"/workspace/autoskillit-runs/impl-20261011-200500-587134/.autoskillit/temp/plan-set-authority/planset-407e119e8ff1c06eeaf93992/0c6efb4faa404ec4fb97f4c0.json"}}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":3378,"cache_creation_input_tokens":0,"cache_read_input_tokens":160144,"output_tokens":534,"output_tokens_details":{"thinking_tokens":131},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"run_python","type":"assistant","uuid":"b02275c1-d87d-4a7b-a28e-c87b020aac8d","timestamp":"2026-10-11T20:11:56.730Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"b02275c1-d87d-4a7b-a28e-c87b020aac8d","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c97eda97598bed1c282","type":"tool_result","content":"run_skill: FAIL [zero_writes]\nsession_id: 54f18144-1188-45d7-a392-17b4529f721e\nreceipt_id: 18af52568f97412b9eecc304d8a75e7c\nexit_code: 0\nneeds_retry: True\nretry_reason: zero_writes\nhas_progress_evidence: True\nadjudication_verdict: {'reason_kind': 'zero_writes', 'subtype': 'zero_writes', 'detail': 'expected implementation writes were not observed', 'outcome_fields': None, 'defects': []}\n\nresult:\nex...[fixture-truncated 46 chars]"}]},"uuid":"e945da13-1cc3-48a0-9ce5-06578b883602","timestamp":"2026-10-11T20:14:47.218Z","toolUseResult":"run_skill: FAIL [zero_writes]\nsession_id: 54f18144-1188-45d7-a392-17b4529f721e\nreceipt_id: 18af52568f97412b9eecc304d8a75e7c\nexit_code: 0\nneeds_retry: True\nretry_reason: zero_writes\nhas_progress_evidence: True\nadjudication_verdict: {'reason_kind': 'zero_writes', 'subtype': 'zero_writes', 'detail': 'expected implementation writes were not observed', 'outcome_fields': None, 'defects': []}\n\nresult:\nex...[fixture-truncated 46 chars]","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"success\":false,\"result\":\"expected implementation writes were not observed\",\"session_id\":\"54f18144-1188-45d7-a392-17b4529f721e\",\"subtype\":\"zero_writes\",\"cli_subtype\":\"success\",\"is_error\":false,\"exit_code\":0,\"kill_reason\":\"natural_exit\",\"needs_retry\":true,\"retry_reason\":\"zero_writes\",\"stderr\":\"\",\"token_usage\":{\"backend\":\"claude-code\",\"provider_used\":\"anthropic\",\"input_tokens\":{\"state\":\"measured\",\"value\":48030},\"output_tokens\":{\"state\":\"measured\",\"value\":4349},\"cache_write_tokens\":{\"state\":\"measured_zero\",\"value\":0},\"cache_read_tokens\":{\"state\":\"measured\",\"value\":166797},\"model_breakdown\":{\"MiniMax-M3\":{\"input_tokens\":{\"state\":\"measured_zero\",\"value\":0},\"output_tokens\":{\"state\":\"measured_zero\",\"value\":0},\"cache_read_tokens\":{\"state\":\"unknown\",\"value\":null},\"cache_write_tokens\":{\"state\":\"unknown\",\"value\":null}}},\"peak_context\":{\"state\":\"unknown\",\"value\":null},\"turn_count\":5},\"write_path_warnings\":[],\"write_call_count\":0,\"fs_writes_detected\":true,\"git_writes_detected\":false,\"file_changes_count\":0,\"has_progress_evidence\":true,\"has_implementation_progress\":false,\"completion_required\":false,\"outcome_fields\":null,\"outcome_invariant_violated\":false,\"outcome_qualifier\":null,\"adjudication_verdict\":{\"reason_kind\":\"zero_writes\",\"subtype\":\"zero_writes\",\"detail\":\"expected implementation writes were not observed\",\"outcome_fields\":null,\"defects\":[]},\"last_stop_reason\":\"\",\"lifespan_started\":true,\"provider_fallback\":false,\"provider_used\":\"anthropic\",\"infra_exit_category\":\"completed\",\"infra_cleanup_incomplete\":false,\"infra_fault_domain\":\"logic\",\"api_retry_count\":0,\"api_retry_last_error\":\"\",\"api_retry_last_status\":null,\"api_retry_exhausted\":false,\"api_error_status\":null,\"api_terminal_reason\":\"completed\",\"api_error_code\":\"\",\"api_error_message_seen\":false,\"candidate_exhausted\":false,\"execution_selection\":{\"selection_id\":\"77495ee8add84a1b8334155a172ef192\",\"campaign_id\":\"48915f19-d220-4069-8d55-b81efff84c05\",\"attempts\":[{\"candidate_id\":\"77495ee8add84a1b8334155a172ef192:0\",\"ordinal\":0,\"attempt\":1,\"parent_backend\":\"claude-code\",\"parent_provider\":\"anthropic\",\"parent_model\":\"sonnet\",\"requested_backend\":\"\",\"requested_provider\":\"\",\"requested_model\":\"\",\"effective_backend\":\"claude-code\",\"effective_provider\":\"anthropic\",\"effective_model\":\"sonnet\",\"backend_source_path\":\"agent_backend.backend\",\"provider_source_path\":\"backend.native_provider\",\"model_source_path\":\"backend.claude-code.model\",\"admission_status\":\"started\",\"admission_at_epoch\":1791749517,\"admission_cache_status\":\"quota_guard_disabled\",\"admission_cache_age_seconds\":null,\"rate_limit_status\":\"\",\"rate_limit_type\":\"\",\"rate_limit_resets_at_epoch\":null,\"rejection_reason\":null,\"transition\":\"primary\",\"execution_started\":true,\"child_session_id\":\"54f18144-1188-45d7-a392-17b4529f721e\"}],\"terminal_candidate_id\":\"77495ee8add84a1b8334155a172ef192:0\",\"manifest_ref\":\"execution-candidates/77495ee8add84a1b8334155a172ef192.json\",\"invocation_deadline_epoch\":1791756276,\"remaining_retry_budget\":2,\"backend_rerouted\":false,\"candidate_fallback\":false,\"provider_fallback\":false,\"continuation\":null,\"completed\":true},\"rate_limit_status\":\"\",\"rate_limit_type\":\"\",\"rate_limit_resets_at_epoch\":null,\"pre_contamination_retry_reason\":\"none\",\"pre_contamination_subtype\":\"\",\"ndjson_unknown_event_count\":0,\"ndjson_unknown_item_count\":0,\"session_errors\":[],\"audit_status\":null,\"audit_verdict\":null,\"audit_cycle_path\":null,\"audit_attempt_id\":null,\"execution_identity\":{\"requested_parent_backend\":\"claude-code\",\"effective_parent_backend\":\"\",\"requested_parent_model\":\"sonnet\",\"effective_parent_model\":\"\",\"requested_parent_effort\":\"\",\"effective_parent_effort\":\"\",\"cli_version\":\"\",\"override_tier\":\"\",\"override_key_path\":\"\",\"parent_session_id\":\"\",\"children\":[]},\"order_id\":\"4666c3fd-486f-4ce1-b51f-9d9d660a5560\",\"receipt_id\":\"18af52568f97412b9eecc304d8a75e7c\"}"}},"sourceToolAssistantUUID":"b02275c1-d87d-4a7b-a28e-c87b020aac8d","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"3a518c19-addd-4ba2-a391-10c4149a6cd4","isSidechain":false,"message":{"id":"071b21375dfedbd6689fd160582e771b","type":"message","role":"assistant","content":[{"type":"tool_use","id":"call_01a12c9a935971faa349a923","name":"mcp__plugin_autoskillit_autoskillit__complete_run_skill_result","input":{"receipt_id":"18af52568f97412b9eecc304d8a75e7c"}}],"model":"MiniMax-M3","stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":727,"cache_creation_input_tokens":0,"cache_read_input_tokens":163522,"output_tokens":385,"output_tokens_details":{"thinking_tokens":319},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":1,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"run_skill","type":"assistant","uuid":"d5d574c9-e7be-4957-b3c9-4f1786d9de2a","timestamp":"2026-10-11T20:14:50.215Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":"d5d574c9-e7be-4957-b3c9-4f1786d9de2a","isSidechain":false,"promptId":"886212ea-29c8-475f-a4bb-05be860f18dd","type":"user","message":{"role":"user","content":[{"tool_use_id":"call_01a12c9a935971faa349a923","type":"tool_result","content":"## complete_run_skill_result\n\nsuccess: True\nreceipt_id: 18af52568f97412b9eecc304d8a75e7c\nrun_skill_success: False\nclassification: zero_writes\nsession_id: 54f18144-1188-45d7-a392-17b4529f721e\ntracker:\n success: True\n status: not_applied\ntracker_repairable: False\n\n--- RECIPE SEGMENT ---\n{\"kind\":\"recovery\",\"pull_closure\":[\"renew_plan_set\",\"create_impl_worktree\",\"implement\",\"release_issue_failure\",\"...[fixture-truncated 1059 chars]"}]},"uuid":"fc21e657-3713-460e-b508-f14b28a65117","timestamp":"2026-10-11T20:14:50.513Z","toolUseResult":"## complete_run_skill_result\n\nsuccess: True\nreceipt_id: 18af52568f97412b9eecc304d8a75e7c\nrun_skill_success: False\nclassification: zero_writes\nsession_id: 54f18144-1188-45d7-a392-17b4529f721e\ntracker:\n success: True\n status: not_applied\ntracker_repairable: False\n\n--- RECIPE SEGMENT ---\n{\"kind\":\"recovery\",\"pull_closure\":[\"renew_plan_set\",\"create_impl_worktree\",\"implement\",\"release_issue_failure\",\"...[fixture-truncated 1059 chars]","mcpMeta":{"_meta":{"fastmcp":{"wrap_result":true}},"structuredContent":{"result":"{\"success\":true,\"receipt_id\":\"18af52568f97412b9eecc304d8a75e7c\",\"run_skill_success\":false,\"classification\":\"zero_writes\",\"session_id\":\"54f18144-1188-45d7-a392-17b4529f721e\",\"tracker\":{\"success\":true,\"status\":\"not_applied\"},\"tracker_repairable\":false,\"recipe_segment\":{\"kind\":\"recovery\",\"source_step\":\"verify\",\"target_steps\":[\"renew_plan_set\",\"release_issue_failure\"],\"pull_closure\":[\"renew_plan_set\",\"create_impl_worktree\",\"implement\",\"release_issue_failure\",\"register_clone_failure\",\"escalate_stop\"],\"pull_requests\":[{\"section\":\"renew_plan_set\",\"part\":0},{\"section\":\"create_impl_worktree\",\"part\":0},{\"section\":\"implement\",\"part\":0},{\"section\":\"release_issue_failure\",\"part\":0},{\"section\":\"register_clone_failure\",\"part\":0},{\"section\":\"escalate_stop\",\"part\":0}],\"recipe_pull\":{\"producer_tool\":\"open_kitchen\",\"recipe_name\":\"implementation\",\"descriptor_version\":2,\"schema_version\":2,\"payload_sha256\":\"sha256:ea2e62f28435745f27a7a742cbb98d2e34cdb00d82eef53117118c3b574d9497\",\"artifact_blob_sha256\":\"sha256:cd5e7fff1ca8145532aa9ce8ca2a26ddbfe48fb704316310789b8123319f8d63\",\"artifact_blob_size_bytes\":576427,\"body_sha256\":\"sha256:3f14e54f1505f8a5c4d13c2bc5658394855e4c7dc624cc527ddb4bf0711954d5\",\"body_size_bytes\":115869,\"flow_schema_version\":2,\"flow_sha256\":\"sha256:3b9ff899035aad7555d5863de6c70c6ed95e6ffac5f19cea0c46cc17261f4f2a\",\"flow_size_bytes\":101558,\"flow_record_count\":694,\"pull_tool\":\"get_recipe_section\"}}}"}},"sourceToolAssistantUUID":"d5d574c9-e7be-4957-b3c9-4f1786d9de2a","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main"} +{"parentUuid":null,"logicalParentUuid":"282278b8-9dd2-4a6f-bbaa-1105acc27bfb","isSidechain":false,"type":"system","subtype":"compact_boundary","content":"Conversation compacted","level":"info","compactMetadata":{"trigger":"auto","preTokens":166958,"postTokens":3121,"cumulativeDroppedTokens":163837,"durationMs":21279,"preservedSegment":{"headUuid":"b00cd523-124f-4830-9333-a77e74e0f7f5","anchorUuid":"c73a3283-5a21-4482-8124-50d7d3251b94","tailUuid":"282278b8-9dd2-4a6f-bbaa-1105acc27bfb"},"preservedMessages":{"anchorUuid":"c73a3283-5a21-4482-8124-50d7d3251b94","uuids":["b00cd523-124f-4830-9333-a77e74e0f7f5","5c92e852-aedc-4502-aabd-4ecc5cb13843","f2cc56c1-0cfa-4ba2-bf38-64115b02adea","282278b8-9dd2-4a6f-bbaa-1105acc27bfb"],"allUuids":["b00cd523-124f-4830-9333-a77e74e0f7f5","2aac9b5f-a477-43f9-bb2f-bf74d1f49f8c","05edcb98-33b3-4462-b670-bca0163361fe","c9579bc1-e850-46cf-8f41-586f87a894ae","8afdb5bf-7d28-49c8-af1a-08f5a3c343fe","62b75b38-cc5e-4922-b5a5-398281472724","52c3981f-6147-4b1f-8256-531acee3828c","4f22a38f-6841-42e3-ba27-40a45b71387f","d09734c1-860a-4015-8590-1c8491e75c14","18b90c12-6bfb-4bd0-8c79-f7d3c7602c87","0287af2b-1d5a-4266-8e75-29c3e96ec868","0a3f287e-7607-4ba9-b042-dd1701ebea9d","0e407ba0-f60d-4f94-9ba8-4543674e2e3f","cb4e9a54-e7ed-433e-950c-e93211025deb","6aa3d78b-4514-456f-a42a-a76e136692d6","04dd0b8b-f81f-4fdd-a777-59a57d69eeb5","5c92e852-aedc-4502-aabd-4ecc5cb13843","fb3efad1-a773-4c1d-9d47-b6760694a153","f2cc56c1-0cfa-4ba2-bf38-64115b02adea","282278b8-9dd2-4a6f-bbaa-1105acc27bfb"]}},"uuid":"9a99fc5b-0ab0-4f91-a84b-4b2196d3b702","timestamp":"2026-10-11T20:15:22.874Z","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main","slug":"shimmying-tinkering-dream"} +{"parentUuid":"a286e33c-9ae5-4118-90ea-99d928d70a15","isSidechain":false,"message":{"id":"071b2190376d0b42f31507604c7b85ae","type":"message","role":"assistant","content":[{"type":"text","text":"The analyze_pipeline_health_error is `optional: true` with `skip_when_false: inputs.pipeline_health`. Since pipeline_health wasn't set, this will skip to escalate_stop, which is the stop action.\n\nThe verify step's zero_writes failure has been correctly routed to release_issue_failure → register_clone_failure → escalate_stop. The pipeline is now at the terminal stop point. Per the message:\n\n> \"Pipe...[fixture-truncated 1476 chars]"}],"model":"MiniMax-M3","stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":455,"cache_creation_input_tokens":0,"cache_read_input_tokens":41427,"output_tokens":472,"output_tokens_details":{"thinking_tokens":0},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[],"speed":"standard"},"service_tier":"standard","stop_details":null},"apiBlockIndex":0,"attributionMcpServer":"plugin:autoskillit:autoskillit","attributionMcpTool":"run_cmd","type":"assistant","uuid":"8136c073-129f-45bf-a17c-71a2d795dd38","timestamp":"2026-10-11T20:16:20.582Z","effort":"high","perTurnEffort":null,"userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main","slug":"shimmying-tinkering-dream"} +{"parentUuid":"8136c073-129f-45bf-a17c-71a2d795dd38","isSidechain":false,"type":"system","subtype":"stop_hook_summary","hookCount":2,"hookInfos":[{"command":"python3 -B \"${CLAUDE_PLUGIN_ROOT}/hooks/_dispatch.py\" guards/join_stop_guard","durationMs":117},{"command":"python3 -B \"${CLAUDE_PLUGIN_ROOT}/hooks/_dispatch.py\" session_lifetime_notice_hook","durationMs":99}],"hookErrors":[],"hookAdditionalContext":[],"preventedContinuation":false,"stopReason":"","hasOutput":false,"level":"suggestion","timestamp":"2026-10-11T20:16:20.704Z","uuid":"28868d68-9669-4c8c-874c-7b76c09003b0","toolUseID":"fdb4c1c9-f5b5-4999-8095-64dd918d9388","userType":"external","entrypoint":"sdk-cli","cwd":"/workspace/sandbox","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","version":"2.1.280","gitBranch":"main","slug":"shimmying-tinkering-dream"} +{"type":"cost-state","sessionId":"2292658a-34cf-44a0-a0b1-f7a20218874b","totalCostUSD":3.1986767999999994,"totalAPIDuration":314689,"totalAPIDurationWithoutRetries":314423,"totalToolDuration":300139,"totalLinesAdded":0,"totalLinesRemoved":0,"totalDuration":704309,"startTime":1791749076557,"modelUsage":{"claude-sonnet-5":{"inputTokens":206383,"outputTokens":43841,"thinkingTokens":17593,"cacheReadInputTokens":11737504,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":3.1986767999999994}},"hasUnknownModelCost":false} diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index c2236f0ef3..291f3c9030 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -5,6 +5,7 @@ import json import math import os +import shlex import stat import subprocess import tempfile @@ -12,10 +13,14 @@ from dataclasses import dataclass, replace from pathlib import Path from types import SimpleNamespace +from typing import Any +from urllib.parse import quote import pytest +from tests._hook_protocol_oracle import claude_matcher_admits from tests.infra._complexity_helpers import load_check_script +from tests.infra._e2e_evidence import load_provenance, materialize_run pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] @@ -259,9 +264,13 @@ def test_pipeline_runs_directory_matches_clone_source_of_truth() -> None: "message": "model reported failure", "issue": "https://github.com/TalonT-Org/AutoSkillit/issues/5232", } +_IMPLEMENTATION_ISSUE = "https://github.com/owner/sandbox/issues/5233" +_IMPLEMENTATION_EXPECTED_ISSUE = "https://github.com/TalonT-Org/AutoSkillit/issues/5233" +_PIPELINE_HEAD = "0123456789abcdef" +_PIPELINE_BRANCH = "codex/issue-5233" -@dataclass +@dataclass(eq=False) class RecordedCall: argv: list[str] cwd: Path | None @@ -295,6 +304,39 @@ def _completed(argv, stdout: str = "", returncode: int = 0, stderr: str = ""): return subprocess.CompletedProcess(list(argv), returncode, stdout, stderr) +def _recorded_calls( + calls: list[RecordedCall], + *, + prefix: list[str] | None = None, + exact: list[str] | None = None, + contains: str | None = None, +) -> list[RecordedCall]: + return [ + call + for call in calls + if (prefix is None or call.argv[: len(prefix)] == prefix) + and (exact is None or call.argv == exact) + and (contains is None or contains in " ".join(call.argv)) + ] + + +def _recorded_pr_list_calls(calls: list[RecordedCall], *, baseline: bool) -> list[RecordedCall]: + lists = _recorded_calls(calls, prefix=["gh", "pr", "list"]) + selected = [] + for call in lists: + is_baseline = call.argv[call.argv.index("--limit") + 1] == "1" + if (baseline and is_baseline) or (not baseline and not is_baseline): + selected.append(call) + return selected + + +def _recorded_pr_detail_calls(calls: list[RecordedCall], number: int) -> list[RecordedCall]: + suffix = f"/pulls/{number}" + return [ + call for call in calls if call.argv[:2] == ["gh", "api"] and call.argv[2].endswith(suffix) + ] + + def _pr(number: int, state: str, additions: int = 3, deletions: int = 1) -> dict: return {"number": number, "state": state, "additions": additions, "deletions": deletions} @@ -382,6 +424,319 @@ def _env(**overrides: str) -> dict[str, str]: return {name: value for name, value in env.items() if value} +def _pipeline_catalog( + expected_failures: list[dict[str, str]] | None = None, *, repository=SANDBOX +): + entry = { + "name": "implementation", + "kind": "recipe", + "pipeline": True, + "peak_sessions": 6, + "timeout_sec": 600, + "trigger_paths": ["src/*"], + "recipe": "implementation", + "ingredients": {"source_dir": "/workspace/sandbox", "open_pr": "true"}, + "expected_pull_request_state": "open", + "issue_url_env": "E2E_IMPLEMENTATION_ISSUE_URL", + "required_changed_paths": ["sandbox/", "tests/"], + "test_command": ["task", "test-check"], + "peak_sessions_evidence": "recorded session events", + } + if expected_failures is not None: + entry["expected_failures"] = expected_failures + return e2e_catalog.parse_catalog({"sandbox_repository": repository, "tests": [entry]}) + + +def _pipeline_pr(number: int = 8, *, branch: str = _PIPELINE_BRANCH) -> dict[str, Any]: + return { + "number": number, + "state": "open", + "merged": False, + "merged_at": None, + "auto_merge": None, + "body": f"Implements {_IMPLEMENTATION_ISSUE}", + "additions": 5, + "deletions": 2, + "head": { + "ref": branch, + "sha": _PIPELINE_HEAD, + "repo": {"full_name": SANDBOX}, + }, + "base": {"ref": "main", "repo": {"full_name": SANDBOX}}, + } + + +def _pipeline_setup_response(argv: list[str], state: dict[str, Any]): + if argv[:3] == ["autoskillit", "config", "show"]: + return _completed(argv, json.dumps(state["config"])) + if argv[:3] == ["gh", "issue", "view"]: + payload = ( + state["seed"] if "number,url,state,title,body,labels" in argv else state["final_issue"] + ) + return _completed(argv, json.dumps(payload)) + if argv[:3] == ["gh", "repo", "view"]: + payload = {"defaultBranchRef": {"name": state["default_branch"]}} + return _completed(argv, json.dumps(payload)) + if argv[:3] == ["gh", "pr", "list"]: + rows = [{"number": 7}] if argv[argv.index("--limit") + 1] == "1" else state["listed_prs"] + return _completed(argv, json.dumps(rows)) + return None + + +def _pipeline_execution_response(argv: list[str], state: dict[str, Any]): + if argv[:3] == ["autoskillit", "fleet", "run"]: + if state["fleet_exception"] is not None: + raise state["fleet_exception"] + state["head_refs"].extend(state["published_refs"]) + return _completed(argv, json.dumps(state["fleet_envelope"]) + "\n") + if argv[:2] == ["task", "test-check"]: + state["test_check_count"] += 1 + if state["test_check_count"] > 1 and state["pr_test_returncode"]: + return _completed(argv, returncode=state["pr_test_returncode"], stderr="tests failed") + return None + if argv[:2] == ["git", "rev-parse"]: + return _completed(argv, state["fetch_oid"] + "\n") + return None + + +def _pipeline_ci_api_response(argv: list[str], endpoint: str, state: dict[str, Any]): + if endpoint.endswith("/actions/workflows?per_page=100"): + workflows = [{"id": 1, "name": "CI", "state": "active"}] + return _completed(argv, json.dumps([{"workflows": workflows}])) + if "/actions/workflows/1/runs?" in endpoint: + run = { + "id": 2, + "status": "completed", + "conclusion": state["workflow_conclusion"], + "pull_requests": [ + { + "number": 8, + "head": {"sha": _PIPELINE_HEAD, "repo": {"full_name": SANDBOX}}, + "base": {"repo": {"full_name": SANDBOX}}, + } + ], + } + return _completed(argv, json.dumps([{"workflow_runs": [run]}])) + if endpoint.endswith("/actions/runs/2/jobs?per_page=100"): + job = {"name": "test", "status": "completed", "conclusion": state["job_conclusion"]} + return _completed(argv, json.dumps([{"jobs": [job]}])) + return None + + +def _pipeline_pull_request_api_response(argv: list[str], endpoint: str, state: dict[str, Any]): + if "/pulls/" not in endpoint: + return None + if endpoint.endswith("/files?per_page=100"): + return _completed(argv, json.dumps([state["files"]])) + number = int(endpoint.split("/pulls/", 1)[1].split("/", 1)[0]) + return _completed(argv, json.dumps(state["pr_details"][number])) + + +def _pipeline_refs_response(argv: list[str], endpoint: str, state: dict[str, Any]): + if "/git/matching-refs/heads/" not in endpoint: + return None + state["refs_list_calls"] += 1 + error = state["refs_list_faults"].get(state["refs_list_calls"]) + if error is not None: + raise error + response = state["refs_list_response"] + if response is None: + response = [[{"ref": f"refs/heads/{name}"} for name in state["head_refs"]]] + return _completed(argv, json.dumps(response)) + + +def _pipeline_cleanup_api_response(argv: list[str], endpoint: str, state: dict[str, Any]): + if argv[2] == "--method": + error = state["cleanup_errors"].get("branch_delete") + if error is not None: + raise error + if argv[3] == "DELETE" and "/git/refs/heads/" in endpoint: + ref_name = endpoint.split("/git/refs/heads/", 1)[1] + state["head_refs"] = [ + name for name in state["head_refs"] if quote(name, safe="") != ref_name + ] + return None + if "/git/refs/heads/" in endpoint: + error = state["cleanup_errors"].get("branch_get") + if error is not None: + raise error + return _completed(argv, "{}") + return None + + +def _pipeline_api_response(argv: list[str], state: dict[str, Any]): + if argv[:2] != ["gh", "api"]: + return None + endpoint = argv[4] if argv[2] == "--method" else argv[2] + for route in ( + _pipeline_ci_api_response, + _pipeline_pull_request_api_response, + _pipeline_refs_response, + _pipeline_cleanup_api_response, + ): + response = route(argv, endpoint, state) + if response is not None: + return response + return None + + +def _pipeline_cleanup_response(argv: list[str], state: dict[str, Any]): + if argv[:3] == ["gh", "pr", "close"]: + error = state["cleanup_errors"].get("pr_close") + if error is not None: + raise error + return None + + +def _pipeline_response(argv: list[str], state: dict[str, Any]): + for route in ( + _pipeline_setup_response, + _pipeline_execution_response, + _pipeline_api_response, + _pipeline_cleanup_response, + ): + response = route(argv, state) + if response is not None: + return response + return None + + +def _pipeline_runner() -> tuple[FakeRunner, dict[str, Any]]: + seed = { + "number": 5233, + "url": _IMPLEMENTATION_ISSUE, + "state": "OPEN", + "title": "Add a greeting", + "body": "Implement the requested behavior.", + "labels": [], + } + listed = { + "number": 8, + "state": "OPEN", + "additions": 5, + "deletions": 2, + "body": f"Implements {_IMPLEMENTATION_ISSUE}", + "headRefOid": _PIPELINE_HEAD, + "headRefName": _PIPELINE_BRANCH, + "headRepository": {"nameWithOwner": SANDBOX}, + "baseRefName": "main", + "autoMergeRequest": None, + "mergedAt": None, + } + state: dict[str, Any] = { + "seed": seed, + "default_branch": "main", + "config": { + "github": {"in_progress_label": "in-progress"}, + "branching": {"default_base_branch": "main"}, + }, + "listed_prs": [listed], + "pr_details": {8: _pipeline_pr()}, + "files": [ + {"filename": "sandbox/worker.py", "status": "added", "additions": 3, "deletions": 0}, + { + "filename": "tests/test_worker.py", + "status": "modified", + "additions": 2, + "deletions": 1, + }, + ], + "final_issue": {"state": "OPEN", "labels": [{"name": "in-progress"}]}, + "fetch_oid": _PIPELINE_HEAD, + "test_check_count": 0, + "pr_test_returncode": 0, + "workflow_conclusion": "success", + "job_conclusion": "success", + "fleet_envelope": { + "success": True, + "dispatch_status": "success", + "dispatch_id": "dispatch-5233", + "dispatched_session_id": "session-5233", + "l3_payload": {"reason": "implementation_complete"}, + }, + "fleet_exception": None, + "head_refs": ["main"], + "published_refs": [], + "refs_list_calls": 0, + "refs_list_faults": {}, + "refs_list_response": None, + "cleanup_errors": {}, + "proof": {"branches": [_PIPELINE_BRANCH], "claimed": True, "violations": []}, + "summary": {"coverage": {"complete": True}, "peak_sessions": 4, "violations": []}, + } + + return FakeRunner(lambda argv: _pipeline_response(argv, state)), state + + +def _patch_pipeline_measurement(monkeypatch, state: dict[str, Any]): + observed: dict[str, list[tuple]] = {"ownership": [], "summaries": []} + + def collect(trace_path, envelope, log_root, issue_url, repository, *, claim_label=None): + observed["ownership"].append( + (trace_path, envelope, log_root, issue_url, repository, claim_label) + ) + return state["proof"] + + def summarize(trace_path, envelope, log_root): + observed["summaries"].append((trace_path, envelope, log_root)) + return state["summary"] + + monkeypatch.setattr(harness.e2e_sessions, "collect_ownership", collect) + monkeypatch.setattr(harness.e2e_sessions, "summarize", summarize) + monkeypatch.setattr(harness.time, "sleep", lambda _seconds: None) + return observed + + +def _run_pipeline( + tmp_path: Path, + runner: FakeRunner, + state: dict[str, Any], + monkeypatch, + *, + env=None, + expected_failures=None, + real_measurement=False, + repository=SANDBOX, +): + catalog = _pipeline_catalog(expected_failures, repository=repository) + original_handler = runner.handler + + def handle(argv: list[str]): + if argv[:3] == ["autoskillit", "fleet", "run"]: + _seed_sessions( + _log_dir(tmp_path), + state.get( + "sessions", {"pipeline-session": [_EXPECTED_REQUIRED_CONTEXT_WINDOW_TOKENS]} + ), + ) + return original_handler(argv) + + runner.handler = handle + out = tmp_path / "out" + home = tmp_path / "home" + if real_measurement: + monkeypatch.setattr(harness.time, "sleep", lambda _seconds: None) + observed = None + else: + observed = _patch_pipeline_measurement(monkeypatch, state) + failures = harness.run_test( + catalog.get("implementation"), + catalog, + out=out, + home=home, + env=env if env is not None else _env(E2E_IMPLEMENTATION_ISSUE_URL=_IMPLEMENTATION_ISSUE), + runner=runner, + ) + return ( + failures, + json.loads((out / "result.json").read_text(encoding="utf-8")), + out, + home, + catalog, + observed, + ) + + def _run_recipe( tmp_path: Path, runner: FakeRunner, @@ -1935,7 +2290,9 @@ def reject_login(argv: list[str]): assert ("autoskillit", "fleet", "run") not in prefixes def test_fleet_timeout_and_window_failures_keep_cleanup(self, tmp_path: Path) -> None: - timeout = subprocess.TimeoutExpired(["autoskillit"], 720) + timeout = subprocess.TimeoutExpired( + ["autoskillit"], 720, output=b"partial stdout", stderr=b"partial stderr" + ) runner = FakeRunner( _recipe_handler( log_dir=_log_dir(tmp_path), @@ -1950,6 +2307,8 @@ def test_fleet_timeout_and_window_failures_keep_cleanup(self, tmp_path: Path) -> f"session windows: timed-out: observed [200000] != required " f"{_EXPECTED_REQUIRED_CONTEXT_WINDOW_TOKENS}" ) in failures + assert (tmp_path / "out" / "fleet-run.stdout.log").read_text() == "partial stdout" + assert (tmp_path / "out" / "fleet-run.stderr.log").read_text() == "partial stderr" assert runner.calls[-1].argv[:4] == ["gh", "pr", "close", "8"] def test_unexpected_fleet_error_records_failure_and_still_cleans_up( @@ -2078,6 +2437,741 @@ def test_expected_failure_keeps_a_window_failure(self, tmp_path: Path) -> None: assert result["expected_findings"] == [expected] +class TestPipelineRecipeFlow: + def test_pipeline_rejects_small_context_window_and_still_cleans_up( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["sessions"] = {"pipeline-session": [200000]} + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["passed"] is False + assert ( + "session windows: pipeline-session: observed [200000] != required " + f"{_EXPECTED_REQUIRED_CONTEXT_WINDOW_TOKENS}" + ) in failures + assert ("gh", "pr", "close", "8") in runner.argv_prefixes(4) + + def test_session_hooks_install_the_declared_policy_and_keep_existing_settings( + self, tmp_path: Path + ) -> None: + home = tmp_path / "home" + settings_path = home / ".claude" / "settings.json" + settings_path.parent.mkdir(parents=True, exist_ok=True) + foreign_subagent_stop = { + "matcher": "ForeignAgent", + "hooks": [{"type": "command", "command": "foreign"}], + } + other_pre_tool_use = { + "matcher": "Bash", + "hooks": [{"type": "command", "command": "other"}], + } + settings_path.write_text( + json.dumps( + { + "theme": "dark", + "env": {"KEEP1": "1"}, + "hooks": { + "SubagentStop": [foreign_subagent_stop], + "PreToolUse": [other_pre_tool_use], + }, + } + ), + encoding="utf-8", + ) + trace = tmp_path / "trace with space" / "events.jsonl" + + harness.install_session_hooks(home, trace) + + settings = json.loads(settings_path.read_text(encoding="utf-8")) + lifecycle_events = {"SessionStart", "SessionEnd", "SubagentStart", "SubagentStop"} + declarations = harness.e2e_sessions.LIFECYCLE_HOOK_MATCHERS + assert set(declarations) == lifecycle_events + assert set(settings["hooks"]) == lifecycle_events | {"PreToolUse"} + assert settings["theme"] == "dark" + assert settings["env"] == {"KEEP1": "1"} + assert settings["hooks"]["PreToolUse"] == [other_pre_tool_use] + assert settings["hooks"]["SubagentStop"][0] == foreign_subagent_stop + + expected_command_hook = { + "type": "command", + "command": f"python3 /opt/e2e/e2e_sessions.py {shlex.quote(str(trace))}", + "timeout": 10, + } + for event in lifecycle_events: + installed = settings["hooks"][event] + if event == "SubagentStop": + installed = installed[1:] + assert len(installed) == 1 + entry = installed[0] + assert entry["hooks"] == [expected_command_hook] + declared_matcher = declarations[event] + if declared_matcher is None: + assert "matcher" not in entry + else: + assert entry["matcher"] == declared_matcher + assert not claude_matcher_admits(entry["matcher"], "") + + def test_session_hooks_refuse_a_custom_main_agent(self, tmp_path: Path) -> None: + home = tmp_path / "home" + settings_path = home / ".claude" / "settings.json" + settings_path.parent.mkdir(parents=True, exist_ok=True) + settings_path.write_bytes(b'{"agent":"custom-main"}') + original = settings_path.read_bytes() + + with pytest.raises(ValueError, match="main agent"): + harness.install_session_hooks(home, tmp_path / "events.jsonl") + + assert settings_path.read_bytes() == original + + def test_custom_main_agent_setting_fails_preflight(self, tmp_path: Path, monkeypatch) -> None: + settings_path = tmp_path / "home" / ".claude" / "settings.json" + settings_path.parent.mkdir(parents=True, exist_ok=True) + settings_path.write_text('{"agent":"custom-main"}', encoding="utf-8") + runner, state = _pipeline_runner() + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert failures == ["preflight: session hooks: settings.json sets a main agent"] + assert result["outcome"] == "failed" + assert not any(call.argv[:3] == ["autoskillit", "fleet", "run"] for call in runner.calls) + + def test_pipeline_uses_the_seed_issue_and_cleans_only_its_owned_pr( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + unrelated = {**state["listed_prs"][0], "number": 9, "headRefName": "other/branch"} + state["listed_prs"].append(unrelated) + state["pr_details"][9] = _pipeline_pr(9, branch="other/branch") + failures, result, out, home, catalog, observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert failures == [] + assert result == { + "test": "implementation", + "passed": True, + "failures": [], + "outcome": "passed", + "expected_findings": [], + } + test = catalog.get("implementation") + assert test.pipeline is True + assert test.issue_url_env == "E2E_IMPLEMENTATION_ISSUE_URL" + assert test.required_changed_paths == ("sandbox/", "tests/") + assert test.test_command == ("task", "test-check") + + calls = runner.calls + config_call = _recorded_calls(calls, prefix=["autoskillit", "config", "show"])[0] + assert config_call.cwd == harness.SANDBOX_CLONE + assert config_call.env == { + **harness.child_env(_env(E2E_IMPLEMENTATION_ISSUE_URL=_IMPLEMENTATION_ISSUE)), + "AUTOSKILLIT_LOG_DIR": str(_log_dir(tmp_path)), + } + seed_call = _recorded_calls(calls, prefix=["gh", "issue", "view"])[0] + repo_call = _recorded_calls(calls, prefix=["gh", "repo", "view"])[0] + baseline_install = _recorded_calls(calls, exact=["task", "install-worktree"])[0] + checks = _recorded_calls(calls, exact=["task", "test-check"]) + baseline_list = _recorded_pr_list_calls(calls, baseline=True)[0] + fleet_call = _recorded_calls(calls, prefix=["autoskillit", "fleet", "run"])[0] + assert ( + calls.index(config_call) + < calls.index(seed_call) + < calls.index(repo_call) + < calls.index(baseline_install) + < calls.index(checks[0]) + < calls.index(baseline_list) + < calls.index(fleet_call) + ) + assert baseline_install.cwd == checks[0].cwd == harness.SANDBOX_CLONE + assert len(checks) == 2 and checks[1].cwd == harness.SANDBOX_CLONE + + listed_calls = _recorded_pr_list_calls(calls, baseline=False) + file_call = _recorded_calls(calls, contains="/files?per_page=100")[0] + fetch_call = _recorded_calls(calls, prefix=["git", "fetch"])[0] + oid_call = _recorded_calls(calls, prefix=["git", "rev-parse"])[0] + checkout_call = _recorded_calls(calls, prefix=["git", "checkout", "--detach"])[0] + install_calls = _recorded_calls(calls, exact=["task", "install-worktree"]) + workflow_call = _recorded_calls( + calls, prefix=["gh", "api"], contains="/actions/workflows?per_page=100" + )[0] + runs_call = _recorded_calls(calls, contains="/actions/workflows/1/runs?")[0] + jobs_call = _recorded_calls(calls, contains="/actions/runs/2/jobs?")[0] + pr_reads = _recorded_pr_detail_calls(calls, 8) + close = _recorded_calls(calls, prefix=["gh", "pr", "close"])[0] + branch_delete = _recorded_calls(calls, prefix=["gh", "api", "--method", "DELETE"])[0] + final_issue = _recorded_calls( + calls, prefix=["gh", "issue", "view"], contains="state,labels" + )[-1] + issue_edit = _recorded_calls(calls, prefix=["gh", "issue", "edit"])[0] + assert len(listed_calls) == 2 and len(install_calls) == 2 and len(pr_reads) == 3 + assert ( + calls.index(fleet_call) + < calls.index(listed_calls[0]) + < calls.index(file_call) + < calls.index(fetch_call) + < calls.index(oid_call) + < calls.index(checkout_call) + < calls.index(install_calls[1]) + < calls.index(checks[1]) + < calls.index(workflow_call) + < calls.index(runs_call) + < calls.index(jobs_call) + < calls.index(pr_reads[1]) + < calls.index(listed_calls[1]) + < calls.index(close) + < calls.index(branch_delete) + < calls.index(final_issue) + < calls.index(issue_edit) + ) + + ingredient_args = [ + fleet_call.argv[index + 1] + for index, arg in enumerate(fleet_call.argv[:-1]) + if arg == "-i" + ] + assert ingredient_args.count(f"issue_url={_IMPLEMENTATION_ISSUE}") == 1 + assert ( + ingredient_args.count("task=Add a greeting\n\nImplement the requested behavior.") == 1 + ) + ingredient_names = {key for key, _value in test.ingredients} + assert "issue_url" not in ingredient_names and "task" not in ingredient_names + assert fleet_call.timeout == test.timeout_sec + e2e_catalog.HARNESS_GRACE_SEC + assert fleet_call.cwd == harness.SANDBOX_CLONE + assert [call.input for call in calls if call.input] == [SANDBOX_TOKEN] + assert all(SANDBOX_TOKEN not in " ".join(call.argv) for call in calls) + assert all(call.env == config_call.env for call in calls) + + trace = out / "session-events.jsonl" + log_root = home / ".local" / "share" / "autoskillit" / "logs" + envelope = json.loads((out / "envelope.json").read_text(encoding="utf-8")) + assert observed["ownership"] == [ + (trace, envelope, log_root, _IMPLEMENTATION_ISSUE, SANDBOX, "in-progress") + ] + assert observed["summaries"] == [(trace, envelope, log_root)] + settings = json.loads((home / ".claude" / "settings.json").read_text(encoding="utf-8")) + assert settings["env"] == _EXPECTED_SETTINGS_ENV + assert all( + len(settings["hooks"][event]) == 1 + for event in harness.e2e_sessions.LIFECYCLE_HOOK_MATCHERS + ) + + close = next(call for call in calls if call.argv[:3] == ["gh", "pr", "close"]) + assert close.argv == ["gh", "pr", "close", "8", "--repo", SANDBOX] + assert not any(call.argv[:4] == ["gh", "pr", "close", "9"] for call in calls) + assert any( + call.argv[:4] == ["gh", "api", "--method", "DELETE"] + and call.argv[4].endswith("codex%2Fissue-5233") + for call in calls + ) + assert any( + call.argv[:3] == ["gh", "issue", "edit"] + and call.argv[-2:] == ["--remove-label", "in-progress"] + for call in calls + ) + + @pytest.mark.parametrize( + ("problem", "expected"), + [ + ("malformed", "seed: expected the configured OPEN sandbox issue"), + ("closed", "seed: expected the configured OPEN sandbox issue"), + ("claimed", "seed: issue is already claimed"), + ], + ) + def test_invalid_seed_stops_before_baseline_or_fleet( + self, tmp_path: Path, monkeypatch, problem: str, expected: str + ) -> None: + runner, state = _pipeline_runner() + if problem == "malformed": + state["seed"].pop("title") + elif problem == "closed": + state["seed"]["state"] = "CLOSED" + else: + state["seed"]["labels"] = [{"name": "in-progress"}] + + failures, result, _out, _home, _catalog, observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert any(expected in failure for failure in failures) + assert result["outcome"] == "failed" + assert not any(call.argv[:2] == ["task", "install-worktree"] for call in runner.calls) + assert not any(call.argv[:2] == ["autoskillit", "fleet"] for call in runner.calls) + assert not any(call.argv[:3] == ["gh", "pr", "list"] for call in runner.calls) + assert observed["ownership"] == observed["summaries"] == [] + + def test_missing_seed_url_stops_before_github_issue_lookup( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, + runner, + state, + monkeypatch, + env=_env(E2E_IMPLEMENTATION_ISSUE_URL=""), + ) + + assert failures == ["preflight: seed: missing or noncanonical sandbox issue URL"] + assert result["outcome"] == "failed" + assert not any(call.argv[:3] == ["gh", "issue", "view"] for call in runner.calls) + assert not any(call.argv[:3] == ["autoskillit", "fleet", "run"] for call in runner.calls) + + @pytest.mark.parametrize("wrong_base", ["sandbox", "effective-config"]) + def test_seed_requires_the_main_branch_in_repository_and_config( + self, tmp_path: Path, monkeypatch, wrong_base: str + ) -> None: + runner, state = _pipeline_runner() + if wrong_base == "sandbox": + state["default_branch"] = "develop" + else: + state["config"]["branching"]["default_base_branch"] = "trunk" + + failures, _result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert any( + "sandbox and effective configured base branch must both be main" in item + for item in failures + ) + assert not any(call.argv[:2] == ["task", "install-worktree"] for call in runner.calls) + assert not any(call.argv[:3] == ["autoskillit", "fleet", "run"] for call in runner.calls) + + def test_linked_stable_finding_is_recorded_and_skips_pr_assertions( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["fleet_envelope"] = { + "success": False, + "dispatch_status": "retry_exhausted", + "dispatch_id": "dispatch-5233", + "dispatched_session_id": "session-5233", + "reason": "known orchestration failure", + } + state["listed_prs"] = [] + state["pr_details"] = {} + state["final_issue"] = {"state": "OPEN", "labels": []} + state["proof"] = {"branches": [], "claimed": False, "violations": []} + expected = { + "check": "fleet_envelope", + "message": "retry_exhausted: known orchestration failure", + "issue": _IMPLEMENTATION_EXPECTED_ISSUE, + } + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch, expected_failures=[expected] + ) + + assert failures == [] + assert result["outcome"] == "expected_failure" + assert result["passed"] is False + assert result["expected_findings"] == [expected] + assert not any(call.argv[:2] == ["git", "fetch"] for call in runner.calls) + assert not any( + "/pulls/" in " ".join(call.argv) and call.argv[0:2] == ["gh", "api"] + for call in runner.calls + ) + assert not any("actions/workflows" in " ".join(call.argv) for call in runner.calls) + + @pytest.mark.parametrize("envelope_kind", ["stale", "fatal"]) + def test_stale_or_fatal_pipeline_failure_cannot_be_allowlisted( + self, tmp_path: Path, monkeypatch, envelope_kind: str + ) -> None: + runner, state = _pipeline_runner() + expected = { + "check": "fleet_envelope", + "message": "retry_exhausted: known orchestration failure", + "issue": _IMPLEMENTATION_EXPECTED_ISSUE, + } + state["listed_prs"] = [] + state["pr_details"] = {} + state["final_issue"] = {"state": "OPEN", "labels": []} + state["proof"] = {"branches": [], "claimed": False, "violations": []} + if envelope_kind == "fatal": + state["fleet_envelope"] = { + "success": False, + "dispatch_status": "failed", + "dispatch_id": "dispatch-5233", + "dispatched_session_id": "session-5233", + "l3_payload": {}, + } + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch, expected_failures=[expected] + ) + + assert result["outcome"] == "failed" + assert result["expected_findings"] == [] + if envelope_kind == "stale": + assert any("expected recipe finding absent" in failure for failure in failures) + else: + assert any( + "unsuccessful envelope has no specific failure reason" in f for f in failures + ) + assert any("expected recipe finding absent" in failure for failure in failures) + assert not any(call.argv[:2] == ["git", "fetch"] for call in runner.calls) + + @pytest.mark.parametrize( + ("problem", "expected"), + [ + ("pr", "implementation PR: expected an open, unmerged sandbox PR"), + ("body", "implementation PR: body does not reference the canonical seed URL"), + ("paths", "implementation PR: missing non-removed changes under tests/"), + ("ci", "sandbox CI: CI run and test job must succeed"), + ("head", "implementation PR: fetched head differs from captured head"), + ("ci-job", "sandbox CI: CI run and test job must succeed"), + ("test", "pr-test: exit 1: tests failed"), + ], + ) + def test_pipeline_rejects_unverified_pr_or_validation_failure( + self, tmp_path: Path, monkeypatch, problem: str, expected: str + ) -> None: + runner, state = _pipeline_runner() + pr = state["pr_details"][8] + if problem == "pr": + pr["base"]["ref"] = "develop" + elif problem == "body": + pr["body"] = "No linked issue" + elif problem == "paths": + state["files"] = state["files"][:1] + elif problem == "ci": + state["workflow_conclusion"] = "failure" + elif problem == "ci-job": + state["job_conclusion"] = "failure" + elif problem == "head": + state["fetch_oid"] = "fetched-a-different-head" + else: + state["pr_test_returncode"] = 1 + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert any(expected in failure for failure in failures) + + def test_listed_pr_head_must_match_rest_head_before_validation( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["listed_prs"][0]["headRefOid"] = "stale-listed-head" + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert any("head" in failure.lower() or "oid" in failure.lower() for failure in failures) + assert not any("/files?" in " ".join(call.argv) for call in runner.calls) + assert not any("actions/workflows" in " ".join(call.argv) for call in runner.calls) + + @pytest.mark.parametrize( + ("problem", "expected"), + [ + ("coverage", "session measurement: incomplete lifecycle coverage"), + ("capacity", "session measurement: observed peak exceeds catalog capacity"), + ], + ) + def test_session_evidence_must_be_complete_and_within_catalog_capacity( + self, tmp_path: Path, monkeypatch, problem: str, expected: str + ) -> None: + runner, state = _pipeline_runner() + if problem == "coverage": + state["summary"]["coverage"]["complete"] = False + else: + state["summary"]["peak_sessions"] = 7 + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert any(expected in failure for failure in failures) + + def test_ownership_violations_fail_the_pipeline_result( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["proof"]["violations"] = ["transcript evidence is incomplete"] + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert "ownership: transcript evidence is incomplete" in failures + + def test_cleanup_errors_do_not_prevent_remaining_owned_cleanup( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["cleanup_errors"] = { + "pr_close": RuntimeError("close failed"), + "branch_delete": RuntimeError("branch delete failed"), + } + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert any("cleanup PR: close failed" in failure for failure in failures) + assert any("cleanup branch: branch delete failed" in failure for failure in failures) + assert any(call.argv[:3] == ["gh", "issue", "edit"] for call in runner.calls) + + def test_fleet_exception_and_cleanup_exception_are_both_preserved( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["fleet_exception"] = RuntimeError("fleet launch crashed") + state["cleanup_errors"] = {"branch_delete": RuntimeError("branch delete failed")} + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert any("harness error: fleet launch crashed" in failure for failure in failures) + assert any("cleanup branch: branch delete failed" in failure for failure in failures) + assert any(call.argv[:3] == ["gh", "issue", "edit"] for call in runner.calls) + + def test_branch_created_without_ownership_proof_fails_cleanup_and_is_preserved( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["fleet_envelope"] = { + "success": False, + "dispatch_status": "retry_exhausted", + "dispatch_id": "dispatch-5233", + "dispatched_session_id": "session-5233", + "reason": "known orchestration failure", + } + state["listed_prs"] = [] + state["pr_details"] = {} + state["final_issue"] = {"state": "OPEN", "labels": []} + state["proof"] = {"branches": [], "claimed": False, "violations": []} + state["published_refs"] = ["impl/leaked"] + + failures, _result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert ( + "cleanup: branch impl/leaked was created during this run without ownership proof" + in failures + ) + assert not any( + call.argv[:4] == ["gh", "api", "--method", "DELETE"] for call in runner.calls + ) + assert "impl/leaked" in state["head_refs"] + + def test_proven_branch_is_deleted_and_leaves_nothing_unexplained( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["published_refs"] = [_PIPELINE_BRANCH] + original_list_head_refs = harness.list_head_refs + snapshots = {} + + def record_head_refs(repository, out, env, runner, stage): + refs = original_list_head_refs(repository, out, env, runner, stage) + snapshots[stage] = refs + return refs + + monkeypatch.setattr(harness, "list_head_refs", record_head_refs) + failures, _result, out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + delete = [ + "gh", + "api", + "--method", + "DELETE", + f"repos/{SANDBOX}/git/refs/heads/{quote(_PIPELINE_BRANCH, safe='')}", + ] + assert not any(failure.startswith("cleanup") for failure in failures) + assert any(call.argv == delete for call in runner.calls) + assert state["head_refs"] == ["main"] + assert snapshots["before"] == snapshots["after"] == frozenset({"main"}) + for stage in ("before", "after"): + assert json.loads( + (out / f"head-refs-{stage}.stdout.txt").read_text(encoding="utf-8") + ) == [[{"ref": "refs/heads/main"}]] + + def test_branch_present_before_the_run_is_never_reported(self, tmp_path, monkeypatch) -> None: + runner, state = _pipeline_runner() + state["fleet_envelope"] = { + "success": False, + "dispatch_status": "retry_exhausted", + "dispatch_id": "dispatch-5233", + "dispatched_session_id": "session-5233", + "reason": "known orchestration failure", + } + state["listed_prs"] = [] + state["pr_details"] = {} + state["final_issue"] = {"state": "OPEN", "labels": []} + state["head_refs"] = ["main", "impl/old-leftover"] + state["proof"] = {"branches": [], "claimed": False, "violations": []} + + failures, _result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert not any("impl/old-leftover" in failure for failure in failures) + assert not any( + call.argv[:4] == ["gh", "api", "--method", "DELETE"] for call in runner.calls + ) + + @pytest.mark.parametrize( + ("call_number", "expected_prefix"), [(1, "preflight:"), (2, "cleanup refs:")] + ) + def test_ref_listing_failure_is_reported( + self, tmp_path: Path, monkeypatch, call_number: int, expected_prefix: str + ) -> None: + runner, state = _pipeline_runner() + state["published_refs"] = [_PIPELINE_BRANCH] + state["refs_list_faults"] = {call_number: RuntimeError("refs unavailable")} + + failures, _result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + listing_failures = [failure for failure in failures if failure.startswith(expected_prefix)] + assert len(listing_failures) == 1 + assert "refs unavailable" in listing_failures[0] + if call_number == 1: + assert not any( + call.argv[:3] == ["autoskillit", "fleet", "run"] for call in runner.calls + ) + else: + delete = [ + "gh", + "api", + "--method", + "DELETE", + f"repos/{SANDBOX}/git/refs/heads/{quote(_PIPELINE_BRANCH, safe='')}", + ] + assert any(call.argv == delete for call in runner.calls) + assert any( + call.argv[:3] == ["gh", "issue", "edit"] + and call.argv[-2:] == ["--remove-label", "in-progress"] + for call in runner.calls + ) + + @pytest.mark.parametrize( + "response", + [[], [[{}]], [[{"ref": "refs/tags/v1"}]]], + ) + def test_malformed_ref_listing_fails_preflight( + self, tmp_path: Path, monkeypatch, response + ) -> None: + runner, state = _pipeline_runner() + state["refs_list_response"] = response + + failures, result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert result["outcome"] == "failed" + assert sum(failure.startswith("preflight:") for failure in failures) == 1 + assert not any(call.argv[:3] == ["autoskillit", "fleet", "run"] for call in runner.calls) + + def test_proven_branch_whose_delete_fails_is_not_reported_as_unexplained( + self, tmp_path: Path, monkeypatch + ) -> None: + runner, state = _pipeline_runner() + state["published_refs"] = [_PIPELINE_BRANCH] + state["cleanup_errors"] = {"branch_delete": RuntimeError("branch delete failed")} + + failures, _result, _out, _home, _catalog, _observed = _run_pipeline( + tmp_path, runner, state, monkeypatch + ) + + assert sum("cleanup branch: branch delete failed" in failure for failure in failures) == 1 + assert not any("without ownership proof" in failure for failure in failures) + + def test_failed_run_teardown_deletes_the_branch_proven_by_captured_evidence( + self, tmp_path: Path, monkeypatch + ) -> None: + provenance = load_provenance("run_38170521660_attempt_1") + repository = provenance["repository"] + issue_url = provenance["issue_url"] + claim_label = provenance["claim_label"] + branch = "e2e-implementation-fixture-normalize-whitespace/1-3" + runner, state = _pipeline_runner() + state["seed"] = { + **state["seed"], + "number": int(issue_url.rsplit("/", 1)[1]), + "url": issue_url, + } + state["config"]["github"]["in_progress_label"] = claim_label + state["listed_prs"] = [] + state["pr_details"] = {} + state["final_issue"] = {"state": "OPEN", "labels": []} + state["published_refs"] = [branch] + out = tmp_path / "out" + home = tmp_path / "home" + original_handler = runner.handler + + def materialize_before_fleet(argv): + if argv[:3] == ["autoskillit", "fleet", "run"]: + state["fleet_envelope"] = materialize_run( + "run_38170521660_attempt_1", + trace=out / "session-events.jsonl", + log_root=home / ".local" / "share" / "autoskillit" / "logs", + projects=home / ".claude" / "projects", + ) + return original_handler(argv) + + runner.handler = materialize_before_fleet + failures, result, out, _home, _catalog, _observed = _run_pipeline( + tmp_path, + runner, + state, + monkeypatch, + env=_env(E2E_IMPLEMENTATION_ISSUE_URL=issue_url), + real_measurement=True, + repository=repository, + ) + + delete = [ + "gh", + "api", + "--method", + "DELETE", + f"repos/{repository}/git/refs/heads/{quote(branch, safe='')}", + ] + assert any(call.argv == delete for call in runner.calls) + assert state["head_refs"] == ["main"] + assert not any(failure.startswith(("ownership:", "cleanup")) for failure in failures) + assert result["outcome"] == "failed" + assert ( + sum( + failure.startswith("fleet_envelope:") + and "resumable: fleet_l3_no_result_block" in failure + for failure in failures + ) + == 1 + ) + assert ( + sum( + failure.startswith("session measurement: incomplete lifecycle coverage") + for failure in failures + ) + == 1 + ) + assert json.loads((out / "ownership.json").read_text(encoding="utf-8"))["branches"] == [ + branch + ] + + class TestCanaryFlow: def test_canary_runs_one_claude_prompt(self, tmp_path: Path, monkeypatch) -> None: runner = FakeRunner(lambda argv: _completed(argv, _CANARY_REPLY)) diff --git a/tests/infra/test_e2e_select.py b/tests/infra/test_e2e_select.py index ecec41bf95..1c2610d36b 100644 --- a/tests/infra/test_e2e_select.py +++ b/tests/infra/test_e2e_select.py @@ -23,6 +23,13 @@ REPOSITORY = "TalonT-Org/AutoSkillit" SHA_ZERO = "0" * 40 SYNTHETIC_SHAS = [hashlib.sha1(str(index).encode()).hexdigest() for index in range(2000)] +_IMPLEMENTATION_PATHS = ( + "src/autoskillit/recipes/", + "src/autoskillit/skills/", + "src/autoskillit/skills_extended/", + "src/autoskillit/server/", + "src/autoskillit/execution/", +) def _facts( @@ -188,6 +195,44 @@ def test_deleted_head_repository_is_not_selected(self) -> None: assert pr.head_repository is None assert not selection.selected + @pytest.mark.parametrize( + ("changed_files", "additions", "deletions"), + [(10, 0, 0), (1, 300, 0), (1, 100, 200)], + ) + def test_large_change_with_matching_root_selects_pipeline( + self, changed_files: int, additions: int, deletions: int + ) -> None: + pr = _facts( + head_sha=_sha_with_sample(5230, sampled=True), + changed_files=changed_files, + additions=additions, + deletions=deletions, + ) + selection = select.select_for_pull_request( + pr, + ["src/autoskillit/recipes/changed.py"], + _catalog(_pipeline_entry()), + REPOSITORY, + ) + assert selection.tests == ("implementation",) + + def test_label_does_not_make_a_small_change_pipeline_eligible(self) -> None: + pr = _facts(changed_files=1, additions=10, labels=("e2e",)) + selection = select.select_for_pull_request( + pr, + ["src/autoskillit/recipes/changed.py"], + _catalog(_pipeline_entry()), + REPOSITORY, + ) + assert not selection.selected + + def test_pipeline_is_not_an_unrelated_fallback(self) -> None: + pr = _facts(head_sha=_sha_with_sample(5230, sampled=True), changed_files=10) + selection = select.select_for_pull_request( + pr, ["README.md"], _catalog(_pipeline_entry()), REPOSITORY + ) + assert not selection.selected + class TestPickTests: _SEED = f"5230:{SHA_ZERO}" @@ -228,6 +273,25 @@ def test_star_matches_across_directories(self) -> None: catalog = _catalog(_entry("deep", ("src/*",)), _entry("other", ("docs/*",))) assert select.pick_tests(catalog, ["src/a/b/c.py"], self._SEED) == ("deep",) + @pytest.mark.parametrize("root", _IMPLEMENTATION_PATHS) + def test_pipeline_trigger_roots_match(self, root: str) -> None: + assert select.pick_tests( + _catalog(_pipeline_entry()), + [f"{root}changed.py"], + self._SEED, + large_change=True, + ) == ("implementation",) + + def test_at_most_one_pipeline_is_selected(self) -> None: + catalog = _catalog( + _pipeline_entry("pipeline-a", trigger_paths=["src/*"]), + _pipeline_entry("pipeline-b", trigger_paths=["src/*"]), + _entry("ordinary", ("src/*",)), + ) + picked = select.pick_tests(catalog, ["src/changed.py"], self._SEED, large_change=True) + assert len(picked) == select.MAX_TESTS + assert sum(catalog.get(name).pipeline for name in picked) == 1 + class TestSelectForEvent: def test_merge_group_is_never_selected(self) -> None: @@ -301,6 +365,18 @@ def test_matrix_includes_kind_from_each_catalog_entry() -> None: "ingredients": {"task": "Add a greeting"}, "expected_pull_request_state": "merged", } +_RECIPE_EXPECTED_FAILURE = { + "check": "implementation-check", + "message": "expected implementation failure", + "issue": "https://github.com/TalonT-Org/AutoSkillit/issues/123", +} +_IMPLEMENTATION_INGREDIENTS = { + "source_dir": "/workspace/sandbox", + "open_pr": "true", + "auto_merge": "false", + "arch_lenses": "false", + "audit_impl": "true", +} _MAX_TIMEOUT_SEC = ( e2e_catalog.MAX_JOB_MINUTES - e2e_catalog.JOB_OVERHEAD_MINUTES ) * 60 - e2e_catalog.HARNESS_GRACE_SEC @@ -311,6 +387,28 @@ def _recipe_entry_without(field: str) -> dict: return _entry("impl", kind="recipe", **fields) +def _pipeline_entry(name: str = "implementation", **overrides) -> dict: + entry = _entry( + name, + tuple(f"{path}*" for path in _IMPLEMENTATION_PATHS), + kind="recipe", + peak_sessions=8, + timeout_sec=7200, + recipe="implementation", + ingredients=dict(_IMPLEMENTATION_INGREDIENTS), + expected_pull_request_state="open", + pipeline=True, + issue_url_env="E2E_IMPLEMENTATION_ISSUE_URL", + required_changed_paths=list(_IMPLEMENTATION_PATHS), + test_command=["task", "test-check"], + peak_sessions_evidence=( + "configured capacity limit of eight concurrent sessions; no measured peak supplied" + ), + ) + entry.update(overrides) + return entry + + _INVALID_CATALOGS = { "duplicate-names": _raw_catalog(_entry("a"), _entry("a")), "bad-name": _raw_catalog(_entry("Bad_Name")), @@ -352,6 +450,9 @@ def _recipe_entry_without(field: str) -> dict: **_RECIPE_FIELDS, ) ), + "expected-failures-on-recipe": _raw_catalog( + _pipeline_entry(expected_failures=[_EXPECTED_FAILURE]) + ), "expected-failures-missing-field": _raw_catalog( _clean_install_entry( expected_failures=[ @@ -465,6 +566,7 @@ def test_recipe_entry_parses(self) -> None: "impl", kind="recipe", recipe_fixture="sandbox-smoke.yaml", + expected_failures=[{"severity": "error", **_RECIPE_EXPECTED_FAILURE}], **_RECIPE_FIELDS, ) ) @@ -473,6 +575,12 @@ def test_recipe_entry_parses(self) -> None: assert test.ingredients == (("task", "Add a greeting"),) assert test.expected_pull_request_state == "merged" assert test.recipe_fixture == "sandbox-smoke.yaml" + assert test.pipeline is False + assert test.issue_url_env is None + assert test.required_changed_paths == () + assert test.test_command == () + assert test.peak_sessions_evidence is None + assert test.expected_failures == ({"severity": "error", **_RECIPE_EXPECTED_FAILURE},) def test_recipe_without_fixture_defaults_to_none(self) -> None: test = _catalog(_entry("impl", kind="recipe", **_RECIPE_FIELDS)).get("impl") @@ -524,3 +632,111 @@ def test_unknown_test_name_raises_catalog_error(self) -> None: def test_invalid_catalog_raises(self, raw: dict) -> None: with pytest.raises(e2e_catalog.CatalogError): e2e_catalog.parse_catalog(raw) + + +class TestPipelineCatalog: + def test_shipped_implementation_targets_sandbox_source_and_tests_without_a_measurement( + self, + ) -> None: + test = e2e_catalog.load_catalog().get("implementation") + assert test.required_changed_paths == ("sandbox/", "tests/") + assert test.peak_sessions == 8 + assert test.peak_sessions_evidence is None + assert test.timeout_sec == 7200 + assert dict(test.ingredients) == _IMPLEMENTATION_INGREDIENTS + + def test_pipeline_recipe_parses_its_contract_and_expected_failures(self) -> None: + rows = [ + _RECIPE_EXPECTED_FAILURE, + {**_RECIPE_EXPECTED_FAILURE, "check": "another-check"}, + ] + test = _catalog(_pipeline_entry(expected_failures=rows)).get("implementation") + + assert test.pipeline is True + assert test.issue_url_env == "E2E_IMPLEMENTATION_ISSUE_URL" + assert test.required_changed_paths == _IMPLEMENTATION_PATHS + assert test.test_command == ("task", "test-check") + assert test.peak_sessions_evidence == ( + "configured capacity limit of eight concurrent sessions; no measured peak supplied" + ) + assert test.expected_failures == tuple(rows) + with pytest.raises(TypeError): + cast(MutableMapping[str, str], test.expected_failures[0])["check"] = "changed" + + def test_recipe_options_are_independently_optional(self) -> None: + entry = _pipeline_entry() + for key in ( + "issue_url_env", + "required_changed_paths", + "test_command", + "peak_sessions_evidence", + ): + entry.pop(key) + test = _catalog(entry).get("implementation") + assert test.pipeline is True + assert test.issue_url_env is None + assert test.required_changed_paths == () + assert test.test_command == () + assert test.peak_sessions_evidence is None + + def test_pipeline_rejects_unknown_keys(self) -> None: + with pytest.raises(e2e_catalog.CatalogError, match="unknown keys"): + _catalog(_pipeline_entry(unrecognized=True)) + + @pytest.mark.parametrize("ingredient", ["issue_url", "task"]) + def test_issue_url_env_conflicts_with_recipe_ingredient(self, ingredient: str) -> None: + ingredients = {**_IMPLEMENTATION_INGREDIENTS, ingredient: "value"} + with pytest.raises(e2e_catalog.CatalogError, match="cannot contain issue_url or task"): + _catalog(_pipeline_entry(ingredients=ingredients)) + + def test_recipe_options_do_not_require_pipeline_true(self) -> None: + entry = _pipeline_entry(pipeline=False) + assert _catalog(entry).get("implementation").pipeline is False + + @pytest.mark.parametrize("value", [None, "true", 1]) + def test_pipeline_flag_must_be_boolean(self, value: object) -> None: + with pytest.raises(e2e_catalog.CatalogError, match="pipeline must be a boolean"): + _catalog(_pipeline_entry(pipeline=value)) + + @pytest.mark.parametrize("value", ["", " ", 123]) + def test_issue_url_environment_name_must_be_nonempty(self, value: object) -> None: + with pytest.raises(e2e_catalog.CatalogError, match="issue_url_env"): + _catalog(_pipeline_entry(issue_url_env=value)) + + @pytest.mark.parametrize( + "prefix", + ["", " ", 123, "/absolute/path", "../parent", "src/../other", "C:\\absolute"], + ) + def test_required_changed_paths_must_be_relative_prefixes(self, prefix: object) -> None: + with pytest.raises(e2e_catalog.CatalogError): + _catalog(_pipeline_entry(required_changed_paths=[prefix])) + + @pytest.mark.parametrize("command", [[], [""], [1]]) + def test_pipeline_test_command_must_be_nonempty_argv(self, command: object) -> None: + with pytest.raises(e2e_catalog.CatalogError, match="test_command"): + _catalog(_pipeline_entry(test_command=command)) + + @pytest.mark.parametrize("evidence", ["", " ", 123]) + def test_pipeline_peak_sessions_evidence_must_be_nonempty(self, evidence: object) -> None: + with pytest.raises(e2e_catalog.CatalogError, match="peak_sessions_evidence"): + _catalog(_pipeline_entry(peak_sessions_evidence=evidence)) + + @pytest.mark.parametrize( + "row", + [ + {key: value for key, value in _RECIPE_EXPECTED_FAILURE.items() if key != "check"}, + {**_RECIPE_EXPECTED_FAILURE, "severity": "warning"}, + {**_RECIPE_EXPECTED_FAILURE, "issue": "https://github.com/other/repo/issues/123"}, + ], + ) + def test_recipe_expected_failures_have_exact_canonical_rows(self, row: dict) -> None: + with pytest.raises(e2e_catalog.CatalogError): + _catalog(_pipeline_entry(expected_failures=[row])) + + def test_recipe_expected_failures_reject_duplicate_check_and_message(self) -> None: + duplicate = { + **_RECIPE_EXPECTED_FAILURE, + "issue": "https://github.com/TalonT-Org/AutoSkillit/issues/124", + } + with pytest.raises(e2e_catalog.CatalogError, match="duplicate diagnostic"): + _catalog(_pipeline_entry(expected_failures=[_RECIPE_EXPECTED_FAILURE, duplicate])) diff --git a/tests/infra/test_e2e_select_cli.py b/tests/infra/test_e2e_select_cli.py index 25a7f2c1e2..0f0fb0f317 100644 --- a/tests/infra/test_e2e_select_cli.py +++ b/tests/infra/test_e2e_select_cli.py @@ -24,6 +24,9 @@ _HEADLESS_SMOKE_MATRIX = ( '{"include":[{"test":"headless-smoke","kind":"recipe","timeout_minutes":47}]}' ) +_IMPLEMENTATION_MATRIX = ( + '{"include":[{"test":"implementation","kind":"recipe","timeout_minutes":152}]}' +) def _run( @@ -112,6 +115,19 @@ def test_small_unlabelled_pull_request_selects_nothing(tmp_path: Path) -> None: assert outputs["matrix"] == '{"include":[]}' +def test_large_pull_request_selects_implementation_pipeline(tmp_path: Path) -> None: + changed = tmp_path / "changed-paths.txt" + changed.write_text("src/autoskillit/recipes/changed.py\n", encoding="utf-8") + payload = _pull_request(changed_files=10, additions=0, labels=("e2e",)) + + result = _select(tmp_path, changed, "pull_request", payload) + + assert result.returncode == 0, result.stderr + outputs = _outputs(result.stdout) + assert outputs["selected"] == "true" + assert outputs["matrix"] == _IMPLEMENTATION_MATRIX + + def test_merge_group_never_reads_the_changed_paths_file(tmp_path: Path) -> None: result = _select(tmp_path, tmp_path / "missing.txt", "merge_group", {"merge_group": {}}) assert result.returncode == 0, result.stderr @@ -151,6 +167,19 @@ def test_dispatch_selects_both_named_tests_in_request_order(tmp_path: Path) -> N ) +def test_dispatch_selects_named_pipeline_without_pull_request_eligibility(tmp_path: Path) -> None: + result = _select( + tmp_path, + tmp_path / "missing.txt", + "workflow_dispatch", + {"inputs": {"tests": "implementation"}}, + ) + assert result.returncode == 0, result.stderr + outputs = _outputs(result.stdout) + assert outputs["selected"] == "true" + assert outputs["matrix"] == _IMPLEMENTATION_MATRIX + + def test_dispatch_of_an_unknown_test_fails(tmp_path: Path) -> None: payload = {"inputs": {"tests": "nope"}} result = _select(tmp_path, tmp_path / "missing.txt", "workflow_dispatch", payload) diff --git a/tests/infra/test_e2e_sessions.py b/tests/infra/test_e2e_sessions.py new file mode 100644 index 0000000000..465de639c6 --- /dev/null +++ b/tests/infra/test_e2e_sessions.py @@ -0,0 +1,1343 @@ +"""Tests for the E2E session lifecycle and ownership evidence helper.""" + +from __future__ import annotations + +import json +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any + +import pytest + +from autoskillit.core import DIRECT_PREFIX, PLUGIN_PREFIX +from tests.infra._complexity_helpers import load_check_script + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] + +REPO_ROOT = Path(__file__).resolve().parents[2] +_SCRIPT = REPO_ROOT / "scripts" / "e2e" / "e2e_sessions.py" +sessions = load_check_script("_autoskillit_e2e_sessions", _SCRIPT) + +DISPATCH_ID = "dispatch-5233" +ROOT_ID = "session-root" +ISSUE_URL = "https://github.com/owner/repo/issues/5233" +REPOSITORY = "owner/repo" + + +def _jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + "".join(json.dumps(row, separators=(",", ":")) + "\n" for row in rows), + encoding="utf-8", + ) + + +def _session( + log_root: Path, + session_id: str, + *, + parent_session_id: str = "", + caller_session_id: str = "", + records: list[dict[str, Any]] | None = None, +) -> Path: + transcript = log_root / "transcripts" / f"{session_id}.jsonl" + _jsonl(transcript, records or []) + session_dir = log_root / "sessions" / session_id + session_dir.mkdir(parents=True, exist_ok=True) + (session_dir / "summary.json").write_text( + json.dumps( + { + "session_id": session_id, + "dispatch_id": DISPATCH_ID, + "execution_identity": {"parent_session_id": parent_session_id}, + } + ), + encoding="utf-8", + ) + return transcript + + +def _index(log_root: Path, entries: list[dict[str, Any]]) -> None: + _jsonl(log_root / "sessions.jsonl", entries) + + +def _entry( + session_id: str, + transcript: Path, + *, + parent_session_id: str = "", + caller_session_id: str = "", +) -> dict[str, Any]: + return { + "session_id": session_id, + "dir_name": session_id, + "dispatch_id": DISPATCH_ID, + "parent_session_id": parent_session_id, + "caller_session_id": caller_session_id, + "claude_code_log": str(transcript), + } + + +def _event( + name: str, + session_id: str, + timestamp: float, + *, + transcript: Path | None = None, + agent_id: str | None = None, + agent_type: str | None = None, + agent_transcript: Path | None = None, +) -> dict[str, Any]: + row: dict[str, Any] = { + "event": name, + "timestamp": timestamp, + "session_id": session_id, + } + if transcript is not None: + row["transcript_path"] = str(transcript) + if agent_id is not None: + row["agent_id"] = agent_id + if agent_type is not None: + row["agent_type"] = agent_type + if agent_transcript is not None: + row["agent_transcript_path"] = str(agent_transcript) + return row + + +def _trace(path: Path, events: list[dict[str, Any]]) -> None: + _jsonl(path, events) + + +def _envelope(session_id: str = ROOT_ID) -> dict[str, str]: + return {"dispatched_session_id": session_id, "dispatch_id": DISPATCH_ID} + + +def _base_sessions( + tmp_path: Path, + *, + child_id: str | None = None, + child_records: list[dict[str, Any]] | None = None, +) -> tuple[Path, Path, Path | None]: + log_root = tmp_path / "logs" + root_transcript = _session(log_root, ROOT_ID) + entries = [_entry(ROOT_ID, root_transcript)] + child_transcript = None + if child_id is not None: + child_transcript = _session( + log_root, + child_id, + parent_session_id=ROOT_ID, + records=child_records, + ) + entries.append( + _entry( + child_id, + child_transcript, + parent_session_id=ROOT_ID, + caller_session_id=ROOT_ID, + ) + ) + _index(log_root, entries) + trace = tmp_path / "run" / "session-lifecycle.jsonl" + return log_root, trace, child_transcript + + +def _worker(log_root: Path, session_id: str) -> Path: + transcript = _session(log_root, session_id) + index_path = log_root / "sessions.jsonl" + entries = [json.loads(line) for line in index_path.read_text(encoding="utf-8").splitlines()] + entries.append(_entry(session_id, transcript)) + _index(log_root, entries) + return transcript + + +def _trace_root_and_worker( + trace: Path, + log_root: Path, + worker_id: str, + worker_transcript: Path, +) -> None: + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionStart", worker_id, 2, transcript=worker_transcript), + _event("SessionEnd", worker_id, 3, transcript=worker_transcript), + _event("SessionEnd", ROOT_ID, 4, transcript=root_transcript), + ], + ) + + +def _tool_use( + tool_id: str, + name: str, + tool_input: dict[str, Any], + *, + prefix: str = PLUGIN_PREFIX, +) -> dict[str, Any]: + return { + "type": "assistant", + "message": { + "content": [ + { + "type": "tool_use", + "id": tool_id, + "name": f"{prefix}{name}", + "input": tool_input, + } + ] + }, + } + + +def _tool_result( + tool_id: str, + result: dict[str, Any], + *, + carrier: str = "mcp_meta", + is_error: bool = False, +) -> dict[str, Any]: + display = "tool reply\n\nsuccess: True" + if carrier == "mcp_meta" or carrier == "display_only": + content = display + elif carrier == "block_json": + content = json.dumps(result) + else: + raise ValueError(f"unsupported tool result carrier: {carrier}") + + row: dict[str, Any] = { + "type": "user", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": tool_id, + "content": content, + "is_error": is_error, + } + ] + }, + } + if carrier == "mcp_meta": + row["mcpMeta"] = { + "_meta": {"fastmcp": {"wrap_result": True}}, + "structuredContent": {"result": json.dumps(result)}, + } + return row + + +def _launch( + tool_id: str, + reply: dict[str, Any], + *, + tool: str = "run_skill", + is_error: bool = False, +) -> list[dict[str, Any]]: + return [ + _tool_use(tool_id, tool, {}), + _tool_result(tool_id, reply, is_error=is_error), + ] + + +def _root_ownership_proof( + tmp_path: Path, + records: list[dict[str, Any]], +) -> dict[str, Any]: + log_root, trace, _ = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl(root_transcript, records) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=root_transcript), + ], + ) + return sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) + + +def test_hook_capture_appends_concurrent_allow_listed_records(tmp_path: Path) -> None: + trace = tmp_path / "events.jsonl" + with ThreadPoolExecutor(max_workers=8) as pool: + results = list( + pool.map( + lambda index: sessions.capture_hook_event( + trace, + { + "hook_event_name": "SessionStart", + "session_id": f"session-{index}", + "source": "startup", + "transcript_path": f"/tmp/{index}.jsonl", + "prompt": "must not be persisted", + "environment": {"TOKEN": "must not be persisted"}, + }, + ), + range(32), + ) + ) + + records = [json.loads(line) for line in trace.read_text(encoding="utf-8").splitlines()] + assert all(results) + assert len(records) == 32 + assert {record["session_id"] for record in records} == { + f"session-{index}" for index in range(32) + } + assert all(isinstance(record["timestamp"], (int, float)) for record in records) + assert all( + set(record) <= {"event", "timestamp", "session_id", "source", "transcript_path"} + for record in records + ) + assert ( + sessions.capture_hook_event(trace, {"event": "OtherHook", "session_id": "ignored"}) + is False + ) + assert len(trace.read_text(encoding="utf-8").splitlines()) == 32 + + +@pytest.mark.parametrize("agent_type", ["general-purpose", ""]) +def test_hook_capture_preserves_agent_type_and_strips_stop_message( + tmp_path: Path, agent_type: str +) -> None: + trace = tmp_path / "events.jsonl" + captured = sessions.capture_hook_event( + trace, + { + "hook_event_name": "SubagentStop", + "session_id": ROOT_ID, + "agent_id": "agent-x", + "agent_type": agent_type, + "transcript_path": "/tmp/root.jsonl", + "agent_transcript_path": "/tmp/agent-x.jsonl", + "last_assistant_message": "must not be persisted", + }, + ) + + assert captured is True + [record] = [json.loads(line) for line in trace.read_text(encoding="utf-8").splitlines()] + assert record["agent_type"] == agent_type + assert set(record) == { + "event", + "timestamp", + "session_id", + "agent_id", + "agent_type", + "transcript_path", + "agent_transcript_path", + } + + +def test_summary_collapses_repeated_events_excludes_unrelated_and_sweeps_half_open( + tmp_path: Path, +) -> None: + log_root, trace, child_transcript = _base_sessions(tmp_path, child_id="session-child") + assert child_transcript is not None + unrelated = _session(log_root, "session-unrelated", parent_session_id="other-parent") + rows = json.loads((log_root / "sessions.jsonl").read_text(encoding="utf-8").splitlines()[0]) + entries = [ + rows, + _entry( + "session-child", child_transcript, parent_session_id=ROOT_ID, caller_session_id=ROOT_ID + ), + ] + entries.append( + _entry( + "session-unrelated", + unrelated, + parent_session_id="other-parent", + caller_session_id="other-parent", + ) + ) + _index(log_root, entries) + _trace( + trace, + [ + _event( + "SessionStart", + ROOT_ID, + 1, + transcript=log_root / "transcripts" / f"{ROOT_ID}.jsonl", + ), + _event( + "SessionStart", + ROOT_ID, + 1.5, + transcript=log_root / "transcripts" / f"{ROOT_ID}.jsonl", + ), + _event("SessionStart", "session-child", 3, transcript=child_transcript), + _event("SessionStart", "session-unrelated", 1, transcript=unrelated), + _event( + "SessionEnd", ROOT_ID, 2, transcript=log_root / "transcripts" / f"{ROOT_ID}.jsonl" + ), + _event( + "SessionEnd", ROOT_ID, 3, transcript=log_root / "transcripts" / f"{ROOT_ID}.jsonl" + ), + _event("SessionEnd", "session-child", 4, transcript=child_transcript), + _event("SessionEnd", "session-unrelated", 5, transcript=unrelated), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["peak_sessions"] == 1 + assert summary["coverage"]["complete"] is True + assert summary["violations"] == [] + assert summary["coverage"]["managed_session_ids"] == ["session-child"] + assert [identity["session_id"] for identity in summary["identities"]] == [ + ROOT_ID, + "session-child", + ] + root_interval = summary["identities"][0] + assert (root_interval["start"], root_interval["end"]) == (1.0, 3.0) + assert (trace.parent / "session-concurrency.json").is_file() + + +@pytest.mark.parametrize( + ("child_event", "expected_violation"), + [("SessionStart", "missing_end"), ("SessionEnd", "orphan_end")], +) +def test_summary_does_not_infer_child_end( + tmp_path: Path, child_event: str, expected_violation: str +) -> None: + log_root, trace, child_transcript = _base_sessions(tmp_path, child_id="session-child") + assert child_transcript is not None + _trace( + trace, + [ + _event( + "SessionStart", + ROOT_ID, + 1, + transcript=log_root / "transcripts" / f"{ROOT_ID}.jsonl", + ), + _event(child_event, "session-child", 2, transcript=child_transcript), + _event( + "SessionEnd", ROOT_ID, 5, transcript=log_root / "transcripts" / f"{ROOT_ID}.jsonl" + ), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["peak_sessions"] == 1 + assert summary["coverage"]["complete"] is False + assert f"lifecycle:session:session-child:{expected_violation}" in summary["violations"] + + +def test_summary_enumerates_native_agent_transcripts_and_requires_endpoints( + tmp_path: Path, +) -> None: + log_root, trace, _child = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + agent_transcript = log_root / "transcripts" / ROOT_ID / "subagents" / "agent-agent-a.jsonl" + agent_transcript.parent.mkdir(parents=True) + agent_transcript.write_text("{}\n", encoding="utf-8") + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event( + "SubagentStart", + ROOT_ID, + 2, + agent_id="agent-a", + agent_type="general-purpose", + ), + _event( + "SubagentStop", + ROOT_ID, + 4, + agent_id="agent-a", + agent_type="general-purpose", + agent_transcript=agent_transcript, + ), + _event("SessionEnd", ROOT_ID, 5, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["complete"] is True + assert summary["peak_sessions"] == 2 + assert summary["coverage"]["native_agents"] == [ + { + "session_id": ROOT_ID, + "agent_id": "agent-a", + "transcript_path": str(agent_transcript.resolve()), + } + ] + + +@pytest.mark.parametrize( + ("agent_type", "extra_violation"), + [ + pytest.param("custom-main", None, id="named-custom-main"), + pytest.param("", f"capture:{ROOT_ID}:agent-x:agent_type_missing", id="blank-type"), + pytest.param(None, f"capture:{ROOT_ID}:agent-x:agent_type_missing", id="missing-type"), + ], +) +def test_orphan_agent_stop_is_reported_even_without_an_admitted_type( + tmp_path: Path, + agent_type: str | None, + extra_violation: str | None, +) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + missing_agent_transcript = ( + log_root / "transcripts" / ROOT_ID / "subagents" / "agent-agent-x.jsonl" + ) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event( + "SubagentStop", + ROOT_ID, + 2, + agent_id="agent-x", + agent_type=agent_type, + agent_transcript=missing_agent_transcript, + ), + _event("SessionEnd", ROOT_ID, 3, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + expected_violations = [ + f"lifecycle:agent:{ROOT_ID}:agent-x:orphan_end", + f"native:{ROOT_ID}:agent-x:lifecycle_incomplete", + f"native:{ROOT_ID}:agent-x:transcript_unenumerated", + ] + if extra_violation is not None: + expected_violations.append(extra_violation) + assert summary["coverage"]["complete"] is False + assert summary["violations"] == sorted(expected_violations) + assert summary["peak_sessions"] == 1 + + +def test_enumerated_native_transcript_without_agent_hooks_is_incomplete( + tmp_path: Path, +) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + agent_transcript = log_root / "transcripts" / ROOT_ID / "subagents" / "agent-agent-a.jsonl" + agent_transcript.parent.mkdir(parents=True) + agent_transcript.write_text("{}\n", encoding="utf-8") + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 3, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["complete"] is False + assert f"native:{ROOT_ID}:agent-a:lifecycle_incomplete" in summary["violations"] + + +def test_native_transcript_symlink_loop_is_reported_missing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + transcript = tmp_path / "agent-agent-a.jsonl" + transcript.symlink_to(transcript.name) + + def resolve_loop(_path: Path) -> Path: + raise RuntimeError("Symlink loop") + + monkeypatch.setattr(Path, "resolve", resolve_loop) + violations: list[str] = [] + proof = sessions._verify_native_transcript( + ROOT_ID, + "agent-a", + transcript, + [{"event": "SubagentStart"}, {"event": "SubagentStop"}], + violations, + ) + + assert proof is None + assert violations == [f"native:{ROOT_ID}:agent-a:transcript_missing"] + + +def test_indexed_owned_child_without_hooks_is_incomplete(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path, child_id="missing-hooks") + transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=transcript), + _event("SessionEnd", ROOT_ID, 5, transcript=transcript), + ], + ) + summary = sessions.summarize(trace, _envelope(), log_root) + assert not summary["coverage"]["complete"] + assert "lifecycle:missing-hooks:missing_endpoints" in summary["violations"] + + +def test_resumed_identity_requires_a_final_end(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=transcript), + _event("SessionStart", ROOT_ID, 3, transcript=transcript), + ], + ) + summary = sessions.summarize(trace, _envelope(), log_root) + assert not summary["coverage"]["complete"] + assert f"lifecycle:session:{ROOT_ID}:missing_final_end" in summary["violations"] + + +def test_empty_half_open_interval_does_not_reduce_concurrent_peak() -> None: + assert ( + sessions._peak_sessions( + [{"start": 1, "end": 3}, {"start": 2, "end": 2}, {"start": 2, "end": 4}] + ) + == 2 + ) + + +def test_no_attempted_resource_calls_leave_cleanup_proof_empty(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=transcript), + ], + ) + proof = sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) + assert proof["branches"] == [] + assert proof["claimed"] is False + assert proof["violations"] == [] + + +def test_missing_root_hooks_do_not_discard_available_branch_proof(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + transcript, + [ + _tool_use( + "branch", + "create_and_publish_branch", + {"issue_number": "5233", "remote_url": "https://github.com/owner/repo.git"}, + ), + _tool_result("branch", {"merge_target": "impl/5233-fix", "was_unique": True}), + ], + ) + _trace(trace, []) + proof = sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) + assert proof["branches"] == ["impl/5233-fix"] + assert "trace:root_session_missing" in proof["violations"] + + +def test_missing_root_and_malformed_trace_are_incomplete(tmp_path: Path) -> None: + log_root = tmp_path / "logs" + trace = tmp_path / "run" / "session-lifecycle.jsonl" + trace.parent.mkdir(parents=True) + trace.write_text("{malformed\n", encoding="utf-8") + + summary = sessions.summarize(trace, None, log_root) + + assert summary["coverage"]["complete"] is False + assert "envelope:missing" in summary["violations"] + assert "sessions_index:unavailable" in summary["violations"] + assert "trace:invalid_json:1" in summary["violations"] + + +@pytest.mark.parametrize("dir_name", [".", ".."]) +def test_summary_rejects_dot_session_directories(tmp_path: Path, dir_name: str) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + entry = _entry(ROOT_ID, root_transcript) + entry["dir_name"] = dir_name + _index(log_root, [entry]) + _trace(trace, [_event("SessionStart", ROOT_ID, 1, transcript=root_transcript)]) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["complete"] is False + assert f"sessions_index:{ROOT_ID}:invalid_dir_name" in summary["violations"] + + +def test_summary_rejects_malformed_owned_index_summary_and_transcript( + tmp_path: Path, +) -> None: + log_root, trace, _child = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + index_path = log_root / "sessions.jsonl" + index_path.write_text( + index_path.read_text(encoding="utf-8") + "{malformed\n", encoding="utf-8" + ) + (log_root / "sessions" / ROOT_ID / "summary.json").write_text("{malformed\n", encoding="utf-8") + root_transcript.write_text("{malformed\n", encoding="utf-8") + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["complete"] is False + assert "sessions_index:invalid_json:2" in summary["violations"] + assert f"summary:{ROOT_ID}:missing_or_malformed" in summary["violations"] + assert f"transcript:{ROOT_ID}:invalid_json:1" in summary["violations"] + + +@pytest.mark.parametrize( + ("claim_input_label", "expected_claimed"), + [(None, True), ("in-progress", True), ("different-label", False)], +) +def test_ownership_pairs_root_branch_with_managed_claim_and_keeps_compact_proof( + tmp_path: Path, + claim_input_label: str | None, + expected_claimed: bool, +) -> None: + claim_input: dict[str, Any] = {"issue_url": ISSUE_URL, "allow_reentry": False} + if claim_input_label is not None: + claim_input["label"] = claim_input_label + claim_records = [ + _tool_use("claim-1", "claim_and_resolve_issue", claim_input), + _tool_result( + "claim-1", + { + "success": True, + "claimed": True, + "issue_number": 5233, + "issue_title": "private title", + }, + ), + ] + log_root, trace, child_transcript = _base_sessions( + tmp_path, + child_id="session-child", + child_records=claim_records, + ) + assert child_transcript is not None + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + root_transcript, + [ + _tool_use( + "branch-1", + "create_and_publish_branch", + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + ), + _tool_result("branch-1", {"merge_target": "impl/5233-fix", "was_unique": True}), + ], + ) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionStart", "session-child", 2, transcript=child_transcript), + _event("SessionEnd", "session-child", 4, transcript=child_transcript), + _event("SessionEnd", ROOT_ID, 5, transcript=root_transcript), + ], + ) + + proof = sessions.collect_ownership( + trace, _envelope(), log_root, ISSUE_URL, REPOSITORY, claim_label="in-progress" + ) + + assert proof["branches"] == ["impl/5233-fix"] + assert proof["claimed"] is expected_claimed + if expected_claimed: + assert proof["violations"] == [] + else: + assert "ownership:claim_result_failed" in proof["violations"] + assert {item["session_id"] for item in proof["transcripts"]} == {ROOT_ID, "session-child"} + + +def test_ownership_ignores_unhashable_tool_names(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + record = _tool_use("unrelated", "ignored", {}) + record["message"]["content"][0]["name"] = [] + _jsonl(root_transcript, [record]) + _trace(trace, [_event("SessionStart", ROOT_ID, 1, transcript=root_transcript)]) + + proof = sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) + + assert proof["branches"] == [] + assert proof["claimed"] is False + assert proof["violations"] == [] + + +@pytest.mark.parametrize("scenario", ["child_branch", "unpaired_root_branch", "claim_reentry"]) +def test_ownership_isolates_branch_and_claim_scenarios( + tmp_path: Path, + scenario: str, +) -> None: + child_records = [ + _tool_use( + "child-branch", + "create_and_publish_branch", + {"issue_number": "5233", "remote_url": "git@github.com:owner/repo.git"}, + ), + _tool_result("child-branch", {"merge_target": "wrong-owner/branch"}), + ] + log_root, trace, child_transcript = _base_sessions( + tmp_path, + child_id="session-child" if scenario == "child_branch" else None, + child_records=child_records if scenario == "child_branch" else None, + ) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + if scenario == "claim_reentry": + root_records = [ + _tool_use( + "claim-1", + "claim_and_resolve_issue", + {"issue_url": ISSUE_URL, "allow_reentry": False}, + ), + _tool_result("claim-1", {"success": True, "claimed": True, "issue_number": 5233}), + _tool_use( + "claim-2", + "claim_and_resolve_issue", + {"issue_url": ISSUE_URL, "allow_reentry": False}, + ), + _tool_result("claim-2", {"success": True, "claimed": True, "issue_number": 5233}), + ] + else: + root_records = [ + _tool_use( + "root-branch", + "create_and_publish_branch", + {"issue_number": "5233", "remote_url": "git@github.com:owner/repo.git"}, + ), + _tool_result( + "root-branch" if scenario == "child_branch" else "wrong-branch-id", + {"merge_target": "impl/5233-root"}, + ), + ] + _jsonl(root_transcript, root_records) + _trace(trace, [_event("SessionStart", ROOT_ID, 1, transcript=root_transcript)]) + + proof = sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) + + assert proof["branches"] == (["impl/5233-root"] if scenario == "child_branch" else []) + assert proof["claimed"] is False + if scenario == "claim_reentry": + assert proof["violations"] == ["ownership:claim_reentry"] + elif scenario == "unpaired_root_branch": + assert proof["violations"] == [ + "ownership:branch_result_failed", + f"transcript:{ROOT_ID}:unpaired_tool_use:root-branch", + ] + else: + assert proof["violations"] == [] + + +@pytest.mark.parametrize("prefix", [DIRECT_PREFIX, PLUGIN_PREFIX]) +def test_ownership_accepts_both_canonical_namespaces(tmp_path: Path, prefix: str) -> None: + proof = _root_ownership_proof( + tmp_path, + [ + _tool_use( + "claim", + "claim_and_resolve_issue", + {"issue_url": ISSUE_URL, "allow_reentry": False}, + prefix=prefix, + ), + _tool_result("claim", {"success": True, "claimed": True, "issue_number": 5233}), + _tool_use( + "branch", + "create_and_publish_branch", + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + prefix=prefix, + ), + _tool_result("branch", {"merge_target": "impl/5233-fix", "was_unique": True}), + ], + ) + + assert proof["branches"] == ["impl/5233-fix"] + assert proof["claimed"] is True + assert proof["violations"] == [] + + +@pytest.mark.parametrize( + "name", + [ + "create_and_publish_branch", + "mcp__other__create_and_publish_branch", + f"{PLUGIN_PREFIX}extra__create_and_publish_branch", + ], +) +def test_tracked_tool_under_another_namespace_is_reported(tmp_path: Path, name: str) -> None: + proof = _root_ownership_proof( + tmp_path, + [ + _tool_use( + "branch", + name, + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + prefix="", + ), + _tool_result("branch", {"merge_target": "impl/5233-fix"}), + ], + ) + + assert proof["branches"] == [] + assert f"transcript:{ROOT_ID}:unrecognised_tool_namespace:1" in proof["violations"] + + +@pytest.mark.parametrize("carrier", ["mcp_meta", "block_json"]) +def test_reply_is_read_from_each_supported_carrier(tmp_path: Path, carrier: str) -> None: + proof = _root_ownership_proof( + tmp_path, + [ + _tool_use( + "branch", + "create_and_publish_branch", + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + ), + _tool_result("branch", {"merge_target": "impl/5233-fix"}, carrier=carrier), + ], + ) + + assert proof["branches"] == ["impl/5233-fix"] + assert proof["violations"] == [] + + +def test_non_json_display_text_proves_no_reply(tmp_path: Path) -> None: + result_row = _tool_result( + "branch", + {"merge_target": "impl/evil"}, + carrier="display_only", + ) + result_row["message"]["content"][0]["content"] += "\nmerge_target: impl/evil" + proof = _root_ownership_proof( + tmp_path, + [ + _tool_use( + "branch", + "create_and_publish_branch", + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + ), + result_row, + ], + ) + + assert proof["branches"] == [] + assert "ownership:branch_result_failed" in proof["violations"] + + +def test_invalid_row_metadata_does_not_fall_back_to_block_json(tmp_path: Path) -> None: + result_row = _tool_result( + "branch", + {"merge_target": "impl/5233-fix"}, + carrier="block_json", + ) + result_row["mcpMeta"] = { + "_meta": {"fastmcp": {"wrap_result": True}}, + "structuredContent": {"result": "not json"}, + } + proof = _root_ownership_proof( + tmp_path, + [ + _tool_use( + "branch", + "create_and_publish_branch", + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + ), + result_row, + ], + ) + + assert proof["branches"] == [] + assert "ownership:branch_result_failed" in proof["violations"] + + +def test_row_level_reply_is_ignored_when_a_row_holds_several_results(tmp_path: Path) -> None: + result = {"merge_target": "impl/5233-fix"} + proof = _root_ownership_proof( + tmp_path, + [ + _tool_use( + "branch", + "create_and_publish_branch", + { + "issue_number": "5233", + "remote_url": "https://github.com/owner/repo.git", + }, + ), + _tool_use("claim", "claim_and_resolve_issue", {"issue_url": "https://example.com"}), + { + "type": "user", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": "branch", + "content": "tool reply\n\nsuccess: True", + }, + { + "type": "tool_result", + "tool_use_id": "claim", + "content": "tool reply\n\nsuccess: True", + }, + ] + }, + "mcpMeta": { + "_meta": {"fastmcp": {"wrap_result": True}}, + "structuredContent": {"result": json.dumps(result)}, + }, + }, + ], + ) + + assert proof["branches"] == [] + assert "ownership:branch_result_failed" in proof["violations"] + + +def test_untracked_merge_target_reply_is_not_a_branch(tmp_path: Path) -> None: + proof = _root_ownership_proof( + tmp_path, + [ + _tool_use("clone", "bootstrap_clone", {}), + _tool_result("clone", {"merge_target": "main"}), + ], + ) + + assert proof["branches"] == [] + assert proof["violations"] == [] + + +@pytest.mark.parametrize( + ("tool", "success"), + [ + ("run_skill", True), + ("complete_run_skill_result", True), + ("recover_run_skill_result", True), + ("run_skill", False), + ], +) +def test_launch_receipt_adopts_a_worker_with_empty_parent_fields( + tmp_path: Path, + tool: str, + success: bool, +) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + root_transcript, + _launch( + "launch", + {"success": success, "receipt_id": "r1", "session_id": worker_id}, + tool=tool, + ), + ) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["managed_session_ids"] == [worker_id] + assert summary["coverage"]["complete"] is True + + +def test_acknowledgement_alone_adopts_a_worker(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + root_transcript, + [ + *_launch( + "launch", + {"success": True, "receipt_id": "r1", "audit_status": "published"}, + ), + *_launch( + "ack", + {"success": True, "receipt_id": "r1", "session_id": worker_id}, + tool="complete_run_skill_result", + ), + ], + ) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["managed_session_ids"] == [worker_id] + assert summary["violations"] == [] + + +@pytest.mark.parametrize("worker_dispatch", [DISPATCH_ID, "another-dispatch"]) +def test_overlapping_session_without_receipt_or_parent_is_reported_not_adopted( + tmp_path: Path, + worker_dispatch: str, +) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + index_path = log_root / "sessions.jsonl" + entries = [json.loads(line) for line in index_path.read_text(encoding="utf-8").splitlines()] + next(row for row in entries if row["session_id"] == worker_id)["dispatch_id"] = worker_dispatch + _index(log_root, entries) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + proof = sessions.collect_ownership(trace, _envelope(), log_root, ISSUE_URL, REPOSITORY) + + violation = f"lineage:{worker_id}:unattributed" + assert worker_id not in summary["coverage"]["managed_session_ids"] + assert violation in summary["violations"] + assert summary["coverage"]["complete"] is False + assert violation in proof["violations"] + + +def test_receipt_in_an_unowned_transcript_does_not_adopt(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + foreign_id = "session-foreign" + foreign_transcript = _session( + log_root, + foreign_id, + parent_session_id="other-parent", + records=_launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": worker_id}, + ), + ) + index_path = log_root / "sessions.jsonl" + entries = [json.loads(line) for line in index_path.read_text(encoding="utf-8").splitlines()] + entries.append( + _entry( + foreign_id, + foreign_transcript, + parent_session_id="other-parent", + caller_session_id="other-parent", + ) + ) + _index(log_root, entries) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert worker_id not in summary["coverage"]["managed_session_ids"] + assert f"lineage:{worker_id}:unattributed" in summary["violations"] + + +@pytest.mark.parametrize( + "reply", + [ + { + "success": False, + "is_error": True, + "error": "permission denied", + "stage": "preflight", + "retriable": False, + }, + {"success": False, "session_id": ""}, + {"success": False, "session_id": "headless-0123456789ab"}, + ], +) +def test_reply_without_receipt_id_proves_no_launch( + tmp_path: Path, + reply: dict[str, Any], +) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl(root_transcript, _launch("launch", reply)) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["managed_session_ids"] == [] + assert summary["violations"] == [] + + +def test_launch_reply_marked_is_error_is_not_evidence(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + root_transcript, + _launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": worker_id}, + is_error=True, + ), + ) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert worker_id not in summary["coverage"]["managed_session_ids"] + assert f"lineage:{worker_id}:unattributed" in summary["violations"] + + +def test_launched_session_absent_from_trace_and_index_is_reported(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + root_transcript, + _launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": "session-ghost"}, + ), + ) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert "lineage:session-ghost:launched_session_unobserved" in summary["violations"] + + +@pytest.mark.parametrize( + ("index_count", "receipt_names_worker"), + [(0, False), (0, True), (2, False), (2, True)], +) +def test_candidate_without_exactly_one_index_row_is_reported( + tmp_path: Path, + index_count: int, + receipt_names_worker: bool, +) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + index_path = log_root / "sessions.jsonl" + entries = [json.loads(line) for line in index_path.read_text(encoding="utf-8").splitlines()] + worker_entry = next(row for row in entries if row["session_id"] == worker_id) + entries = [row for row in entries if row["session_id"] != worker_id] + if index_count == 2: + entries.extend([worker_entry, dict(worker_entry)]) + _index(log_root, entries) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + root_transcript, + _launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": worker_id}, + ) + if receipt_names_worker + else [], + ) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert worker_id not in summary["coverage"]["managed_session_ids"] + assert f"sessions_index:{worker_id}:missing_or_ambiguous" in summary["violations"] + + +def test_receipt_named_worker_from_another_dispatch_is_not_adopted(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + index_path = log_root / "sessions.jsonl" + entries = [json.loads(line) for line in index_path.read_text(encoding="utf-8").splitlines()] + next(row for row in entries if row["session_id"] == worker_id)["dispatch_id"] = ( + "another-dispatch" + ) + _index(log_root, entries) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + root_transcript, + _launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": worker_id}, + ), + ) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert worker_id not in summary["coverage"]["managed_session_ids"] + assert f"sessions_index:{worker_id}:dispatch_mismatch" in summary["violations"] + + +def test_receipt_adopted_worker_with_a_contradicting_declared_parent_is_reported( + tmp_path: Path, +) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + worker_id = "session-worker" + worker_transcript = _worker(log_root, worker_id) + summary_path = log_root / "sessions" / worker_id / "summary.json" + worker_summary = json.loads(summary_path.read_text(encoding="utf-8")) + worker_summary["execution_identity"]["parent_session_id"] = "other-parent" + summary_path.write_text(json.dumps(worker_summary), encoding="utf-8") + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + root_transcript, + _launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": worker_id}, + ), + ) + _trace_root_and_worker(trace, log_root, worker_id, worker_transcript) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["managed_session_ids"] == [worker_id] + assert f"summary:{worker_id}:parent_mismatch" in summary["violations"] + + +def test_receipt_adopted_worker_without_lifecycle_endpoints_is_incomplete(tmp_path: Path) -> None: + log_root, trace, _ = _base_sessions(tmp_path) + worker_id = "session-worker" + _worker(log_root, worker_id) + root_transcript = log_root / "transcripts" / f"{ROOT_ID}.jsonl" + _jsonl( + root_transcript, + _launch( + "launch", + {"success": True, "receipt_id": "r1", "session_id": worker_id}, + ), + ) + _trace( + trace, + [ + _event("SessionStart", ROOT_ID, 1, transcript=root_transcript), + _event("SessionEnd", ROOT_ID, 2, transcript=root_transcript), + ], + ) + + summary = sessions.summarize(trace, _envelope(), log_root) + + assert summary["coverage"]["managed_session_ids"] == [worker_id] + assert summary["coverage"]["complete"] is False + assert f"lifecycle:{worker_id}:missing_endpoints" in summary["violations"] diff --git a/tests/infra/test_e2e_sessions_captured.py b/tests/infra/test_e2e_sessions_captured.py new file mode 100644 index 0000000000..03b4d4b52d --- /dev/null +++ b/tests/infra/test_e2e_sessions_captured.py @@ -0,0 +1,256 @@ +"""Tests for sanitized, captured E2E session evidence.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any + +import pytest + +from autoskillit.core import PLUGIN_PREFIX +from tests._hook_protocol_oracle import claude_matcher_admits +from tests.infra._complexity_helpers import load_check_script +from tests.infra._e2e_evidence import load_provenance, materialize_run + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] + +REPO_ROOT = Path(__file__).resolve().parents[2] +FIXTURE_NAME = "run_38170521660_attempt_1" +FIXTURE_DIR = Path(__file__).with_name("fixtures") / "e2e_sessions" / FIXTURE_NAME +_SCRIPT = REPO_ROOT / "scripts" / "e2e" / "e2e_sessions.py" +sessions = load_check_script("_autoskillit_e2e_sessions_captured", _SCRIPT) +ROOT_SESSION_ID = "2292658a-34cf-44a0-a0b1-f7a20218874b" +WORKER_SESSION_IDS = ( + "8ef8f993-8caa-47dd-83b0-26f9e5b51139", + "54f18144-1188-45d7-a392-17b4529f721e", +) +UNMATCHED_AGENT_ID = "a610a4bda1f28ca35" +NATIVE_AGENT_IDS = { + "a516cbae85151dc7c", + "a9ed51b7c02b828dd", + "aeb96cc0ddbdab294", + "a7e613554b849d6b2", +} + +_INPUTS_TO_PRESERVE = { + PLUGIN_PREFIX + "claim_and_resolve_issue", + PLUGIN_PREFIX + "create_and_publish_branch", +} +_TRUNCATION = re.compile(r"\.\.\.\[fixture-truncated (\d+) chars\]$") + + +def _rows(path: Path) -> list[dict[str, Any]]: + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + + +def _tool_reply(rows: list[dict[str, Any]], suffix: str) -> tuple[dict[str, Any], dict[str, Any]]: + matches = [ + (row, block) + for row in rows + for block in row.get("message", {}).get("content", []) + if isinstance(block, dict) + and block.get("type") == "tool_use" + and isinstance(block.get("name"), str) + and block["name"].endswith(suffix) + ] + assert len(matches) == 1 + use_row, use_block = matches[0] + reply = next(row for row in rows if row.get("sourceToolAssistantUUID") == use_row["uuid"]) + result = next( + block + for block in reply.get("message", {}).get("content", []) + if isinstance(block, dict) + and block.get("type") == "tool_result" + and block.get("tool_use_id") == use_block.get("id") + ) + return reply, result + + +def _assert_truncated_prose(value: Any, *, preserve_input: bool = False) -> None: + if isinstance(value, str): + stripped = value.lstrip() + if not preserve_input and stripped.startswith(("{", "[")): + try: + parsed = json.loads(value) + except json.JSONDecodeError: + parsed = None + if isinstance(parsed, (dict, list)): + _assert_truncated_prose(parsed) + return + if not preserve_input and len(value) > 400: + marker = _TRUNCATION.search(value) + assert marker is not None + assert marker.start() == 400 + assert int(marker.group(1)) > 0 + return + if isinstance(value, list): + for item in value: + _assert_truncated_prose(item, preserve_input=preserve_input) + return + if isinstance(value, dict): + keep_full_input = ( + value.get("type") == "tool_use" and value.get("name") in _INPUTS_TO_PRESERVE + ) + for key, item in value.items(): + _assert_truncated_prose( + item, + preserve_input=preserve_input or (key == "input" and keep_full_input), + ) + + +def test_captured_fixture_is_sanitized_and_keeps_the_real_wire_shape() -> None: + provenance = load_provenance(FIXTURE_NAME) + for path in FIXTURE_DIR.rglob("*"): + if path.is_file() and path.name != "PROVENANCE.json": + assert b"/home/" not in path.read_bytes(), path.relative_to(FIXTURE_DIR) + + envelope = json.loads((FIXTURE_DIR / "out/envelope.json").read_text(encoding="utf-8")) + root_transcript = ( + FIXTURE_DIR + / "projects" + / "-workspace-sandbox" + / f"{envelope['dispatched_session_id']}.jsonl" + ) + rows = _rows(root_transcript) + version_rows = [row["version"] for row in rows if "version" in row] + assert version_rows + assert all(version == provenance["cli_version"] for version in version_rows) + + tool_uses = [ + block + for row in rows + for block in row.get("message", {}).get("content", []) + if isinstance(block, dict) and block.get("type") == "tool_use" + ] + autoskillit_names = [ + block["name"] + for block in tool_uses + if isinstance(block.get("name"), str) and "autoskillit" in block["name"] + ] + assert autoskillit_names + assert all(name.startswith(PLUGIN_PREFIX) for name in autoskillit_names) + + for suffix in ("claim_and_resolve_issue", "create_and_publish_branch"): + reply, result_block = _tool_reply(rows, suffix) + structured = reply["mcpMeta"]["structuredContent"]["result"] + assert isinstance(json.loads(structured), dict) + with pytest.raises(json.JSONDecodeError): + json.loads(result_block["content"]) + + bootstrap_reply, _ = _tool_reply(rows, "bootstrap_clone") + bootstrap_result = json.loads(bootstrap_reply["mcpMeta"]["structuredContent"]["result"]) + assert bootstrap_result["merge_target"] == "main" + _assert_truncated_prose(rows) + + +def test_captured_run_proves_claim_and_published_branch(tmp_path: Path) -> None: + provenance = load_provenance(FIXTURE_NAME) + trace = tmp_path / "trace" / "nested" / "session-events.jsonl" + log_root = tmp_path / "log-root" + projects = tmp_path / "projects" + envelope = materialize_run( + FIXTURE_NAME, + trace=trace, + log_root=log_root, + projects=projects, + ) + + proof = sessions.collect_ownership( + trace, + envelope, + log_root, + provenance["issue_url"], + provenance["repository"], + claim_label=provenance["claim_label"], + ) + assert proof["branches"] == ["e2e-implementation-fixture-normalize-whitespace/1-3"] + assert proof["claimed"] is True + assert proof["violations"] == [] + assert envelope["l3_payload"] is None + + +def test_captured_run_accounts_for_successful_and_failed_workers( + tmp_path: Path, +) -> None: + provenance = load_provenance(FIXTURE_NAME) + trace = tmp_path / "trace" / "session-events.jsonl" + log_root = tmp_path / "log-root" + envelope = materialize_run( + FIXTURE_NAME, + trace=trace, + log_root=log_root, + projects=tmp_path / "projects", + ) + + summary = sessions.summarize(trace, envelope, log_root) + coverage = summary["coverage"] + assert coverage["managed_session_ids"] == sorted(WORKER_SESSION_IDS) + assert coverage["owned_session_ids"] == sorted((ROOT_SESSION_ID, *WORKER_SESSION_IDS)) + assert summary["peak_sessions"] == 5 + assert {agent["agent_id"] for agent in coverage["native_agents"]} == NATIVE_AGENT_IDS + assert coverage["complete"] is False + assert summary["violations"] == sorted( + [ + f"lifecycle:agent:{ROOT_SESSION_ID}:{UNMATCHED_AGENT_ID}:orphan_end", + f"native:{ROOT_SESSION_ID}:{UNMATCHED_AGENT_ID}:lifecycle_incomplete", + f"native:{ROOT_SESSION_ID}:{UNMATCHED_AGENT_ID}:transcript_unenumerated", + f"capture:{ROOT_SESSION_ID}:{UNMATCHED_AGENT_ID}:agent_type_missing", + ] + ) + assert envelope["l3_payload"] is None + + ownership = sessions.collect_ownership( + trace, + envelope, + log_root, + provenance["issue_url"], + provenance["repository"], + claim_label=provenance["claim_label"], + ) + assert ownership["branches"] == ["e2e-implementation-fixture-normalize-whitespace/1-3"] + assert ownership["claimed"] is True + assert ownership["violations"] == [] + assert [item["session_id"] for item in ownership["transcripts"]] == sorted( + (ROOT_SESSION_ID, *WORKER_SESSION_IDS) + ) + + +def test_captured_run_is_complete_under_the_declared_capture_policy( + tmp_path: Path, +) -> None: + trace = tmp_path / "trace" / "session-events.jsonl" + log_root = tmp_path / "log-root" + envelope = materialize_run( + FIXTURE_NAME, + trace=trace, + log_root=log_root, + projects=tmp_path / "projects", + ) + events = _rows(trace) + admitted = [] + excluded: list[tuple[str, str, str]] = [] + for event in events: + matcher = sessions.LIFECYCLE_HOOK_MATCHERS[event["event"]] + agent_type = event.get("agent_type", "") + if claude_matcher_admits(matcher, agent_type): + admitted.append(event) + else: + excluded.append((event["event"], event.get("agent_id", ""), agent_type)) + assert excluded == [("SubagentStop", UNMATCHED_AGENT_ID, "")] + trace.write_text( + "".join( + json.dumps(event, separators=(",", ":"), ensure_ascii=False) + "\n" + for event in admitted + ), + encoding="utf-8", + ) + + summary = sessions.summarize(trace, envelope, log_root) + assert summary["coverage"]["complete"] is True + assert summary["violations"] == [] + assert summary["peak_sessions"] == 5 + assert { + agent["agent_id"] for agent in summary["coverage"]["native_agents"] + } == NATIVE_AGENT_IDS diff --git a/tests/infra/test_e2e_workflow.py b/tests/infra/test_e2e_workflow.py index ede505cd35..942f39f6de 100644 --- a/tests/infra/test_e2e_workflow.py +++ b/tests/infra/test_e2e_workflow.py @@ -382,9 +382,17 @@ def test_docker_run_forwards_secrets_by_name_only(self, workflow: dict, step_id: forwarded = re.findall(r"--env ([A-Z0-9_]+)", run) expected = ["MINIMAX_API_KEY", "E2E_SANDBOX_TOKEN"] if step_id == "e2e": - expected.append("E2E_SANDBOX_TOKEN") + expected.extend(["E2E_SANDBOX_TOKEN", "E2E_IMPLEMENTATION_ISSUE_URL"]) assert Counter(forwarded) == Counter(expected) + def test_seed_repository_variable_reaches_test_container(self, workflow: dict) -> None: + step = _step(workflow, "run", step_id="e2e") + assert ( + step["env"]["E2E_IMPLEMENTATION_ISSUE_URL"] + == "${{ vars.E2E_IMPLEMENTATION_ISSUE_URL }}" + ) + assert "--env E2E_IMPLEMENTATION_ISSUE_URL" in step["run"] + def test_redaction_keeps_required_secret_names(self, workflow: dict) -> None: run = _step(workflow, "run", step_id="redact")["run"] branch, outside = _model_branch(run) @@ -433,10 +441,10 @@ def test_clean_install_uses_the_common_user_image_and_artifacts( ] assert len(credential_lines) == 1 assert credential_lines[0].startswith("docker_args+=(") - assert re.findall(r"--env ([A-Z0-9_]+)", credential_lines[0]) == [ - "MINIMAX_API_KEY", - "E2E_SANDBOX_TOKEN", - ] + forwarded = ["MINIMAX_API_KEY", "E2E_SANDBOX_TOKEN"] + if step_id == "e2e": + forwarded.append("E2E_IMPLEMENTATION_ISSUE_URL") + assert re.findall(r"--env ([A-Z0-9_]+)", credential_lines[0]) == forwarded if step_id == "e2e": assert not re.search(r"--env MINIMAX_API_KEY\b", outside) assert re.findall(r"--env ([A-Z0-9_]+)", outside) == ["E2E_SANDBOX_TOKEN"]