From 333c94845ee128d438835d443501425fb61b88f8 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 4 Oct 2026 09:38:02 -0700 Subject: [PATCH 01/11] feat(e2e): add the E2E test catalog and deterministic selector Adds scripts/e2e/catalog.json with its stdlib validator e2e_catalog.py and e2e_select.py, which decides the tests a pull_request, workflow_dispatch or merge_group event runs (seeded sampling, touched-test preference, at most two tests) and judges the required e2e-gate check. Routes scripts/e2e/* to the infra test directory. Co-Authored-By: Claude Opus 5.5 (1M context) --- .autoskillit/test-filter-manifest.yaml | 3 + scripts/e2e/catalog.json | 16 + scripts/e2e/e2e_catalog.py | 175 ++++++++++ scripts/e2e/e2e_select.py | 249 ++++++++++++++ tests/infra/_complexity_helpers.py | 3 + tests/infra/test_e2e_select.py | 310 ++++++++++++++++++ tests/infra/test_e2e_select_cli.py | 149 +++++++++ .../test_manifest_directory_completeness.py | 1 + 8 files changed, 906 insertions(+) create mode 100644 scripts/e2e/catalog.json create mode 100644 scripts/e2e/e2e_catalog.py create mode 100644 scripts/e2e/e2e_select.py create mode 100644 tests/infra/test_e2e_select.py create mode 100644 tests/infra/test_e2e_select_cli.py diff --git a/.autoskillit/test-filter-manifest.yaml b/.autoskillit/test-filter-manifest.yaml index 8524746eee..de5edf7a36 100644 --- a/.autoskillit/test-filter-manifest.yaml +++ b/.autoskillit/test-filter-manifest.yaml @@ -169,6 +169,9 @@ Taskfile.yml: scripts/docker/*: - infra/ +scripts/e2e/*: + - infra/ + .gitattributes: - infra/ diff --git a/scripts/e2e/catalog.json b/scripts/e2e/catalog.json new file mode 100644 index 0000000000..e0a967c5bf --- /dev/null +++ b/scripts/e2e/catalog.json @@ -0,0 +1,16 @@ +{ + "sandbox_repository": "TalonT-Org/autoskillit-e2e-sandbox", + "tests": [ + { + "name": "canary", + "kind": "canary", + "peak_sessions": 1, + "timeout_sec": 300, + "trigger_paths": [ + "scripts/e2e/*", + "scripts/docker/*", + ".github/workflows/e2e.yml" + ] + } + ] +} diff --git a/scripts/e2e/e2e_catalog.py b/scripts/e2e/e2e_catalog.py new file mode 100644 index 0000000000..f206f25f5f --- /dev/null +++ b/scripts/e2e/e2e_catalog.py @@ -0,0 +1,175 @@ +"""Load and validate the E2E test catalog shared by the selector and the harness.""" + +from __future__ import annotations + +import json +import math +import re +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path + +CATALOG_PATH = Path(__file__).with_name("catalog.json") +# One step fans out to at most six subagents beside its orchestrator and step session, and one +# run at a time keeps CI at or below eight of the roughly twelve concurrent MiniMax sessions. +MAX_PEAK_SESSIONS = 8 +# Setup, assertions and cleanup around the test's own budget, so every harness timeout fires +# before the job ceiling. +HARNESS_GRACE_SEC = 120 +# Image build and load, redaction and artifact upload. +JOB_OVERHEAD_MINUTES = 30 +MAX_JOB_MINUTES = 360 +KINDS = ("canary", "recipe") +PULL_REQUEST_STATES = ("open", "merged", "closed") + +_NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9-]*$") +_CATALOG_KEYS = frozenset({"sandbox_repository", "tests"}) +_COMMON_KEYS = frozenset({"name", "kind", "peak_sessions", "timeout_sec", "trigger_paths"}) +_RECIPE_KEYS = frozenset({"recipe", "ingredients", "expected_pull_request_state"}) + + +class CatalogError(ValueError): + """The catalog violates its schema.""" + + +@dataclass(frozen=True, slots=True) +class CatalogTest: + """One live E2E test and the resources it is allowed to use.""" + + name: str + kind: str + peak_sessions: int + timeout_sec: int + trigger_paths: tuple[str, ...] + recipe: str | None + ingredients: tuple[tuple[str, str], ...] + expected_pull_request_state: str | None + + +@dataclass(frozen=True, slots=True) +class Catalog: + """Every registered E2E test and the sandbox repository recipe tests run against.""" + + sandbox_repository: str + tests: tuple[CatalogTest, ...] + + def get(self, name: str) -> CatalogTest: + for test in self.tests: + if test.name == name: + return test + raise CatalogError(f"unknown E2E test: {name!r}") + + +def job_timeout_minutes(test: CatalogTest) -> int: + return math.ceil((test.timeout_sec + HARNESS_GRACE_SEC) / 60) + JOB_OVERHEAD_MINUTES + + +def _require_keys(raw: Mapping[str, object], expected: frozenset[str], where: str) -> None: + missing = sorted(expected - raw.keys()) + unknown = sorted(raw.keys() - expected) + if missing or unknown: + raise CatalogError(f"{where}: missing keys {missing}, unknown keys {unknown}") + + +def _require_str(value: object, where: str) -> str: + if not isinstance(value, str) or not value: + raise CatalogError(f"{where} must be a non-empty string") + return value + + +def _require_int(value: object, where: str, *, low: int, high: int | None = None) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise CatalogError(f"{where} must be an integer") + if value < low or (high is not None and value > high): + bound = f"between {low} and {high}" if high is not None else f"at least {low}" + raise CatalogError(f"{where} must be {bound}") + return value + + +def _parse_trigger_paths(value: object, where: str) -> tuple[str, ...]: + if not isinstance(value, list) or not value: + raise CatalogError(f"{where} must be a non-empty list") + return tuple(_require_str(item, f"{where}[]") for item in value) + + +def _parse_ingredients(value: object, where: str) -> tuple[tuple[str, str], ...]: + if not isinstance(value, dict): + raise CatalogError(f"{where} must be an object") + pairs: list[tuple[str, str]] = [] + for key, item in value.items(): + if not isinstance(item, str): + raise CatalogError(f"{where}.{key} must be a string") + pairs.append((_require_str(key, f"{where} key"), item)) + return tuple(pairs) + + +def _parse_recipe_fields( + raw: Mapping[str, object], where: str +) -> tuple[str, tuple[tuple[str, str], ...], str]: + recipe = _require_str(raw["recipe"], f"{where}.recipe") + ingredients = _parse_ingredients(raw["ingredients"], f"{where}.ingredients") + state = _require_str( + raw["expected_pull_request_state"], f"{where}.expected_pull_request_state" + ) + if state not in PULL_REQUEST_STATES: + raise CatalogError( + f"{where}.expected_pull_request_state must be one of {PULL_REQUEST_STATES}" + ) + return recipe, ingredients, state + + +def _parse_test(raw: object, index: int) -> CatalogTest: + where = f"tests[{index}]" + if not isinstance(raw, dict): + raise CatalogError(f"{where} must be an object") + kind = raw.get("kind") + if kind not in KINDS: + raise CatalogError(f"{where}.kind must be one of {KINDS}") + _require_keys(raw, _COMMON_KEYS | _RECIPE_KEYS if kind == "recipe" else _COMMON_KEYS, where) + name = _require_str(raw["name"], f"{where}.name") + if not _NAME_PATTERN.fullmatch(name): + raise CatalogError(f"{where}.name must match {_NAME_PATTERN.pattern}") + recipe, ingredients, state = ( + _parse_recipe_fields(raw, where) if kind == "recipe" else (None, (), None) + ) + test = CatalogTest( + name=name, + kind=kind, + peak_sessions=_require_int( + raw["peak_sessions"], f"{where}.peak_sessions", low=1, high=MAX_PEAK_SESSIONS + ), + timeout_sec=_require_int(raw["timeout_sec"], f"{where}.timeout_sec", low=1), + trigger_paths=_parse_trigger_paths(raw["trigger_paths"], f"{where}.trigger_paths"), + recipe=recipe, + ingredients=ingredients, + expected_pull_request_state=state, + ) + if job_timeout_minutes(test) > MAX_JOB_MINUTES: + raise CatalogError(f"{where}.timeout_sec exceeds the {MAX_JOB_MINUTES}-minute job limit") + return test + + +def parse_catalog(raw: object) -> Catalog: + if not isinstance(raw, dict): + raise CatalogError("catalog must be an object") + _require_keys(raw, _CATALOG_KEYS, "catalog") + repository = _require_str(raw["sandbox_repository"], "sandbox_repository") + if repository.count("/") != 1: + raise CatalogError("sandbox_repository must be OWNER/REPO") + entries = raw["tests"] + if not isinstance(entries, list) or not entries: + raise CatalogError("tests must be a non-empty list") + tests = tuple(_parse_test(entry, index) for index, entry in enumerate(entries)) + names = [test.name for test in tests] + duplicates = sorted({name for name in names if names.count(name) > 1}) + if duplicates: + raise CatalogError(f"duplicate test names: {duplicates}") + return Catalog(sandbox_repository=repository, tests=tests) + + +def load_catalog(path: Path = CATALOG_PATH) -> Catalog: + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise CatalogError(f"{path}: {exc}") from exc + return parse_catalog(raw) diff --git a/scripts/e2e/e2e_select.py b/scripts/e2e/e2e_select.py new file mode 100644 index 0000000000..a5192dd515 --- /dev/null +++ b/scripts/e2e/e2e_select.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +"""Decide which live E2E tests a GitHub Actions event runs, and judge the required gate.""" + +from __future__ import annotations + +import argparse +import fnmatch +import hashlib +import json +import os +import sys +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import e2e_catalog + +E2E_LABEL = "e2e" +MIN_CHANGED_FILES = 10 +MIN_CHANGED_LINES = 300 +SAMPLE_PROBABILITY = 0.5 +MAX_TESTS = 2 + + +@dataclass(frozen=True, slots=True) +class PullRequestFacts: + """The fields of a ``pull_request`` event payload that drive selection.""" + + number: int + head_sha: str + head_repository: str | None + changed_files: int + additions: int + deletions: int + labels: frozenset[str] + + @classmethod + def from_payload(cls, pr: Mapping[str, Any]) -> PullRequestFacts: + try: + head = pr["head"] + head_repo = head.get("repo") + return cls( + number=int(pr["number"]), + head_sha=str(head["sha"]), + head_repository=( + head_repo.get("full_name") if isinstance(head_repo, dict) else None + ), + changed_files=int(pr["changed_files"]), + additions=int(pr["additions"]), + deletions=int(pr["deletions"]), + labels=frozenset(str(label["name"]) for label in pr.get("labels") or ()), + ) + except (KeyError, TypeError, AttributeError) as exc: + raise ValueError(f"malformed pull_request payload: {exc!r}") from exc + + +@dataclass(frozen=True, slots=True) +class Selection: + """The tests one event runs, with the reason the selector reached that decision.""" + + tests: tuple[str, ...] + reason: str + + @property + def selected(self) -> bool: + return bool(self.tests) + + +def is_eligible(pr: PullRequestFacts) -> bool: + return ( + E2E_LABEL in pr.labels + or pr.changed_files >= MIN_CHANGED_FILES + or pr.additions + pr.deletions >= MIN_CHANGED_LINES + ) + + +def sample_score(number: int, head_sha: str) -> float: + digest = hashlib.sha256(f"{number}:{head_sha}".encode()).digest() + return int.from_bytes(digest[:8], "big") / 2**64 + + +def is_sampled(pr: PullRequestFacts, probability: float) -> bool: + return sample_score(pr.number, pr.head_sha) < probability + + +def _rank(seed: str, name: str) -> str: + return hashlib.sha256(f"{seed}:{name}".encode()).hexdigest() + + +def _touches(test: e2e_catalog.CatalogTest, changed_paths: Sequence[str]) -> bool: + return any( + fnmatch.fnmatchcase(path, pattern) + for path in changed_paths + for pattern in test.trigger_paths + ) + + +def pick_tests( + catalog: e2e_catalog.Catalog, changed_paths: Sequence[str], seed: str +) -> tuple[str, ...]: + ranked = sorted(catalog.tests, key=lambda test: _rank(seed, test.name)) + touched = [test.name for test in ranked if _touches(test, changed_paths)] + if touched: + return tuple(touched[:MAX_TESTS]) + return (ranked[0].name,) + + +def select_for_pull_request( + pr: PullRequestFacts, + changed_paths: Sequence[str], + catalog: e2e_catalog.Catalog, + repository: str, +) -> Selection: + if pr.head_repository != repository: + return Selection((), f"head repository {pr.head_repository} is not {repository}") + if not is_eligible(pr): + lines = pr.additions + pr.deletions + return Selection( + (), + f"below thresholds ({pr.changed_files} files, {lines} lines) and no {E2E_LABEL} label", + ) + seed = f"{pr.number}:{pr.head_sha}" + if E2E_LABEL in pr.labels: + return Selection(pick_tests(catalog, changed_paths, seed), f"{E2E_LABEL} label") + if not is_sampled(pr, SAMPLE_PROBABILITY): + return Selection((), "eligible but not sampled for this head commit") + return Selection(pick_tests(catalog, changed_paths, seed), "eligible and sampled") + + +def select_for_dispatch(requested: str, catalog: e2e_catalog.Catalog) -> Selection: + names = tuple(dict.fromkeys(part.strip() for part in requested.split(",") if part.strip())) + if not 1 <= len(names) <= MAX_TESTS: + raise ValueError(f"workflow_dispatch must name 1 to {MAX_TESTS} tests, got {len(names)}") + for name in names: + catalog.get(name) + return Selection(names, "workflow_dispatch") + + +def _dispatch_request(payload: Mapping[str, Any]) -> str: + inputs = payload.get("inputs") + requested = inputs.get("tests") if isinstance(inputs, dict) else None + if not isinstance(requested, str): + raise ValueError("workflow_dispatch payload has no string inputs.tests") + return requested + + +def select_for_event( + event_name: str, + payload: Mapping[str, Any], + changed_paths: Sequence[str], + catalog: e2e_catalog.Catalog, + repository: str, +) -> Selection: + if event_name == "pull_request": + pr = payload.get("pull_request") + if not isinstance(pr, dict): + raise ValueError("pull_request payload has no pull_request object") + facts = PullRequestFacts.from_payload(pr) + return select_for_pull_request(facts, changed_paths, catalog, repository) + if event_name == "workflow_dispatch": + return select_for_dispatch(_dispatch_request(payload), catalog) + if event_name == "merge_group": + return Selection((), "merge queue: E2E is decided on the pull request") + raise ValueError(f"unsupported event: {event_name}") + + +def matrix_json(selection: Selection, catalog: e2e_catalog.Catalog) -> str: + include = [ + {"test": name, "timeout_minutes": e2e_catalog.job_timeout_minutes(catalog.get(name))} + for name in selection.tests + ] + return json.dumps({"include": include}, separators=(",", ":")) + + +def gate_verdict(select_result: str, selected: str, run_result: str) -> tuple[bool, str]: + if select_result == "success" and selected == "true" and run_result == "success": + return True, "selected E2E tests passed" + if select_result == "success" and selected == "false" and run_result == "skipped": + return True, "no E2E test selected" + return False, ( + f"E2E gate failed: select={select_result or ''} " + f"selected={selected or ''} run={run_result or ''}" + ) + + +def _read_event() -> tuple[str, Mapping[str, Any], str]: + event_name = os.environ.get("GITHUB_EVENT_NAME") + event_path = os.environ.get("GITHUB_EVENT_PATH") + repository = os.environ.get("GITHUB_REPOSITORY") + if not event_name or not event_path or not repository: + raise ValueError("GITHUB_EVENT_NAME, GITHUB_EVENT_PATH and GITHUB_REPOSITORY are required") + payload = json.loads(Path(event_path).read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError("GitHub event payload must be an object") + return event_name, payload, repository + + +def _read_changed_paths(path: Path) -> tuple[str, ...]: + lines = path.read_text(encoding="utf-8").splitlines() + return tuple(line.strip() for line in lines if line.strip()) + + +def _select(catalog_path: Path, changed_paths_file: Path) -> int: + try: + catalog = e2e_catalog.load_catalog(catalog_path) + event_name, payload, repository = _read_event() + changed_paths = ( + _read_changed_paths(changed_paths_file) if event_name == "pull_request" else () + ) + selection = select_for_event(event_name, payload, changed_paths, catalog, repository) + matrix = matrix_json(selection, catalog) + except (OSError, ValueError) as exc: + print(f"e2e_select: {exc}", file=sys.stderr) + return 2 + print(f"e2e_select: {selection.reason}", file=sys.stderr) + sys.stdout.write( + f"selected={'true' if selection.selected else 'false'}\n" + f"matrix={matrix}\n" + f"reason={selection.reason}\n" + ) + return 0 + + +def main(argv: Sequence[str]) -> int: + """Print a GitHub Actions output record for ``select``; exit 0/1 on the ``gate`` verdict.""" + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + select = commands.add_parser("select", help="Choose the tests this event runs.") + select.add_argument("--catalog", default=str(e2e_catalog.CATALOG_PATH)) + select.add_argument( + "--changed-paths", + required=True, + help="File listing the pull request's changed paths, one per line.", + ) + gate = commands.add_parser("gate", help="Judge the required e2e-gate check.") + gate.add_argument("--select-result", required=True) + gate.add_argument("--selected", required=True) + gate.add_argument("--run-result", required=True) + args = parser.parse_args(argv) + if args.command == "select": + return _select(Path(args.catalog), Path(args.changed_paths)) + passed, reason = gate_verdict(args.select_result, args.selected, args.run_result) + print(reason) + return 0 if passed else 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/infra/_complexity_helpers.py b/tests/infra/_complexity_helpers.py index 6c13bab2e7..45676f01c2 100644 --- a/tests/infra/_complexity_helpers.py +++ b/tests/infra/_complexity_helpers.py @@ -6,6 +6,9 @@ sys.modules isolation stays a parameter, not shared state) -- only the byte-identical bodies live here, matching the tests/infra/_pretty_output_helpers.py convention of a dedicated underscore-prefixed helper module for one feature's split test files. + +tests/infra/test_e2e_select.py also loads scripts/e2e/e2e_select.py through +load_check_script, so its sibling ``import e2e_catalog`` resolves. """ from __future__ import annotations diff --git a/tests/infra/test_e2e_select.py b/tests/infra/test_e2e_select.py new file mode 100644 index 0000000000..d21b5f7f25 --- /dev/null +++ b/tests/infra/test_e2e_select.py @@ -0,0 +1,310 @@ +"""Unit tests for scripts/e2e/e2e_select.py and the catalog schema it reads.""" + +from __future__ import annotations + +import hashlib +import itertools +from pathlib import Path + +import pytest + +from tests.infra._complexity_helpers import load_check_script + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.small] + +REPO_ROOT = Path(__file__).resolve().parents[2] +_SELECT_SCRIPT = REPO_ROOT / "scripts" / "e2e" / "e2e_select.py" + +select = load_check_script("_autoskillit_e2e_select", _SELECT_SCRIPT) +e2e_catalog = select.e2e_catalog + +REPOSITORY = "TalonT-Org/AutoSkillit" +SHA_ZERO = "0" * 40 +SYNTHETIC_SHAS = [hashlib.sha1(str(index).encode()).hexdigest() for index in range(2000)] + + +def _facts( + *, + number: int = 5230, + head_sha: str = SHA_ZERO, + head_repository: str | None = REPOSITORY, + changed_files: int = 10, + additions: int = 0, + deletions: int = 0, + labels: tuple[str, ...] = (), +): + return select.PullRequestFacts( + number=number, + head_sha=head_sha, + head_repository=head_repository, + changed_files=changed_files, + additions=additions, + deletions=deletions, + labels=frozenset(labels), + ) + + +def _entry(name: str, trigger_paths: tuple[str, ...] = ("src/*",), **overrides) -> dict: + entry = { + "name": name, + "kind": "canary", + "peak_sessions": 1, + "timeout_sec": 300, + "trigger_paths": list(trigger_paths), + } + entry.update(overrides) + return entry + + +def _raw_catalog(*entries: dict) -> dict: + return {"sandbox_repository": "owner/sandbox", "tests": list(entries)} + + +def _catalog(*entries: dict): + return e2e_catalog.parse_catalog(_raw_catalog(*entries)) + + +def _sha_with_sample(number: int, *, sampled: bool) -> str: + for index in itertools.count(): + sha = f"{index:040x}" + if (select.sample_score(number, sha) < select.SAMPLE_PROBABILITY) == sampled: + return sha + raise AssertionError("unreachable") + + +_FIVE_TESTS = tuple(_entry(f"t{index}", (f"area{index}/*",)) for index in range(5)) + + +class TestEligibility: + def test_ten_changed_files_is_eligible(self) -> None: + assert select.is_eligible(_facts(changed_files=10)) + + def test_three_hundred_changed_lines_is_eligible(self) -> None: + assert select.is_eligible(_facts(changed_files=1, additions=200, deletions=100)) + + def test_below_both_thresholds_is_not_eligible(self) -> None: + assert not select.is_eligible(_facts(changed_files=9, additions=200, deletions=99)) + + def test_e2e_label_makes_a_one_file_change_eligible(self) -> None: + assert select.is_eligible(_facts(changed_files=1, labels=("e2e",))) + + +class TestSampleScore: + def test_score_is_the_first_eight_digest_bytes_over_two_to_the_sixty_four(self) -> None: + digest = hashlib.sha256(b"5230:" + b"0" * 40).digest() + expected = int.from_bytes(digest[:8], "big") / 2**64 + assert select.sample_score(5230, SHA_ZERO) == expected + + def test_score_is_deterministic(self) -> None: + assert select.sample_score(5230, SHA_ZERO) == select.sample_score(5230, SHA_ZERO) + + def test_half_probability_samples_about_half(self) -> None: + sampled = sum(select.is_sampled(_facts(head_sha=sha), 0.5) for sha in SYNTHETIC_SHAS) + assert 0.45 <= sampled / len(SYNTHETIC_SHAS) <= 0.55 + + def test_zero_probability_never_samples(self) -> None: + assert not any(select.is_sampled(_facts(head_sha=sha), 0.0) for sha in SYNTHETIC_SHAS) + + def test_unit_probability_always_samples(self) -> None: + assert all(select.is_sampled(_facts(head_sha=sha), 1.0) for sha in SYNTHETIC_SHAS) + + +class TestSelectForPullRequest: + def test_eligible_and_sampled_is_selected(self) -> None: + pr = _facts(head_sha=_sha_with_sample(5230, sampled=True)) + selection = select.select_for_pull_request(pr, (), _catalog(_entry("a")), REPOSITORY) + assert selection.tests == ("a",) + assert selection.reason == "eligible and sampled" + + def test_eligible_but_sampled_out_is_not_selected(self) -> None: + pr = _facts(head_sha=_sha_with_sample(5230, sampled=False)) + selection = select.select_for_pull_request(pr, (), _catalog(_entry("a")), REPOSITORY) + assert not selection.selected + assert "not sampled" in selection.reason + + def test_e2e_label_bypasses_sampling(self) -> None: + pr = _facts(head_sha=_sha_with_sample(5230, sampled=False), labels=("e2e",)) + selection = select.select_for_pull_request(pr, (), _catalog(_entry("a")), REPOSITORY) + assert selection.tests == ("a",) + + def test_fork_head_is_not_selected(self) -> None: + pr = _facts(head_repository="someone/AutoSkillit", labels=("e2e",)) + selection = select.select_for_pull_request(pr, (), _catalog(_entry("a")), REPOSITORY) + assert not selection.selected + assert "someone/AutoSkillit" in selection.reason + + def test_deleted_head_repository_is_not_selected(self) -> None: + payload = { + "number": 5230, + "head": {"sha": SHA_ZERO, "repo": None}, + "changed_files": 50, + "additions": 1000, + "deletions": 0, + "labels": [{"name": "e2e"}], + } + pr = select.PullRequestFacts.from_payload(payload) + selection = select.select_for_pull_request(pr, (), _catalog(_entry("a")), REPOSITORY) + assert pr.head_repository is None + assert not selection.selected + + +class TestPickTests: + _SEED = f"5230:{SHA_ZERO}" + + def test_all_touched_caps_at_two(self) -> None: + changed = [f"area{index}/x.py" for index in range(5)] + assert len(select.pick_tests(_catalog(*_FIVE_TESTS), changed, self._SEED)) == 2 + + def test_none_touched_picks_exactly_one(self) -> None: + assert len(select.pick_tests(_catalog(*_FIVE_TESTS), ["docs/x.md"], self._SEED)) == 1 + + @pytest.mark.parametrize("seed_index", range(20)) + def test_one_touched_picks_only_that_test(self, seed_index: int) -> None: + picked = select.pick_tests(_catalog(*_FIVE_TESTS), ["area3/x.py"], f"{seed_index}:sha") + assert picked == ("t3",) + + def test_single_test_catalog_picks_it(self) -> None: + assert select.pick_tests(_catalog(_entry("only")), [], self._SEED) == ("only",) + + @pytest.mark.parametrize( + "changed", + [[], ["docs/x.md"], ["area0/x.py"], [f"area{index}/x.py" for index in range(5)]], + ) + def test_selection_is_never_empty_and_deterministic(self, changed: list[str]) -> None: + first = select.pick_tests(_catalog(*_FIVE_TESTS), changed, self._SEED) + assert first + assert first == select.pick_tests(_catalog(*_FIVE_TESTS), changed, self._SEED) + + def test_rank_is_the_sha256_hex_digest_of_seed_and_name(self) -> None: + catalog = _catalog(_entry("alpha"), _entry("beta")) + expected = sorted( + ["alpha", "beta"], + key=lambda name: hashlib.sha256(f"{self._SEED}:{name}".encode()).hexdigest(), + ) + assert select.pick_tests(catalog, ["src/x.py"], self._SEED) == tuple(expected) + + def test_star_matches_across_directories(self) -> None: + catalog = _catalog(_entry("deep", ("src/*",)), _entry("other", ("docs/*",))) + assert select.pick_tests(catalog, ["src/a/b/c.py"], self._SEED) == ("deep",) + + +class TestSelectForEvent: + def test_merge_group_is_never_selected(self) -> None: + selection = select.select_for_event( + "merge_group", {}, (), _catalog(_entry("a")), REPOSITORY + ) + assert not selection.selected + assert "merge queue" in selection.reason + + def test_unsupported_event_raises(self) -> None: + with pytest.raises(ValueError, match="unsupported event"): + select.select_for_event("push", {}, (), _catalog(_entry("a")), REPOSITORY) + + def test_pull_request_without_pull_request_object_raises(self) -> None: + with pytest.raises(ValueError): + select.select_for_event("pull_request", {}, (), _catalog(_entry("a")), REPOSITORY) + + def test_dispatch_reads_inputs_tests(self) -> None: + catalog = e2e_catalog.load_catalog() + payload = {"inputs": {"tests": "canary"}} + selection = select.select_for_event("workflow_dispatch", payload, (), catalog, REPOSITORY) + assert selection.tests == ("canary",) + + def test_dispatch_deduplicates_names(self) -> None: + catalog = e2e_catalog.load_catalog() + assert select.select_for_dispatch("canary, canary", catalog).tests == ("canary",) + + @pytest.mark.parametrize("requested", ["t0,t1,t2", "t0,nope", "", " , "]) + def test_dispatch_rejects_invalid_requests(self, requested: str) -> None: + with pytest.raises(ValueError): + select.select_for_dispatch(requested, _catalog(*_FIVE_TESTS)) + + def test_dispatch_without_inputs_raises(self) -> None: + with pytest.raises(ValueError, match="inputs.tests"): + select.select_for_event("workflow_dispatch", {}, (), _catalog(_entry("a")), REPOSITORY) + + +_PASSING_GATE_ROWS = {("success", "false", "skipped"), ("success", "true", "success")} +_RESULTS = ("success", "failure", "cancelled", "skipped", "") + + +@pytest.mark.parametrize( + ("select_result", "selected", "run_result"), + list(itertools.product(_RESULTS, ("true", "false", ""), _RESULTS)), +) +def test_gate_passes_only_on_the_two_healthy_shapes( + select_result: str, selected: str, run_result: str +) -> None: + passed, reason = select.gate_verdict(select_result, selected, run_result) + assert passed is ((select_result, selected, run_result) in _PASSING_GATE_ROWS) + assert reason + + +def test_matrix_for_an_empty_selection_is_parseable() -> None: + selection = select.Selection((), "nothing selected") + assert select.matrix_json(selection, _catalog(_entry("a"))) == '{"include":[]}' + + +_RECIPE_FIELDS = { + "recipe": "implementation", + "ingredients": {"task": "Add a greeting"}, + "expected_pull_request_state": "merged", +} +_MAX_TIMEOUT_SEC = ( + e2e_catalog.MAX_JOB_MINUTES - e2e_catalog.JOB_OVERHEAD_MINUTES +) * 60 - e2e_catalog.HARNESS_GRACE_SEC + + +def _recipe_entry_without(field: str) -> dict: + fields = {key: value for key, value in _RECIPE_FIELDS.items() if key != field} + return _entry("impl", kind="recipe", **fields) + + +_INVALID_CATALOGS = { + "duplicate-names": _raw_catalog(_entry("a"), _entry("a")), + "bad-name": _raw_catalog(_entry("Bad_Name")), + "unknown-kind": _raw_catalog(_entry("a", kind="smoke")), + "zero-peak-sessions": _raw_catalog(_entry("a", peak_sessions=0)), + "nine-peak-sessions": _raw_catalog(_entry("a", peak_sessions=9)), + "timeout-over-job-limit": _raw_catalog(_entry("a", timeout_sec=_MAX_TIMEOUT_SEC + 1)), + "recipe-without-recipe": _raw_catalog(_recipe_entry_without("recipe")), + "recipe-without-ingredients": _raw_catalog(_recipe_entry_without("ingredients")), + "recipe-without-state": _raw_catalog(_recipe_entry_without("expected_pull_request_state")), + "recipe-unknown-state": _raw_catalog( + _entry("a", kind="recipe", **{**_RECIPE_FIELDS, "expected_pull_request_state": "draft"}) + ), + "canary-with-recipe-fields": _raw_catalog(_entry("a", **_RECIPE_FIELDS)), + "empty-trigger-paths": _raw_catalog(_entry("a", ())), + "unknown-key": _raw_catalog(_entry("a", extra=1)), + "unknown-top-level-key": {**_raw_catalog(_entry("a")), "extra": 1}, + "no-tests": _raw_catalog(), +} + + +class TestCatalog: + def test_shipped_catalog_holds_the_canary(self) -> None: + catalog = e2e_catalog.load_catalog() + canary = catalog.get("canary") + assert canary.kind == "canary" + assert canary.recipe is None + assert e2e_catalog.job_timeout_minutes(canary) == 37 + + def test_recipe_entry_parses(self) -> None: + catalog = _catalog(_entry("impl", kind="recipe", **_RECIPE_FIELDS)) + test = catalog.get("impl") + assert test.recipe == "implementation" + assert test.ingredients == (("task", "Add a greeting"),) + assert test.expected_pull_request_state == "merged" + + def test_longest_timeout_within_the_job_limit_parses(self) -> None: + catalog = _catalog(_entry("slow", timeout_sec=_MAX_TIMEOUT_SEC)) + assert e2e_catalog.job_timeout_minutes(catalog.get("slow")) == e2e_catalog.MAX_JOB_MINUTES + + def test_unknown_test_name_raises_catalog_error(self) -> None: + with pytest.raises(e2e_catalog.CatalogError): + _catalog(_entry("a")).get("b") + + @pytest.mark.parametrize("raw", _INVALID_CATALOGS.values(), ids=list(_INVALID_CATALOGS)) + def test_invalid_catalog_raises(self, raw: dict) -> None: + with pytest.raises(e2e_catalog.CatalogError): + e2e_catalog.parse_catalog(raw) diff --git a/tests/infra/test_e2e_select_cli.py b/tests/infra/test_e2e_select_cli.py new file mode 100644 index 0000000000..a0ed198a84 --- /dev/null +++ b/tests/infra/test_e2e_select_cli.py @@ -0,0 +1,149 @@ +"""Subprocess tests for the scripts/e2e/e2e_select.py command line the E2E workflow runs.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from tests.conftest import production_interpreter_env + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] + +REPO_ROOT = Path(__file__).resolve().parents[2] +_SCRIPT = REPO_ROOT / "scripts" / "e2e" / "e2e_select.py" +REPOSITORY = "TalonT-Org/AutoSkillit" +_CANARY_MATRIX = '{"include":[{"test":"canary","timeout_minutes":37}]}' + + +def _run( + *args: str, tmp_path: Path, event_name: str, payload: object +) -> subprocess.CompletedProcess[str]: + event_path = tmp_path / "event.json" + event_path.write_text(json.dumps(payload), encoding="utf-8") + env = production_interpreter_env() + env["GITHUB_EVENT_NAME"] = event_name + env["GITHUB_EVENT_PATH"] = str(event_path) + env["GITHUB_REPOSITORY"] = REPOSITORY + return subprocess.run( + [sys.executable, str(_SCRIPT), *args], + capture_output=True, + text=True, + env=env, + check=False, + ) + + +def _outputs(stdout: str) -> dict[str, str]: + return dict(line.split("=", 1) for line in stdout.splitlines()) + + +def _pull_request(*, changed_files: int, additions: int, labels: tuple[str, ...] = ()) -> dict: + return { + "pull_request": { + "number": 5230, + "head": {"sha": "0" * 40, "repo": {"full_name": REPOSITORY}}, + "changed_files": changed_files, + "additions": additions, + "deletions": 0, + "labels": [{"name": label} for label in labels], + } + } + + +def _select(tmp_path: Path, changed_paths: Path, event_name: str, payload: object): + return _run( + "select", + "--changed-paths", + str(changed_paths), + tmp_path=tmp_path, + event_name=event_name, + payload=payload, + ) + + +def test_labelled_pull_request_selects_the_touched_canary(tmp_path: Path) -> None: + changed = tmp_path / "changed-paths.txt" + changed.write_text("scripts/e2e/e2e_harness.py\n", encoding="utf-8") + payload = _pull_request(changed_files=1, additions=5, labels=("e2e",)) + result = _select(tmp_path, changed, "pull_request", payload) + assert result.returncode == 0, result.stderr + outputs = _outputs(result.stdout) + assert outputs["selected"] == "true" + assert outputs["matrix"] == _CANARY_MATRIX + + +def test_small_unlabelled_pull_request_selects_nothing(tmp_path: Path) -> None: + changed = tmp_path / "changed-paths.txt" + changed.write_text("README.md\n", encoding="utf-8") + result = _select( + tmp_path, changed, "pull_request", _pull_request(changed_files=3, additions=15) + ) + assert result.returncode == 0, result.stderr + outputs = _outputs(result.stdout) + assert outputs["selected"] == "false" + assert outputs["matrix"] == '{"include":[]}' + + +def test_merge_group_never_reads_the_changed_paths_file(tmp_path: Path) -> None: + result = _select(tmp_path, tmp_path / "missing.txt", "merge_group", {"merge_group": {}}) + assert result.returncode == 0, result.stderr + outputs = _outputs(result.stdout) + assert outputs["selected"] == "false" + assert outputs["matrix"] == '{"include":[]}' + + +def test_dispatch_selects_the_named_test(tmp_path: Path) -> None: + payload = {"inputs": {"tests": "canary"}} + result = _select(tmp_path, tmp_path / "missing.txt", "workflow_dispatch", payload) + assert result.returncode == 0, result.stderr + outputs = _outputs(result.stdout) + assert outputs["selected"] == "true" + assert outputs["matrix"] == _CANARY_MATRIX + + +def test_dispatch_of_an_unknown_test_fails(tmp_path: Path) -> None: + payload = {"inputs": {"tests": "nope"}} + result = _select(tmp_path, tmp_path / "missing.txt", "workflow_dispatch", payload) + assert result.returncode != 0 + assert result.stdout == "" + + +def test_pull_request_without_changed_paths_file_fails(tmp_path: Path) -> None: + payload = _pull_request(changed_files=20, additions=500, labels=("e2e",)) + result = _select(tmp_path, tmp_path / "missing.txt", "pull_request", payload) + assert result.returncode != 0 + assert result.stdout == "" + + +@pytest.mark.parametrize( + ("select_result", "selected", "run_result", "expected"), + [ + ("success", "false", "skipped", 0), + ("success", "true", "success", 0), + ("success", "true", "failure", 1), + ("success", "true", "skipped", 1), + ("failure", "", "skipped", 1), + ("success", "", "", 1), + ("cancelled", "true", "cancelled", 1), + ], +) +def test_gate_exit_code( + tmp_path: Path, select_result: str, selected: str, run_result: str, expected: int +) -> None: + result = _run( + "gate", + "--select-result", + select_result, + "--selected", + selected, + "--run-result", + run_result, + tmp_path=tmp_path, + event_name="pull_request", + payload={}, + ) + assert result.returncode == expected, result.stdout + result.stderr diff --git a/tests/infra/test_manifest_directory_completeness.py b/tests/infra/test_manifest_directory_completeness.py index 445ffa4390..03d583692d 100644 --- a/tests/infra/test_manifest_directory_completeness.py +++ b/tests/infra/test_manifest_directory_completeness.py @@ -34,6 +34,7 @@ "scripts/check_complexity.py", "scripts/_git_plumbing.py", "scripts/docker/*", + "scripts/e2e/*", } ), "workspace": frozenset( From fa6d08efca662b58be13b3763f6496acc4b086a1 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 4 Oct 2026 09:40:23 -0700 Subject: [PATCH 02/11] feat(e2e): add the MiniMax E2E harness and its AutoSkillit user config e2e_harness.py runs one catalog test inside the user image: it routes Claude Code to MiniMax only through ~/.claude/settings.json env, installs the user config that disables the quota guard, enables fleet headless runs and pins every default Codex route to claude-code, runs the canary or a fleet recipe with structural assertions, and always records result.json. Its redact subcommand copies artifacts world-readable with every secret replaced and fails if any secret survives. Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/e2e/autoskillit-config.yaml | 31 ++ scripts/e2e/e2e_harness.py | 477 ++++++++++++++++++++++++++++ tests/infra/_complexity_helpers.py | 5 +- tests/infra/test_e2e_harness.py | 466 +++++++++++++++++++++++++++ 4 files changed, 977 insertions(+), 2 deletions(-) create mode 100644 scripts/e2e/autoskillit-config.yaml create mode 100644 scripts/e2e/e2e_harness.py create mode 100644 tests/infra/test_e2e_harness.py diff --git a/scripts/e2e/autoskillit-config.yaml b/scripts/e2e/autoskillit-config.yaml new file mode 100644 index 0000000000..577e8df598 --- /dev/null +++ b/scripts/e2e/autoskillit-config.yaml @@ -0,0 +1,31 @@ +# AutoSkillit user-layer config the E2E harness installs as ~/.autoskillit/config.yaml. +# See scripts/e2e/README.md for why each key is set. +quota_guard: + enabled: false + +features: + fleet: true + fleet_headless_run: true + +agent_backend: + recipe_overrides: + implementation: + run_arch_lenses: claude-code + implementation-groups: + run_arch_lenses: claude-code + planner: + analyze: claude-code + extract_domain: claude-code + elaborate_phases: claude-code + remediation: + investigate: claude-code + run_arch_lenses: claude-code + research: + run_experiment_lenses: claude-code + scope: claude-code + vis_apply: claude-code + research-design: + scope: claude-code + vis_apply: claude-code + research-review: + run_experiment_lenses: claude-code diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py new file mode 100644 index 0000000000..4c8502cc0d --- /dev/null +++ b/scripts/e2e/e2e_harness.py @@ -0,0 +1,477 @@ +#!/usr/bin/env python3 +"""Run one catalog E2E test inside the AutoSkillit user image, and redact its artifacts.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +from collections.abc import Callable, Mapping, Sequence +from pathlib import Path +from typing import Any + +import e2e_catalog + +MINIMAX_BASE_URL = "https://api.minimax.io/anthropic" +MINIMAX_MODEL = "MiniMax-M3[1m]" +MINIMAX_COMPACT_WINDOW_TOKENS = "1000000" +# implement runs four concurrent children and resolve-review three to six; a spawn past the +# limit fails without retry. +MAX_CONCURRENT_SUBAGENTS = "6" +CANARY_TOKEN = "AUTOSKILLIT-E2E-CANARY-OK" +CANARY_PROMPT = f"Reply with exactly {CANARY_TOKEN} and nothing else." +GIT_AUTHOR = ("AutoSkillit E2E", "autoskillit-e2e@users.noreply.github.com") +SETUP_COMMAND_TIMEOUT_SEC = 120 +SECRET_ENV = ("MINIMAX_API_KEY", "E2E_SANDBOX_TOKEN") +REDACTED = b"[REDACTED]" +CONFIG_TEMPLATE = Path(__file__).with_name("autoskillit-config.yaml") +SANDBOX_CLONE = Path("/workspace/sandbox") + +_SCRUBBED_ENV = frozenset({"CLAUDE_CODE_OAUTH_TOKEN", *SECRET_ENV}) +_STDERR_TAIL_CHARS = 2000 + +Runner = Callable[..., subprocess.CompletedProcess[str]] +CallResult = tuple[subprocess.CompletedProcess[str] | None, str | None] + + +def run_command( + argv: Sequence[str], + *, + cwd: Path | None = None, + env: Mapping[str, str] | None = None, + input: str | None = None, + timeout: float | None = None, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + list(argv), + cwd=cwd, + env=env, + input=input, + timeout=timeout, + capture_output=True, + text=True, + check=False, + ) + + +def _call(runner: Runner, argv: Sequence[str], *, what: str, **kwargs: Any) -> CallResult: + """Run *argv*, turning a nonzero exit, a timeout or a launch error into a failure string.""" + try: + result = runner(argv, **kwargs) + except subprocess.TimeoutExpired as exc: + return None, f"{what}: timed out after {exc.timeout}s" + except OSError as exc: + return None, f"{what}: {exc}" + if result.returncode != 0: + stderr = (result.stderr or "").strip()[-_STDERR_TAIL_CHARS:] + return result, f"{what}: exit {result.returncode}: {stderr}" + return result, None + + +def child_env(env: Mapping[str, str]) -> dict[str, str]: + return { + name: value + for name, value in env.items() + if name not in _SCRUBBED_ENV and not name.startswith("ANTHROPIC_") + } + + +def claude_settings_env(api_key: str) -> dict[str, str]: + return { + "ANTHROPIC_BASE_URL": MINIMAX_BASE_URL, + "ANTHROPIC_AUTH_TOKEN": api_key, + "ANTHROPIC_MODEL": MINIMAX_MODEL, + "ANTHROPIC_DEFAULT_SONNET_MODEL": MINIMAX_MODEL, + "ANTHROPIC_DEFAULT_OPUS_MODEL": MINIMAX_MODEL, + "ANTHROPIC_DEFAULT_HAIKU_MODEL": MINIMAX_MODEL, + "CLAUDE_CODE_AUTO_COMPACT_WINDOW": MINIMAX_COMPACT_WINDOW_TOKENS, + "CLAUDE_CODE_MAX_CONCURRENT_SUBAGENTS": MAX_CONCURRENT_SUBAGENTS, + } + + +def write_claude_settings(home: Path, api_key: str) -> Path: + path = home / ".claude" / "settings.json" + settings: dict[str, Any] = {} + if path.is_file(): + existing = json.loads(path.read_text(encoding="utf-8")) + if isinstance(existing, dict): + settings = existing + settings["env"] = claude_settings_env(api_key) + path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + os.fchmod(fd, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(settings, handle, indent=2) + return path + + +def install_autoskillit_config(home: Path) -> Path: + target = home / ".autoskillit" / "config.yaml" + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(CONFIG_TEMPLATE, target) + return target + + +def check_canary_output(stdout: str) -> list[str]: + try: + payload = json.loads(stdout) + except json.JSONDecodeError: + return ["canary: claude output is not JSON"] + if not isinstance(payload, dict): + return ["canary: claude output is not a JSON object"] + if payload.get("is_error") is not False: + return [f"canary: claude reported an error: {str(payload.get('result'))[:500]}"] + reply = payload.get("result") + if not isinstance(reply, str) or CANARY_TOKEN not in reply: + return [f"canary: reply does not contain {CANARY_TOKEN}"] + return [] + + +def run_canary( + test: e2e_catalog.CatalogTest, out: Path, env: Mapping[str, str], runner: Runner +) -> list[str]: + workdir = Path(tempfile.mkdtemp(prefix="e2e-canary-")) + result, failure = _call( + runner, + ["claude", "-p", CANARY_PROMPT, "--output-format", "json"], + what="claude -p", + cwd=workdir, + env=env, + timeout=test.timeout_sec, + ) + if result is not None: + (out / "canary.json").write_text(result.stdout, encoding="utf-8") + (out / "canary.stderr.log").write_text(result.stderr, encoding="utf-8") + if failure is not None or result is None: + return [str(failure)] + return check_canary_output(result.stdout) + + +def _setup_call( + runner: Runner, argv: Sequence[str], env: Mapping[str, str], **kw: Any +) -> CallResult: + return _call( + runner, argv, what=" ".join(argv[:3]), env=env, timeout=SETUP_COMMAND_TIMEOUT_SEC, **kw + ) + + +def configure_git(env: Mapping[str, str], runner: Runner) -> str | None: + for key, value in (("user.name", GIT_AUTHOR[0]), ("user.email", GIT_AUTHOR[1])): + _, failure = _setup_call(runner, ["git", "config", "--global", key, value], env) + if failure is not None: + return failure + return None + + +def authenticate_gh(token: str, env: Mapping[str, str], runner: Runner) -> str | None: + login = ["gh", "auth", "login", "--hostname", "github.com", "--with-token"] + _, failure = _setup_call(runner, login, env, input=token) + if failure is not None: + return failure + _, failure = _setup_call(runner, ["gh", "auth", "setup-git"], env) + return failure + + +def clone_sandbox(repository: str, env: Mapping[str, str], runner: Runner) -> str | None: + _, failure = _setup_call(runner, ["gh", "repo", "clone", repository, str(SANDBOX_CLONE)], env) + return failure + + +def _list_pull_requests( + repository: str, limit: int, fields: str, env: Mapping[str, str], runner: Runner +) -> tuple[list[dict[str, Any]] | None, str | None]: + argv = ["gh", "pr", "list", "--repo", repository, "--state", "all"] + argv += ["--limit", str(limit), "--json", fields] + result, failure = _setup_call(runner, argv, env) + if failure is not None or result is None: + return None, failure + try: + rows = json.loads(result.stdout) + except json.JSONDecodeError: + return None, "gh pr list: output is not JSON" + if not isinstance(rows, list) or not all(isinstance(row, dict) for row in rows): + return None, "gh pr list: output is not a list of objects" + return rows, None + + +def latest_pull_request_number( + repository: str, env: Mapping[str, str], runner: Runner +) -> tuple[int | None, str | None]: + rows, failure = _list_pull_requests(repository, 1, "number", env, runner) + if rows is None: + return None, failure + return (int(rows[0]["number"]) if rows else 0), None + + +def new_pull_requests( + repository: str, baseline: int, env: Mapping[str, str], runner: Runner +) -> tuple[list[dict[str, Any]] | None, str | None]: + rows, failure = _list_pull_requests( + repository, 20, "number,state,additions,deletions", env, runner + ) + if rows is None: + return None, failure + return [row for row in rows if int(row["number"]) > baseline], None + + +def fleet_run_argv(test: e2e_catalog.CatalogTest) -> list[str]: + ingredients = [arg for key, value in test.ingredients for arg in ("-i", f"{key}={value}")] + return [ + "autoskillit", + "fleet", + "run", + str(test.recipe), + *ingredients, + "--disable-quota-guard", + "--timeout-sec", + str(test.timeout_sec), + ] + + +def last_line(stdout: str) -> str: + lines = [line for line in stdout.splitlines() if line.strip()] + return lines[-1] if lines else "" + + +def check_envelope(returncode: int, stdout: str) -> list[str]: + failures = [] if returncode == 0 else [f"fleet run: exit {returncode}"] + line = last_line(stdout) + try: + envelope = json.loads(line) + except json.JSONDecodeError: + return [*failures, "fleet run: the last stdout line is not a JSON envelope"] + if not isinstance(envelope, dict): + failures.append("fleet run: the envelope is not a JSON object") + elif envelope.get("success") is not True: + failures.append("fleet run: the envelope does not report success") + return failures + + +def run_fleet( + test: e2e_catalog.CatalogTest, out: Path, env: Mapping[str, str], runner: Runner +) -> list[str]: + result, failure = _call( + runner, + fleet_run_argv(test), + what="fleet run", + cwd=SANDBOX_CLONE, + env=env, + timeout=test.timeout_sec + e2e_catalog.HARNESS_GRACE_SEC, + ) + if result is None: + return [str(failure)] + (out / "fleet-run.stdout.log").write_text(result.stdout, encoding="utf-8") + (out / "fleet-run.stderr.log").write_text(result.stderr, encoding="utf-8") + (out / "envelope.json").write_text(last_line(result.stdout) + "\n", encoding="utf-8") + return check_envelope(result.returncode, result.stdout) + + +def check_pull_requests(prs: Sequence[Mapping[str, Any]], expected_state: str | None) -> list[str]: + if len(prs) != 1: + return [f"expected exactly one new sandbox pull request, found {len(prs)}"] + pr = prs[0] + failures: list[str] = [] + state = str(pr.get("state", "")).lower() + if state != expected_state: + failures.append(f"pull request #{pr.get('number')} is {state}, expected {expected_state}") + if int(pr.get("additions", 0)) + int(pr.get("deletions", 0)) <= 0: + failures.append(f"pull request #{pr.get('number')} has an empty diff") + return failures + + +def close_open_pull_requests( + repository: str, + baseline: int, + listed: Sequence[Mapping[str, Any]] | None, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + if listed is None: + listed, failure = new_pull_requests(repository, baseline, env, runner) + if listed is None: + return [f"cleanup: {failure}"] + failures: list[str] = [] + for pr in listed: + if str(pr.get("state", "")).lower() != "open": + continue + argv = ["gh", "pr", "close", str(pr["number"]), "--repo", repository, "--delete-branch"] + _, failure = _setup_call(runner, argv, env) + if failure is not None: + failures.append(f"cleanup: {failure}") + return failures + + +def run_recipe( + test: e2e_catalog.CatalogTest, + repository: str, + token: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + failure = ( + configure_git(env, runner) + or authenticate_gh(token, env, runner) + or clone_sandbox(repository, env, runner) + ) + if failure is not None: + return [failure] + baseline, failure = latest_pull_request_number(repository, env, runner) + if baseline is None: + return [str(failure)] + failures: list[str] = [] + listed: list[dict[str, Any]] | None = None + try: + failures += run_fleet(test, out, env, runner) + listed, failure = new_pull_requests(repository, baseline, env, runner) + if listed is None: + failures.append(str(failure)) + else: + failures += check_pull_requests(listed, test.expected_pull_request_state) + finally: + failures += close_open_pull_requests(repository, baseline, listed, env, runner) + return failures + + +def _run_test( + test: e2e_catalog.CatalogTest, + catalog: e2e_catalog.Catalog, + *, + out: Path, + home: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + required = SECRET_ENV if test.kind == "recipe" else ("MINIMAX_API_KEY",) + missing = [f"{name} is not set" for name in required if not env.get(name)] + if missing: + return missing + write_claude_settings(home, env["MINIMAX_API_KEY"]) + install_autoskillit_config(home) + scrubbed = child_env(env) + if test.kind == "canary": + return run_canary(test, out, scrubbed, runner) + token = env["E2E_SANDBOX_TOKEN"] + return run_recipe(test, catalog.sandbox_repository, token, out, scrubbed, runner) + + +def run_test( + test: e2e_catalog.CatalogTest, + catalog: e2e_catalog.Catalog, + *, + out: Path, + home: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + """Run *test* and always record ``out/result.json``; return the failures.""" + out.mkdir(parents=True, exist_ok=True) + try: + failures = _run_test(test, catalog, out=out, home=home, env=env, runner=runner) + except (OSError, ValueError) as exc: + failures = [f"harness error: {exc}"] + result = {"test": test.name, "passed": not failures, "failures": failures} + (out / "result.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return failures + + +def _redact(data: bytes, needles: Sequence[bytes]) -> bytes: + for needle in needles: + data = data.replace(needle, REDACTED) + return data + + +def _copy_redacted(source: Path, target: Path, needles: Sequence[bytes]) -> None: + target.write_bytes(_redact(source.read_bytes(), needles)) + target.chmod(0o644) + + +def _copy_tree_redacted(source: Path, target: Path, needles: Sequence[bytes]) -> None: + for root, _dirs, files in os.walk(source, followlinks=False): + root_path = Path(root) + target_dir = target / root_path.relative_to(source) + target_dir.mkdir(parents=True, exist_ok=True) + target_dir.chmod(0o755) + for name in files: + path = root_path / name + if not path.is_symlink() and path.is_file(): + _copy_redacted(path, target_dir / name, needles) + + +def _find_leak(dest: Path, needles: Sequence[bytes]) -> Path | None: + for root, _dirs, files in os.walk(dest, followlinks=False): + for name in files: + path = Path(root) / name + data = path.read_bytes() + if any(needle in data for needle in needles): + return path + return None + + +def redact_tree(sources: Sequence[Path], dest: Path, secrets: Sequence[str]) -> None: + """Copy *sources* into *dest* world-readable with every secret replaced; verify the copy.""" + needles = [secret.encode() for secret in secrets if secret] + dest.mkdir(parents=True, exist_ok=True) + for source in sources: + if source.is_symlink() or not source.exists(): + continue + target = dest / source.name + if source.is_dir(): + _copy_tree_redacted(source, target, needles) + else: + _copy_redacted(source, target, needles) + leak = _find_leak(dest, needles) + if leak is not None: + raise RuntimeError(f"a secret survived redaction in {leak}") + + +def _run(test_name: str, out: Path, catalog_path: Path) -> int: + try: + catalog = e2e_catalog.load_catalog(catalog_path) + test = catalog.get(test_name) + except e2e_catalog.CatalogError as exc: + print(f"e2e_harness: {exc}", file=sys.stderr) + return 2 + failures = run_test( + test, catalog, out=out, home=Path.home(), env=os.environ, runner=run_command + ) + for failure in failures: + print(f"e2e_harness: FAIL {failure}", file=sys.stderr) + print(f"e2e_harness: {test.name} {'failed' if failures else 'passed'}") + return 1 if failures else 0 + + +def _redact_command(dest: Path, secret_env: Sequence[str], sources: Sequence[str]) -> int: + secrets = [os.environ.get(name, "") for name in secret_env] + try: + redact_tree([Path(source) for source in sources], dest, secrets) + except (OSError, RuntimeError) as exc: + print(f"e2e_harness: redaction failed: {exc}", file=sys.stderr) + return 1 + return 0 + + +def main(argv: Sequence[str]) -> int: + """``run`` exits 0 iff the test passed; ``redact`` exits 0 iff the redacted copy is clean.""" + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + run = commands.add_parser("run", help="Run one catalog test.") + run.add_argument("--test", required=True) + run.add_argument("--out", required=True) + run.add_argument("--catalog", default=str(e2e_catalog.CATALOG_PATH)) + redact = commands.add_parser("redact", help="Copy artifacts with every secret redacted.") + redact.add_argument("--dest", required=True) + redact.add_argument("--secret-env", action="append", required=True) + redact.add_argument("sources", nargs="+") + args = parser.parse_args(argv) + if args.command == "run": + return _run(args.test, Path(args.out), Path(args.catalog)) + return _redact_command(Path(args.dest), args.secret_env, args.sources) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/infra/_complexity_helpers.py b/tests/infra/_complexity_helpers.py index 45676f01c2..eb30befa04 100644 --- a/tests/infra/_complexity_helpers.py +++ b/tests/infra/_complexity_helpers.py @@ -7,8 +7,9 @@ bodies live here, matching the tests/infra/_pretty_output_helpers.py convention of a dedicated underscore-prefixed helper module for one feature's split test files. -tests/infra/test_e2e_select.py also loads scripts/e2e/e2e_select.py through -load_check_script, so its sibling ``import e2e_catalog`` resolves. +tests/infra/test_e2e_select.py and tests/infra/test_e2e_harness.py also load +scripts/e2e/e2e_select.py and scripts/e2e/e2e_harness.py through load_check_script, so +their sibling ``import e2e_catalog`` resolves. """ from __future__ import annotations diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py new file mode 100644 index 0000000000..4bec77a72d --- /dev/null +++ b/tests/infra/test_e2e_harness.py @@ -0,0 +1,466 @@ +"""Tests for scripts/e2e/e2e_harness.py with a recording fake in place of real processes.""" + +from __future__ import annotations + +import json +import os +import stat +import subprocess +import tempfile +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from tests.infra._complexity_helpers import load_check_script + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] + +REPO_ROOT = Path(__file__).resolve().parents[2] +_HARNESS_SCRIPT = REPO_ROOT / "scripts" / "e2e" / "e2e_harness.py" + +harness = load_check_script("_autoskillit_e2e_harness", _HARNESS_SCRIPT) +e2e_catalog = harness.e2e_catalog + +MINIMAX_KEY = "minimax-test-key" +SANDBOX_TOKEN = "sandbox-test-token" +SANDBOX = "owner/sandbox" +_MINIMAX_MODEL = "MiniMax-M3[1m]" +_EXPECTED_SETTINGS_ENV = { + "ANTHROPIC_BASE_URL": "https://api.minimax.io/anthropic", + "ANTHROPIC_AUTH_TOKEN": MINIMAX_KEY, + "ANTHROPIC_MODEL": _MINIMAX_MODEL, + "ANTHROPIC_DEFAULT_SONNET_MODEL": _MINIMAX_MODEL, + "ANTHROPIC_DEFAULT_OPUS_MODEL": _MINIMAX_MODEL, + "ANTHROPIC_DEFAULT_HAIKU_MODEL": _MINIMAX_MODEL, + "CLAUDE_CODE_AUTO_COMPACT_WINDOW": "1000000", + "CLAUDE_CODE_MAX_CONCURRENT_SUBAGENTS": "6", +} +_CANARY_REPLY = json.dumps( + {"type": "result", "is_error": False, "result": "AUTOSKILLIT-E2E-CANARY-OK"} +) +_SKILL_NOTICE = "2 skills unavailable on this backend (codex): a, b" + + +@dataclass +class RecordedCall: + argv: list[str] + cwd: Path | None + env: dict[str, str] | None + input: str | None + timeout: float | None + + +Handler = Callable[[list[str]], "subprocess.CompletedProcess[str] | BaseException | None"] + + +class FakeRunner: + """Record every launch; answer from *handler*, defaulting to a clean exit.""" + + def __init__(self, handler: Handler) -> None: + self.handler = handler + self.calls: list[RecordedCall] = [] + + def __call__(self, argv, *, cwd=None, env=None, input=None, timeout=None): + self.calls.append(RecordedCall(list(argv), cwd, env, input, timeout)) + response = self.handler(list(argv)) + if isinstance(response, BaseException): + raise response + return response if response is not None else _completed(argv) + + def argv_prefixes(self, length: int = 3) -> list[tuple[str, ...]]: + return [tuple(call.argv[:length]) for call in self.calls] + + +def _completed(argv, stdout: str = "", returncode: int = 0, stderr: str = ""): + return subprocess.CompletedProcess(list(argv), returncode, stdout, stderr) + + +def _pr(number: int, state: str, additions: int = 3, deletions: int = 1) -> dict: + return {"number": number, "state": state, "additions": additions, "deletions": deletions} + + +def _recipe_handler( + *, + new_prs: list[dict] | None = None, + fleet: subprocess.CompletedProcess[str] | BaseException | None = None, + extra: Handler | None = None, +) -> Handler: + listed = [_pr(8, "OPEN"), _pr(7, "MERGED")] if new_prs is None else new_prs + + def handle(argv: list[str]): + if extra is not None and (response := extra(argv)) is not None: + return response + if argv[:3] == ["gh", "pr", "list"]: + baseline = argv[argv.index("--limit") + 1] == "1" + return _completed(argv, json.dumps([{"number": 7}] if baseline else listed)) + if argv[:3] == ["autoskillit", "fleet", "run"]: + return fleet or _completed(argv, f'{_SKILL_NOTICE}\n{{"success": true}}\n') + return None + + return handle + + +def _recipe_catalog(state: str = "open"): + return e2e_catalog.parse_catalog( + { + "sandbox_repository": SANDBOX, + "tests": [ + { + "name": "impl", + "kind": "recipe", + "peak_sessions": 6, + "timeout_sec": 600, + "trigger_paths": ["src/*"], + "recipe": "implementation", + "ingredients": {"task": "Add a greeting"}, + "expected_pull_request_state": state, + } + ], + } + ) + + +def _env(**overrides: str) -> dict[str, str]: + env = { + "PATH": "/usr/bin", + "HOME": "/home/autoskillit", + "MINIMAX_API_KEY": MINIMAX_KEY, + "E2E_SANDBOX_TOKEN": SANDBOX_TOKEN, + "ANTHROPIC_API_KEY": "anthropic-test-key", + "CLAUDE_CODE_OAUTH_TOKEN": "oauth-test-token", + } + env.update(overrides) + return {name: value for name, value in env.items() if value} + + +def _run_recipe(tmp_path: Path, runner: FakeRunner, *, state: str = "open", env=None): + catalog = _recipe_catalog(state) + out = tmp_path / "out" + failures = harness.run_test( + catalog.get("impl"), + catalog, + out=out, + home=tmp_path / "home", + env=env if env is not None else _env(), + runner=runner, + ) + return failures, json.loads((out / "result.json").read_text(encoding="utf-8")) + + +def _run_canary(tmp_path: Path, runner: FakeRunner, monkeypatch, env=None): + monkeypatch.setattr(tempfile, "tempdir", str(tmp_path)) + catalog = e2e_catalog.load_catalog() + out = tmp_path / "out" + failures = harness.run_test( + catalog.get("canary"), + catalog, + out=out, + home=tmp_path / "home", + env=env if env is not None else _env(), + runner=runner, + ) + return failures, json.loads((out / "result.json").read_text(encoding="utf-8")) + + +class TestProviderSetup: + def test_settings_env_is_the_minimax_map_and_keeps_other_keys(self, tmp_path: Path) -> None: + settings_path = tmp_path / ".claude" / "settings.json" + settings_path.parent.mkdir() + settings_path.write_text(json.dumps({"theme": "dark", "env": {"OLD": "1"}})) + harness.write_claude_settings(tmp_path, MINIMAX_KEY) + settings = json.loads(settings_path.read_text(encoding="utf-8")) + assert settings["env"] == _EXPECTED_SETTINGS_ENV + assert settings["theme"] == "dark" + assert stat.S_IMODE(settings_path.stat().st_mode) == 0o600 + + def test_new_settings_file_is_private(self, tmp_path: Path) -> None: + path = harness.write_claude_settings(tmp_path, MINIMAX_KEY) + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + + def test_child_env_drops_every_credential(self) -> None: + env = _env(ANTHROPIC_BASE_URL="https://example.invalid") + scrubbed = harness.child_env(env) + assert scrubbed == {"PATH": "/usr/bin", "HOME": "/home/autoskillit"} + + +def test_shipped_config_routes_every_default_codex_step_to_claude_code( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from autoskillit.config import load_config + from autoskillit.core.io import load_yaml + from autoskillit.core.paths import pkg_root + + home = tmp_path / "home" + monkeypatch.setattr("pathlib.Path.home", lambda: home) + harness.install_autoskillit_config(home) + project = tmp_path / "project" + project.mkdir() + cfg = load_config(project) + + defaults = load_yaml(pkg_root() / "config" / "defaults.yaml") + routes = cfg.agent_backend.recipe_overrides + unrouted = sorted( + (recipe, step) + for recipe, steps in defaults["agent_backend"]["recipe_overrides"].items() + for step, backend in steps.items() + if backend == "codex" and routes.get(recipe, {}).get(step) != "claude-code" + ) + assert unrouted == [] + assert "codex" not in {backend for steps in routes.values() for backend in steps.values()} + assert cfg.agent_backend.backend == "claude-code" + assert cfg.quota_guard.enabled is False + assert cfg.features["fleet"] is True + assert cfg.features["fleet_headless_run"] is True + + +class TestCanaryOutput: + def test_successful_reply_passes(self) -> None: + assert harness.check_canary_output(_CANARY_REPLY) == [] + + @pytest.mark.parametrize( + "stdout", + [ + json.dumps({"is_error": True, "result": "AUTOSKILLIT-E2E-CANARY-OK"}), + json.dumps({"is_error": False, "result": "something else"}), + "not json", + json.dumps(["is_error", False]), + ], + ids=["is-error", "missing-token", "not-json", "not-an-object"], + ) + def test_bad_reply_fails_once(self, stdout: str) -> None: + assert len(harness.check_canary_output(stdout)) == 1 + + +class TestEnvelope: + def test_success_envelope_passes(self) -> None: + assert harness.check_envelope(0, '{"success": true}\n') == [] + + def test_envelope_is_the_last_line_after_plain_text(self) -> None: + stdout = f'{_SKILL_NOTICE}\n{{"success": true}}\n\n' + assert harness.last_line(stdout) == '{"success": true}' + assert harness.check_envelope(0, stdout) == [] + + @pytest.mark.parametrize( + ("returncode", "stdout"), + [ + (1, '{"success": true}'), + (0, '{"success": false}'), + (0, "[true]"), + (0, '{"success": true}\nnot json'), + (0, ""), + ], + ids=["nonzero-exit", "unsuccessful", "list", "non-json-last-line", "empty"], + ) + def test_bad_envelope_fails(self, returncode: int, stdout: str) -> None: + assert harness.check_envelope(returncode, stdout) + + +class TestPullRequestAssertions: + def test_one_merged_pull_request_matches_lower_case_expectation(self) -> None: + assert harness.check_pull_requests([_pr(8, "MERGED")], "merged") == [] + + @pytest.mark.parametrize( + "prs", + [[], [_pr(8, "MERGED"), _pr(9, "MERGED")], [_pr(8, "OPEN")], [_pr(8, "MERGED", 0, 0)]], + ids=["none", "two", "state-mismatch", "empty-diff"], + ) + def test_unexpected_pull_requests_fail(self, prs: list[dict]) -> None: + assert harness.check_pull_requests(prs, "merged") + + +class TestRecipeFlow: + def test_steps_run_in_order_with_the_token_only_on_stdin(self, tmp_path: Path) -> None: + runner = FakeRunner(_recipe_handler()) + failures, result = _run_recipe(tmp_path, runner) + + assert failures == [] + assert result == {"test": "impl", "passed": True, "failures": []} + assert [call.argv for call in runner.calls] == [ + ["git", "config", "--global", "user.name", "AutoSkillit E2E"], + [ + "git", + "config", + "--global", + "user.email", + "autoskillit-e2e@users.noreply.github.com", + ], + ["gh", "auth", "login", "--hostname", "github.com", "--with-token"], + ["gh", "auth", "setup-git"], + ["gh", "repo", "clone", SANDBOX, "/workspace/sandbox"], + ["gh", "pr", "list", "--repo", SANDBOX, "--state", "all"] + + ["--limit", "1", "--json", "number"], + ["autoskillit", "fleet", "run", "implementation", "-i", "task=Add a greeting"] + + ["--disable-quota-guard", "--timeout-sec", "600"], + ["gh", "pr", "list", "--repo", SANDBOX, "--state", "all"] + + ["--limit", "20", "--json", "number,state,additions,deletions"], + ["gh", "pr", "close", "8", "--repo", SANDBOX, "--delete-branch"], + ] + assert [call.input for call in runner.calls if call.input] == [SANDBOX_TOKEN] + assert all(SANDBOX_TOKEN not in " ".join(call.argv) for call in runner.calls) + assert all(call.env == harness.child_env(_env()) for call in runner.calls) + fleet_call = runner.calls[6] + assert fleet_call.cwd == harness.SANDBOX_CLONE + assert fleet_call.timeout == 600 + e2e_catalog.HARNESS_GRACE_SEC + + def test_outputs_keep_full_stdout_and_only_the_envelope_line(self, tmp_path: Path) -> None: + _run_recipe(tmp_path, FakeRunner(_recipe_handler())) + out = tmp_path / "out" + stdout = (out / "fleet-run.stdout.log").read_text(encoding="utf-8") + assert stdout == f'{_SKILL_NOTICE}\n{{"success": true}}\n' + assert (out / "envelope.json").read_text(encoding="utf-8") == '{"success": true}\n' + assert (out / "fleet-run.stderr.log").exists() + + def test_harness_writes_provider_settings_and_user_config(self, tmp_path: Path) -> None: + _run_recipe(tmp_path, FakeRunner(_recipe_handler())) + home = tmp_path / "home" + settings = json.loads((home / ".claude" / "settings.json").read_text(encoding="utf-8")) + assert settings["env"] == _EXPECTED_SETTINGS_ENV + installed = (home / ".autoskillit" / "config.yaml").read_bytes() + assert installed == harness.CONFIG_TEMPLATE.read_bytes() + + def test_merged_pull_request_is_not_closed(self, tmp_path: Path) -> None: + runner = FakeRunner(_recipe_handler(new_prs=[_pr(8, "MERGED")])) + failures, _ = _run_recipe(tmp_path, runner, state="merged") + assert failures == [] + assert ("gh", "pr", "close") not in runner.argv_prefixes() + + def test_auth_failure_stops_before_the_baseline(self, tmp_path: Path) -> None: + def reject_login(argv: list[str]): + if argv[:3] == ["gh", "auth", "login"]: + return _completed(argv, returncode=1, stderr="bad credentials") + return None + + runner = FakeRunner(_recipe_handler(extra=reject_login)) + failures, result = _run_recipe(tmp_path, runner) + prefixes = runner.argv_prefixes() + assert result["passed"] is False + assert "bad credentials" in failures[0] + assert ("gh", "pr", "list") not in prefixes + assert ("gh", "pr", "close") not in prefixes + assert ("autoskillit", "fleet", "run") not in prefixes + + def test_fleet_timeout_is_a_failure_and_cleanup_still_runs(self, tmp_path: Path) -> None: + timeout = subprocess.TimeoutExpired(["autoskillit"], 720) + runner = FakeRunner(_recipe_handler(fleet=timeout)) + failures, result = _run_recipe(tmp_path, runner) + assert result["passed"] is False + assert any("fleet run: timed out" in failure for failure in failures) + assert runner.calls[-1].argv[:4] == ["gh", "pr", "close", "8"] + + def test_missing_gh_is_a_failure_string(self, tmp_path: Path) -> None: + def no_gh(argv: list[str]): + return ( + FileNotFoundError(2, "No such file or directory", "gh") + if argv[0] == "gh" + else None + ) + + runner = FakeRunner(_recipe_handler(extra=no_gh)) + failures, result = _run_recipe(tmp_path, runner) + assert result["passed"] is False + assert failures[0].startswith("gh auth login:") + assert ("autoskillit", "fleet", "run") not in runner.argv_prefixes() + + def test_failed_listing_after_the_run_is_retried_for_cleanup(self, tmp_path: Path) -> None: + listings = iter(["broken", json.dumps([_pr(8, "OPEN")])]) + + def flaky_listing(argv: list[str]): + if argv[:3] == ["gh", "pr", "list"] and "20" in argv: + return _completed(argv, next(listings)) + return None + + runner = FakeRunner(_recipe_handler(extra=flaky_listing)) + failures, _ = _run_recipe(tmp_path, runner) + assert failures == ["gh pr list: output is not JSON"] + assert runner.calls[-1].argv[:4] == ["gh", "pr", "close", "8"] + + def test_missing_sandbox_token_fails_before_any_launch(self, tmp_path: Path) -> None: + runner = FakeRunner(_recipe_handler()) + failures, result = _run_recipe(tmp_path, runner, env=_env(E2E_SANDBOX_TOKEN="")) + assert failures == ["E2E_SANDBOX_TOKEN is not set"] + assert result["passed"] is False + assert runner.calls == [] + + +class TestCanaryFlow: + def test_canary_runs_one_claude_prompt(self, tmp_path: Path, monkeypatch) -> None: + runner = FakeRunner(lambda argv: _completed(argv, _CANARY_REPLY)) + failures, result = _run_canary(tmp_path, runner, monkeypatch) + assert failures == [] + assert result == {"test": "canary", "passed": True, "failures": []} + assert [call.argv for call in runner.calls] == [ + ["claude", "-p", harness.CANARY_PROMPT, "--output-format", "json"] + ] + assert runner.calls[0].env == harness.child_env(_env()) + assert (tmp_path / "out" / "canary.json").read_text(encoding="utf-8") == _CANARY_REPLY + + def test_missing_minimax_key_fails_before_any_launch( + self, tmp_path: Path, monkeypatch + ) -> None: + runner = FakeRunner(lambda argv: _completed(argv, _CANARY_REPLY)) + failures, result = _run_canary(tmp_path, runner, monkeypatch, env=_env(MINIMAX_API_KEY="")) + assert failures == ["MINIMAX_API_KEY is not set"] + assert result["passed"] is False + assert runner.calls == [] + + +class TestRedaction: + def test_secrets_are_replaced_in_nested_text_and_binary_files(self, tmp_path: Path) -> None: + source = tmp_path / "out" + (source / "nested").mkdir(parents=True) + (source / "result.json").write_text(f"key={MINIMAX_KEY}\n", encoding="utf-8") + (source / "nested" / "blob.bin").write_bytes(b"\x00" + SANDBOX_TOKEN.encode() + b"\xff") + (source / "nested" / "blob.bin").chmod(0o600) + dest = tmp_path / "upload" + + harness.redact_tree([source, tmp_path / "missing"], dest, [MINIMAX_KEY, SANDBOX_TOKEN, ""]) + + assert (dest / "out" / "result.json").read_text(encoding="utf-8") == "key=[REDACTED]\n" + assert (dest / "out" / "nested" / "blob.bin").read_bytes() == b"\x00[REDACTED]\xff" + assert stat.S_IMODE((dest / "out" / "nested" / "blob.bin").stat().st_mode) == 0o644 + assert stat.S_IMODE((dest / "out" / "nested").stat().st_mode) == 0o755 + assert not (dest / "missing").exists() + + def test_symlinks_are_never_copied(self, tmp_path: Path) -> None: + secret_file = tmp_path / "secret.txt" + secret_file.write_text(MINIMAX_KEY, encoding="utf-8") + source = tmp_path / "out" + source.mkdir() + (source / "kept.txt").write_text("kept", encoding="utf-8") + (source / "file-link").symlink_to(secret_file) + (source / "dir-link").symlink_to(tmp_path) + linked_source = tmp_path / "linked" + linked_source.symlink_to(source) + dest = tmp_path / "upload" + + harness.redact_tree([source, linked_source], dest, [MINIMAX_KEY]) + + assert sorted(os.listdir(dest)) == ["out"] + assert sorted(os.listdir(dest / "out")) == ["kept.txt"] + + def test_surviving_secret_fails_the_post_scan( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + source = tmp_path / "out" + source.mkdir() + (source / "log.txt").write_text(f"token {SANDBOX_TOKEN}", encoding="utf-8") + monkeypatch.setattr(harness, "_redact", lambda data, needles: data) + with pytest.raises(RuntimeError, match="survived redaction"): + harness.redact_tree([source], tmp_path / "upload", [SANDBOX_TOKEN]) + + def test_redact_command_reads_secrets_from_the_environment( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + source = tmp_path / "out" + source.mkdir() + (source / "log.txt").write_text(f"key {MINIMAX_KEY}", encoding="utf-8") + monkeypatch.setenv("MINIMAX_API_KEY", MINIMAX_KEY) + monkeypatch.delenv("E2E_SANDBOX_TOKEN", raising=False) + argv = ["redact", "--dest", str(tmp_path / "upload")] + argv += ["--secret-env", "MINIMAX_API_KEY", "--secret-env", "E2E_SANDBOX_TOKEN"] + assert harness.main([*argv, str(source)]) == 0 + assert (tmp_path / "upload" / "out" / "log.txt").read_text(encoding="utf-8") == ( + "key [REDACTED]" + ) From a0ae812e33985508fdc77794b4011b5c697e9b75 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 4 Oct 2026 09:43:58 -0700 Subject: [PATCH 03/11] feat(e2e): file E2E failures to an append-only tracking issue CanaryIssueUpdater gains append_to_issue, which appends a section to the open issue for a title, opens one when none exists, and closes a full issue in favour of a successor that links it before an append would cross GitHub's body limit. The new post-e2e-failure subcommand and scripts/e2e/ post-e2e-failure.sh report a failed E2E test or redaction stage with the pull request, commit and run URL. Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/e2e/post-e2e-failure.sh | 36 ++++ src/autoskillit/_probe_canary.py | 114 +++++++++++- tests/execution/backends/test_probe_canary.py | 170 ++++++++++++++++++ tests/infra/test_probe_scripts.py | 21 +++ 4 files changed, 337 insertions(+), 4 deletions(-) create mode 100755 scripts/e2e/post-e2e-failure.sh diff --git a/scripts/e2e/post-e2e-failure.sh b/scripts/e2e/post-e2e-failure.sh new file mode 100755 index 0000000000..7ef0863609 --- /dev/null +++ b/scripts/e2e/post-e2e-failure.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Append an E2E test failure to that test's tracking issue. +# +# Usage: scripts/e2e/post-e2e-failure.sh TEST STAGE PULL_REQUEST HEAD_SHA WORKFLOW_RUN_URL +# +# Args: $1=TEST $2=STAGE (test|redaction) $3=PULL_REQUEST (empty for workflow_dispatch) +# $4=HEAD_SHA $5=WORKFLOW_RUN_URL +# +# Runs inside the AutoSkillit user image, against its uv tool installation. +# +# Environment: +# GH_TOKEN — token with issues: write (required by gh) +# GITHUB_REPOSITORY — owner/repo for the issue (required by Python) + +set -euo pipefail +export LC_ALL=C + +USAGE="Usage: $0 TEST STAGE PULL_REQUEST HEAD_SHA WORKFLOW_RUN_URL" +TEST="${1:?$USAGE}" +STAGE="${2:?$USAGE}" +PULL_REQUEST="${3?$USAGE}" +HEAD_SHA="${4:?$USAGE}" +WORKFLOW_RUN_URL="${5:?$USAGE}" + +PYTHON="$(uv tool dir)/autoskillit/bin/python" +if [[ ! -x "${PYTHON}" ]]; then + echo "ERROR: ${PYTHON} not found; run this inside the AutoSkillit user image." >&2 + exit 1 +fi + +exec "${PYTHON}" -m autoskillit._probe_canary post-e2e-failure \ + --test "${TEST}" \ + --stage "${STAGE}" \ + --pull-request "${PULL_REQUEST}" \ + --head-sha "${HEAD_SHA}" \ + --workflow-run-url "${WORKFLOW_RUN_URL}" diff --git a/src/autoskillit/_probe_canary.py b/src/autoskillit/_probe_canary.py index 5a966bed1e..c954cb0c6d 100644 --- a/src/autoskillit/_probe_canary.py +++ b/src/autoskillit/_probe_canary.py @@ -1,7 +1,7 @@ -"""Reusable canary state machine and GitHub issue updater for live probes. +"""Reusable canary state machine and GitHub issue updater for live probes and E2E runs. -IL-1 module: imports only stdlib and `autoskillit.core`. Provides the -persistence + flake-guard primitives that live probe classes build on. +IL-1 module: imports only stdlib and `autoskillit.core`. Provides the persistence + +flake-guard primitives live probes build on and the E2E workflow's failure issue. """ from __future__ import annotations @@ -79,6 +79,10 @@ def _run_gh_with_body_file(args: list[str], body: str) -> subprocess.CompletedPr logger.debug("canary_body_file_unlink_failed", path=body_path) +# GitHub rejects an issue body longer than this many characters. +ISSUE_BODY_MAX_CHARS = 65536 + + class CanaryIssueUpdater: def __init__(self, *, owner: str, repo: str) -> None: self._owner = owner @@ -105,6 +109,34 @@ def ensure_issue(self, state: CanaryState, title: str, body: str) -> int: ) state.last_issue_number = existing return existing + issue_number = self._create(title, body) + state.last_issue_number = issue_number + return issue_number + + def append_to_issue(self, title: str, header: str, section: str) -> int: + """Append *section* to the open issue titled *title*; open or rotate it as needed. + + An append that would cross ``ISSUE_BODY_MAX_CHARS`` closes the full issue and + opens a successor linking it, so the title keeps one open reporting channel. + """ + existing = self._find_existing(title) + if existing is None: + return self._create(title, f"{header}\n\n{section}") + repo = f"{self._owner}/{self._repo}" + body = self._current_body(existing) + if len(body) + 2 + len(section) > ISSUE_BODY_MAX_CHARS: + result = run_gh(["issue", "close", str(existing), "--repo", repo]) + if result.returncode != 0: + raise RuntimeError(f"gh issue close failed: {result.stderr}") + return self._create(title, f"{header}\n\nContinues #{existing}.\n\n{section}") + result = _run_gh_with_body_file( + ["issue", "edit", str(existing), "--repo", repo], f"{body}\n\n{section}" + ) + if result.returncode != 0: + raise RuntimeError(f"gh issue edit failed: {result.stderr}") + return existing + + def _create(self, title: str, body: str) -> int: result = _run_gh_with_body_file( [ "issue", @@ -126,9 +158,22 @@ def ensure_issue(self, state: CanaryState, title: str, body: str) -> int: except (json.JSONDecodeError, KeyError) as exc: msg = f"gh issue create returned unexpected output: {result.stdout!r}" raise RuntimeError(msg) from exc - state.last_issue_number = issue_number return issue_number + def _current_body(self, number: int) -> str: + repo = f"{self._owner}/{self._repo}" + result = run_gh(["issue", "view", str(number), "--repo", repo, "--json", "body"]) + if result.returncode != 0: + raise RuntimeError(f"gh issue view failed: {result.stderr}") + try: + body = json.loads(result.stdout)["body"] + except (json.JSONDecodeError, KeyError, TypeError) as exc: + msg = f"gh issue view returned unexpected output: {result.stdout!r}" + raise RuntimeError(msg) from exc + if not isinstance(body, str): + raise RuntimeError(f"gh issue view returned a non-string body: {body!r}") + return body + def _find_existing(self, title: str) -> int | None: result = run_gh( [ @@ -210,6 +255,42 @@ def _handle_post_failure( return 0 +def _handle_post_e2e_failure( + *, + test: str, + stage: str, + pull_request: str, + head_sha: str, + workflow_run_url: str, +) -> int: + repo_slug = os.environ.get("GITHUB_REPOSITORY", "") + if not repo_slug or "/" not in repo_slug: + logger.error("post_e2e_failure_missing_github_repository") + return 1 + + owner, repo = repo_slug.split("/", 1) + title = f"[E2E] {test} failure" + header = ( + f"Tracks failures of the `{test}` test in `.github/workflows/e2e.yml`. " + "Each failed run appends a section below." + ) + trigger = f"pull request #{pull_request}" if pull_request else "workflow_dispatch" + section = ( + f"### Failure at {head_sha[:12]}\n\n" + f"**Stage:** {stage}\n" + f"**Trigger:** {trigger}\n" + f"**Commit:** {head_sha}\n" + f"**Workflow Run:** {workflow_run_url}\n" + ) + updater = CanaryIssueUpdater(owner=owner, repo=repo) + try: + updater.append_to_issue(title, header, section) + except Exception as exc: + logger.error("e2e_failure_issue_failed", error=str(exc)) + return 1 + return 0 + + def _cli_main(argv: list[str] | None = None) -> int: import argparse @@ -230,6 +311,22 @@ def _cli_main(argv: list[str] | None = None) -> int: ) post.add_argument("--workflow-run-url", required=True, help="GitHub Actions workflow run URL") + e2e = sub.add_parser( + "post-e2e-failure", help="Append an E2E test failure to that test's tracking issue" + ) + e2e.add_argument("--test", required=True, help="Catalog name of the failed E2E test") + e2e.add_argument( + "--stage", + required=True, + choices=("test", "redaction"), + help="Workflow stage that failed", + ) + e2e.add_argument( + "--pull-request", default="", help="Pull request number; empty for workflow_dispatch" + ) + e2e.add_argument("--head-sha", required=True, help="Commit the E2E run tested") + e2e.add_argument("--workflow-run-url", required=True, help="GitHub Actions workflow run URL") + args = parser.parse_args(argv) if args.command is None: @@ -245,6 +342,15 @@ def _cli_main(argv: list[str] | None = None) -> int: workflow_run_url=args.workflow_run_url, ) + if args.command == "post-e2e-failure": + return _handle_post_e2e_failure( + test=args.test, + stage=args.stage, + pull_request=args.pull_request, + head_sha=args.head_sha, + workflow_run_url=args.workflow_run_url, + ) + return 1 diff --git a/tests/execution/backends/test_probe_canary.py b/tests/execution/backends/test_probe_canary.py index f4e1e66a47..28201d2f86 100644 --- a/tests/execution/backends/test_probe_canary.py +++ b/tests/execution/backends/test_probe_canary.py @@ -9,6 +9,7 @@ import autoskillit._probe_canary as _patch_autoskillit__probe_canary from autoskillit._probe_canary import ( + ISSUE_BODY_MAX_CHARS, N_CONSECUTIVE_FLAKE_GUARD, CanaryIssueUpdater, CanaryState, @@ -206,6 +207,175 @@ def mock_run_gh(args, **kwargs): assert any(e.get("event") == "canary_issue_edit_failed" for e in cap_logs) +_E2E_TITLE = "[E2E] canary failure" + + +class _FakeGh: + """Answer gh issue commands for one optional open issue; record argv and body files.""" + + def __init__( + self, + *, + existing: int | None = None, + body: str = "old body", + failing: frozenset[str] = frozenset(), + ) -> None: + self.existing = existing + self.body = body + self.failing = failing + self.calls: list[list[str]] = [] + self.bodies: dict[str, str] = {} + + def __call__(self, args, **kwargs): + args = list(args) + self.calls.append(args) + command = args[1] + if "--body-file" in args: + self.bodies[command] = Path(args[args.index("--body-file") + 1]).read_text() + if command in self.failing: + return CompletedProcess(args=args, returncode=1, stdout="", stderr=f"{command} boom") + if command == "list": + issues = ( + [] if self.existing is None else [{"number": self.existing, "title": _E2E_TITLE}] + ) + return CompletedProcess(args=args, returncode=0, stdout=json.dumps(issues), stderr="") + if command == "view": + stdout = json.dumps({"body": self.body}) + return CompletedProcess(args=args, returncode=0, stdout=stdout, stderr="") + if command == "create": + stdout = json.dumps({"number": 99}) + return CompletedProcess(args=args, returncode=0, stdout=stdout, stderr="") + return CompletedProcess(args=args, returncode=0, stdout="", stderr="") + + def commands(self) -> list[str]: + return [call[1] for call in self.calls] + + +def _append(monkeypatch: pytest.MonkeyPatch, gh: _FakeGh, section: str = "### Section") -> int: + monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", gh) + updater = CanaryIssueUpdater(owner="test-org", repo="test-repo") + return updater.append_to_issue(_E2E_TITLE, "Header", section) + + +class TestAppendToIssue: + def test_no_open_issue_creates_one_with_header_and_section( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + gh = _FakeGh() + assert _append(monkeypatch, gh) == 99 + assert gh.commands() == ["list", "create"] + assert gh.bodies["create"] == "Header\n\n### Section" + + def test_open_issue_gets_the_section_appended(self, monkeypatch: pytest.MonkeyPatch) -> None: + gh = _FakeGh(existing=42) + assert _append(monkeypatch, gh) == 42 + assert gh.calls[1] == ["issue", "view", "42", "--repo", "test-org/test-repo"] + [ + "--json", + "body", + ] + assert gh.calls[2][:5] == ["issue", "edit", "42", "--repo", "test-org/test-repo"] + assert gh.bodies["edit"] == "old body\n\n### Section" + + def test_view_failure_raises_without_editing(self, monkeypatch: pytest.MonkeyPatch) -> None: + gh = _FakeGh(existing=42, failing=frozenset({"view"})) + with pytest.raises(RuntimeError, match="gh issue view failed"): + _append(monkeypatch, gh) + assert "edit" not in gh.commands() + + def test_edit_failure_raises(self, monkeypatch: pytest.MonkeyPatch) -> None: + gh = _FakeGh(existing=42, failing=frozenset({"edit"})) + with pytest.raises(RuntimeError, match="gh issue edit failed"): + _append(monkeypatch, gh) + + def test_append_exactly_at_the_limit_edits(self, monkeypatch: pytest.MonkeyPatch) -> None: + section = "### Section" + gh = _FakeGh(existing=42, body="x" * (ISSUE_BODY_MAX_CHARS - 2 - len(section))) + assert _append(monkeypatch, gh, section) == 42 + assert gh.commands() == ["list", "view", "edit"] + + def test_append_past_the_limit_rotates_to_a_successor( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + section = "### Section" + gh = _FakeGh(existing=42, body="x" * (ISSUE_BODY_MAX_CHARS - 1 - len(section))) + assert _append(monkeypatch, gh, section) == 99 + assert gh.commands() == ["list", "view", "close", "create"] + assert gh.calls[2] == ["issue", "close", "42", "--repo", "test-org/test-repo"] + assert "#42" in gh.bodies["create"] + assert gh.bodies["create"].endswith(section) + + def test_close_failure_raises_without_creating(self, monkeypatch: pytest.MonkeyPatch) -> None: + section = "### Section" + gh = _FakeGh( + existing=42, + body="x" * ISSUE_BODY_MAX_CHARS, + failing=frozenset({"close"}), + ) + with pytest.raises(RuntimeError, match="gh issue close failed"): + _append(monkeypatch, gh, section) + assert "create" not in gh.commands() + + +def _post_e2e_failure(*extra: str) -> int: + return _cli_main( + [ + "post-e2e-failure", + "--test", + "canary", + "--head-sha", + "abcdef0123456789abcdef0123456789abcdef01", + "--workflow-run-url", + "https://example.com/run/9", + *extra, + ] + ) + + +class TestPostE2eFailure: + def test_missing_github_repository_returns_one(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("GITHUB_REPOSITORY", raising=False) + assert _post_e2e_failure("--stage", "test", "--pull-request", "12") == 1 + + @pytest.mark.parametrize("stage", ["test", "redaction"]) + def test_section_names_pull_request_commit_run_and_stage( + self, monkeypatch: pytest.MonkeyPatch, stage: str + ) -> None: + monkeypatch.setenv("GITHUB_REPOSITORY", "test-org/test-repo") + gh = _FakeGh() + monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", gh) + assert _post_e2e_failure("--stage", stage, "--pull-request", "12") == 0 + create = gh.calls[-1] + assert create[create.index("--title") + 1] == _E2E_TITLE + body = gh.bodies["create"] + assert "#12" in body + assert "abcdef0123456789abcdef0123456789abcdef01" in body + assert "### Failure at abcdef012345" in body + assert "https://example.com/run/9" in body + assert f"**Stage:** {stage}" in body + + def test_empty_pull_request_reports_workflow_dispatch( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("GITHUB_REPOSITORY", "test-org/test-repo") + gh = _FakeGh() + monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", gh) + assert _post_e2e_failure("--stage", "test", "--pull-request", "") == 0 + assert "workflow_dispatch" in gh.bodies["create"] + + def test_unknown_stage_is_rejected(self) -> None: + with pytest.raises(SystemExit): + _post_e2e_failure("--stage", "other") + + def test_issue_failure_returns_one(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GITHUB_REPOSITORY", "test-org/test-repo") + monkeypatch.setattr( + _patch_autoskillit__probe_canary, "run_gh", _FakeGh(failing=frozenset({"create"})) + ) + with structlog.testing.capture_logs() as cap_logs: + assert _post_e2e_failure("--stage", "test") == 1 + assert any(e.get("event") == "e2e_failure_issue_failed" for e in cap_logs) + + class TestCliMain: def test_post_failure_records_and_saves( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch diff --git a/tests/infra/test_probe_scripts.py b/tests/infra/test_probe_scripts.py index d4aaad703f..e1a3433621 100644 --- a/tests/infra/test_probe_scripts.py +++ b/tests/infra/test_probe_scripts.py @@ -52,6 +52,27 @@ def test_rejects_missing_args(self) -> None: assert "Usage" in result.stderr +class TestPostE2eFailure: + _SCRIPT = SCRIPTS_DIR / "e2e" / "post-e2e-failure.sh" + + def test_syntax(self) -> None: + result = subprocess.run(["bash", "-n", str(self._SCRIPT)], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + + def test_executable(self) -> None: + assert os.access(self._SCRIPT, os.X_OK) + + def test_rejects_missing_args(self) -> None: + result = subprocess.run( + ["bash", str(self._SCRIPT)], + capture_output=True, + text=True, + env={"PATH": os.environ["PATH"], "HOME": os.environ.get("HOME", "/tmp")}, + ) + assert result.returncode != 0 + assert "Usage" in result.stderr + + class TestCreateProbeCanaryIssueValidation: def test_rejects_missing_github_repository(self) -> None: """Exits non-zero when GITHUB_REPOSITORY is not set.""" From 27524440ceacb25d73d0908840f4d020fb2af55c Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 4 Oct 2026 09:46:36 -0700 Subject: [PATCH 04/11] ci(e2e): add the selective E2E workflow with a single required gate e2e.yml selects tests on pull_request, workflow_dispatch and merge_group, runs each selected test serially in one repository-wide queue inside the freshly built user image, redacts and uploads its artifacts, files a failure issue with only the workflow token, and reports one always-running e2e-gate check. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/e2e.yml | 183 +++++++++++++++++++++++++++++++ tests/infra/test_e2e_workflow.py | 158 ++++++++++++++++++++++++++ 2 files changed, 341 insertions(+) create mode 100644 .github/workflows/e2e.yml create mode 100644 tests/infra/test_e2e_workflow.py diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000000..85439c97eb --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,183 @@ +name: E2E + +# Live end-to-end recipe runs on MiniMax. scripts/e2e/README.md documents the selection +# policy, the secrets and the sandbox repository this workflow relies on. + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + merge_group: + workflow_dispatch: + inputs: + tests: + description: Comma-separated catalog test names (at most two) + type: string + required: true + default: canary + +permissions: + contents: read + +jobs: + select: + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + outputs: + selected: ${{ steps.select.outputs.selected }} + matrix: ${{ steps.select.outputs.matrix }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + sparse-checkout: scripts/e2e + + - name: List changed paths + if: github.event_name == 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \ + --jq '.[].filename' > "$RUNNER_TEMP/changed-paths.txt" + + - name: Select tests + id: select + run: | + python3 scripts/e2e/e2e_select.py select \ + --changed-paths "$RUNNER_TEMP/changed-paths.txt" >> "$GITHUB_OUTPUT" + + run: + needs: select + if: needs.select.outputs.selected == 'true' + runs-on: ubuntu-latest + timeout-minutes: ${{ matrix.timeout_minutes }} + permissions: + contents: read + issues: write + # One repository-wide queue: every selected test of every run waits its turn, so + # concurrent MiniMax sessions stay within one test's peak. + concurrency: + group: e2e + queue: max + cancel-in-progress: false + strategy: + fail-fast: false + max-parallel: 1 + matrix: ${{ fromJSON(needs.select.outputs.matrix) }} + env: + IMAGE: autoskillit-e2e:${{ github.sha }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Prepare directories + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP"/e2e/{out,data,upload} + # The container runs as uid 1000, which is not the runner's uid. + chmod 0777 "$RUNNER_TEMP"/e2e/{out,data,upload} + + - name: Export the archived build context + run: | + set -euo pipefail + mkdir "$RUNNER_TEMP/build-context" + git archive "$GITHUB_SHA" | tar -x -C "$RUNNER_TEMP/build-context" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Build the user image + id: build + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: ${{ runner.temp }}/build-context + file: ${{ runner.temp }}/build-context/scripts/docker/Dockerfile + target: user + load: true + push: false + tags: ${{ env.IMAGE }} + cache-from: type=gha,scope=docker-image-user + + - name: Run the E2E test + id: e2e + env: + MINIMAX_API_KEY: ${{ secrets.MINIMAX_API_KEY }} + E2E_SANDBOX_TOKEN: ${{ secrets.E2E_SANDBOX_TOKEN }} + TEST: ${{ matrix.test }} + run: | + docker run --rm \ + --env MINIMAX_API_KEY \ + --env E2E_SANDBOX_TOKEN \ + --volume "$GITHUB_WORKSPACE/scripts/e2e:/opt/e2e:ro" \ + --volume "$RUNNER_TEMP/e2e/out:/artifacts" \ + --volume "$RUNNER_TEMP/e2e/data:/home/autoskillit/.local/share/autoskillit" \ + "$IMAGE" \ + python3 /opt/e2e/e2e_harness.py run --test "$TEST" --out /artifacts + + - name: Redact secrets from the artifacts + id: redact + if: always() && steps.build.outcome == 'success' + env: + MINIMAX_API_KEY: ${{ secrets.MINIMAX_API_KEY }} + E2E_SANDBOX_TOKEN: ${{ secrets.E2E_SANDBOX_TOKEN }} + run: | + docker run --rm \ + --env MINIMAX_API_KEY \ + --env E2E_SANDBOX_TOKEN \ + --volume "$GITHUB_WORKSPACE/scripts/e2e:/opt/e2e:ro" \ + --volume "$RUNNER_TEMP/e2e:/e2e" \ + "$IMAGE" \ + python3 /opt/e2e/e2e_harness.py redact --dest /e2e/upload \ + --secret-env MINIMAX_API_KEY --secret-env E2E_SANDBOX_TOKEN \ + /e2e/out /e2e/data/logs + + - name: Upload the redacted artifacts + if: always() && steps.redact.outcome == 'success' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: e2e-${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/e2e/upload + include-hidden-files: true + if-no-files-found: warn + retention-days: 14 + + - name: Report failure + if: failure() && (steps.e2e.outcome == 'failure' || steps.redact.outcome == 'failure') + env: + GH_TOKEN: ${{ github.token }} + TEST: ${{ matrix.test }} + STAGE: ${{ steps.e2e.outcome == 'failure' && 'test' || 'redaction' }} + PULL_REQUEST: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + docker run --rm \ + --env GH_TOKEN \ + --env GITHUB_REPOSITORY \ + --volume "$GITHUB_WORKSPACE/scripts/e2e:/opt/e2e:ro" \ + "$IMAGE" \ + /opt/e2e/post-e2e-failure.sh "$TEST" "$STAGE" "$PULL_REQUEST" "$HEAD_SHA" "$RUN_URL" + + e2e-gate: + needs: [select, run] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + sparse-checkout: scripts/e2e + + - name: Judge the E2E gate + env: + SELECT_RESULT: ${{ needs.select.result }} + SELECTED: ${{ needs.select.outputs.selected }} + RUN_RESULT: ${{ needs.run.result }} + run: | + python3 scripts/e2e/e2e_select.py gate \ + --select-result "$SELECT_RESULT" \ + --selected "$SELECTED" \ + --run-result "$RUN_RESULT" diff --git a/tests/infra/test_e2e_workflow.py b/tests/infra/test_e2e_workflow.py new file mode 100644 index 0000000000..a83f7b05f9 --- /dev/null +++ b/tests/infra/test_e2e_workflow.py @@ -0,0 +1,158 @@ +"""Structural tests for the .github/workflows/e2e.yml workflow.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +from autoskillit.core.io import load_yaml + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.small] + +REPO_ROOT = Path(__file__).resolve().parents[2] +_WORKFLOWS = REPO_ROOT / ".github" / "workflows" +_WORKFLOW_PATH = _WORKFLOWS / "e2e.yml" +_DOCKERFILE = REPO_ROOT / "scripts" / "docker" / "Dockerfile" +_SHA_PIN = re.compile(r"@[0-9a-f]{40}\b") + + +def _on(workflow: dict) -> dict: + """Return the triggers; PyYAML (YAML 1.1) parses a bare ``on:`` key as ``True``.""" + return workflow.get("on") or workflow.get(True) or {} + + +@pytest.fixture(scope="module") +def workflow() -> dict: + return load_yaml(_WORKFLOW_PATH) + + +@pytest.fixture(scope="module") +def text() -> str: + return _WORKFLOW_PATH.read_text(encoding="utf-8") + + +def _step(workflow: dict, job: str, *, step_id: str | None = None, name: str | None = None): + return next( + step + for step in workflow["jobs"][job]["steps"] + if (step_id is not None and step.get("id") == step_id) + or (name is not None and step.get("name") == name) + ) + + +def _all_steps(workflow: dict): + for job in workflow["jobs"].values(): + yield from job["steps"] + + +class TestTriggers: + def test_pull_request_types_include_labeled(self, workflow: dict) -> None: + types = _on(workflow)["pull_request"]["types"] + assert types == ["opened", "synchronize", "reopened", "labeled"] + + def test_merge_group_is_a_trigger(self, workflow: dict) -> None: + assert "merge_group" in _on(workflow) + + def test_dispatch_takes_a_string_tests_input(self, workflow: dict) -> None: + assert _on(workflow)["workflow_dispatch"]["inputs"]["tests"]["type"] == "string" + + +class TestSecrets: + def test_only_the_minimax_key_and_sandbox_token_are_used(self, text: str) -> None: + assert set(re.findall(r"secrets\.(\w+)", text)) == {"MINIMAX_API_KEY", "E2E_SANDBOX_TOKEN"} + + @pytest.mark.parametrize("name", ["ANTHROPIC", "CLAUDE_CODE_OAUTH_TOKEN", "OPENAI_API_KEY"]) + def test_no_other_provider_credential_is_referenced(self, text: str, name: str) -> None: + assert name not in text + + def test_secrets_reach_only_the_test_and_redaction_steps(self, workflow: dict) -> None: + holders = { + step.get("id") + for step in _all_steps(workflow) + if "secrets." in str(step.get("env", {})) + } + assert holders == {"e2e", "redact"} + assert all("secrets." not in str(job.get("env", {})) for job in workflow["jobs"].values()) + + def test_report_step_uses_only_the_workflow_token(self, workflow: dict) -> None: + env = _step(workflow, "run", name="Report failure")["env"] + assert env["GH_TOKEN"] == "${{ github.token }}" + assert "secrets." not in str(env) + + @pytest.mark.parametrize("step_id", ["e2e", "redact"]) + def test_docker_run_forwards_secrets_by_name_only(self, workflow: dict, step_id: str) -> None: + run = _step(workflow, "run", step_id=step_id)["run"] + forwarded = re.findall(r"--env (\S+)", run) + assert forwarded == ["MINIMAX_API_KEY", "E2E_SANDBOX_TOKEN"] + + +class TestRunJob: + def test_one_repository_wide_queue(self, workflow: dict) -> None: + assert workflow["jobs"]["run"]["concurrency"] == { + "group": "e2e", + "queue": "max", + "cancel-in-progress": False, + } + + def test_matrix_runs_one_test_at_a_time(self, workflow: dict) -> None: + strategy = workflow["jobs"]["run"]["strategy"] + assert strategy["max-parallel"] == 1 + assert strategy["fail-fast"] is False + + def test_runs_only_when_selected_and_times_out_per_test(self, workflow: dict) -> None: + run = workflow["jobs"]["run"] + assert "needs.select.outputs.selected" in run["if"] + assert "matrix.timeout_minutes" in str(run["timeout-minutes"]) + + def test_permissions(self, workflow: dict) -> None: + assert workflow["jobs"]["run"]["permissions"] == {"contents": "read", "issues": "write"} + + def test_data_mount_targets_the_user_image_autoskillit_data_dir(self, workflow: dict) -> None: + stage = _DOCKERFILE.read_text(encoding="utf-8").split("FROM toolchain AS user", 1)[1] + home = re.search(r"ENV HOME=(\S+)", stage) + assert home is not None + run = _step(workflow, "run", step_id="e2e")["run"] + mounts = re.findall(r'--volume "\$RUNNER_TEMP/e2e/data:([^"]+)"', run) + assert mounts == [f"{home.group(1)}/.local/share/autoskillit"] + + def test_upload_sends_only_a_successfully_redacted_copy(self, workflow: dict) -> None: + upload = next( + step for step in _all_steps(workflow) if "upload-artifact" in step.get("uses", "") + ) + assert "steps.redact.outcome == 'success'" in upload["if"] + assert upload["with"]["path"].endswith("e2e/upload") + assert upload["with"]["include-hidden-files"] is True + + def test_report_runs_for_a_failed_test_or_redaction(self, workflow: dict) -> None: + report = _step(workflow, "run", name="Report failure") + assert report["if"] == ( + "failure() && (steps.e2e.outcome == 'failure' || steps.redact.outcome == 'failure')" + ) + assert "STAGE" in report["env"] + assert '"$STAGE"' in report["run"] + + def test_build_context_export_matches_the_docker_image_workflow(self, workflow: dict) -> None: + name = "Export the archived build context" + docker_image = load_yaml(_WORKFLOWS / "docker-image.yml") + ours = _step(workflow, "run", name=name)["run"] + assert ours == _step(docker_image, "publish", name=name)["run"] + + +class TestGate: + def test_gate_always_runs_after_select_and_run(self, workflow: dict) -> None: + gate = workflow["jobs"]["e2e-gate"] + assert gate["if"] == "always()" + assert gate["needs"] == ["select", "run"] + assert "name" not in gate + + def test_gate_judges_with_the_selector(self, workflow: dict) -> None: + runs = [step.get("run", "") for step in workflow["jobs"]["e2e-gate"]["steps"]] + assert any("e2e_select.py gate" in run for run in runs) + + +def test_every_action_is_sha_pinned(workflow: dict) -> None: + for step in _all_steps(workflow): + if "uses" in step: + assert _SHA_PIN.search(step["uses"]), step["uses"] From f133283bbea1fa631fef68e3fc9f220c6de43981 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 4 Oct 2026 09:47:42 -0700 Subject: [PATCH 05/11] docs(e2e): document E2E setup, selection policy and the MiniMax route Covers maintainer and sandbox setup with the token's permission set, how a run reaches MiniMax through Claude settings env, why autoskillit install is skipped, the user config, selection and gate policy, concurrency and timeouts, redaction, failure issues, the catalog schema and local reproduction. Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/e2e/README.md | 227 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 scripts/e2e/README.md diff --git a/scripts/e2e/README.md b/scripts/e2e/README.md new file mode 100644 index 0000000000..31f586a540 --- /dev/null +++ b/scripts/e2e/README.md @@ -0,0 +1,227 @@ +# Live E2E runs on MiniMax + +`.github/workflows/e2e.yml` runs AutoSkillit recipes end to end against a real model — +MiniMax, through its Anthropic-compatible endpoint — inside the published `user` Docker +image. This directory holds everything the workflow runs: + +| File | Role | +|------|------| +| `catalog.json` | Registered E2E tests (the only source of test names, budgets and trigger paths) | +| `e2e_catalog.py` | Stdlib loader and validator for the catalog | +| `e2e_select.py` | Picks the tests an event runs (`select`) and judges the required check (`gate`) | +| `e2e_harness.py` | Runs one test inside the image (`run`) and redacts its artifacts (`redact`) | +| `autoskillit-config.yaml` | AutoSkillit user-layer config the harness installs | +| `post-e2e-failure.sh` | Appends a failure to the test's tracking issue | + +`e2e_select.py` runs on the runner's system `python3`; `e2e_harness.py` runs inside the +image as `/usr/bin/python3`. Both are stdlib-only, which is why the catalog is JSON. + +## Maintainer setup + +1. **`MINIMAX_API_KEY`** — an organization or repository secret holding a MiniMax API key. +2. **`E2E_SANDBOX_TOKEN`** — a repository secret holding a fine-grained personal access + token scoped to the sandbox repository (`sandbox_repository` in `catalog.json`) only. + Start with these permissions and grow them only when a recipe needs more: + + | Permission | Needed by | + |------------|-----------| + | Contents: read and write | `gh repo clone`, and recipes pushing branches | + | Pull requests: read and write | recipes opening PRs; the harness listing and closing them | + | Issues: read and write | recipes editing issue bodies | + | Actions: read | `wait_for_ci` | + | Commit statuses: read | `wait_for_ci` | + + The canary needs only `MINIMAX_API_KEY`; no Anthropic credential is used anywhere. +3. Create the `e2e` label in this repository. +4. In branch protection, mark only `e2e-gate` as required — never `select` or `run (…)`. + +## Sandbox repository requirements + +Recipe tests clone the sandbox and let a recipe open a real pull request there. The sandbox +must have: + +- a pull-request CI workflow, so `wait_for_ci` has something to wait for; +- a `Taskfile.yml` with `install-worktree` and `test-check` tasks; +- a `.autoskillit/config.yaml` that sets none of `features`, `quota_guard` or + `agent_backend` — as the project layer it outranks the harness's user-layer config; +- no committed `.claude/settings.json` or `.claude/settings.local.json` with `env` provider + or model keys — project and local settings outrank user settings. + +## How a run reaches MiniMax + +AutoSkillit strips `ANTHROPIC_API_KEY`, `ANTHROPIC_AUTH_TOKEN` and `ANTHROPIC_BASE_URL` from +every session it launches and never passes `--settings`, `--setting-sources` or +`CLAUDE_CONFIG_DIR`. The only route left is the `env` block of `~/.claude/settings.json`, +which Claude Code applies over inherited variables and passes to every subprocess it starts. +The harness writes that block (mode `0600`): + +| Variable | Value | +|----------|-------| +| `ANTHROPIC_BASE_URL` | `https://api.minimax.io/anthropic` | +| `ANTHROPIC_AUTH_TOKEN` | `MINIMAX_API_KEY` | +| `ANTHROPIC_MODEL`, `ANTHROPIC_DEFAULT_{SONNET,OPUS,HAIKU}_MODEL` | `MiniMax-M3[1m]` | +| `CLAUDE_CODE_AUTO_COMPACT_WINDOW` | `1000000` | +| `CLAUDE_CODE_MAX_CONCURRENT_SUBAGENTS` | `6` | + +The values follow MiniMax's Claude Code guide +() and the repository's own +`providers.profiles.minimax`. The `[1m]` suffix makes Claude Code assume a 1M-token window +for the unrecognized model id; the compact window is capped at it. Six concurrent subagents +covers `implement` (four children) and `resolve-review` (three to six); a spawn past the +limit fails without retry. + +The harness relies on this provider contract: the Anthropic Messages API at +`ANTHROPIC_BASE_URL`, Bearer authentication from `ANTHROPIC_AUTH_TOKEN`, tool use, and +acceptance of the explicit `claude-*` model ids AutoSkillit passes with `--model`. The +canary passes no `--model`, so `ANTHROPIC_MODEL` covers it; `ANTHROPIC_DEFAULT_HAIKU_MODEL` +covers the bare `haiku` alias. + +Every child process the harness starts gets an environment with `MINIMAX_API_KEY`, +`E2E_SANDBOX_TOKEN`, `CLAUDE_CODE_OAUTH_TOKEN` and every `ANTHROPIC_*` variable removed, so +the settings file is the single source of provider configuration. The sandbox token reaches +`gh auth login` on stdin only. Because settings `env` reaches every agent subprocess, the key +can still surface in session output, so redaction is mandatory (see below). + +## Why `autoskillit install` is not run + +`fleet run` builds its context with `make_context(cfg, project_dir=Path.cwd(), ...)` +(`src/autoskillit/cli/fleet/_fleet_run.py`). Without an installed plugin it falls back to +`project_default_plugin_authority()` → `DirectInstall(plugin_dir=pkg_root())` +(`src/autoskillit/workspace/_projected_artifact/authority.py`), and `acquire_launch_binding` +publishes the projection on demand. The food-truck errors for a missing installation fire +only for backends that inject skills without plugin installs (Codex); the `claude-code` +backend receives the projection as `--plugin-dir`, and the harness routes every step to +`claude-code`. `install` also defers under `CLAUDECODE`, and the image never runs it. + +## AutoSkillit user config + +`autoskillit-config.yaml` becomes `~/.autoskillit/config.yaml` in the container: + +- `quota_guard.enabled: false` — the guard measures Anthropic account quota through Claude + credentials (`~/.claude/.credentials.json`) the container does not have; `fleet run` also + gets `--disable-quota-guard`; +- `features.fleet` and `features.fleet_headless_run` — `fleet run` requires both; +- `agent_backend.recipe_overrides` — every (recipe, step) the packaged defaults route to + `codex` is set to `claude-code`. Config layers deep-merge, so each pair is listed + explicitly; `tests/infra/test_e2e_harness.py` derives the set from + `src/autoskillit/config/defaults.yaml`, so a new Codex default fails CI until it is added. + +## Selection policy + +`e2e_select.py select` emits `selected`, `matrix` and `reason` as GitHub Actions outputs. The +matrix is always parseable (`{"include":[]}` when nothing is selected). + +| Event | Decision | +|-------|----------| +| `pull_request` from a fork, or with a deleted head repository | nothing | +| `pull_request` below both thresholds without the `e2e` label | nothing | +| `pull_request` with the `e2e` label | selected, bypassing sampling | +| `pull_request` at or above a threshold | selected when sampled | +| `workflow_dispatch` | the named tests (one or two, all in the catalog) | +| `merge_group` | nothing — the pull request already decided | + +- **Thresholds:** at least `MIN_CHANGED_FILES` (10) changed files or `MIN_CHANGED_LINES` + (300) added plus deleted lines. +- **Sampling:** probability `SAMPLE_PROBABILITY` (0.5), seeded by `"{pr number}:{head sha}"` + — the first eight bytes of its SHA-256 over 2⁶⁴ must fall below the probability. The label + bypasses sampling because a labelled pull request whose sample fell out would otherwise + never run for that commit. +- **Picking:** tests whose `trigger_paths` match a changed path, up to `MAX_TESTS` (2); when + none match, exactly one test. Ties are ordered by the SHA-256 hex digest of + `"{seed}:{name}"`, so the decision is a pure function of the payload and the changed paths. + Patterns use `fnmatch.fnmatchcase`, where `*` also matches `/` (`scripts/e2e/*` matches + every file below `scripts/e2e/`). +- **Relabelling:** any `labeled` event recomputes the same decision, because a new run's + `e2e-gate` supersedes the previous one for the commit; skipping it would let an unrelated + label turn a failed gate green. + +`e2e_select.py gate` passes only when `select` succeeded and either a selected `run` +succeeded or nothing was selected and `run` was skipped. Every other combination fails. + +## Concurrency + +The `run` job uses one repository-wide concurrency group (`group: e2e`, `queue: max`, +`cancel-in-progress: false`) and `max-parallel: 1`, so every selected test of every run +waits its turn and never cancels another. `select` and `e2e-gate` are not queued. A +catalog entry's `peak_sessions` is capped at `MAX_PEAK_SESSIONS` (8): one step's fan-out of +up to six subagents plus the orchestrator and step session. With one test running at a +time, CI stays at or below eight of the roughly twelve concurrent MiniMax sessions. + +Every harness subprocess has a timeout that fires before the job's: `fleet run` gets the +test's `timeout_sec` plus `HARNESS_GRACE_SEC`, and the job gets that budget in minutes plus +`JOB_OVERHEAD_MINUTES` for building the image, redaction and upload. So `result.json`, PR +cleanup and redaction never depend on GitHub cancelling the job. + +## Artifacts and redaction + +The container's AutoSkillit data directory (`~/.local/share/autoskillit`) is bind-mounted +from the runner, so session logs survive a killed container. Those files belong to uid 1000 +and are often `0600`, so a second container run copies `out/` and `data/logs/` into +`upload/` world-readable with both secrets replaced by `[REDACTED]`, skips symlinks, and +fails if any secret survives. Only `upload/` is uploaded, and only when redaction +succeeded. + +## Failure issues + +When the test or redaction step fails, `post-e2e-failure.sh` runs inside the built image +with only the workflow token (`issues: write`) and appends a section — stage, pull request +or `workflow_dispatch`, commit and run URL — to the open issue titled +`[E2E] failure`, opening it when none exists. An append that would cross GitHub's +65,536-character body limit closes the full issue and opens a successor that links it. +There is no flake guard: Actions caches are ref-scoped, so no streak survives across pull +requests. The test container never sees the workflow token. + +## Recipe test assertions + +A recipe test passes when `fleet run` exits 0, its envelope — the last non-empty stdout +line, since earlier lines can be plain-text notices — reports `"success": true`, and exactly +one sandbox pull request numbered above the pre-run baseline exists in the catalog's +`expected_pull_request_state` with a non-empty diff. `gh` reports states in upper case; the +harness lower-cases them. Afterwards the harness closes, with `--delete-branch`, every new +pull request still open — but only once it captured the baseline. + +## Catalog schema + +```json +{ + "sandbox_repository": "OWNER/REPO", + "tests": [ + { + "name": "lower-case-with-dashes", + "kind": "canary | recipe", + "peak_sessions": 1, + "timeout_sec": 300, + "trigger_paths": ["scripts/e2e/*"], + "recipe": "implementation", + "ingredients": {"task": "..."}, + "expected_pull_request_state": "open | merged | closed" + } + ] +} +``` + +`recipe`, `ingredients` and `expected_pull_request_state` are required for `recipe` tests +and forbidden for `canary` tests; any other key is rejected. `peak_sessions` must be 1–8 and +`timeout_sec` must keep the derived job timeout within 360 minutes. + +### Adding a test + +1. Add the entry to `catalog.json` with a realistic `peak_sessions` and `timeout_sec`. +2. Point `trigger_paths` at the code the test exercises. +3. Check the sandbox repository supports the recipe (CI, Taskfile tasks, token permissions). +4. Run it once with `workflow_dispatch` (available once `e2e.yml` is on `main`) or locally. + +## Local reproduction + +```bash +git archive HEAD | docker build --file scripts/docker/Dockerfile --target user \ + --tag autoskillit-e2e:local - +mkdir -p /tmp/e2e/out /tmp/e2e/data && chmod 0777 /tmp/e2e/out /tmp/e2e/data +MINIMAX_API_KEY=... docker run --rm \ + --env MINIMAX_API_KEY --env E2E_SANDBOX_TOKEN \ + --volume "$PWD/scripts/e2e:/opt/e2e:ro" \ + --volume /tmp/e2e/out:/artifacts \ + --volume /tmp/e2e/data:/home/autoskillit/.local/share/autoskillit \ + autoskillit-e2e:local \ + python3 /opt/e2e/e2e_harness.py run --test canary --out /artifacts +``` From 742400a8913570fed247cd68ce0ec9c7e9d3b177 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 4 Oct 2026 09:54:53 -0700 Subject: [PATCH 06/11] fix(e2e): skip files that vanish during the redaction post-scan The enumeration-read guard requires a walked path's content read to handle a file disappearing after the walk. A vanished file cannot be uploaded, so the post-scan skips it; every other read error still fails redaction. Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/e2e/e2e_harness.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index 4c8502cc0d..6d22fffc49 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -406,7 +406,11 @@ def _find_leak(dest: Path, needles: Sequence[bytes]) -> Path | None: for root, _dirs, files in os.walk(dest, followlinks=False): for name in files: path = Path(root) / name - data = path.read_bytes() + try: + data = path.read_bytes() + except (FileNotFoundError, NotADirectoryError): + # A file that vanished after the walk cannot be uploaded, so it cannot leak. + continue if any(needle in data for needle in needles): return path return None From c60888a6c2f83158851aba135563982dbf302d63 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 4 Oct 2026 11:17:29 -0700 Subject: [PATCH 07/11] fix(review): record failed E2E results for unexpected test errors --- scripts/e2e/e2e_harness.py | 2 +- tests/infra/test_e2e_harness.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index 6d22fffc49..5a9e1b016e 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -372,7 +372,7 @@ def run_test( out.mkdir(parents=True, exist_ok=True) try: failures = _run_test(test, catalog, out=out, home=home, env=env, runner=runner) - except (OSError, ValueError) as exc: + except Exception as exc: failures = [f"harness error: {exc}"] result = {"test": test.name, "passed": not failures, "failures": failures} (out / "result.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index 4bec77a72d..48018b6e9b 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -349,6 +349,15 @@ def test_fleet_timeout_is_a_failure_and_cleanup_still_runs(self, tmp_path: Path) assert any("fleet run: timed out" in failure for failure in failures) assert runner.calls[-1].argv[:4] == ["gh", "pr", "close", "8"] + def test_unexpected_fleet_error_records_failure_and_still_cleans_up( + self, tmp_path: Path + ) -> None: + runner = FakeRunner(_recipe_handler(fleet=RuntimeError("unexpected fleet error"))) + failures, result = _run_recipe(tmp_path, runner) + assert failures == ["harness error: unexpected fleet error"] + assert result == {"test": "impl", "passed": False, "failures": failures} + assert runner.calls[-1].argv[:4] == ["gh", "pr", "close", "8"] + def test_missing_gh_is_a_failure_string(self, tmp_path: Path) -> None: def no_gh(argv: list[str]): return ( From 7e5db069c1073b95a264c60248fc5a97006caee0 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 4 Oct 2026 11:18:23 -0700 Subject: [PATCH 08/11] fix(review): validate sandbox PR numbers before consuming listings --- scripts/e2e/e2e_harness.py | 2 ++ tests/infra/test_e2e_harness.py | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index 5a9e1b016e..071dcf2aad 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -195,6 +195,8 @@ def _list_pull_requests( return None, "gh pr list: output is not JSON" if not isinstance(rows, list) or not all(isinstance(row, dict) for row in rows): return None, "gh pr list: output is not a list of objects" + if any(type(row.get("number")) is not int or row["number"] <= 0 for row in rows): + return None, "gh pr list: pull request number is missing or invalid" return rows, None diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index 48018b6e9b..d15179b334 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -385,6 +385,19 @@ def flaky_listing(argv: list[str]): assert failures == ["gh pr list: output is not JSON"] assert runner.calls[-1].argv[:4] == ["gh", "pr", "close", "8"] + @pytest.mark.parametrize("limit", ["1", "20"]) + def test_missing_pull_request_number_records_failure(self, tmp_path: Path, limit: str) -> None: + def malformed_listing(argv: list[str]): + if argv[:3] == ["gh", "pr", "list"] and limit in argv: + return _completed(argv, json.dumps([{"state": "OPEN"}])) + return None + + runner = FakeRunner(_recipe_handler(extra=malformed_listing)) + failures, result = _run_recipe(tmp_path, runner) + assert "gh pr list: pull request number is missing or invalid" in failures + assert result == {"test": "impl", "passed": False, "failures": failures} + assert ("gh", "pr", "close") not in runner.argv_prefixes() + def test_missing_sandbox_token_fails_before_any_launch(self, tmp_path: Path) -> None: runner = FakeRunner(_recipe_handler()) failures, result = _run_recipe(tmp_path, runner, env=_env(E2E_SANDBOX_TOKEN="")) From a7eb714894d925ea5bea79ea20b08a10f2e16464 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 4 Oct 2026 11:19:26 -0700 Subject: [PATCH 09/11] fix(review): retain traceback details when E2E issue reporting fails --- src/autoskillit/_probe_canary.py | 2 +- tests/execution/backends/test_probe_canary.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/autoskillit/_probe_canary.py b/src/autoskillit/_probe_canary.py index c954cb0c6d..666bd3868c 100644 --- a/src/autoskillit/_probe_canary.py +++ b/src/autoskillit/_probe_canary.py @@ -286,7 +286,7 @@ def _handle_post_e2e_failure( try: updater.append_to_issue(title, header, section) except Exception as exc: - logger.error("e2e_failure_issue_failed", error=str(exc)) + logger.error("e2e_failure_issue_failed", error=str(exc), exc_info=True) return 1 return 0 diff --git a/tests/execution/backends/test_probe_canary.py b/tests/execution/backends/test_probe_canary.py index 28201d2f86..471d22fd77 100644 --- a/tests/execution/backends/test_probe_canary.py +++ b/tests/execution/backends/test_probe_canary.py @@ -373,7 +373,8 @@ def test_issue_failure_returns_one(self, monkeypatch: pytest.MonkeyPatch) -> Non ) with structlog.testing.capture_logs() as cap_logs: assert _post_e2e_failure("--stage", "test") == 1 - assert any(e.get("event") == "e2e_failure_issue_failed" for e in cap_logs) + failure = next(e for e in cap_logs if e.get("event") == "e2e_failure_issue_failed") + assert failure["exc_info"] is True class TestCliMain: From 701faa8da9ac2e9849ba492d12e83ee94b283eaa Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 4 Oct 2026 11:20:39 -0700 Subject: [PATCH 10/11] docs(review): clarify issue headers are used only for creation and rotation --- src/autoskillit/_probe_canary.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/autoskillit/_probe_canary.py b/src/autoskillit/_probe_canary.py index 666bd3868c..dd99584134 100644 --- a/src/autoskillit/_probe_canary.py +++ b/src/autoskillit/_probe_canary.py @@ -116,6 +116,9 @@ def ensure_issue(self, state: CanaryState, title: str, body: str) -> int: def append_to_issue(self, title: str, header: str, section: str) -> int: """Append *section* to the open issue titled *title*; open or rotate it as needed. + *header* seeds new and successor issues; ordinary appends preserve the + existing body and add only *section*. + An append that would cross ``ISSUE_BODY_MAX_CHARS`` closes the full issue and opens a successor linking it, so the title keeps one open reporting channel. """ From 89e4bf27c8c7c980fac2c6d217b6ba1baa729d90 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 4 Oct 2026 11:22:20 -0700 Subject: [PATCH 11/11] refactor(review): name the shared GitHub issue updater for both callers --- src/autoskillit/_probe_canary.py | 6 +++--- .../backends/test_cli_conformance_probes.py | 4 ++-- tests/execution/backends/test_probe_canary.py | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/autoskillit/_probe_canary.py b/src/autoskillit/_probe_canary.py index dd99584134..2bbf8d492d 100644 --- a/src/autoskillit/_probe_canary.py +++ b/src/autoskillit/_probe_canary.py @@ -83,7 +83,7 @@ def _run_gh_with_body_file(args: list[str], body: str) -> subprocess.CompletedPr ISSUE_BODY_MAX_CHARS = 65536 -class CanaryIssueUpdater: +class IssueUpdater: def __init__(self, *, owner: str, repo: str) -> None: self._owner = owner self._repo = repo @@ -248,7 +248,7 @@ def _handle_post_failure( f"**Network Streak:** {state.network_streak}\n" f"**Schema Streak:** {state.schema_streak}\n" ) - updater = CanaryIssueUpdater(owner=owner, repo=repo) + updater = IssueUpdater(owner=owner, repo=repo) try: updater.ensure_issue(state, title, body) except Exception as exc: @@ -285,7 +285,7 @@ def _handle_post_e2e_failure( f"**Commit:** {head_sha}\n" f"**Workflow Run:** {workflow_run_url}\n" ) - updater = CanaryIssueUpdater(owner=owner, repo=repo) + updater = IssueUpdater(owner=owner, repo=repo) try: updater.append_to_issue(title, header, section) except Exception as exc: diff --git a/tests/execution/backends/test_cli_conformance_probes.py b/tests/execution/backends/test_cli_conformance_probes.py index b99d686848..5d3211080f 100644 --- a/tests/execution/backends/test_cli_conformance_probes.py +++ b/tests/execution/backends/test_cli_conformance_probes.py @@ -33,9 +33,9 @@ import pytest from autoskillit._probe_canary import ( - CanaryIssueUpdater, CanaryState, ErrorKind, + IssueUpdater, ) from autoskillit.config import OutputBudgetConfig from autoskillit.core import ( @@ -388,7 +388,7 @@ def _record_failure( repo_slug = os.environ.get("GITHUB_REPOSITORY", "") if repo_slug and "/" in repo_slug: owner, repo = repo_slug.split("/", 1) - updater = CanaryIssueUpdater(owner=owner, repo=repo) + updater = IssueUpdater(owner=owner, repo=repo) title = f"{_CANARY_TITLE_PREFIX}: {probe_name}" body = _make_canary_body(probe_name, kind, cli_version, detail) updater.ensure_issue(state, title, body) diff --git a/tests/execution/backends/test_probe_canary.py b/tests/execution/backends/test_probe_canary.py index 471d22fd77..06ef2f2389 100644 --- a/tests/execution/backends/test_probe_canary.py +++ b/tests/execution/backends/test_probe_canary.py @@ -11,9 +11,9 @@ from autoskillit._probe_canary import ( ISSUE_BODY_MAX_CHARS, N_CONSECUTIVE_FLAKE_GUARD, - CanaryIssueUpdater, CanaryState, ErrorKind, + IssueUpdater, _cli_main, ) @@ -126,7 +126,7 @@ def test_values(self) -> None: assert set(ErrorKind) == {ErrorKind.NETWORK, ErrorKind.SCHEMA} -class TestCanaryIssueUpdater: +class TestIssueUpdater: def test_ensure_issue_creates_new(self, monkeypatch: pytest.MonkeyPatch) -> None: def mock_run_gh(args, **kwargs): if args[0:2] == ["issue", "list"]: @@ -142,7 +142,7 @@ def mock_run_gh(args, **kwargs): return CompletedProcess(args=args, returncode=1, stdout="", stderr="") monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", mock_run_gh) - updater = CanaryIssueUpdater(owner="test-org", repo="test-repo") + updater = IssueUpdater(owner="test-org", repo="test-repo") state = CanaryState() num = updater.ensure_issue(state, "Probe failure", "Details here") assert num == 123 @@ -163,7 +163,7 @@ def mock_run_gh(args, **kwargs): return CompletedProcess(args=args, returncode=1, stdout="", stderr="") monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", mock_run_gh) - updater = CanaryIssueUpdater(owner="test-org", repo="test-repo") + updater = IssueUpdater(owner="test-org", repo="test-repo") state = CanaryState() num = updater.ensure_issue(state, "Probe failure", "Updated body") assert num == 42 @@ -178,7 +178,7 @@ def mock_run_gh(args, **kwargs): raise AssertionError(f"Unexpected gh call: {args}") monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", mock_run_gh) - updater = CanaryIssueUpdater(owner="test-org", repo="test-repo") + updater = IssueUpdater(owner="test-org", repo="test-repo") state = CanaryState() with pytest.raises(RuntimeError, match="gh issue create failed"): updater.ensure_issue(state, "Probe failure", "Details") @@ -198,7 +198,7 @@ def mock_run_gh(args, **kwargs): raise AssertionError(f"Unexpected gh call: {args}") monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", mock_run_gh) - updater = CanaryIssueUpdater(owner="test-org", repo="test-repo") + updater = IssueUpdater(owner="test-org", repo="test-repo") state = CanaryState() with structlog.testing.capture_logs() as cap_logs: num = updater.ensure_issue(state, "Probe failure", "Updated body") @@ -253,7 +253,7 @@ def commands(self) -> list[str]: def _append(monkeypatch: pytest.MonkeyPatch, gh: _FakeGh, section: str = "### Section") -> int: monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", gh) - updater = CanaryIssueUpdater(owner="test-org", repo="test-repo") + updater = IssueUpdater(owner="test-org", repo="test-repo") return updater.append_to_issue(_E2E_TITLE, "Header", section)