diff --git a/.autoskillit/test-filter-manifest.yaml b/.autoskillit/test-filter-manifest.yaml index 8524746eee..de5edf7a36 100644 --- a/.autoskillit/test-filter-manifest.yaml +++ b/.autoskillit/test-filter-manifest.yaml @@ -169,6 +169,9 @@ Taskfile.yml: scripts/docker/*: - infra/ +scripts/e2e/*: + - infra/ + .gitattributes: - infra/ diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000000..85439c97eb --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,183 @@ +name: E2E + +# Live end-to-end recipe runs on MiniMax. scripts/e2e/README.md documents the selection +# policy, the secrets and the sandbox repository this workflow relies on. + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + merge_group: + workflow_dispatch: + inputs: + tests: + description: Comma-separated catalog test names (at most two) + type: string + required: true + default: canary + +permissions: + contents: read + +jobs: + select: + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + outputs: + selected: ${{ steps.select.outputs.selected }} + matrix: ${{ steps.select.outputs.matrix }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + sparse-checkout: scripts/e2e + + - name: List changed paths + if: github.event_name == 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \ + --jq '.[].filename' > "$RUNNER_TEMP/changed-paths.txt" + + - name: Select tests + id: select + run: | + python3 scripts/e2e/e2e_select.py select \ + --changed-paths "$RUNNER_TEMP/changed-paths.txt" >> "$GITHUB_OUTPUT" + + run: + needs: select + if: needs.select.outputs.selected == 'true' + runs-on: ubuntu-latest + timeout-minutes: ${{ matrix.timeout_minutes }} + permissions: + contents: read + issues: write + # One repository-wide queue: every selected test of every run waits its turn, so + # concurrent MiniMax sessions stay within one test's peak. + concurrency: + group: e2e + queue: max + cancel-in-progress: false + strategy: + fail-fast: false + max-parallel: 1 + matrix: ${{ fromJSON(needs.select.outputs.matrix) }} + env: + IMAGE: autoskillit-e2e:${{ github.sha }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Prepare directories + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP"/e2e/{out,data,upload} + # The container runs as uid 1000, which is not the runner's uid. + chmod 0777 "$RUNNER_TEMP"/e2e/{out,data,upload} + + - name: Export the archived build context + run: | + set -euo pipefail + mkdir "$RUNNER_TEMP/build-context" + git archive "$GITHUB_SHA" | tar -x -C "$RUNNER_TEMP/build-context" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Build the user image + id: build + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: ${{ runner.temp }}/build-context + file: ${{ runner.temp }}/build-context/scripts/docker/Dockerfile + target: user + load: true + push: false + tags: ${{ env.IMAGE }} + cache-from: type=gha,scope=docker-image-user + + - name: Run the E2E test + id: e2e + env: + MINIMAX_API_KEY: ${{ secrets.MINIMAX_API_KEY }} + E2E_SANDBOX_TOKEN: ${{ secrets.E2E_SANDBOX_TOKEN }} + TEST: ${{ matrix.test }} + run: | + docker run --rm \ + --env MINIMAX_API_KEY \ + --env E2E_SANDBOX_TOKEN \ + --volume "$GITHUB_WORKSPACE/scripts/e2e:/opt/e2e:ro" \ + --volume "$RUNNER_TEMP/e2e/out:/artifacts" \ + --volume "$RUNNER_TEMP/e2e/data:/home/autoskillit/.local/share/autoskillit" \ + "$IMAGE" \ + python3 /opt/e2e/e2e_harness.py run --test "$TEST" --out /artifacts + + - name: Redact secrets from the artifacts + id: redact + if: always() && steps.build.outcome == 'success' + env: + MINIMAX_API_KEY: ${{ secrets.MINIMAX_API_KEY }} + E2E_SANDBOX_TOKEN: ${{ secrets.E2E_SANDBOX_TOKEN }} + run: | + docker run --rm \ + --env MINIMAX_API_KEY \ + --env E2E_SANDBOX_TOKEN \ + --volume "$GITHUB_WORKSPACE/scripts/e2e:/opt/e2e:ro" \ + --volume "$RUNNER_TEMP/e2e:/e2e" \ + "$IMAGE" \ + python3 /opt/e2e/e2e_harness.py redact --dest /e2e/upload \ + --secret-env MINIMAX_API_KEY --secret-env E2E_SANDBOX_TOKEN \ + /e2e/out /e2e/data/logs + + - name: Upload the redacted artifacts + if: always() && steps.redact.outcome == 'success' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: e2e-${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/e2e/upload + include-hidden-files: true + if-no-files-found: warn + retention-days: 14 + + - name: Report failure + if: failure() && (steps.e2e.outcome == 'failure' || steps.redact.outcome == 'failure') + env: + GH_TOKEN: ${{ github.token }} + TEST: ${{ matrix.test }} + STAGE: ${{ steps.e2e.outcome == 'failure' && 'test' || 'redaction' }} + PULL_REQUEST: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + docker run --rm \ + --env GH_TOKEN \ + --env GITHUB_REPOSITORY \ + --volume "$GITHUB_WORKSPACE/scripts/e2e:/opt/e2e:ro" \ + "$IMAGE" \ + /opt/e2e/post-e2e-failure.sh "$TEST" "$STAGE" "$PULL_REQUEST" "$HEAD_SHA" "$RUN_URL" + + e2e-gate: + needs: [select, run] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + sparse-checkout: scripts/e2e + + - name: Judge the E2E gate + env: + SELECT_RESULT: ${{ needs.select.result }} + SELECTED: ${{ needs.select.outputs.selected }} + RUN_RESULT: ${{ needs.run.result }} + run: | + python3 scripts/e2e/e2e_select.py gate \ + --select-result "$SELECT_RESULT" \ + --selected "$SELECTED" \ + --run-result "$RUN_RESULT" diff --git a/scripts/e2e/README.md b/scripts/e2e/README.md new file mode 100644 index 0000000000..31f586a540 --- /dev/null +++ b/scripts/e2e/README.md @@ -0,0 +1,227 @@ +# Live E2E runs on MiniMax + +`.github/workflows/e2e.yml` runs AutoSkillit recipes end to end against a real model — +MiniMax, through its Anthropic-compatible endpoint — inside the published `user` Docker +image. This directory holds everything the workflow runs: + +| File | Role | +|------|------| +| `catalog.json` | Registered E2E tests (the only source of test names, budgets and trigger paths) | +| `e2e_catalog.py` | Stdlib loader and validator for the catalog | +| `e2e_select.py` | Picks the tests an event runs (`select`) and judges the required check (`gate`) | +| `e2e_harness.py` | Runs one test inside the image (`run`) and redacts its artifacts (`redact`) | +| `autoskillit-config.yaml` | AutoSkillit user-layer config the harness installs | +| `post-e2e-failure.sh` | Appends a failure to the test's tracking issue | + +`e2e_select.py` runs on the runner's system `python3`; `e2e_harness.py` runs inside the +image as `/usr/bin/python3`. Both are stdlib-only, which is why the catalog is JSON. + +## Maintainer setup + +1. **`MINIMAX_API_KEY`** — an organization or repository secret holding a MiniMax API key. +2. **`E2E_SANDBOX_TOKEN`** — a repository secret holding a fine-grained personal access + token scoped to the sandbox repository (`sandbox_repository` in `catalog.json`) only. + Start with these permissions and grow them only when a recipe needs more: + + | Permission | Needed by | + |------------|-----------| + | Contents: read and write | `gh repo clone`, and recipes pushing branches | + | Pull requests: read and write | recipes opening PRs; the harness listing and closing them | + | Issues: read and write | recipes editing issue bodies | + | Actions: read | `wait_for_ci` | + | Commit statuses: read | `wait_for_ci` | + + The canary needs only `MINIMAX_API_KEY`; no Anthropic credential is used anywhere. +3. Create the `e2e` label in this repository. +4. In branch protection, mark only `e2e-gate` as required — never `select` or `run (…)`. + +## Sandbox repository requirements + +Recipe tests clone the sandbox and let a recipe open a real pull request there. The sandbox +must have: + +- a pull-request CI workflow, so `wait_for_ci` has something to wait for; +- a `Taskfile.yml` with `install-worktree` and `test-check` tasks; +- a `.autoskillit/config.yaml` that sets none of `features`, `quota_guard` or + `agent_backend` — as the project layer it outranks the harness's user-layer config; +- no committed `.claude/settings.json` or `.claude/settings.local.json` with `env` provider + or model keys — project and local settings outrank user settings. + +## How a run reaches MiniMax + +AutoSkillit strips `ANTHROPIC_API_KEY`, `ANTHROPIC_AUTH_TOKEN` and `ANTHROPIC_BASE_URL` from +every session it launches and never passes `--settings`, `--setting-sources` or +`CLAUDE_CONFIG_DIR`. The only route left is the `env` block of `~/.claude/settings.json`, +which Claude Code applies over inherited variables and passes to every subprocess it starts. +The harness writes that block (mode `0600`): + +| Variable | Value | +|----------|-------| +| `ANTHROPIC_BASE_URL` | `https://api.minimax.io/anthropic` | +| `ANTHROPIC_AUTH_TOKEN` | `MINIMAX_API_KEY` | +| `ANTHROPIC_MODEL`, `ANTHROPIC_DEFAULT_{SONNET,OPUS,HAIKU}_MODEL` | `MiniMax-M3[1m]` | +| `CLAUDE_CODE_AUTO_COMPACT_WINDOW` | `1000000` | +| `CLAUDE_CODE_MAX_CONCURRENT_SUBAGENTS` | `6` | + +The values follow MiniMax's Claude Code guide +() and the repository's own +`providers.profiles.minimax`. The `[1m]` suffix makes Claude Code assume a 1M-token window +for the unrecognized model id; the compact window is capped at it. Six concurrent subagents +covers `implement` (four children) and `resolve-review` (three to six); a spawn past the +limit fails without retry. + +The harness relies on this provider contract: the Anthropic Messages API at +`ANTHROPIC_BASE_URL`, Bearer authentication from `ANTHROPIC_AUTH_TOKEN`, tool use, and +acceptance of the explicit `claude-*` model ids AutoSkillit passes with `--model`. The +canary passes no `--model`, so `ANTHROPIC_MODEL` covers it; `ANTHROPIC_DEFAULT_HAIKU_MODEL` +covers the bare `haiku` alias. + +Every child process the harness starts gets an environment with `MINIMAX_API_KEY`, +`E2E_SANDBOX_TOKEN`, `CLAUDE_CODE_OAUTH_TOKEN` and every `ANTHROPIC_*` variable removed, so +the settings file is the single source of provider configuration. The sandbox token reaches +`gh auth login` on stdin only. Because settings `env` reaches every agent subprocess, the key +can still surface in session output, so redaction is mandatory (see below). + +## Why `autoskillit install` is not run + +`fleet run` builds its context with `make_context(cfg, project_dir=Path.cwd(), ...)` +(`src/autoskillit/cli/fleet/_fleet_run.py`). Without an installed plugin it falls back to +`project_default_plugin_authority()` → `DirectInstall(plugin_dir=pkg_root())` +(`src/autoskillit/workspace/_projected_artifact/authority.py`), and `acquire_launch_binding` +publishes the projection on demand. The food-truck errors for a missing installation fire +only for backends that inject skills without plugin installs (Codex); the `claude-code` +backend receives the projection as `--plugin-dir`, and the harness routes every step to +`claude-code`. `install` also defers under `CLAUDECODE`, and the image never runs it. + +## AutoSkillit user config + +`autoskillit-config.yaml` becomes `~/.autoskillit/config.yaml` in the container: + +- `quota_guard.enabled: false` — the guard measures Anthropic account quota through Claude + credentials (`~/.claude/.credentials.json`) the container does not have; `fleet run` also + gets `--disable-quota-guard`; +- `features.fleet` and `features.fleet_headless_run` — `fleet run` requires both; +- `agent_backend.recipe_overrides` — every (recipe, step) the packaged defaults route to + `codex` is set to `claude-code`. Config layers deep-merge, so each pair is listed + explicitly; `tests/infra/test_e2e_harness.py` derives the set from + `src/autoskillit/config/defaults.yaml`, so a new Codex default fails CI until it is added. + +## Selection policy + +`e2e_select.py select` emits `selected`, `matrix` and `reason` as GitHub Actions outputs. The +matrix is always parseable (`{"include":[]}` when nothing is selected). + +| Event | Decision | +|-------|----------| +| `pull_request` from a fork, or with a deleted head repository | nothing | +| `pull_request` below both thresholds without the `e2e` label | nothing | +| `pull_request` with the `e2e` label | selected, bypassing sampling | +| `pull_request` at or above a threshold | selected when sampled | +| `workflow_dispatch` | the named tests (one or two, all in the catalog) | +| `merge_group` | nothing — the pull request already decided | + +- **Thresholds:** at least `MIN_CHANGED_FILES` (10) changed files or `MIN_CHANGED_LINES` + (300) added plus deleted lines. +- **Sampling:** probability `SAMPLE_PROBABILITY` (0.5), seeded by `"{pr number}:{head sha}"` + — the first eight bytes of its SHA-256 over 2⁶⁴ must fall below the probability. The label + bypasses sampling because a labelled pull request whose sample fell out would otherwise + never run for that commit. +- **Picking:** tests whose `trigger_paths` match a changed path, up to `MAX_TESTS` (2); when + none match, exactly one test. Ties are ordered by the SHA-256 hex digest of + `"{seed}:{name}"`, so the decision is a pure function of the payload and the changed paths. + Patterns use `fnmatch.fnmatchcase`, where `*` also matches `/` (`scripts/e2e/*` matches + every file below `scripts/e2e/`). +- **Relabelling:** any `labeled` event recomputes the same decision, because a new run's + `e2e-gate` supersedes the previous one for the commit; skipping it would let an unrelated + label turn a failed gate green. + +`e2e_select.py gate` passes only when `select` succeeded and either a selected `run` +succeeded or nothing was selected and `run` was skipped. Every other combination fails. + +## Concurrency + +The `run` job uses one repository-wide concurrency group (`group: e2e`, `queue: max`, +`cancel-in-progress: false`) and `max-parallel: 1`, so every selected test of every run +waits its turn and never cancels another. `select` and `e2e-gate` are not queued. A +catalog entry's `peak_sessions` is capped at `MAX_PEAK_SESSIONS` (8): one step's fan-out of +up to six subagents plus the orchestrator and step session. With one test running at a +time, CI stays at or below eight of the roughly twelve concurrent MiniMax sessions. + +Every harness subprocess has a timeout that fires before the job's: `fleet run` gets the +test's `timeout_sec` plus `HARNESS_GRACE_SEC`, and the job gets that budget in minutes plus +`JOB_OVERHEAD_MINUTES` for building the image, redaction and upload. So `result.json`, PR +cleanup and redaction never depend on GitHub cancelling the job. + +## Artifacts and redaction + +The container's AutoSkillit data directory (`~/.local/share/autoskillit`) is bind-mounted +from the runner, so session logs survive a killed container. Those files belong to uid 1000 +and are often `0600`, so a second container run copies `out/` and `data/logs/` into +`upload/` world-readable with both secrets replaced by `[REDACTED]`, skips symlinks, and +fails if any secret survives. Only `upload/` is uploaded, and only when redaction +succeeded. + +## Failure issues + +When the test or redaction step fails, `post-e2e-failure.sh` runs inside the built image +with only the workflow token (`issues: write`) and appends a section — stage, pull request +or `workflow_dispatch`, commit and run URL — to the open issue titled +`[E2E] failure`, opening it when none exists. An append that would cross GitHub's +65,536-character body limit closes the full issue and opens a successor that links it. +There is no flake guard: Actions caches are ref-scoped, so no streak survives across pull +requests. The test container never sees the workflow token. + +## Recipe test assertions + +A recipe test passes when `fleet run` exits 0, its envelope — the last non-empty stdout +line, since earlier lines can be plain-text notices — reports `"success": true`, and exactly +one sandbox pull request numbered above the pre-run baseline exists in the catalog's +`expected_pull_request_state` with a non-empty diff. `gh` reports states in upper case; the +harness lower-cases them. Afterwards the harness closes, with `--delete-branch`, every new +pull request still open — but only once it captured the baseline. + +## Catalog schema + +```json +{ + "sandbox_repository": "OWNER/REPO", + "tests": [ + { + "name": "lower-case-with-dashes", + "kind": "canary | recipe", + "peak_sessions": 1, + "timeout_sec": 300, + "trigger_paths": ["scripts/e2e/*"], + "recipe": "implementation", + "ingredients": {"task": "..."}, + "expected_pull_request_state": "open | merged | closed" + } + ] +} +``` + +`recipe`, `ingredients` and `expected_pull_request_state` are required for `recipe` tests +and forbidden for `canary` tests; any other key is rejected. `peak_sessions` must be 1–8 and +`timeout_sec` must keep the derived job timeout within 360 minutes. + +### Adding a test + +1. Add the entry to `catalog.json` with a realistic `peak_sessions` and `timeout_sec`. +2. Point `trigger_paths` at the code the test exercises. +3. Check the sandbox repository supports the recipe (CI, Taskfile tasks, token permissions). +4. Run it once with `workflow_dispatch` (available once `e2e.yml` is on `main`) or locally. + +## Local reproduction + +```bash +git archive HEAD | docker build --file scripts/docker/Dockerfile --target user \ + --tag autoskillit-e2e:local - +mkdir -p /tmp/e2e/out /tmp/e2e/data && chmod 0777 /tmp/e2e/out /tmp/e2e/data +MINIMAX_API_KEY=... docker run --rm \ + --env MINIMAX_API_KEY --env E2E_SANDBOX_TOKEN \ + --volume "$PWD/scripts/e2e:/opt/e2e:ro" \ + --volume /tmp/e2e/out:/artifacts \ + --volume /tmp/e2e/data:/home/autoskillit/.local/share/autoskillit \ + autoskillit-e2e:local \ + python3 /opt/e2e/e2e_harness.py run --test canary --out /artifacts +``` diff --git a/scripts/e2e/autoskillit-config.yaml b/scripts/e2e/autoskillit-config.yaml new file mode 100644 index 0000000000..577e8df598 --- /dev/null +++ b/scripts/e2e/autoskillit-config.yaml @@ -0,0 +1,31 @@ +# AutoSkillit user-layer config the E2E harness installs as ~/.autoskillit/config.yaml. +# See scripts/e2e/README.md for why each key is set. +quota_guard: + enabled: false + +features: + fleet: true + fleet_headless_run: true + +agent_backend: + recipe_overrides: + implementation: + run_arch_lenses: claude-code + implementation-groups: + run_arch_lenses: claude-code + planner: + analyze: claude-code + extract_domain: claude-code + elaborate_phases: claude-code + remediation: + investigate: claude-code + run_arch_lenses: claude-code + research: + run_experiment_lenses: claude-code + scope: claude-code + vis_apply: claude-code + research-design: + scope: claude-code + vis_apply: claude-code + research-review: + run_experiment_lenses: claude-code diff --git a/scripts/e2e/catalog.json b/scripts/e2e/catalog.json new file mode 100644 index 0000000000..e0a967c5bf --- /dev/null +++ b/scripts/e2e/catalog.json @@ -0,0 +1,16 @@ +{ + "sandbox_repository": "TalonT-Org/autoskillit-e2e-sandbox", + "tests": [ + { + "name": "canary", + "kind": "canary", + "peak_sessions": 1, + "timeout_sec": 300, + "trigger_paths": [ + "scripts/e2e/*", + "scripts/docker/*", + ".github/workflows/e2e.yml" + ] + } + ] +} diff --git a/scripts/e2e/e2e_catalog.py b/scripts/e2e/e2e_catalog.py new file mode 100644 index 0000000000..f206f25f5f --- /dev/null +++ b/scripts/e2e/e2e_catalog.py @@ -0,0 +1,175 @@ +"""Load and validate the E2E test catalog shared by the selector and the harness.""" + +from __future__ import annotations + +import json +import math +import re +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path + +CATALOG_PATH = Path(__file__).with_name("catalog.json") +# One step fans out to at most six subagents beside its orchestrator and step session, and one +# run at a time keeps CI at or below eight of the roughly twelve concurrent MiniMax sessions. +MAX_PEAK_SESSIONS = 8 +# Setup, assertions and cleanup around the test's own budget, so every harness timeout fires +# before the job ceiling. +HARNESS_GRACE_SEC = 120 +# Image build and load, redaction and artifact upload. +JOB_OVERHEAD_MINUTES = 30 +MAX_JOB_MINUTES = 360 +KINDS = ("canary", "recipe") +PULL_REQUEST_STATES = ("open", "merged", "closed") + +_NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9-]*$") +_CATALOG_KEYS = frozenset({"sandbox_repository", "tests"}) +_COMMON_KEYS = frozenset({"name", "kind", "peak_sessions", "timeout_sec", "trigger_paths"}) +_RECIPE_KEYS = frozenset({"recipe", "ingredients", "expected_pull_request_state"}) + + +class CatalogError(ValueError): + """The catalog violates its schema.""" + + +@dataclass(frozen=True, slots=True) +class CatalogTest: + """One live E2E test and the resources it is allowed to use.""" + + name: str + kind: str + peak_sessions: int + timeout_sec: int + trigger_paths: tuple[str, ...] + recipe: str | None + ingredients: tuple[tuple[str, str], ...] + expected_pull_request_state: str | None + + +@dataclass(frozen=True, slots=True) +class Catalog: + """Every registered E2E test and the sandbox repository recipe tests run against.""" + + sandbox_repository: str + tests: tuple[CatalogTest, ...] + + def get(self, name: str) -> CatalogTest: + for test in self.tests: + if test.name == name: + return test + raise CatalogError(f"unknown E2E test: {name!r}") + + +def job_timeout_minutes(test: CatalogTest) -> int: + return math.ceil((test.timeout_sec + HARNESS_GRACE_SEC) / 60) + JOB_OVERHEAD_MINUTES + + +def _require_keys(raw: Mapping[str, object], expected: frozenset[str], where: str) -> None: + missing = sorted(expected - raw.keys()) + unknown = sorted(raw.keys() - expected) + if missing or unknown: + raise CatalogError(f"{where}: missing keys {missing}, unknown keys {unknown}") + + +def _require_str(value: object, where: str) -> str: + if not isinstance(value, str) or not value: + raise CatalogError(f"{where} must be a non-empty string") + return value + + +def _require_int(value: object, where: str, *, low: int, high: int | None = None) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise CatalogError(f"{where} must be an integer") + if value < low or (high is not None and value > high): + bound = f"between {low} and {high}" if high is not None else f"at least {low}" + raise CatalogError(f"{where} must be {bound}") + return value + + +def _parse_trigger_paths(value: object, where: str) -> tuple[str, ...]: + if not isinstance(value, list) or not value: + raise CatalogError(f"{where} must be a non-empty list") + return tuple(_require_str(item, f"{where}[]") for item in value) + + +def _parse_ingredients(value: object, where: str) -> tuple[tuple[str, str], ...]: + if not isinstance(value, dict): + raise CatalogError(f"{where} must be an object") + pairs: list[tuple[str, str]] = [] + for key, item in value.items(): + if not isinstance(item, str): + raise CatalogError(f"{where}.{key} must be a string") + pairs.append((_require_str(key, f"{where} key"), item)) + return tuple(pairs) + + +def _parse_recipe_fields( + raw: Mapping[str, object], where: str +) -> tuple[str, tuple[tuple[str, str], ...], str]: + recipe = _require_str(raw["recipe"], f"{where}.recipe") + ingredients = _parse_ingredients(raw["ingredients"], f"{where}.ingredients") + state = _require_str( + raw["expected_pull_request_state"], f"{where}.expected_pull_request_state" + ) + if state not in PULL_REQUEST_STATES: + raise CatalogError( + f"{where}.expected_pull_request_state must be one of {PULL_REQUEST_STATES}" + ) + return recipe, ingredients, state + + +def _parse_test(raw: object, index: int) -> CatalogTest: + where = f"tests[{index}]" + if not isinstance(raw, dict): + raise CatalogError(f"{where} must be an object") + kind = raw.get("kind") + if kind not in KINDS: + raise CatalogError(f"{where}.kind must be one of {KINDS}") + _require_keys(raw, _COMMON_KEYS | _RECIPE_KEYS if kind == "recipe" else _COMMON_KEYS, where) + name = _require_str(raw["name"], f"{where}.name") + if not _NAME_PATTERN.fullmatch(name): + raise CatalogError(f"{where}.name must match {_NAME_PATTERN.pattern}") + recipe, ingredients, state = ( + _parse_recipe_fields(raw, where) if kind == "recipe" else (None, (), None) + ) + test = CatalogTest( + name=name, + kind=kind, + peak_sessions=_require_int( + raw["peak_sessions"], f"{where}.peak_sessions", low=1, high=MAX_PEAK_SESSIONS + ), + timeout_sec=_require_int(raw["timeout_sec"], f"{where}.timeout_sec", low=1), + trigger_paths=_parse_trigger_paths(raw["trigger_paths"], f"{where}.trigger_paths"), + recipe=recipe, + ingredients=ingredients, + expected_pull_request_state=state, + ) + if job_timeout_minutes(test) > MAX_JOB_MINUTES: + raise CatalogError(f"{where}.timeout_sec exceeds the {MAX_JOB_MINUTES}-minute job limit") + return test + + +def parse_catalog(raw: object) -> Catalog: + if not isinstance(raw, dict): + raise CatalogError("catalog must be an object") + _require_keys(raw, _CATALOG_KEYS, "catalog") + repository = _require_str(raw["sandbox_repository"], "sandbox_repository") + if repository.count("/") != 1: + raise CatalogError("sandbox_repository must be OWNER/REPO") + entries = raw["tests"] + if not isinstance(entries, list) or not entries: + raise CatalogError("tests must be a non-empty list") + tests = tuple(_parse_test(entry, index) for index, entry in enumerate(entries)) + names = [test.name for test in tests] + duplicates = sorted({name for name in names if names.count(name) > 1}) + if duplicates: + raise CatalogError(f"duplicate test names: {duplicates}") + return Catalog(sandbox_repository=repository, tests=tests) + + +def load_catalog(path: Path = CATALOG_PATH) -> Catalog: + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise CatalogError(f"{path}: {exc}") from exc + return parse_catalog(raw) diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py new file mode 100644 index 0000000000..071dcf2aad --- /dev/null +++ b/scripts/e2e/e2e_harness.py @@ -0,0 +1,483 @@ +#!/usr/bin/env python3 +"""Run one catalog E2E test inside the AutoSkillit user image, and redact its artifacts.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +from collections.abc import Callable, Mapping, Sequence +from pathlib import Path +from typing import Any + +import e2e_catalog + +MINIMAX_BASE_URL = "https://api.minimax.io/anthropic" +MINIMAX_MODEL = "MiniMax-M3[1m]" +MINIMAX_COMPACT_WINDOW_TOKENS = "1000000" +# implement runs four concurrent children and resolve-review three to six; a spawn past the +# limit fails without retry. +MAX_CONCURRENT_SUBAGENTS = "6" +CANARY_TOKEN = "AUTOSKILLIT-E2E-CANARY-OK" +CANARY_PROMPT = f"Reply with exactly {CANARY_TOKEN} and nothing else." +GIT_AUTHOR = ("AutoSkillit E2E", "autoskillit-e2e@users.noreply.github.com") +SETUP_COMMAND_TIMEOUT_SEC = 120 +SECRET_ENV = ("MINIMAX_API_KEY", "E2E_SANDBOX_TOKEN") +REDACTED = b"[REDACTED]" +CONFIG_TEMPLATE = Path(__file__).with_name("autoskillit-config.yaml") +SANDBOX_CLONE = Path("/workspace/sandbox") + +_SCRUBBED_ENV = frozenset({"CLAUDE_CODE_OAUTH_TOKEN", *SECRET_ENV}) +_STDERR_TAIL_CHARS = 2000 + +Runner = Callable[..., subprocess.CompletedProcess[str]] +CallResult = tuple[subprocess.CompletedProcess[str] | None, str | None] + + +def run_command( + argv: Sequence[str], + *, + cwd: Path | None = None, + env: Mapping[str, str] | None = None, + input: str | None = None, + timeout: float | None = None, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + list(argv), + cwd=cwd, + env=env, + input=input, + timeout=timeout, + capture_output=True, + text=True, + check=False, + ) + + +def _call(runner: Runner, argv: Sequence[str], *, what: str, **kwargs: Any) -> CallResult: + """Run *argv*, turning a nonzero exit, a timeout or a launch error into a failure string.""" + try: + result = runner(argv, **kwargs) + except subprocess.TimeoutExpired as exc: + return None, f"{what}: timed out after {exc.timeout}s" + except OSError as exc: + return None, f"{what}: {exc}" + if result.returncode != 0: + stderr = (result.stderr or "").strip()[-_STDERR_TAIL_CHARS:] + return result, f"{what}: exit {result.returncode}: {stderr}" + return result, None + + +def child_env(env: Mapping[str, str]) -> dict[str, str]: + return { + name: value + for name, value in env.items() + if name not in _SCRUBBED_ENV and not name.startswith("ANTHROPIC_") + } + + +def claude_settings_env(api_key: str) -> dict[str, str]: + return { + "ANTHROPIC_BASE_URL": MINIMAX_BASE_URL, + "ANTHROPIC_AUTH_TOKEN": api_key, + "ANTHROPIC_MODEL": MINIMAX_MODEL, + "ANTHROPIC_DEFAULT_SONNET_MODEL": MINIMAX_MODEL, + "ANTHROPIC_DEFAULT_OPUS_MODEL": MINIMAX_MODEL, + "ANTHROPIC_DEFAULT_HAIKU_MODEL": MINIMAX_MODEL, + "CLAUDE_CODE_AUTO_COMPACT_WINDOW": MINIMAX_COMPACT_WINDOW_TOKENS, + "CLAUDE_CODE_MAX_CONCURRENT_SUBAGENTS": MAX_CONCURRENT_SUBAGENTS, + } + + +def write_claude_settings(home: Path, api_key: str) -> Path: + path = home / ".claude" / "settings.json" + settings: dict[str, Any] = {} + if path.is_file(): + existing = json.loads(path.read_text(encoding="utf-8")) + if isinstance(existing, dict): + settings = existing + settings["env"] = claude_settings_env(api_key) + path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + os.fchmod(fd, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(settings, handle, indent=2) + return path + + +def install_autoskillit_config(home: Path) -> Path: + target = home / ".autoskillit" / "config.yaml" + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(CONFIG_TEMPLATE, target) + return target + + +def check_canary_output(stdout: str) -> list[str]: + try: + payload = json.loads(stdout) + except json.JSONDecodeError: + return ["canary: claude output is not JSON"] + if not isinstance(payload, dict): + return ["canary: claude output is not a JSON object"] + if payload.get("is_error") is not False: + return [f"canary: claude reported an error: {str(payload.get('result'))[:500]}"] + reply = payload.get("result") + if not isinstance(reply, str) or CANARY_TOKEN not in reply: + return [f"canary: reply does not contain {CANARY_TOKEN}"] + return [] + + +def run_canary( + test: e2e_catalog.CatalogTest, out: Path, env: Mapping[str, str], runner: Runner +) -> list[str]: + workdir = Path(tempfile.mkdtemp(prefix="e2e-canary-")) + result, failure = _call( + runner, + ["claude", "-p", CANARY_PROMPT, "--output-format", "json"], + what="claude -p", + cwd=workdir, + env=env, + timeout=test.timeout_sec, + ) + if result is not None: + (out / "canary.json").write_text(result.stdout, encoding="utf-8") + (out / "canary.stderr.log").write_text(result.stderr, encoding="utf-8") + if failure is not None or result is None: + return [str(failure)] + return check_canary_output(result.stdout) + + +def _setup_call( + runner: Runner, argv: Sequence[str], env: Mapping[str, str], **kw: Any +) -> CallResult: + return _call( + runner, argv, what=" ".join(argv[:3]), env=env, timeout=SETUP_COMMAND_TIMEOUT_SEC, **kw + ) + + +def configure_git(env: Mapping[str, str], runner: Runner) -> str | None: + for key, value in (("user.name", GIT_AUTHOR[0]), ("user.email", GIT_AUTHOR[1])): + _, failure = _setup_call(runner, ["git", "config", "--global", key, value], env) + if failure is not None: + return failure + return None + + +def authenticate_gh(token: str, env: Mapping[str, str], runner: Runner) -> str | None: + login = ["gh", "auth", "login", "--hostname", "github.com", "--with-token"] + _, failure = _setup_call(runner, login, env, input=token) + if failure is not None: + return failure + _, failure = _setup_call(runner, ["gh", "auth", "setup-git"], env) + return failure + + +def clone_sandbox(repository: str, env: Mapping[str, str], runner: Runner) -> str | None: + _, failure = _setup_call(runner, ["gh", "repo", "clone", repository, str(SANDBOX_CLONE)], env) + return failure + + +def _list_pull_requests( + repository: str, limit: int, fields: str, env: Mapping[str, str], runner: Runner +) -> tuple[list[dict[str, Any]] | None, str | None]: + argv = ["gh", "pr", "list", "--repo", repository, "--state", "all"] + argv += ["--limit", str(limit), "--json", fields] + result, failure = _setup_call(runner, argv, env) + if failure is not None or result is None: + return None, failure + try: + rows = json.loads(result.stdout) + except json.JSONDecodeError: + return None, "gh pr list: output is not JSON" + if not isinstance(rows, list) or not all(isinstance(row, dict) for row in rows): + return None, "gh pr list: output is not a list of objects" + if any(type(row.get("number")) is not int or row["number"] <= 0 for row in rows): + return None, "gh pr list: pull request number is missing or invalid" + return rows, None + + +def latest_pull_request_number( + repository: str, env: Mapping[str, str], runner: Runner +) -> tuple[int | None, str | None]: + rows, failure = _list_pull_requests(repository, 1, "number", env, runner) + if rows is None: + return None, failure + return (int(rows[0]["number"]) if rows else 0), None + + +def new_pull_requests( + repository: str, baseline: int, env: Mapping[str, str], runner: Runner +) -> tuple[list[dict[str, Any]] | None, str | None]: + rows, failure = _list_pull_requests( + repository, 20, "number,state,additions,deletions", env, runner + ) + if rows is None: + return None, failure + return [row for row in rows if int(row["number"]) > baseline], None + + +def fleet_run_argv(test: e2e_catalog.CatalogTest) -> list[str]: + ingredients = [arg for key, value in test.ingredients for arg in ("-i", f"{key}={value}")] + return [ + "autoskillit", + "fleet", + "run", + str(test.recipe), + *ingredients, + "--disable-quota-guard", + "--timeout-sec", + str(test.timeout_sec), + ] + + +def last_line(stdout: str) -> str: + lines = [line for line in stdout.splitlines() if line.strip()] + return lines[-1] if lines else "" + + +def check_envelope(returncode: int, stdout: str) -> list[str]: + failures = [] if returncode == 0 else [f"fleet run: exit {returncode}"] + line = last_line(stdout) + try: + envelope = json.loads(line) + except json.JSONDecodeError: + return [*failures, "fleet run: the last stdout line is not a JSON envelope"] + if not isinstance(envelope, dict): + failures.append("fleet run: the envelope is not a JSON object") + elif envelope.get("success") is not True: + failures.append("fleet run: the envelope does not report success") + return failures + + +def run_fleet( + test: e2e_catalog.CatalogTest, out: Path, env: Mapping[str, str], runner: Runner +) -> list[str]: + result, failure = _call( + runner, + fleet_run_argv(test), + what="fleet run", + cwd=SANDBOX_CLONE, + env=env, + timeout=test.timeout_sec + e2e_catalog.HARNESS_GRACE_SEC, + ) + if result is None: + return [str(failure)] + (out / "fleet-run.stdout.log").write_text(result.stdout, encoding="utf-8") + (out / "fleet-run.stderr.log").write_text(result.stderr, encoding="utf-8") + (out / "envelope.json").write_text(last_line(result.stdout) + "\n", encoding="utf-8") + return check_envelope(result.returncode, result.stdout) + + +def check_pull_requests(prs: Sequence[Mapping[str, Any]], expected_state: str | None) -> list[str]: + if len(prs) != 1: + return [f"expected exactly one new sandbox pull request, found {len(prs)}"] + pr = prs[0] + failures: list[str] = [] + state = str(pr.get("state", "")).lower() + if state != expected_state: + failures.append(f"pull request #{pr.get('number')} is {state}, expected {expected_state}") + if int(pr.get("additions", 0)) + int(pr.get("deletions", 0)) <= 0: + failures.append(f"pull request #{pr.get('number')} has an empty diff") + return failures + + +def close_open_pull_requests( + repository: str, + baseline: int, + listed: Sequence[Mapping[str, Any]] | None, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + if listed is None: + listed, failure = new_pull_requests(repository, baseline, env, runner) + if listed is None: + return [f"cleanup: {failure}"] + failures: list[str] = [] + for pr in listed: + if str(pr.get("state", "")).lower() != "open": + continue + argv = ["gh", "pr", "close", str(pr["number"]), "--repo", repository, "--delete-branch"] + _, failure = _setup_call(runner, argv, env) + if failure is not None: + failures.append(f"cleanup: {failure}") + return failures + + +def run_recipe( + test: e2e_catalog.CatalogTest, + repository: str, + token: str, + out: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + failure = ( + configure_git(env, runner) + or authenticate_gh(token, env, runner) + or clone_sandbox(repository, env, runner) + ) + if failure is not None: + return [failure] + baseline, failure = latest_pull_request_number(repository, env, runner) + if baseline is None: + return [str(failure)] + failures: list[str] = [] + listed: list[dict[str, Any]] | None = None + try: + failures += run_fleet(test, out, env, runner) + listed, failure = new_pull_requests(repository, baseline, env, runner) + if listed is None: + failures.append(str(failure)) + else: + failures += check_pull_requests(listed, test.expected_pull_request_state) + finally: + failures += close_open_pull_requests(repository, baseline, listed, env, runner) + return failures + + +def _run_test( + test: e2e_catalog.CatalogTest, + catalog: e2e_catalog.Catalog, + *, + out: Path, + home: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + required = SECRET_ENV if test.kind == "recipe" else ("MINIMAX_API_KEY",) + missing = [f"{name} is not set" for name in required if not env.get(name)] + if missing: + return missing + write_claude_settings(home, env["MINIMAX_API_KEY"]) + install_autoskillit_config(home) + scrubbed = child_env(env) + if test.kind == "canary": + return run_canary(test, out, scrubbed, runner) + token = env["E2E_SANDBOX_TOKEN"] + return run_recipe(test, catalog.sandbox_repository, token, out, scrubbed, runner) + + +def run_test( + test: e2e_catalog.CatalogTest, + catalog: e2e_catalog.Catalog, + *, + out: Path, + home: Path, + env: Mapping[str, str], + runner: Runner, +) -> list[str]: + """Run *test* and always record ``out/result.json``; return the failures.""" + out.mkdir(parents=True, exist_ok=True) + try: + failures = _run_test(test, catalog, out=out, home=home, env=env, runner=runner) + except Exception as exc: + failures = [f"harness error: {exc}"] + result = {"test": test.name, "passed": not failures, "failures": failures} + (out / "result.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return failures + + +def _redact(data: bytes, needles: Sequence[bytes]) -> bytes: + for needle in needles: + data = data.replace(needle, REDACTED) + return data + + +def _copy_redacted(source: Path, target: Path, needles: Sequence[bytes]) -> None: + target.write_bytes(_redact(source.read_bytes(), needles)) + target.chmod(0o644) + + +def _copy_tree_redacted(source: Path, target: Path, needles: Sequence[bytes]) -> None: + for root, _dirs, files in os.walk(source, followlinks=False): + root_path = Path(root) + target_dir = target / root_path.relative_to(source) + target_dir.mkdir(parents=True, exist_ok=True) + target_dir.chmod(0o755) + for name in files: + path = root_path / name + if not path.is_symlink() and path.is_file(): + _copy_redacted(path, target_dir / name, needles) + + +def _find_leak(dest: Path, needles: Sequence[bytes]) -> Path | None: + for root, _dirs, files in os.walk(dest, followlinks=False): + for name in files: + path = Path(root) / name + try: + data = path.read_bytes() + except (FileNotFoundError, NotADirectoryError): + # A file that vanished after the walk cannot be uploaded, so it cannot leak. + continue + if any(needle in data for needle in needles): + return path + return None + + +def redact_tree(sources: Sequence[Path], dest: Path, secrets: Sequence[str]) -> None: + """Copy *sources* into *dest* world-readable with every secret replaced; verify the copy.""" + needles = [secret.encode() for secret in secrets if secret] + dest.mkdir(parents=True, exist_ok=True) + for source in sources: + if source.is_symlink() or not source.exists(): + continue + target = dest / source.name + if source.is_dir(): + _copy_tree_redacted(source, target, needles) + else: + _copy_redacted(source, target, needles) + leak = _find_leak(dest, needles) + if leak is not None: + raise RuntimeError(f"a secret survived redaction in {leak}") + + +def _run(test_name: str, out: Path, catalog_path: Path) -> int: + try: + catalog = e2e_catalog.load_catalog(catalog_path) + test = catalog.get(test_name) + except e2e_catalog.CatalogError as exc: + print(f"e2e_harness: {exc}", file=sys.stderr) + return 2 + failures = run_test( + test, catalog, out=out, home=Path.home(), env=os.environ, runner=run_command + ) + for failure in failures: + print(f"e2e_harness: FAIL {failure}", file=sys.stderr) + print(f"e2e_harness: {test.name} {'failed' if failures else 'passed'}") + return 1 if failures else 0 + + +def _redact_command(dest: Path, secret_env: Sequence[str], sources: Sequence[str]) -> int: + secrets = [os.environ.get(name, "") for name in secret_env] + try: + redact_tree([Path(source) for source in sources], dest, secrets) + except (OSError, RuntimeError) as exc: + print(f"e2e_harness: redaction failed: {exc}", file=sys.stderr) + return 1 + return 0 + + +def main(argv: Sequence[str]) -> int: + """``run`` exits 0 iff the test passed; ``redact`` exits 0 iff the redacted copy is clean.""" + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + run = commands.add_parser("run", help="Run one catalog test.") + run.add_argument("--test", required=True) + run.add_argument("--out", required=True) + run.add_argument("--catalog", default=str(e2e_catalog.CATALOG_PATH)) + redact = commands.add_parser("redact", help="Copy artifacts with every secret redacted.") + redact.add_argument("--dest", required=True) + redact.add_argument("--secret-env", action="append", required=True) + redact.add_argument("sources", nargs="+") + args = parser.parse_args(argv) + if args.command == "run": + return _run(args.test, Path(args.out), Path(args.catalog)) + return _redact_command(Path(args.dest), args.secret_env, args.sources) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/e2e/e2e_select.py b/scripts/e2e/e2e_select.py new file mode 100644 index 0000000000..a5192dd515 --- /dev/null +++ b/scripts/e2e/e2e_select.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +"""Decide which live E2E tests a GitHub Actions event runs, and judge the required gate.""" + +from __future__ import annotations + +import argparse +import fnmatch +import hashlib +import json +import os +import sys +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import e2e_catalog + +E2E_LABEL = "e2e" +MIN_CHANGED_FILES = 10 +MIN_CHANGED_LINES = 300 +SAMPLE_PROBABILITY = 0.5 +MAX_TESTS = 2 + + +@dataclass(frozen=True, slots=True) +class PullRequestFacts: + """The fields of a ``pull_request`` event payload that drive selection.""" + + number: int + head_sha: str + head_repository: str | None + changed_files: int + additions: int + deletions: int + labels: frozenset[str] + + @classmethod + def from_payload(cls, pr: Mapping[str, Any]) -> PullRequestFacts: + try: + head = pr["head"] + head_repo = head.get("repo") + return cls( + number=int(pr["number"]), + head_sha=str(head["sha"]), + head_repository=( + head_repo.get("full_name") if isinstance(head_repo, dict) else None + ), + changed_files=int(pr["changed_files"]), + additions=int(pr["additions"]), + deletions=int(pr["deletions"]), + labels=frozenset(str(label["name"]) for label in pr.get("labels") or ()), + ) + except (KeyError, TypeError, AttributeError) as exc: + raise ValueError(f"malformed pull_request payload: {exc!r}") from exc + + +@dataclass(frozen=True, slots=True) +class Selection: + """The tests one event runs, with the reason the selector reached that decision.""" + + tests: tuple[str, ...] + reason: str + + @property + def selected(self) -> bool: + return bool(self.tests) + + +def is_eligible(pr: PullRequestFacts) -> bool: + return ( + E2E_LABEL in pr.labels + or pr.changed_files >= MIN_CHANGED_FILES + or pr.additions + pr.deletions >= MIN_CHANGED_LINES + ) + + +def sample_score(number: int, head_sha: str) -> float: + digest = hashlib.sha256(f"{number}:{head_sha}".encode()).digest() + return int.from_bytes(digest[:8], "big") / 2**64 + + +def is_sampled(pr: PullRequestFacts, probability: float) -> bool: + return sample_score(pr.number, pr.head_sha) < probability + + +def _rank(seed: str, name: str) -> str: + return hashlib.sha256(f"{seed}:{name}".encode()).hexdigest() + + +def _touches(test: e2e_catalog.CatalogTest, changed_paths: Sequence[str]) -> bool: + return any( + fnmatch.fnmatchcase(path, pattern) + for path in changed_paths + for pattern in test.trigger_paths + ) + + +def pick_tests( + catalog: e2e_catalog.Catalog, changed_paths: Sequence[str], seed: str +) -> tuple[str, ...]: + ranked = sorted(catalog.tests, key=lambda test: _rank(seed, test.name)) + touched = [test.name for test in ranked if _touches(test, changed_paths)] + if touched: + return tuple(touched[:MAX_TESTS]) + return (ranked[0].name,) + + +def select_for_pull_request( + pr: PullRequestFacts, + changed_paths: Sequence[str], + catalog: e2e_catalog.Catalog, + repository: str, +) -> Selection: + if pr.head_repository != repository: + return Selection((), f"head repository {pr.head_repository} is not {repository}") + if not is_eligible(pr): + lines = pr.additions + pr.deletions + return Selection( + (), + f"below thresholds ({pr.changed_files} files, {lines} lines) and no {E2E_LABEL} label", + ) + seed = f"{pr.number}:{pr.head_sha}" + if E2E_LABEL in pr.labels: + return Selection(pick_tests(catalog, changed_paths, seed), f"{E2E_LABEL} label") + if not is_sampled(pr, SAMPLE_PROBABILITY): + return Selection((), "eligible but not sampled for this head commit") + return Selection(pick_tests(catalog, changed_paths, seed), "eligible and sampled") + + +def select_for_dispatch(requested: str, catalog: e2e_catalog.Catalog) -> Selection: + names = tuple(dict.fromkeys(part.strip() for part in requested.split(",") if part.strip())) + if not 1 <= len(names) <= MAX_TESTS: + raise ValueError(f"workflow_dispatch must name 1 to {MAX_TESTS} tests, got {len(names)}") + for name in names: + catalog.get(name) + return Selection(names, "workflow_dispatch") + + +def _dispatch_request(payload: Mapping[str, Any]) -> str: + inputs = payload.get("inputs") + requested = inputs.get("tests") if isinstance(inputs, dict) else None + if not isinstance(requested, str): + raise ValueError("workflow_dispatch payload has no string inputs.tests") + return requested + + +def select_for_event( + event_name: str, + payload: Mapping[str, Any], + changed_paths: Sequence[str], + catalog: e2e_catalog.Catalog, + repository: str, +) -> Selection: + if event_name == "pull_request": + pr = payload.get("pull_request") + if not isinstance(pr, dict): + raise ValueError("pull_request payload has no pull_request object") + facts = PullRequestFacts.from_payload(pr) + return select_for_pull_request(facts, changed_paths, catalog, repository) + if event_name == "workflow_dispatch": + return select_for_dispatch(_dispatch_request(payload), catalog) + if event_name == "merge_group": + return Selection((), "merge queue: E2E is decided on the pull request") + raise ValueError(f"unsupported event: {event_name}") + + +def matrix_json(selection: Selection, catalog: e2e_catalog.Catalog) -> str: + include = [ + {"test": name, "timeout_minutes": e2e_catalog.job_timeout_minutes(catalog.get(name))} + for name in selection.tests + ] + return json.dumps({"include": include}, separators=(",", ":")) + + +def gate_verdict(select_result: str, selected: str, run_result: str) -> tuple[bool, str]: + if select_result == "success" and selected == "true" and run_result == "success": + return True, "selected E2E tests passed" + if select_result == "success" and selected == "false" and run_result == "skipped": + return True, "no E2E test selected" + return False, ( + f"E2E gate failed: select={select_result or ''} " + f"selected={selected or ''} run={run_result or ''}" + ) + + +def _read_event() -> tuple[str, Mapping[str, Any], str]: + event_name = os.environ.get("GITHUB_EVENT_NAME") + event_path = os.environ.get("GITHUB_EVENT_PATH") + repository = os.environ.get("GITHUB_REPOSITORY") + if not event_name or not event_path or not repository: + raise ValueError("GITHUB_EVENT_NAME, GITHUB_EVENT_PATH and GITHUB_REPOSITORY are required") + payload = json.loads(Path(event_path).read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError("GitHub event payload must be an object") + return event_name, payload, repository + + +def _read_changed_paths(path: Path) -> tuple[str, ...]: + lines = path.read_text(encoding="utf-8").splitlines() + return tuple(line.strip() for line in lines if line.strip()) + + +def _select(catalog_path: Path, changed_paths_file: Path) -> int: + try: + catalog = e2e_catalog.load_catalog(catalog_path) + event_name, payload, repository = _read_event() + changed_paths = ( + _read_changed_paths(changed_paths_file) if event_name == "pull_request" else () + ) + selection = select_for_event(event_name, payload, changed_paths, catalog, repository) + matrix = matrix_json(selection, catalog) + except (OSError, ValueError) as exc: + print(f"e2e_select: {exc}", file=sys.stderr) + return 2 + print(f"e2e_select: {selection.reason}", file=sys.stderr) + sys.stdout.write( + f"selected={'true' if selection.selected else 'false'}\n" + f"matrix={matrix}\n" + f"reason={selection.reason}\n" + ) + return 0 + + +def main(argv: Sequence[str]) -> int: + """Print a GitHub Actions output record for ``select``; exit 0/1 on the ``gate`` verdict.""" + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + select = commands.add_parser("select", help="Choose the tests this event runs.") + select.add_argument("--catalog", default=str(e2e_catalog.CATALOG_PATH)) + select.add_argument( + "--changed-paths", + required=True, + help="File listing the pull request's changed paths, one per line.", + ) + gate = commands.add_parser("gate", help="Judge the required e2e-gate check.") + gate.add_argument("--select-result", required=True) + gate.add_argument("--selected", required=True) + gate.add_argument("--run-result", required=True) + args = parser.parse_args(argv) + if args.command == "select": + return _select(Path(args.catalog), Path(args.changed_paths)) + passed, reason = gate_verdict(args.select_result, args.selected, args.run_result) + print(reason) + return 0 if passed else 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/e2e/post-e2e-failure.sh b/scripts/e2e/post-e2e-failure.sh new file mode 100755 index 0000000000..7ef0863609 --- /dev/null +++ b/scripts/e2e/post-e2e-failure.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Append an E2E test failure to that test's tracking issue. +# +# Usage: scripts/e2e/post-e2e-failure.sh TEST STAGE PULL_REQUEST HEAD_SHA WORKFLOW_RUN_URL +# +# Args: $1=TEST $2=STAGE (test|redaction) $3=PULL_REQUEST (empty for workflow_dispatch) +# $4=HEAD_SHA $5=WORKFLOW_RUN_URL +# +# Runs inside the AutoSkillit user image, against its uv tool installation. +# +# Environment: +# GH_TOKEN — token with issues: write (required by gh) +# GITHUB_REPOSITORY — owner/repo for the issue (required by Python) + +set -euo pipefail +export LC_ALL=C + +USAGE="Usage: $0 TEST STAGE PULL_REQUEST HEAD_SHA WORKFLOW_RUN_URL" +TEST="${1:?$USAGE}" +STAGE="${2:?$USAGE}" +PULL_REQUEST="${3?$USAGE}" +HEAD_SHA="${4:?$USAGE}" +WORKFLOW_RUN_URL="${5:?$USAGE}" + +PYTHON="$(uv tool dir)/autoskillit/bin/python" +if [[ ! -x "${PYTHON}" ]]; then + echo "ERROR: ${PYTHON} not found; run this inside the AutoSkillit user image." >&2 + exit 1 +fi + +exec "${PYTHON}" -m autoskillit._probe_canary post-e2e-failure \ + --test "${TEST}" \ + --stage "${STAGE}" \ + --pull-request "${PULL_REQUEST}" \ + --head-sha "${HEAD_SHA}" \ + --workflow-run-url "${WORKFLOW_RUN_URL}" diff --git a/src/autoskillit/_probe_canary.py b/src/autoskillit/_probe_canary.py index 5a966bed1e..2bbf8d492d 100644 --- a/src/autoskillit/_probe_canary.py +++ b/src/autoskillit/_probe_canary.py @@ -1,7 +1,7 @@ -"""Reusable canary state machine and GitHub issue updater for live probes. +"""Reusable canary state machine and GitHub issue updater for live probes and E2E runs. -IL-1 module: imports only stdlib and `autoskillit.core`. Provides the -persistence + flake-guard primitives that live probe classes build on. +IL-1 module: imports only stdlib and `autoskillit.core`. Provides the persistence + +flake-guard primitives live probes build on and the E2E workflow's failure issue. """ from __future__ import annotations @@ -79,7 +79,11 @@ def _run_gh_with_body_file(args: list[str], body: str) -> subprocess.CompletedPr logger.debug("canary_body_file_unlink_failed", path=body_path) -class CanaryIssueUpdater: +# GitHub rejects an issue body longer than this many characters. +ISSUE_BODY_MAX_CHARS = 65536 + + +class IssueUpdater: def __init__(self, *, owner: str, repo: str) -> None: self._owner = owner self._repo = repo @@ -105,6 +109,37 @@ def ensure_issue(self, state: CanaryState, title: str, body: str) -> int: ) state.last_issue_number = existing return existing + issue_number = self._create(title, body) + state.last_issue_number = issue_number + return issue_number + + def append_to_issue(self, title: str, header: str, section: str) -> int: + """Append *section* to the open issue titled *title*; open or rotate it as needed. + + *header* seeds new and successor issues; ordinary appends preserve the + existing body and add only *section*. + + An append that would cross ``ISSUE_BODY_MAX_CHARS`` closes the full issue and + opens a successor linking it, so the title keeps one open reporting channel. + """ + existing = self._find_existing(title) + if existing is None: + return self._create(title, f"{header}\n\n{section}") + repo = f"{self._owner}/{self._repo}" + body = self._current_body(existing) + if len(body) + 2 + len(section) > ISSUE_BODY_MAX_CHARS: + result = run_gh(["issue", "close", str(existing), "--repo", repo]) + if result.returncode != 0: + raise RuntimeError(f"gh issue close failed: {result.stderr}") + return self._create(title, f"{header}\n\nContinues #{existing}.\n\n{section}") + result = _run_gh_with_body_file( + ["issue", "edit", str(existing), "--repo", repo], f"{body}\n\n{section}" + ) + if result.returncode != 0: + raise RuntimeError(f"gh issue edit failed: {result.stderr}") + return existing + + def _create(self, title: str, body: str) -> int: result = _run_gh_with_body_file( [ "issue", @@ -126,9 +161,22 @@ def ensure_issue(self, state: CanaryState, title: str, body: str) -> int: except (json.JSONDecodeError, KeyError) as exc: msg = f"gh issue create returned unexpected output: {result.stdout!r}" raise RuntimeError(msg) from exc - state.last_issue_number = issue_number return issue_number + def _current_body(self, number: int) -> str: + repo = f"{self._owner}/{self._repo}" + result = run_gh(["issue", "view", str(number), "--repo", repo, "--json", "body"]) + if result.returncode != 0: + raise RuntimeError(f"gh issue view failed: {result.stderr}") + try: + body = json.loads(result.stdout)["body"] + except (json.JSONDecodeError, KeyError, TypeError) as exc: + msg = f"gh issue view returned unexpected output: {result.stdout!r}" + raise RuntimeError(msg) from exc + if not isinstance(body, str): + raise RuntimeError(f"gh issue view returned a non-string body: {body!r}") + return body + def _find_existing(self, title: str) -> int | None: result = run_gh( [ @@ -200,7 +248,7 @@ def _handle_post_failure( f"**Network Streak:** {state.network_streak}\n" f"**Schema Streak:** {state.schema_streak}\n" ) - updater = CanaryIssueUpdater(owner=owner, repo=repo) + updater = IssueUpdater(owner=owner, repo=repo) try: updater.ensure_issue(state, title, body) except Exception as exc: @@ -210,6 +258,42 @@ def _handle_post_failure( return 0 +def _handle_post_e2e_failure( + *, + test: str, + stage: str, + pull_request: str, + head_sha: str, + workflow_run_url: str, +) -> int: + repo_slug = os.environ.get("GITHUB_REPOSITORY", "") + if not repo_slug or "/" not in repo_slug: + logger.error("post_e2e_failure_missing_github_repository") + return 1 + + owner, repo = repo_slug.split("/", 1) + title = f"[E2E] {test} failure" + header = ( + f"Tracks failures of the `{test}` test in `.github/workflows/e2e.yml`. " + "Each failed run appends a section below." + ) + trigger = f"pull request #{pull_request}" if pull_request else "workflow_dispatch" + section = ( + f"### Failure at {head_sha[:12]}\n\n" + f"**Stage:** {stage}\n" + f"**Trigger:** {trigger}\n" + f"**Commit:** {head_sha}\n" + f"**Workflow Run:** {workflow_run_url}\n" + ) + updater = IssueUpdater(owner=owner, repo=repo) + try: + updater.append_to_issue(title, header, section) + except Exception as exc: + logger.error("e2e_failure_issue_failed", error=str(exc), exc_info=True) + return 1 + return 0 + + def _cli_main(argv: list[str] | None = None) -> int: import argparse @@ -230,6 +314,22 @@ def _cli_main(argv: list[str] | None = None) -> int: ) post.add_argument("--workflow-run-url", required=True, help="GitHub Actions workflow run URL") + e2e = sub.add_parser( + "post-e2e-failure", help="Append an E2E test failure to that test's tracking issue" + ) + e2e.add_argument("--test", required=True, help="Catalog name of the failed E2E test") + e2e.add_argument( + "--stage", + required=True, + choices=("test", "redaction"), + help="Workflow stage that failed", + ) + e2e.add_argument( + "--pull-request", default="", help="Pull request number; empty for workflow_dispatch" + ) + e2e.add_argument("--head-sha", required=True, help="Commit the E2E run tested") + e2e.add_argument("--workflow-run-url", required=True, help="GitHub Actions workflow run URL") + args = parser.parse_args(argv) if args.command is None: @@ -245,6 +345,15 @@ def _cli_main(argv: list[str] | None = None) -> int: workflow_run_url=args.workflow_run_url, ) + if args.command == "post-e2e-failure": + return _handle_post_e2e_failure( + test=args.test, + stage=args.stage, + pull_request=args.pull_request, + head_sha=args.head_sha, + workflow_run_url=args.workflow_run_url, + ) + return 1 diff --git a/tests/execution/backends/test_cli_conformance_probes.py b/tests/execution/backends/test_cli_conformance_probes.py index b99d686848..5d3211080f 100644 --- a/tests/execution/backends/test_cli_conformance_probes.py +++ b/tests/execution/backends/test_cli_conformance_probes.py @@ -33,9 +33,9 @@ import pytest from autoskillit._probe_canary import ( - CanaryIssueUpdater, CanaryState, ErrorKind, + IssueUpdater, ) from autoskillit.config import OutputBudgetConfig from autoskillit.core import ( @@ -388,7 +388,7 @@ def _record_failure( repo_slug = os.environ.get("GITHUB_REPOSITORY", "") if repo_slug and "/" in repo_slug: owner, repo = repo_slug.split("/", 1) - updater = CanaryIssueUpdater(owner=owner, repo=repo) + updater = IssueUpdater(owner=owner, repo=repo) title = f"{_CANARY_TITLE_PREFIX}: {probe_name}" body = _make_canary_body(probe_name, kind, cli_version, detail) updater.ensure_issue(state, title, body) diff --git a/tests/execution/backends/test_probe_canary.py b/tests/execution/backends/test_probe_canary.py index f4e1e66a47..06ef2f2389 100644 --- a/tests/execution/backends/test_probe_canary.py +++ b/tests/execution/backends/test_probe_canary.py @@ -9,10 +9,11 @@ import autoskillit._probe_canary as _patch_autoskillit__probe_canary from autoskillit._probe_canary import ( + ISSUE_BODY_MAX_CHARS, N_CONSECUTIVE_FLAKE_GUARD, - CanaryIssueUpdater, CanaryState, ErrorKind, + IssueUpdater, _cli_main, ) @@ -125,7 +126,7 @@ def test_values(self) -> None: assert set(ErrorKind) == {ErrorKind.NETWORK, ErrorKind.SCHEMA} -class TestCanaryIssueUpdater: +class TestIssueUpdater: def test_ensure_issue_creates_new(self, monkeypatch: pytest.MonkeyPatch) -> None: def mock_run_gh(args, **kwargs): if args[0:2] == ["issue", "list"]: @@ -141,7 +142,7 @@ def mock_run_gh(args, **kwargs): return CompletedProcess(args=args, returncode=1, stdout="", stderr="") monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", mock_run_gh) - updater = CanaryIssueUpdater(owner="test-org", repo="test-repo") + updater = IssueUpdater(owner="test-org", repo="test-repo") state = CanaryState() num = updater.ensure_issue(state, "Probe failure", "Details here") assert num == 123 @@ -162,7 +163,7 @@ def mock_run_gh(args, **kwargs): return CompletedProcess(args=args, returncode=1, stdout="", stderr="") monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", mock_run_gh) - updater = CanaryIssueUpdater(owner="test-org", repo="test-repo") + updater = IssueUpdater(owner="test-org", repo="test-repo") state = CanaryState() num = updater.ensure_issue(state, "Probe failure", "Updated body") assert num == 42 @@ -177,7 +178,7 @@ def mock_run_gh(args, **kwargs): raise AssertionError(f"Unexpected gh call: {args}") monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", mock_run_gh) - updater = CanaryIssueUpdater(owner="test-org", repo="test-repo") + updater = IssueUpdater(owner="test-org", repo="test-repo") state = CanaryState() with pytest.raises(RuntimeError, match="gh issue create failed"): updater.ensure_issue(state, "Probe failure", "Details") @@ -197,7 +198,7 @@ def mock_run_gh(args, **kwargs): raise AssertionError(f"Unexpected gh call: {args}") monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", mock_run_gh) - updater = CanaryIssueUpdater(owner="test-org", repo="test-repo") + updater = IssueUpdater(owner="test-org", repo="test-repo") state = CanaryState() with structlog.testing.capture_logs() as cap_logs: num = updater.ensure_issue(state, "Probe failure", "Updated body") @@ -206,6 +207,176 @@ def mock_run_gh(args, **kwargs): assert any(e.get("event") == "canary_issue_edit_failed" for e in cap_logs) +_E2E_TITLE = "[E2E] canary failure" + + +class _FakeGh: + """Answer gh issue commands for one optional open issue; record argv and body files.""" + + def __init__( + self, + *, + existing: int | None = None, + body: str = "old body", + failing: frozenset[str] = frozenset(), + ) -> None: + self.existing = existing + self.body = body + self.failing = failing + self.calls: list[list[str]] = [] + self.bodies: dict[str, str] = {} + + def __call__(self, args, **kwargs): + args = list(args) + self.calls.append(args) + command = args[1] + if "--body-file" in args: + self.bodies[command] = Path(args[args.index("--body-file") + 1]).read_text() + if command in self.failing: + return CompletedProcess(args=args, returncode=1, stdout="", stderr=f"{command} boom") + if command == "list": + issues = ( + [] if self.existing is None else [{"number": self.existing, "title": _E2E_TITLE}] + ) + return CompletedProcess(args=args, returncode=0, stdout=json.dumps(issues), stderr="") + if command == "view": + stdout = json.dumps({"body": self.body}) + return CompletedProcess(args=args, returncode=0, stdout=stdout, stderr="") + if command == "create": + stdout = json.dumps({"number": 99}) + return CompletedProcess(args=args, returncode=0, stdout=stdout, stderr="") + return CompletedProcess(args=args, returncode=0, stdout="", stderr="") + + def commands(self) -> list[str]: + return [call[1] for call in self.calls] + + +def _append(monkeypatch: pytest.MonkeyPatch, gh: _FakeGh, section: str = "### Section") -> int: + monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", gh) + updater = IssueUpdater(owner="test-org", repo="test-repo") + return updater.append_to_issue(_E2E_TITLE, "Header", section) + + +class TestAppendToIssue: + def test_no_open_issue_creates_one_with_header_and_section( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + gh = _FakeGh() + assert _append(monkeypatch, gh) == 99 + assert gh.commands() == ["list", "create"] + assert gh.bodies["create"] == "Header\n\n### Section" + + def test_open_issue_gets_the_section_appended(self, monkeypatch: pytest.MonkeyPatch) -> None: + gh = _FakeGh(existing=42) + assert _append(monkeypatch, gh) == 42 + assert gh.calls[1] == ["issue", "view", "42", "--repo", "test-org/test-repo"] + [ + "--json", + "body", + ] + assert gh.calls[2][:5] == ["issue", "edit", "42", "--repo", "test-org/test-repo"] + assert gh.bodies["edit"] == "old body\n\n### Section" + + def test_view_failure_raises_without_editing(self, monkeypatch: pytest.MonkeyPatch) -> None: + gh = _FakeGh(existing=42, failing=frozenset({"view"})) + with pytest.raises(RuntimeError, match="gh issue view failed"): + _append(monkeypatch, gh) + assert "edit" not in gh.commands() + + def test_edit_failure_raises(self, monkeypatch: pytest.MonkeyPatch) -> None: + gh = _FakeGh(existing=42, failing=frozenset({"edit"})) + with pytest.raises(RuntimeError, match="gh issue edit failed"): + _append(monkeypatch, gh) + + def test_append_exactly_at_the_limit_edits(self, monkeypatch: pytest.MonkeyPatch) -> None: + section = "### Section" + gh = _FakeGh(existing=42, body="x" * (ISSUE_BODY_MAX_CHARS - 2 - len(section))) + assert _append(monkeypatch, gh, section) == 42 + assert gh.commands() == ["list", "view", "edit"] + + def test_append_past_the_limit_rotates_to_a_successor( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + section = "### Section" + gh = _FakeGh(existing=42, body="x" * (ISSUE_BODY_MAX_CHARS - 1 - len(section))) + assert _append(monkeypatch, gh, section) == 99 + assert gh.commands() == ["list", "view", "close", "create"] + assert gh.calls[2] == ["issue", "close", "42", "--repo", "test-org/test-repo"] + assert "#42" in gh.bodies["create"] + assert gh.bodies["create"].endswith(section) + + def test_close_failure_raises_without_creating(self, monkeypatch: pytest.MonkeyPatch) -> None: + section = "### Section" + gh = _FakeGh( + existing=42, + body="x" * ISSUE_BODY_MAX_CHARS, + failing=frozenset({"close"}), + ) + with pytest.raises(RuntimeError, match="gh issue close failed"): + _append(monkeypatch, gh, section) + assert "create" not in gh.commands() + + +def _post_e2e_failure(*extra: str) -> int: + return _cli_main( + [ + "post-e2e-failure", + "--test", + "canary", + "--head-sha", + "abcdef0123456789abcdef0123456789abcdef01", + "--workflow-run-url", + "https://example.com/run/9", + *extra, + ] + ) + + +class TestPostE2eFailure: + def test_missing_github_repository_returns_one(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("GITHUB_REPOSITORY", raising=False) + assert _post_e2e_failure("--stage", "test", "--pull-request", "12") == 1 + + @pytest.mark.parametrize("stage", ["test", "redaction"]) + def test_section_names_pull_request_commit_run_and_stage( + self, monkeypatch: pytest.MonkeyPatch, stage: str + ) -> None: + monkeypatch.setenv("GITHUB_REPOSITORY", "test-org/test-repo") + gh = _FakeGh() + monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", gh) + assert _post_e2e_failure("--stage", stage, "--pull-request", "12") == 0 + create = gh.calls[-1] + assert create[create.index("--title") + 1] == _E2E_TITLE + body = gh.bodies["create"] + assert "#12" in body + assert "abcdef0123456789abcdef0123456789abcdef01" in body + assert "### Failure at abcdef012345" in body + assert "https://example.com/run/9" in body + assert f"**Stage:** {stage}" in body + + def test_empty_pull_request_reports_workflow_dispatch( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("GITHUB_REPOSITORY", "test-org/test-repo") + gh = _FakeGh() + monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", gh) + assert _post_e2e_failure("--stage", "test", "--pull-request", "") == 0 + assert "workflow_dispatch" in gh.bodies["create"] + + def test_unknown_stage_is_rejected(self) -> None: + with pytest.raises(SystemExit): + _post_e2e_failure("--stage", "other") + + def test_issue_failure_returns_one(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GITHUB_REPOSITORY", "test-org/test-repo") + monkeypatch.setattr( + _patch_autoskillit__probe_canary, "run_gh", _FakeGh(failing=frozenset({"create"})) + ) + with structlog.testing.capture_logs() as cap_logs: + assert _post_e2e_failure("--stage", "test") == 1 + failure = next(e for e in cap_logs if e.get("event") == "e2e_failure_issue_failed") + assert failure["exc_info"] is True + + class TestCliMain: def test_post_failure_records_and_saves( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch diff --git a/tests/infra/_complexity_helpers.py b/tests/infra/_complexity_helpers.py index 6c13bab2e7..eb30befa04 100644 --- a/tests/infra/_complexity_helpers.py +++ b/tests/infra/_complexity_helpers.py @@ -6,6 +6,10 @@ sys.modules isolation stays a parameter, not shared state) -- only the byte-identical bodies live here, matching the tests/infra/_pretty_output_helpers.py convention of a dedicated underscore-prefixed helper module for one feature's split test files. + +tests/infra/test_e2e_select.py and tests/infra/test_e2e_harness.py also load +scripts/e2e/e2e_select.py and scripts/e2e/e2e_harness.py through load_check_script, so +their sibling ``import e2e_catalog`` resolves. """ from __future__ import annotations diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py new file mode 100644 index 0000000000..d15179b334 --- /dev/null +++ b/tests/infra/test_e2e_harness.py @@ -0,0 +1,488 @@ +"""Tests for scripts/e2e/e2e_harness.py with a recording fake in place of real processes.""" + +from __future__ import annotations + +import json +import os +import stat +import subprocess +import tempfile +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from tests.infra._complexity_helpers import load_check_script + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] + +REPO_ROOT = Path(__file__).resolve().parents[2] +_HARNESS_SCRIPT = REPO_ROOT / "scripts" / "e2e" / "e2e_harness.py" + +harness = load_check_script("_autoskillit_e2e_harness", _HARNESS_SCRIPT) +e2e_catalog = harness.e2e_catalog + +MINIMAX_KEY = "minimax-test-key" +SANDBOX_TOKEN = "sandbox-test-token" +SANDBOX = "owner/sandbox" +_MINIMAX_MODEL = "MiniMax-M3[1m]" +_EXPECTED_SETTINGS_ENV = { + "ANTHROPIC_BASE_URL": "https://api.minimax.io/anthropic", + "ANTHROPIC_AUTH_TOKEN": MINIMAX_KEY, + "ANTHROPIC_MODEL": _MINIMAX_MODEL, + "ANTHROPIC_DEFAULT_SONNET_MODEL": _MINIMAX_MODEL, + "ANTHROPIC_DEFAULT_OPUS_MODEL": _MINIMAX_MODEL, + "ANTHROPIC_DEFAULT_HAIKU_MODEL": _MINIMAX_MODEL, + "CLAUDE_CODE_AUTO_COMPACT_WINDOW": "1000000", + "CLAUDE_CODE_MAX_CONCURRENT_SUBAGENTS": "6", +} +_CANARY_REPLY = json.dumps( + {"type": "result", "is_error": False, "result": "AUTOSKILLIT-E2E-CANARY-OK"} +) +_SKILL_NOTICE = "2 skills unavailable on this backend (codex): a, b" + + +@dataclass +class RecordedCall: + argv: list[str] + cwd: Path | None + env: dict[str, str] | None + input: str | None + timeout: float | None + + +Handler = Callable[[list[str]], "subprocess.CompletedProcess[str] | BaseException | None"] + + +class FakeRunner: + """Record every launch; answer from *handler*, defaulting to a clean exit.""" + + def __init__(self, handler: Handler) -> None: + self.handler = handler + self.calls: list[RecordedCall] = [] + + def __call__(self, argv, *, cwd=None, env=None, input=None, timeout=None): + self.calls.append(RecordedCall(list(argv), cwd, env, input, timeout)) + response = self.handler(list(argv)) + if isinstance(response, BaseException): + raise response + return response if response is not None else _completed(argv) + + def argv_prefixes(self, length: int = 3) -> list[tuple[str, ...]]: + return [tuple(call.argv[:length]) for call in self.calls] + + +def _completed(argv, stdout: str = "", returncode: int = 0, stderr: str = ""): + return subprocess.CompletedProcess(list(argv), returncode, stdout, stderr) + + +def _pr(number: int, state: str, additions: int = 3, deletions: int = 1) -> dict: + return {"number": number, "state": state, "additions": additions, "deletions": deletions} + + +def _recipe_handler( + *, + new_prs: list[dict] | None = None, + fleet: subprocess.CompletedProcess[str] | BaseException | None = None, + extra: Handler | None = None, +) -> Handler: + listed = [_pr(8, "OPEN"), _pr(7, "MERGED")] if new_prs is None else new_prs + + def handle(argv: list[str]): + if extra is not None and (response := extra(argv)) is not None: + return response + if argv[:3] == ["gh", "pr", "list"]: + baseline = argv[argv.index("--limit") + 1] == "1" + return _completed(argv, json.dumps([{"number": 7}] if baseline else listed)) + if argv[:3] == ["autoskillit", "fleet", "run"]: + return fleet or _completed(argv, f'{_SKILL_NOTICE}\n{{"success": true}}\n') + return None + + return handle + + +def _recipe_catalog(state: str = "open"): + return e2e_catalog.parse_catalog( + { + "sandbox_repository": SANDBOX, + "tests": [ + { + "name": "impl", + "kind": "recipe", + "peak_sessions": 6, + "timeout_sec": 600, + "trigger_paths": ["src/*"], + "recipe": "implementation", + "ingredients": {"task": "Add a greeting"}, + "expected_pull_request_state": state, + } + ], + } + ) + + +def _env(**overrides: str) -> dict[str, str]: + env = { + "PATH": "/usr/bin", + "HOME": "/home/autoskillit", + "MINIMAX_API_KEY": MINIMAX_KEY, + "E2E_SANDBOX_TOKEN": SANDBOX_TOKEN, + "ANTHROPIC_API_KEY": "anthropic-test-key", + "CLAUDE_CODE_OAUTH_TOKEN": "oauth-test-token", + } + env.update(overrides) + return {name: value for name, value in env.items() if value} + + +def _run_recipe(tmp_path: Path, runner: FakeRunner, *, state: str = "open", env=None): + catalog = _recipe_catalog(state) + out = tmp_path / "out" + failures = harness.run_test( + catalog.get("impl"), + catalog, + out=out, + home=tmp_path / "home", + env=env if env is not None else _env(), + runner=runner, + ) + return failures, json.loads((out / "result.json").read_text(encoding="utf-8")) + + +def _run_canary(tmp_path: Path, runner: FakeRunner, monkeypatch, env=None): + monkeypatch.setattr(tempfile, "tempdir", str(tmp_path)) + catalog = e2e_catalog.load_catalog() + out = tmp_path / "out" + failures = harness.run_test( + catalog.get("canary"), + catalog, + out=out, + home=tmp_path / "home", + env=env if env is not None else _env(), + runner=runner, + ) + return failures, json.loads((out / "result.json").read_text(encoding="utf-8")) + + +class TestProviderSetup: + def test_settings_env_is_the_minimax_map_and_keeps_other_keys(self, tmp_path: Path) -> None: + settings_path = tmp_path / ".claude" / "settings.json" + settings_path.parent.mkdir() + settings_path.write_text(json.dumps({"theme": "dark", "env": {"OLD": "1"}})) + harness.write_claude_settings(tmp_path, MINIMAX_KEY) + settings = json.loads(settings_path.read_text(encoding="utf-8")) + assert settings["env"] == _EXPECTED_SETTINGS_ENV + assert settings["theme"] == "dark" + assert stat.S_IMODE(settings_path.stat().st_mode) == 0o600 + + def test_new_settings_file_is_private(self, tmp_path: Path) -> None: + path = harness.write_claude_settings(tmp_path, MINIMAX_KEY) + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + + def test_child_env_drops_every_credential(self) -> None: + env = _env(ANTHROPIC_BASE_URL="https://example.invalid") + scrubbed = harness.child_env(env) + assert scrubbed == {"PATH": "/usr/bin", "HOME": "/home/autoskillit"} + + +def test_shipped_config_routes_every_default_codex_step_to_claude_code( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from autoskillit.config import load_config + from autoskillit.core.io import load_yaml + from autoskillit.core.paths import pkg_root + + home = tmp_path / "home" + monkeypatch.setattr("pathlib.Path.home", lambda: home) + harness.install_autoskillit_config(home) + project = tmp_path / "project" + project.mkdir() + cfg = load_config(project) + + defaults = load_yaml(pkg_root() / "config" / "defaults.yaml") + routes = cfg.agent_backend.recipe_overrides + unrouted = sorted( + (recipe, step) + for recipe, steps in defaults["agent_backend"]["recipe_overrides"].items() + for step, backend in steps.items() + if backend == "codex" and routes.get(recipe, {}).get(step) != "claude-code" + ) + assert unrouted == [] + assert "codex" not in {backend for steps in routes.values() for backend in steps.values()} + assert cfg.agent_backend.backend == "claude-code" + assert cfg.quota_guard.enabled is False + assert cfg.features["fleet"] is True + assert cfg.features["fleet_headless_run"] is True + + +class TestCanaryOutput: + def test_successful_reply_passes(self) -> None: + assert harness.check_canary_output(_CANARY_REPLY) == [] + + @pytest.mark.parametrize( + "stdout", + [ + json.dumps({"is_error": True, "result": "AUTOSKILLIT-E2E-CANARY-OK"}), + json.dumps({"is_error": False, "result": "something else"}), + "not json", + json.dumps(["is_error", False]), + ], + ids=["is-error", "missing-token", "not-json", "not-an-object"], + ) + def test_bad_reply_fails_once(self, stdout: str) -> None: + assert len(harness.check_canary_output(stdout)) == 1 + + +class TestEnvelope: + def test_success_envelope_passes(self) -> None: + assert harness.check_envelope(0, '{"success": true}\n') == [] + + def test_envelope_is_the_last_line_after_plain_text(self) -> None: + stdout = f'{_SKILL_NOTICE}\n{{"success": true}}\n\n' + assert harness.last_line(stdout) == '{"success": true}' + assert harness.check_envelope(0, stdout) == [] + + @pytest.mark.parametrize( + ("returncode", "stdout"), + [ + (1, '{"success": true}'), + (0, '{"success": false}'), + (0, "[true]"), + (0, '{"success": true}\nnot json'), + (0, ""), + ], + ids=["nonzero-exit", "unsuccessful", "list", "non-json-last-line", "empty"], + ) + def test_bad_envelope_fails(self, returncode: int, stdout: str) -> None: + assert harness.check_envelope(returncode, stdout) + + +class TestPullRequestAssertions: + def test_one_merged_pull_request_matches_lower_case_expectation(self) -> None: + assert harness.check_pull_requests([_pr(8, "MERGED")], "merged") == [] + + @pytest.mark.parametrize( + "prs", + [[], [_pr(8, "MERGED"), _pr(9, "MERGED")], [_pr(8, "OPEN")], [_pr(8, "MERGED", 0, 0)]], + ids=["none", "two", "state-mismatch", "empty-diff"], + ) + def test_unexpected_pull_requests_fail(self, prs: list[dict]) -> None: + assert harness.check_pull_requests(prs, "merged") + + +class TestRecipeFlow: + def test_steps_run_in_order_with_the_token_only_on_stdin(self, tmp_path: Path) -> None: + runner = FakeRunner(_recipe_handler()) + failures, result = _run_recipe(tmp_path, runner) + + assert failures == [] + assert result == {"test": "impl", "passed": True, "failures": []} + assert [call.argv for call in runner.calls] == [ + ["git", "config", "--global", "user.name", "AutoSkillit E2E"], + [ + "git", + "config", + "--global", + "user.email", + "autoskillit-e2e@users.noreply.github.com", + ], + ["gh", "auth", "login", "--hostname", "github.com", "--with-token"], + ["gh", "auth", "setup-git"], + ["gh", "repo", "clone", SANDBOX, "/workspace/sandbox"], + ["gh", "pr", "list", "--repo", SANDBOX, "--state", "all"] + + ["--limit", "1", "--json", "number"], + ["autoskillit", "fleet", "run", "implementation", "-i", "task=Add a greeting"] + + ["--disable-quota-guard", "--timeout-sec", "600"], + ["gh", "pr", "list", "--repo", SANDBOX, "--state", "all"] + + ["--limit", "20", "--json", "number,state,additions,deletions"], + ["gh", "pr", "close", "8", "--repo", SANDBOX, "--delete-branch"], + ] + assert [call.input for call in runner.calls if call.input] == [SANDBOX_TOKEN] + assert all(SANDBOX_TOKEN not in " ".join(call.argv) for call in runner.calls) + assert all(call.env == harness.child_env(_env()) for call in runner.calls) + fleet_call = runner.calls[6] + assert fleet_call.cwd == harness.SANDBOX_CLONE + assert fleet_call.timeout == 600 + e2e_catalog.HARNESS_GRACE_SEC + + def test_outputs_keep_full_stdout_and_only_the_envelope_line(self, tmp_path: Path) -> None: + _run_recipe(tmp_path, FakeRunner(_recipe_handler())) + out = tmp_path / "out" + stdout = (out / "fleet-run.stdout.log").read_text(encoding="utf-8") + assert stdout == f'{_SKILL_NOTICE}\n{{"success": true}}\n' + assert (out / "envelope.json").read_text(encoding="utf-8") == '{"success": true}\n' + assert (out / "fleet-run.stderr.log").exists() + + def test_harness_writes_provider_settings_and_user_config(self, tmp_path: Path) -> None: + _run_recipe(tmp_path, FakeRunner(_recipe_handler())) + home = tmp_path / "home" + settings = json.loads((home / ".claude" / "settings.json").read_text(encoding="utf-8")) + assert settings["env"] == _EXPECTED_SETTINGS_ENV + installed = (home / ".autoskillit" / "config.yaml").read_bytes() + assert installed == harness.CONFIG_TEMPLATE.read_bytes() + + def test_merged_pull_request_is_not_closed(self, tmp_path: Path) -> None: + runner = FakeRunner(_recipe_handler(new_prs=[_pr(8, "MERGED")])) + failures, _ = _run_recipe(tmp_path, runner, state="merged") + assert failures == [] + assert ("gh", "pr", "close") not in runner.argv_prefixes() + + def test_auth_failure_stops_before_the_baseline(self, tmp_path: Path) -> None: + def reject_login(argv: list[str]): + if argv[:3] == ["gh", "auth", "login"]: + return _completed(argv, returncode=1, stderr="bad credentials") + return None + + runner = FakeRunner(_recipe_handler(extra=reject_login)) + failures, result = _run_recipe(tmp_path, runner) + prefixes = runner.argv_prefixes() + assert result["passed"] is False + assert "bad credentials" in failures[0] + assert ("gh", "pr", "list") not in prefixes + assert ("gh", "pr", "close") not in prefixes + assert ("autoskillit", "fleet", "run") not in prefixes + + def test_fleet_timeout_is_a_failure_and_cleanup_still_runs(self, tmp_path: Path) -> None: + timeout = subprocess.TimeoutExpired(["autoskillit"], 720) + runner = FakeRunner(_recipe_handler(fleet=timeout)) + failures, result = _run_recipe(tmp_path, runner) + assert result["passed"] is False + assert any("fleet run: timed out" in failure for failure in failures) + assert runner.calls[-1].argv[:4] == ["gh", "pr", "close", "8"] + + def test_unexpected_fleet_error_records_failure_and_still_cleans_up( + self, tmp_path: Path + ) -> None: + runner = FakeRunner(_recipe_handler(fleet=RuntimeError("unexpected fleet error"))) + failures, result = _run_recipe(tmp_path, runner) + assert failures == ["harness error: unexpected fleet error"] + assert result == {"test": "impl", "passed": False, "failures": failures} + assert runner.calls[-1].argv[:4] == ["gh", "pr", "close", "8"] + + def test_missing_gh_is_a_failure_string(self, tmp_path: Path) -> None: + def no_gh(argv: list[str]): + return ( + FileNotFoundError(2, "No such file or directory", "gh") + if argv[0] == "gh" + else None + ) + + runner = FakeRunner(_recipe_handler(extra=no_gh)) + failures, result = _run_recipe(tmp_path, runner) + assert result["passed"] is False + assert failures[0].startswith("gh auth login:") + assert ("autoskillit", "fleet", "run") not in runner.argv_prefixes() + + def test_failed_listing_after_the_run_is_retried_for_cleanup(self, tmp_path: Path) -> None: + listings = iter(["broken", json.dumps([_pr(8, "OPEN")])]) + + def flaky_listing(argv: list[str]): + if argv[:3] == ["gh", "pr", "list"] and "20" in argv: + return _completed(argv, next(listings)) + return None + + runner = FakeRunner(_recipe_handler(extra=flaky_listing)) + failures, _ = _run_recipe(tmp_path, runner) + assert failures == ["gh pr list: output is not JSON"] + assert runner.calls[-1].argv[:4] == ["gh", "pr", "close", "8"] + + @pytest.mark.parametrize("limit", ["1", "20"]) + def test_missing_pull_request_number_records_failure(self, tmp_path: Path, limit: str) -> None: + def malformed_listing(argv: list[str]): + if argv[:3] == ["gh", "pr", "list"] and limit in argv: + return _completed(argv, json.dumps([{"state": "OPEN"}])) + return None + + runner = FakeRunner(_recipe_handler(extra=malformed_listing)) + failures, result = _run_recipe(tmp_path, runner) + assert "gh pr list: pull request number is missing or invalid" in failures + assert result == {"test": "impl", "passed": False, "failures": failures} + assert ("gh", "pr", "close") not in runner.argv_prefixes() + + def test_missing_sandbox_token_fails_before_any_launch(self, tmp_path: Path) -> None: + runner = FakeRunner(_recipe_handler()) + failures, result = _run_recipe(tmp_path, runner, env=_env(E2E_SANDBOX_TOKEN="")) + assert failures == ["E2E_SANDBOX_TOKEN is not set"] + assert result["passed"] is False + assert runner.calls == [] + + +class TestCanaryFlow: + def test_canary_runs_one_claude_prompt(self, tmp_path: Path, monkeypatch) -> None: + runner = FakeRunner(lambda argv: _completed(argv, _CANARY_REPLY)) + failures, result = _run_canary(tmp_path, runner, monkeypatch) + assert failures == [] + assert result == {"test": "canary", "passed": True, "failures": []} + assert [call.argv for call in runner.calls] == [ + ["claude", "-p", harness.CANARY_PROMPT, "--output-format", "json"] + ] + assert runner.calls[0].env == harness.child_env(_env()) + assert (tmp_path / "out" / "canary.json").read_text(encoding="utf-8") == _CANARY_REPLY + + def test_missing_minimax_key_fails_before_any_launch( + self, tmp_path: Path, monkeypatch + ) -> None: + runner = FakeRunner(lambda argv: _completed(argv, _CANARY_REPLY)) + failures, result = _run_canary(tmp_path, runner, monkeypatch, env=_env(MINIMAX_API_KEY="")) + assert failures == ["MINIMAX_API_KEY is not set"] + assert result["passed"] is False + assert runner.calls == [] + + +class TestRedaction: + def test_secrets_are_replaced_in_nested_text_and_binary_files(self, tmp_path: Path) -> None: + source = tmp_path / "out" + (source / "nested").mkdir(parents=True) + (source / "result.json").write_text(f"key={MINIMAX_KEY}\n", encoding="utf-8") + (source / "nested" / "blob.bin").write_bytes(b"\x00" + SANDBOX_TOKEN.encode() + b"\xff") + (source / "nested" / "blob.bin").chmod(0o600) + dest = tmp_path / "upload" + + harness.redact_tree([source, tmp_path / "missing"], dest, [MINIMAX_KEY, SANDBOX_TOKEN, ""]) + + assert (dest / "out" / "result.json").read_text(encoding="utf-8") == "key=[REDACTED]\n" + assert (dest / "out" / "nested" / "blob.bin").read_bytes() == b"\x00[REDACTED]\xff" + assert stat.S_IMODE((dest / "out" / "nested" / "blob.bin").stat().st_mode) == 0o644 + assert stat.S_IMODE((dest / "out" / "nested").stat().st_mode) == 0o755 + assert not (dest / "missing").exists() + + def test_symlinks_are_never_copied(self, tmp_path: Path) -> None: + secret_file = tmp_path / "secret.txt" + secret_file.write_text(MINIMAX_KEY, encoding="utf-8") + source = tmp_path / "out" + source.mkdir() + (source / "kept.txt").write_text("kept", encoding="utf-8") + (source / "file-link").symlink_to(secret_file) + (source / "dir-link").symlink_to(tmp_path) + linked_source = tmp_path / "linked" + linked_source.symlink_to(source) + dest = tmp_path / "upload" + + harness.redact_tree([source, linked_source], dest, [MINIMAX_KEY]) + + assert sorted(os.listdir(dest)) == ["out"] + assert sorted(os.listdir(dest / "out")) == ["kept.txt"] + + def test_surviving_secret_fails_the_post_scan( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + source = tmp_path / "out" + source.mkdir() + (source / "log.txt").write_text(f"token {SANDBOX_TOKEN}", encoding="utf-8") + monkeypatch.setattr(harness, "_redact", lambda data, needles: data) + with pytest.raises(RuntimeError, match="survived redaction"): + harness.redact_tree([source], tmp_path / "upload", [SANDBOX_TOKEN]) + + def test_redact_command_reads_secrets_from_the_environment( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + source = tmp_path / "out" + source.mkdir() + (source / "log.txt").write_text(f"key {MINIMAX_KEY}", encoding="utf-8") + monkeypatch.setenv("MINIMAX_API_KEY", MINIMAX_KEY) + monkeypatch.delenv("E2E_SANDBOX_TOKEN", raising=False) + argv = ["redact", "--dest", str(tmp_path / "upload")] + argv += ["--secret-env", "MINIMAX_API_KEY", "--secret-env", "E2E_SANDBOX_TOKEN"] + assert harness.main([*argv, str(source)]) == 0 + assert (tmp_path / "upload" / "out" / "log.txt").read_text(encoding="utf-8") == ( + "key [REDACTED]" + ) diff --git a/tests/infra/test_e2e_select.py b/tests/infra/test_e2e_select.py new file mode 100644 index 0000000000..d21b5f7f25 --- /dev/null +++ b/tests/infra/test_e2e_select.py @@ -0,0 +1,310 @@ +"""Unit tests for scripts/e2e/e2e_select.py and the catalog schema it reads.""" + +from __future__ import annotations + +import hashlib +import itertools +from pathlib import Path + +import pytest + +from tests.infra._complexity_helpers import load_check_script + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.small] + +REPO_ROOT = Path(__file__).resolve().parents[2] +_SELECT_SCRIPT = REPO_ROOT / "scripts" / "e2e" / "e2e_select.py" + +select = load_check_script("_autoskillit_e2e_select", _SELECT_SCRIPT) +e2e_catalog = select.e2e_catalog + +REPOSITORY = "TalonT-Org/AutoSkillit" +SHA_ZERO = "0" * 40 +SYNTHETIC_SHAS = [hashlib.sha1(str(index).encode()).hexdigest() for index in range(2000)] + + +def _facts( + *, + number: int = 5230, + head_sha: str = SHA_ZERO, + head_repository: str | None = REPOSITORY, + changed_files: int = 10, + additions: int = 0, + deletions: int = 0, + labels: tuple[str, ...] = (), +): + return select.PullRequestFacts( + number=number, + head_sha=head_sha, + head_repository=head_repository, + changed_files=changed_files, + additions=additions, + deletions=deletions, + labels=frozenset(labels), + ) + + +def _entry(name: str, trigger_paths: tuple[str, ...] = ("src/*",), **overrides) -> dict: + entry = { + "name": name, + "kind": "canary", + "peak_sessions": 1, + "timeout_sec": 300, + "trigger_paths": list(trigger_paths), + } + entry.update(overrides) + return entry + + +def _raw_catalog(*entries: dict) -> dict: + return {"sandbox_repository": "owner/sandbox", "tests": list(entries)} + + +def _catalog(*entries: dict): + return e2e_catalog.parse_catalog(_raw_catalog(*entries)) + + +def _sha_with_sample(number: int, *, sampled: bool) -> str: + for index in itertools.count(): + sha = f"{index:040x}" + if (select.sample_score(number, sha) < select.SAMPLE_PROBABILITY) == sampled: + return sha + raise AssertionError("unreachable") + + +_FIVE_TESTS = tuple(_entry(f"t{index}", (f"area{index}/*",)) for index in range(5)) + + +class TestEligibility: + def test_ten_changed_files_is_eligible(self) -> None: + assert select.is_eligible(_facts(changed_files=10)) + + def test_three_hundred_changed_lines_is_eligible(self) -> None: + assert select.is_eligible(_facts(changed_files=1, additions=200, deletions=100)) + + def test_below_both_thresholds_is_not_eligible(self) -> None: + assert not select.is_eligible(_facts(changed_files=9, additions=200, deletions=99)) + + def test_e2e_label_makes_a_one_file_change_eligible(self) -> None: + assert select.is_eligible(_facts(changed_files=1, labels=("e2e",))) + + +class TestSampleScore: + def test_score_is_the_first_eight_digest_bytes_over_two_to_the_sixty_four(self) -> None: + digest = hashlib.sha256(b"5230:" + b"0" * 40).digest() + expected = int.from_bytes(digest[:8], "big") / 2**64 + assert select.sample_score(5230, SHA_ZERO) == expected + + def test_score_is_deterministic(self) -> None: + assert select.sample_score(5230, SHA_ZERO) == select.sample_score(5230, SHA_ZERO) + + def test_half_probability_samples_about_half(self) -> None: + sampled = sum(select.is_sampled(_facts(head_sha=sha), 0.5) for sha in SYNTHETIC_SHAS) + assert 0.45 <= sampled / len(SYNTHETIC_SHAS) <= 0.55 + + def test_zero_probability_never_samples(self) -> None: + assert not any(select.is_sampled(_facts(head_sha=sha), 0.0) for sha in SYNTHETIC_SHAS) + + def test_unit_probability_always_samples(self) -> None: + assert all(select.is_sampled(_facts(head_sha=sha), 1.0) for sha in SYNTHETIC_SHAS) + + +class TestSelectForPullRequest: + def test_eligible_and_sampled_is_selected(self) -> None: + pr = _facts(head_sha=_sha_with_sample(5230, sampled=True)) + selection = select.select_for_pull_request(pr, (), _catalog(_entry("a")), REPOSITORY) + assert selection.tests == ("a",) + assert selection.reason == "eligible and sampled" + + def test_eligible_but_sampled_out_is_not_selected(self) -> None: + pr = _facts(head_sha=_sha_with_sample(5230, sampled=False)) + selection = select.select_for_pull_request(pr, (), _catalog(_entry("a")), REPOSITORY) + assert not selection.selected + assert "not sampled" in selection.reason + + def test_e2e_label_bypasses_sampling(self) -> None: + pr = _facts(head_sha=_sha_with_sample(5230, sampled=False), labels=("e2e",)) + selection = select.select_for_pull_request(pr, (), _catalog(_entry("a")), REPOSITORY) + assert selection.tests == ("a",) + + def test_fork_head_is_not_selected(self) -> None: + pr = _facts(head_repository="someone/AutoSkillit", labels=("e2e",)) + selection = select.select_for_pull_request(pr, (), _catalog(_entry("a")), REPOSITORY) + assert not selection.selected + assert "someone/AutoSkillit" in selection.reason + + def test_deleted_head_repository_is_not_selected(self) -> None: + payload = { + "number": 5230, + "head": {"sha": SHA_ZERO, "repo": None}, + "changed_files": 50, + "additions": 1000, + "deletions": 0, + "labels": [{"name": "e2e"}], + } + pr = select.PullRequestFacts.from_payload(payload) + selection = select.select_for_pull_request(pr, (), _catalog(_entry("a")), REPOSITORY) + assert pr.head_repository is None + assert not selection.selected + + +class TestPickTests: + _SEED = f"5230:{SHA_ZERO}" + + def test_all_touched_caps_at_two(self) -> None: + changed = [f"area{index}/x.py" for index in range(5)] + assert len(select.pick_tests(_catalog(*_FIVE_TESTS), changed, self._SEED)) == 2 + + def test_none_touched_picks_exactly_one(self) -> None: + assert len(select.pick_tests(_catalog(*_FIVE_TESTS), ["docs/x.md"], self._SEED)) == 1 + + @pytest.mark.parametrize("seed_index", range(20)) + def test_one_touched_picks_only_that_test(self, seed_index: int) -> None: + picked = select.pick_tests(_catalog(*_FIVE_TESTS), ["area3/x.py"], f"{seed_index}:sha") + assert picked == ("t3",) + + def test_single_test_catalog_picks_it(self) -> None: + assert select.pick_tests(_catalog(_entry("only")), [], self._SEED) == ("only",) + + @pytest.mark.parametrize( + "changed", + [[], ["docs/x.md"], ["area0/x.py"], [f"area{index}/x.py" for index in range(5)]], + ) + def test_selection_is_never_empty_and_deterministic(self, changed: list[str]) -> None: + first = select.pick_tests(_catalog(*_FIVE_TESTS), changed, self._SEED) + assert first + assert first == select.pick_tests(_catalog(*_FIVE_TESTS), changed, self._SEED) + + def test_rank_is_the_sha256_hex_digest_of_seed_and_name(self) -> None: + catalog = _catalog(_entry("alpha"), _entry("beta")) + expected = sorted( + ["alpha", "beta"], + key=lambda name: hashlib.sha256(f"{self._SEED}:{name}".encode()).hexdigest(), + ) + assert select.pick_tests(catalog, ["src/x.py"], self._SEED) == tuple(expected) + + def test_star_matches_across_directories(self) -> None: + catalog = _catalog(_entry("deep", ("src/*",)), _entry("other", ("docs/*",))) + assert select.pick_tests(catalog, ["src/a/b/c.py"], self._SEED) == ("deep",) + + +class TestSelectForEvent: + def test_merge_group_is_never_selected(self) -> None: + selection = select.select_for_event( + "merge_group", {}, (), _catalog(_entry("a")), REPOSITORY + ) + assert not selection.selected + assert "merge queue" in selection.reason + + def test_unsupported_event_raises(self) -> None: + with pytest.raises(ValueError, match="unsupported event"): + select.select_for_event("push", {}, (), _catalog(_entry("a")), REPOSITORY) + + def test_pull_request_without_pull_request_object_raises(self) -> None: + with pytest.raises(ValueError): + select.select_for_event("pull_request", {}, (), _catalog(_entry("a")), REPOSITORY) + + def test_dispatch_reads_inputs_tests(self) -> None: + catalog = e2e_catalog.load_catalog() + payload = {"inputs": {"tests": "canary"}} + selection = select.select_for_event("workflow_dispatch", payload, (), catalog, REPOSITORY) + assert selection.tests == ("canary",) + + def test_dispatch_deduplicates_names(self) -> None: + catalog = e2e_catalog.load_catalog() + assert select.select_for_dispatch("canary, canary", catalog).tests == ("canary",) + + @pytest.mark.parametrize("requested", ["t0,t1,t2", "t0,nope", "", " , "]) + def test_dispatch_rejects_invalid_requests(self, requested: str) -> None: + with pytest.raises(ValueError): + select.select_for_dispatch(requested, _catalog(*_FIVE_TESTS)) + + def test_dispatch_without_inputs_raises(self) -> None: + with pytest.raises(ValueError, match="inputs.tests"): + select.select_for_event("workflow_dispatch", {}, (), _catalog(_entry("a")), REPOSITORY) + + +_PASSING_GATE_ROWS = {("success", "false", "skipped"), ("success", "true", "success")} +_RESULTS = ("success", "failure", "cancelled", "skipped", "") + + +@pytest.mark.parametrize( + ("select_result", "selected", "run_result"), + list(itertools.product(_RESULTS, ("true", "false", ""), _RESULTS)), +) +def test_gate_passes_only_on_the_two_healthy_shapes( + select_result: str, selected: str, run_result: str +) -> None: + passed, reason = select.gate_verdict(select_result, selected, run_result) + assert passed is ((select_result, selected, run_result) in _PASSING_GATE_ROWS) + assert reason + + +def test_matrix_for_an_empty_selection_is_parseable() -> None: + selection = select.Selection((), "nothing selected") + assert select.matrix_json(selection, _catalog(_entry("a"))) == '{"include":[]}' + + +_RECIPE_FIELDS = { + "recipe": "implementation", + "ingredients": {"task": "Add a greeting"}, + "expected_pull_request_state": "merged", +} +_MAX_TIMEOUT_SEC = ( + e2e_catalog.MAX_JOB_MINUTES - e2e_catalog.JOB_OVERHEAD_MINUTES +) * 60 - e2e_catalog.HARNESS_GRACE_SEC + + +def _recipe_entry_without(field: str) -> dict: + fields = {key: value for key, value in _RECIPE_FIELDS.items() if key != field} + return _entry("impl", kind="recipe", **fields) + + +_INVALID_CATALOGS = { + "duplicate-names": _raw_catalog(_entry("a"), _entry("a")), + "bad-name": _raw_catalog(_entry("Bad_Name")), + "unknown-kind": _raw_catalog(_entry("a", kind="smoke")), + "zero-peak-sessions": _raw_catalog(_entry("a", peak_sessions=0)), + "nine-peak-sessions": _raw_catalog(_entry("a", peak_sessions=9)), + "timeout-over-job-limit": _raw_catalog(_entry("a", timeout_sec=_MAX_TIMEOUT_SEC + 1)), + "recipe-without-recipe": _raw_catalog(_recipe_entry_without("recipe")), + "recipe-without-ingredients": _raw_catalog(_recipe_entry_without("ingredients")), + "recipe-without-state": _raw_catalog(_recipe_entry_without("expected_pull_request_state")), + "recipe-unknown-state": _raw_catalog( + _entry("a", kind="recipe", **{**_RECIPE_FIELDS, "expected_pull_request_state": "draft"}) + ), + "canary-with-recipe-fields": _raw_catalog(_entry("a", **_RECIPE_FIELDS)), + "empty-trigger-paths": _raw_catalog(_entry("a", ())), + "unknown-key": _raw_catalog(_entry("a", extra=1)), + "unknown-top-level-key": {**_raw_catalog(_entry("a")), "extra": 1}, + "no-tests": _raw_catalog(), +} + + +class TestCatalog: + def test_shipped_catalog_holds_the_canary(self) -> None: + catalog = e2e_catalog.load_catalog() + canary = catalog.get("canary") + assert canary.kind == "canary" + assert canary.recipe is None + assert e2e_catalog.job_timeout_minutes(canary) == 37 + + def test_recipe_entry_parses(self) -> None: + catalog = _catalog(_entry("impl", kind="recipe", **_RECIPE_FIELDS)) + test = catalog.get("impl") + assert test.recipe == "implementation" + assert test.ingredients == (("task", "Add a greeting"),) + assert test.expected_pull_request_state == "merged" + + def test_longest_timeout_within_the_job_limit_parses(self) -> None: + catalog = _catalog(_entry("slow", timeout_sec=_MAX_TIMEOUT_SEC)) + assert e2e_catalog.job_timeout_minutes(catalog.get("slow")) == e2e_catalog.MAX_JOB_MINUTES + + def test_unknown_test_name_raises_catalog_error(self) -> None: + with pytest.raises(e2e_catalog.CatalogError): + _catalog(_entry("a")).get("b") + + @pytest.mark.parametrize("raw", _INVALID_CATALOGS.values(), ids=list(_INVALID_CATALOGS)) + def test_invalid_catalog_raises(self, raw: dict) -> None: + with pytest.raises(e2e_catalog.CatalogError): + e2e_catalog.parse_catalog(raw) diff --git a/tests/infra/test_e2e_select_cli.py b/tests/infra/test_e2e_select_cli.py new file mode 100644 index 0000000000..a0ed198a84 --- /dev/null +++ b/tests/infra/test_e2e_select_cli.py @@ -0,0 +1,149 @@ +"""Subprocess tests for the scripts/e2e/e2e_select.py command line the E2E workflow runs.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from tests.conftest import production_interpreter_env + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.medium] + +REPO_ROOT = Path(__file__).resolve().parents[2] +_SCRIPT = REPO_ROOT / "scripts" / "e2e" / "e2e_select.py" +REPOSITORY = "TalonT-Org/AutoSkillit" +_CANARY_MATRIX = '{"include":[{"test":"canary","timeout_minutes":37}]}' + + +def _run( + *args: str, tmp_path: Path, event_name: str, payload: object +) -> subprocess.CompletedProcess[str]: + event_path = tmp_path / "event.json" + event_path.write_text(json.dumps(payload), encoding="utf-8") + env = production_interpreter_env() + env["GITHUB_EVENT_NAME"] = event_name + env["GITHUB_EVENT_PATH"] = str(event_path) + env["GITHUB_REPOSITORY"] = REPOSITORY + return subprocess.run( + [sys.executable, str(_SCRIPT), *args], + capture_output=True, + text=True, + env=env, + check=False, + ) + + +def _outputs(stdout: str) -> dict[str, str]: + return dict(line.split("=", 1) for line in stdout.splitlines()) + + +def _pull_request(*, changed_files: int, additions: int, labels: tuple[str, ...] = ()) -> dict: + return { + "pull_request": { + "number": 5230, + "head": {"sha": "0" * 40, "repo": {"full_name": REPOSITORY}}, + "changed_files": changed_files, + "additions": additions, + "deletions": 0, + "labels": [{"name": label} for label in labels], + } + } + + +def _select(tmp_path: Path, changed_paths: Path, event_name: str, payload: object): + return _run( + "select", + "--changed-paths", + str(changed_paths), + tmp_path=tmp_path, + event_name=event_name, + payload=payload, + ) + + +def test_labelled_pull_request_selects_the_touched_canary(tmp_path: Path) -> None: + changed = tmp_path / "changed-paths.txt" + changed.write_text("scripts/e2e/e2e_harness.py\n", encoding="utf-8") + payload = _pull_request(changed_files=1, additions=5, labels=("e2e",)) + result = _select(tmp_path, changed, "pull_request", payload) + assert result.returncode == 0, result.stderr + outputs = _outputs(result.stdout) + assert outputs["selected"] == "true" + assert outputs["matrix"] == _CANARY_MATRIX + + +def test_small_unlabelled_pull_request_selects_nothing(tmp_path: Path) -> None: + changed = tmp_path / "changed-paths.txt" + changed.write_text("README.md\n", encoding="utf-8") + result = _select( + tmp_path, changed, "pull_request", _pull_request(changed_files=3, additions=15) + ) + assert result.returncode == 0, result.stderr + outputs = _outputs(result.stdout) + assert outputs["selected"] == "false" + assert outputs["matrix"] == '{"include":[]}' + + +def test_merge_group_never_reads_the_changed_paths_file(tmp_path: Path) -> None: + result = _select(tmp_path, tmp_path / "missing.txt", "merge_group", {"merge_group": {}}) + assert result.returncode == 0, result.stderr + outputs = _outputs(result.stdout) + assert outputs["selected"] == "false" + assert outputs["matrix"] == '{"include":[]}' + + +def test_dispatch_selects_the_named_test(tmp_path: Path) -> None: + payload = {"inputs": {"tests": "canary"}} + result = _select(tmp_path, tmp_path / "missing.txt", "workflow_dispatch", payload) + assert result.returncode == 0, result.stderr + outputs = _outputs(result.stdout) + assert outputs["selected"] == "true" + assert outputs["matrix"] == _CANARY_MATRIX + + +def test_dispatch_of_an_unknown_test_fails(tmp_path: Path) -> None: + payload = {"inputs": {"tests": "nope"}} + result = _select(tmp_path, tmp_path / "missing.txt", "workflow_dispatch", payload) + assert result.returncode != 0 + assert result.stdout == "" + + +def test_pull_request_without_changed_paths_file_fails(tmp_path: Path) -> None: + payload = _pull_request(changed_files=20, additions=500, labels=("e2e",)) + result = _select(tmp_path, tmp_path / "missing.txt", "pull_request", payload) + assert result.returncode != 0 + assert result.stdout == "" + + +@pytest.mark.parametrize( + ("select_result", "selected", "run_result", "expected"), + [ + ("success", "false", "skipped", 0), + ("success", "true", "success", 0), + ("success", "true", "failure", 1), + ("success", "true", "skipped", 1), + ("failure", "", "skipped", 1), + ("success", "", "", 1), + ("cancelled", "true", "cancelled", 1), + ], +) +def test_gate_exit_code( + tmp_path: Path, select_result: str, selected: str, run_result: str, expected: int +) -> None: + result = _run( + "gate", + "--select-result", + select_result, + "--selected", + selected, + "--run-result", + run_result, + tmp_path=tmp_path, + event_name="pull_request", + payload={}, + ) + assert result.returncode == expected, result.stdout + result.stderr diff --git a/tests/infra/test_e2e_workflow.py b/tests/infra/test_e2e_workflow.py new file mode 100644 index 0000000000..a83f7b05f9 --- /dev/null +++ b/tests/infra/test_e2e_workflow.py @@ -0,0 +1,158 @@ +"""Structural tests for the .github/workflows/e2e.yml workflow.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +from autoskillit.core.io import load_yaml + +pytestmark = [pytest.mark.layer("infra"), pytest.mark.small] + +REPO_ROOT = Path(__file__).resolve().parents[2] +_WORKFLOWS = REPO_ROOT / ".github" / "workflows" +_WORKFLOW_PATH = _WORKFLOWS / "e2e.yml" +_DOCKERFILE = REPO_ROOT / "scripts" / "docker" / "Dockerfile" +_SHA_PIN = re.compile(r"@[0-9a-f]{40}\b") + + +def _on(workflow: dict) -> dict: + """Return the triggers; PyYAML (YAML 1.1) parses a bare ``on:`` key as ``True``.""" + return workflow.get("on") or workflow.get(True) or {} + + +@pytest.fixture(scope="module") +def workflow() -> dict: + return load_yaml(_WORKFLOW_PATH) + + +@pytest.fixture(scope="module") +def text() -> str: + return _WORKFLOW_PATH.read_text(encoding="utf-8") + + +def _step(workflow: dict, job: str, *, step_id: str | None = None, name: str | None = None): + return next( + step + for step in workflow["jobs"][job]["steps"] + if (step_id is not None and step.get("id") == step_id) + or (name is not None and step.get("name") == name) + ) + + +def _all_steps(workflow: dict): + for job in workflow["jobs"].values(): + yield from job["steps"] + + +class TestTriggers: + def test_pull_request_types_include_labeled(self, workflow: dict) -> None: + types = _on(workflow)["pull_request"]["types"] + assert types == ["opened", "synchronize", "reopened", "labeled"] + + def test_merge_group_is_a_trigger(self, workflow: dict) -> None: + assert "merge_group" in _on(workflow) + + def test_dispatch_takes_a_string_tests_input(self, workflow: dict) -> None: + assert _on(workflow)["workflow_dispatch"]["inputs"]["tests"]["type"] == "string" + + +class TestSecrets: + def test_only_the_minimax_key_and_sandbox_token_are_used(self, text: str) -> None: + assert set(re.findall(r"secrets\.(\w+)", text)) == {"MINIMAX_API_KEY", "E2E_SANDBOX_TOKEN"} + + @pytest.mark.parametrize("name", ["ANTHROPIC", "CLAUDE_CODE_OAUTH_TOKEN", "OPENAI_API_KEY"]) + def test_no_other_provider_credential_is_referenced(self, text: str, name: str) -> None: + assert name not in text + + def test_secrets_reach_only_the_test_and_redaction_steps(self, workflow: dict) -> None: + holders = { + step.get("id") + for step in _all_steps(workflow) + if "secrets." in str(step.get("env", {})) + } + assert holders == {"e2e", "redact"} + assert all("secrets." not in str(job.get("env", {})) for job in workflow["jobs"].values()) + + def test_report_step_uses_only_the_workflow_token(self, workflow: dict) -> None: + env = _step(workflow, "run", name="Report failure")["env"] + assert env["GH_TOKEN"] == "${{ github.token }}" + assert "secrets." not in str(env) + + @pytest.mark.parametrize("step_id", ["e2e", "redact"]) + def test_docker_run_forwards_secrets_by_name_only(self, workflow: dict, step_id: str) -> None: + run = _step(workflow, "run", step_id=step_id)["run"] + forwarded = re.findall(r"--env (\S+)", run) + assert forwarded == ["MINIMAX_API_KEY", "E2E_SANDBOX_TOKEN"] + + +class TestRunJob: + def test_one_repository_wide_queue(self, workflow: dict) -> None: + assert workflow["jobs"]["run"]["concurrency"] == { + "group": "e2e", + "queue": "max", + "cancel-in-progress": False, + } + + def test_matrix_runs_one_test_at_a_time(self, workflow: dict) -> None: + strategy = workflow["jobs"]["run"]["strategy"] + assert strategy["max-parallel"] == 1 + assert strategy["fail-fast"] is False + + def test_runs_only_when_selected_and_times_out_per_test(self, workflow: dict) -> None: + run = workflow["jobs"]["run"] + assert "needs.select.outputs.selected" in run["if"] + assert "matrix.timeout_minutes" in str(run["timeout-minutes"]) + + def test_permissions(self, workflow: dict) -> None: + assert workflow["jobs"]["run"]["permissions"] == {"contents": "read", "issues": "write"} + + def test_data_mount_targets_the_user_image_autoskillit_data_dir(self, workflow: dict) -> None: + stage = _DOCKERFILE.read_text(encoding="utf-8").split("FROM toolchain AS user", 1)[1] + home = re.search(r"ENV HOME=(\S+)", stage) + assert home is not None + run = _step(workflow, "run", step_id="e2e")["run"] + mounts = re.findall(r'--volume "\$RUNNER_TEMP/e2e/data:([^"]+)"', run) + assert mounts == [f"{home.group(1)}/.local/share/autoskillit"] + + def test_upload_sends_only_a_successfully_redacted_copy(self, workflow: dict) -> None: + upload = next( + step for step in _all_steps(workflow) if "upload-artifact" in step.get("uses", "") + ) + assert "steps.redact.outcome == 'success'" in upload["if"] + assert upload["with"]["path"].endswith("e2e/upload") + assert upload["with"]["include-hidden-files"] is True + + def test_report_runs_for_a_failed_test_or_redaction(self, workflow: dict) -> None: + report = _step(workflow, "run", name="Report failure") + assert report["if"] == ( + "failure() && (steps.e2e.outcome == 'failure' || steps.redact.outcome == 'failure')" + ) + assert "STAGE" in report["env"] + assert '"$STAGE"' in report["run"] + + def test_build_context_export_matches_the_docker_image_workflow(self, workflow: dict) -> None: + name = "Export the archived build context" + docker_image = load_yaml(_WORKFLOWS / "docker-image.yml") + ours = _step(workflow, "run", name=name)["run"] + assert ours == _step(docker_image, "publish", name=name)["run"] + + +class TestGate: + def test_gate_always_runs_after_select_and_run(self, workflow: dict) -> None: + gate = workflow["jobs"]["e2e-gate"] + assert gate["if"] == "always()" + assert gate["needs"] == ["select", "run"] + assert "name" not in gate + + def test_gate_judges_with_the_selector(self, workflow: dict) -> None: + runs = [step.get("run", "") for step in workflow["jobs"]["e2e-gate"]["steps"]] + assert any("e2e_select.py gate" in run for run in runs) + + +def test_every_action_is_sha_pinned(workflow: dict) -> None: + for step in _all_steps(workflow): + if "uses" in step: + assert _SHA_PIN.search(step["uses"]), step["uses"] diff --git a/tests/infra/test_manifest_directory_completeness.py b/tests/infra/test_manifest_directory_completeness.py index 445ffa4390..03d583692d 100644 --- a/tests/infra/test_manifest_directory_completeness.py +++ b/tests/infra/test_manifest_directory_completeness.py @@ -34,6 +34,7 @@ "scripts/check_complexity.py", "scripts/_git_plumbing.py", "scripts/docker/*", + "scripts/e2e/*", } ), "workspace": frozenset( diff --git a/tests/infra/test_probe_scripts.py b/tests/infra/test_probe_scripts.py index d4aaad703f..e1a3433621 100644 --- a/tests/infra/test_probe_scripts.py +++ b/tests/infra/test_probe_scripts.py @@ -52,6 +52,27 @@ def test_rejects_missing_args(self) -> None: assert "Usage" in result.stderr +class TestPostE2eFailure: + _SCRIPT = SCRIPTS_DIR / "e2e" / "post-e2e-failure.sh" + + def test_syntax(self) -> None: + result = subprocess.run(["bash", "-n", str(self._SCRIPT)], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + + def test_executable(self) -> None: + assert os.access(self._SCRIPT, os.X_OK) + + def test_rejects_missing_args(self) -> None: + result = subprocess.run( + ["bash", str(self._SCRIPT)], + capture_output=True, + text=True, + env={"PATH": os.environ["PATH"], "HOME": os.environ.get("HOME", "/tmp")}, + ) + assert result.returncode != 0 + assert "Usage" in result.stderr + + class TestCreateProbeCanaryIssueValidation: def test_rejects_missing_github_repository(self) -> None: """Exits non-zero when GITHUB_REPOSITORY is not set."""