diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 85439c97eb..6ee1930e9c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -102,16 +102,18 @@ jobs: - name: Run the E2E test id: e2e env: - MINIMAX_API_KEY: ${{ secrets.MINIMAX_API_KEY }} - E2E_SANDBOX_TOKEN: ${{ secrets.E2E_SANDBOX_TOKEN }} + MINIMAX_API_KEY: ${{ matrix.kind != 'clean-install' && secrets.MINIMAX_API_KEY || '' }} + E2E_SANDBOX_TOKEN: ${{ matrix.kind != 'clean-install' && secrets.E2E_SANDBOX_TOKEN || '' }} TEST: ${{ matrix.test }} run: | - docker run --rm \ - --env MINIMAX_API_KEY \ - --env E2E_SANDBOX_TOKEN \ + docker_args=() + if [[ '${{ matrix.kind }}' != 'clean-install' ]]; then + docker_args+=(--env MINIMAX_API_KEY --env E2E_SANDBOX_TOKEN) + docker_args+=(--volume "$RUNNER_TEMP/e2e/data:/home/autoskillit/.local/share/autoskillit") + fi + docker run --rm "${docker_args[@]}" \ --volume "$GITHUB_WORKSPACE/scripts/e2e:/opt/e2e:ro" \ --volume "$RUNNER_TEMP/e2e/out:/artifacts" \ - --volume "$RUNNER_TEMP/e2e/data:/home/autoskillit/.local/share/autoskillit" \ "$IMAGE" \ python3 /opt/e2e/e2e_harness.py run --test "$TEST" --out /artifacts @@ -119,12 +121,14 @@ jobs: id: redact if: always() && steps.build.outcome == 'success' env: - MINIMAX_API_KEY: ${{ secrets.MINIMAX_API_KEY }} - E2E_SANDBOX_TOKEN: ${{ secrets.E2E_SANDBOX_TOKEN }} + MINIMAX_API_KEY: ${{ matrix.kind != 'clean-install' && secrets.MINIMAX_API_KEY || '' }} + E2E_SANDBOX_TOKEN: ${{ matrix.kind != 'clean-install' && secrets.E2E_SANDBOX_TOKEN || '' }} run: | - docker run --rm \ - --env MINIMAX_API_KEY \ - --env E2E_SANDBOX_TOKEN \ + docker_args=() + if [[ '${{ matrix.kind }}' != 'clean-install' ]]; then + docker_args+=(--env MINIMAX_API_KEY --env E2E_SANDBOX_TOKEN) + fi + docker run --rm "${docker_args[@]}" \ --volume "$GITHUB_WORKSPACE/scripts/e2e:/opt/e2e:ro" \ --volume "$RUNNER_TEMP/e2e:/e2e" \ "$IMAGE" \ diff --git a/scripts/e2e/README.md b/scripts/e2e/README.md index 31f586a540..4f2a458031 100644 --- a/scripts/e2e/README.md +++ b/scripts/e2e/README.md @@ -1,8 +1,9 @@ -# Live E2E runs on MiniMax +# E2E tests in the user image -`.github/workflows/e2e.yml` runs AutoSkillit recipes end to end against a real model — -MiniMax, through its Anthropic-compatible endpoint — inside the published `user` Docker -image. This directory holds everything the workflow runs: +`.github/workflows/e2e.yml` builds the `user` Docker image and runs catalog tests inside it. +`clean-install` checks installation and doctor without credentials or model calls. +Canary and recipe tests use MiniMax through its Anthropic-compatible endpoint. +This directory holds everything the workflow runs: | File | Role | |------|------| @@ -18,6 +19,9 @@ image as `/usr/bin/python3`. Both are stdlib-only, which is why the catalog is J ## Maintainer setup +`clean-install` needs no provider or sandbox secrets. The following secrets support live +model tests: + 1. **`MINIMAX_API_KEY`** — an organization or repository secret holding a MiniMax API key. 2. **`E2E_SANDBOX_TOKEN`** — a repository secret holding a fine-grained personal access token scoped to the sandbox repository (`sandbox_repository` in `catalog.json`) only. @@ -82,7 +86,40 @@ the settings file is the single source of provider configuration. The sandbox to `gh auth login` on stdin only. Because settings `env` reaches every agent subprocess, the key can still surface in session output, so redaction is mandatory (see below). -## Why `autoskillit install` is not run +## Clean installation and doctor + +`clean-install` starts a fresh container with only script and artifact mounts. Its actual +HOME, PATH, XDG values and uv tool layout are preserved, and provider credentials are +removed from every child environment. The harness runs: + +1. `autoskillit install`. +2. `git init` in a fresh artifact-local `.autoskillit/temp/clean-install/` directory. +3. Write a `.pre-commit-config.yaml` using the gitleaks hook pinned to `v8.30.0`. +4. `autoskillit init --test-command "git diff --check"`. +5. `autoskillit doctor --output-json` in that repository. + +One monotonic deadline bounds the command sequence. The scratch repository is removed in +`finally`; command stdout, stderr and status records remain in the artifact directory, +including partial timeout output. `doctor.json` retains the raw doctor stdout. + +A clean pass requires successful commands and a complete nonempty JSON `results` list +with valid `ok`, `info`, `warning` or `error` rows. Warnings require an exact check/message +entry with a documented reason in `CLEAN_INSTALL_ALLOWED_WARNINGS`. Only observed +environmental or advisory warnings qualify; installation, plugin and hook defects need +bug tracking. + +Catalog `expected_failures` match exact severity/check/message diagnostics linked to +open AutoSkillit bug issues. Reproduce and verify the bug before adding its expectation. +Every expected diagnostic must appear and every other problem fails the test. Remove +an expectation when its diagnostic disappears; stale expectations fail. Command failures +and malformed JSON always fail. + +`result.json` records `outcome` and `expected_findings`: a clean `passed` outcome has +`passed: true`; an accepted `expected_failure` has `passed: false` and exits zero for the +CI gate; an unexpected `failed` outcome exits nonzero. The console uses the same outcome. +Select this test directly with workflow dispatch's `tests=clean-install` input. + +## Installation in model tests `fleet run` builds its context with `make_context(cfg, project_dir=Path.cwd(), ...)` (`src/autoskillit/cli/fleet/_fleet_run.py`). Without an installed plugin it falls back to @@ -91,11 +128,12 @@ can still surface in session output, so redaction is mandatory (see below). publishes the projection on demand. The food-truck errors for a missing installation fire only for backends that inject skills without plugin installs (Codex); the `claude-code` backend receives the projection as `--plugin-dir`, and the harness routes every step to -`claude-code`. `install` also defers under `CLAUDECODE`, and the image never runs it. +`claude-code`. These model tests skip `autoskillit install`; the clean-install harness +executes it explicitly. ## AutoSkillit user config -`autoskillit-config.yaml` becomes `~/.autoskillit/config.yaml` in the container: +For model tests, `autoskillit-config.yaml` becomes `~/.autoskillit/config.yaml` in the container: - `quota_guard.enabled: false` — the guard measures Anthropic account quota through Claude credentials (`~/.claude/.credentials.json`) the container does not have; `fleet run` also @@ -146,6 +184,7 @@ waits its turn and never cancels another. `select` and `e2e-gate` are not queued catalog entry's `peak_sessions` is capped at `MAX_PEAK_SESSIONS` (8): one step's fan-out of up to six subagents plus the orchestrator and step session. With one test running at a time, CI stays at or below eight of the roughly twelve concurrent MiniMax sessions. +`clean-install` requires exactly zero `peak_sessions`. Every harness subprocess has a timeout that fires before the job's: `fleet run` gets the test's `timeout_sec` plus `HARNESS_GRACE_SEC`, and the job gets that budget in minutes plus @@ -154,13 +193,17 @@ cleanup and redaction never depend on GitHub cancelling the job. ## Artifacts and redaction -The container's AutoSkillit data directory (`~/.local/share/autoskillit`) is bind-mounted +For model tests, the container's AutoSkillit data directory (`~/.local/share/autoskillit`) is bind-mounted from the runner, so session logs survive a killed container. Those files belong to uid 1000 and are often `0600`, so a second container run copies `out/` and `data/logs/` into `upload/` world-readable with both secrets replaced by `[REDACTED]`, skips symlinks, and fails if any secret survives. Only `upload/` is uploaded, and only when redaction succeeded. +`clean-install` mounts no home directories and retains evidence under `out/`. Its redactor +receives the same required `--secret-env` names with variables unset and skips the absent +home-log tree. Redaction and upload still run after test failures. + ## Failure issues When the test or redaction step fails, `post-e2e-failure.sh` runs inside the built image @@ -187,22 +230,33 @@ pull request still open — but only once it captured the baseline. "sandbox_repository": "OWNER/REPO", "tests": [ { - "name": "lower-case-with-dashes", - "kind": "canary | recipe", + "name": "recipe-example", + "kind": "recipe", "peak_sessions": 1, "timeout_sec": 300, "trigger_paths": ["scripts/e2e/*"], "recipe": "implementation", "ingredients": {"task": "..."}, - "expected_pull_request_state": "open | merged | closed" + "expected_pull_request_state": "open" + }, + { + "name": "clean-install", + "kind": "clean-install", + "peak_sessions": 0, + "timeout_sec": 300, + "trigger_paths": ["src/autoskillit/cli/*", "scripts/e2e/*"] } ] } ``` `recipe`, `ingredients` and `expected_pull_request_state` are required for `recipe` tests -and forbidden for `canary` tests; any other key is rejected. `peak_sessions` must be 1–8 and -`timeout_sec` must keep the derived job timeout within 360 minutes. +and forbidden for `canary` and `clean-install` tests. `expected_failures` is optional only +for `clean-install`: each row contains nonempty `severity` (`error` or `warning`), `check`, +`message`, and `issue` (`https://github.com/TalonT-Org/AutoSkillit/issues/`). +Duplicate diagnostic identities and unknown keys are rejected. `peak_sessions` must be +1–8 for model tests and exactly zero for clean-install. `timeout_sec` must keep the +derived job timeout within 360 minutes. ### Adding a test @@ -216,12 +270,25 @@ and forbidden for `canary` tests; any other key is rejected. `peak_sessions` mus ```bash git archive HEAD | docker build --file scripts/docker/Dockerfile --target user \ --tag autoskillit-e2e:local - -mkdir -p /tmp/e2e/out /tmp/e2e/data && chmod 0777 /tmp/e2e/out /tmp/e2e/data +mkdir -p .autoskillit/temp/e2e/{out,data} +chmod 0777 .autoskillit/temp/e2e/{out,data} MINIMAX_API_KEY=... docker run --rm \ --env MINIMAX_API_KEY --env E2E_SANDBOX_TOKEN \ --volume "$PWD/scripts/e2e:/opt/e2e:ro" \ - --volume /tmp/e2e/out:/artifacts \ - --volume /tmp/e2e/data:/home/autoskillit/.local/share/autoskillit \ + --volume "$PWD/.autoskillit/temp/e2e/out:/artifacts" \ + --volume "$PWD/.autoskillit/temp/e2e/data:/home/autoskillit/.local/share/autoskillit" \ autoskillit-e2e:local \ python3 /opt/e2e/e2e_harness.py run --test canary --out /artifacts ``` + +For clean-install, use a fresh artifact directory and omit credentials and home mounts: + +```bash +mkdir -p .autoskillit/temp/e2e/clean-install-out +chmod 0777 .autoskillit/temp/e2e/clean-install-out +docker run --rm \ + --volume "$PWD/scripts/e2e:/opt/e2e:ro" \ + --volume "$PWD/.autoskillit/temp/e2e/clean-install-out:/artifacts" \ + autoskillit-e2e:local \ + python3 /opt/e2e/e2e_harness.py run --test clean-install --out /artifacts +``` diff --git a/scripts/e2e/catalog.json b/scripts/e2e/catalog.json index e0a967c5bf..8059cae320 100644 --- a/scripts/e2e/catalog.json +++ b/scripts/e2e/catalog.json @@ -11,6 +11,45 @@ "scripts/docker/*", ".github/workflows/e2e.yml" ] + }, + { + "name": "clean-install", + "kind": "clean-install", + "peak_sessions": 0, + "timeout_sec": 300, + "trigger_paths": [ + "src/autoskillit/cli/*", + "src/autoskillit/workspace/*", + "scripts/docker/*", + "scripts/e2e/*", + ".github/workflows/e2e.yml" + ], + "expected_failures": [ + { + "severity": "warning", + "check": "mcp_server_registered", + "message": "autoskillit not registered. Run 'autoskillit install' to install as a plugin, or 'autoskillit init' to register in mcpServers.", + "issue": "https://github.com/TalonT-Org/AutoSkillit/issues/5242" + }, + { + "severity": "warning", + "check": "plugin_cache_exists", + "message": "Plugin cache directory missing: /home/autoskillit/.claude/plugins/cache/autoskillit-local/autoskillit. Run `autoskillit install` to recreate it.", + "issue": "https://github.com/TalonT-Org/AutoSkillit/issues/5243" + }, + { + "severity": "warning", + "check": "plugin_cache_integrity", + "message": "Plugin cache directory is absent, so hook paths could not be verified: /home/autoskillit/.claude/plugins/cache/autoskillit-local/autoskillit. Run `autoskillit install` to rebuild it.", + "issue": "https://github.com/TalonT-Org/AutoSkillit/issues/5243" + }, + { + "severity": "error", + "check": "installation_record_invalid", + "message": "Invalid RECORD path '../../../bin/autoskillit' in /home/autoskillit/.local/share/uv/tools/autoskillit/lib/python3.11/site-packages/autoskillit (active distribution: /home/autoskillit/.local/share/uv/tools/autoskillit/lib/python3.11/site-packages/autoskillit; uv tool: /home/autoskillit/.local/share/uv/tools/autoskillit/lib/python3.11/site-packages/autoskillit)", + "issue": "https://github.com/TalonT-Org/AutoSkillit/issues/5244" + } + ] } ] } diff --git a/scripts/e2e/e2e_catalog.py b/scripts/e2e/e2e_catalog.py index f206f25f5f..e37d106b1d 100644 --- a/scripts/e2e/e2e_catalog.py +++ b/scripts/e2e/e2e_catalog.py @@ -8,6 +8,7 @@ from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path +from types import MappingProxyType CATALOG_PATH = Path(__file__).with_name("catalog.json") # One step fans out to at most six subagents beside its orchestrator and step session, and one @@ -19,13 +20,15 @@ # Image build and load, redaction and artifact upload. JOB_OVERHEAD_MINUTES = 30 MAX_JOB_MINUTES = 360 -KINDS = ("canary", "recipe") +KINDS = ("canary", "recipe", "clean-install") PULL_REQUEST_STATES = ("open", "merged", "closed") _NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9-]*$") _CATALOG_KEYS = frozenset({"sandbox_repository", "tests"}) _COMMON_KEYS = frozenset({"name", "kind", "peak_sessions", "timeout_sec", "trigger_paths"}) _RECIPE_KEYS = frozenset({"recipe", "ingredients", "expected_pull_request_state"}) +_EXPECTED_FAILURE_KEYS = frozenset({"severity", "check", "message", "issue"}) +_BUG_URL_PATTERN = re.compile(r"https://github\.com/TalonT-Org/AutoSkillit/issues/[1-9][0-9]*") class CatalogError(ValueError): @@ -34,7 +37,7 @@ class CatalogError(ValueError): @dataclass(frozen=True, slots=True) class CatalogTest: - """One live E2E test and the resources it is allowed to use.""" + """One E2E test and the resources it is allowed to use.""" name: str kind: str @@ -44,6 +47,7 @@ class CatalogTest: recipe: str | None ingredients: tuple[tuple[str, str], ...] expected_pull_request_state: str | None + expected_failures: tuple[Mapping[str, str], ...] = () @dataclass(frozen=True, slots=True) @@ -118,6 +122,32 @@ def _parse_recipe_fields( return recipe, ingredients, state +def _parse_expected_failures(value: object, where: str) -> tuple[Mapping[str, str], ...]: + where = f"{where}.expected_failures" + if not isinstance(value, list): + raise CatalogError(f"{where} must be a list") + rows: list[Mapping[str, str]] = [] + seen: set[tuple[str, str, str]] = set() + for index, raw in enumerate(value): + location = f"{where}[{index}]" + if not isinstance(raw, dict): + raise CatalogError(f"{location} must be an object") + _require_keys(raw, _EXPECTED_FAILURE_KEYS, location) + row = {key: _require_str(raw[key], f"{location}.{key}") for key in _EXPECTED_FAILURE_KEYS} + if any(not item.strip() for item in row.values()): + raise CatalogError(f"{location} fields must be non-empty") + if row["severity"] not in ("error", "warning"): + raise CatalogError(f"{location}.severity must be error or warning") + if not _BUG_URL_PATTERN.fullmatch(row["issue"]): + raise CatalogError(f"{location}.issue must link to an AutoSkillit bug issue") + identity = (row["severity"], row["check"], row["message"]) + if identity in seen: + raise CatalogError(f"{location}: duplicate diagnostic {identity}") + seen.add(identity) + rows.append(MappingProxyType(row)) + return tuple(rows) + + def _parse_test(raw: object, index: int) -> CatalogTest: where = f"tests[{index}]" if not isinstance(raw, dict): @@ -125,7 +155,10 @@ def _parse_test(raw: object, index: int) -> CatalogTest: kind = raw.get("kind") if kind not in KINDS: raise CatalogError(f"{where}.kind must be one of {KINDS}") - _require_keys(raw, _COMMON_KEYS | _RECIPE_KEYS if kind == "recipe" else _COMMON_KEYS, where) + keys = _COMMON_KEYS | _RECIPE_KEYS if kind == "recipe" else _COMMON_KEYS + if kind == "clean-install" and "expected_failures" in raw: + keys |= {"expected_failures"} + _require_keys(raw, keys, where) name = _require_str(raw["name"], f"{where}.name") if not _NAME_PATTERN.fullmatch(name): raise CatalogError(f"{where}.name must match {_NAME_PATTERN.pattern}") @@ -136,13 +169,17 @@ def _parse_test(raw: object, index: int) -> CatalogTest: name=name, kind=kind, peak_sessions=_require_int( - raw["peak_sessions"], f"{where}.peak_sessions", low=1, high=MAX_PEAK_SESSIONS + raw["peak_sessions"], + f"{where}.peak_sessions", + low=0 if kind == "clean-install" else 1, + high=0 if kind == "clean-install" else MAX_PEAK_SESSIONS, ), timeout_sec=_require_int(raw["timeout_sec"], f"{where}.timeout_sec", low=1), trigger_paths=_parse_trigger_paths(raw["trigger_paths"], f"{where}.trigger_paths"), recipe=recipe, ingredients=ingredients, expected_pull_request_state=state, + expected_failures=_parse_expected_failures(raw.get("expected_failures", []), where), ) if job_timeout_minutes(test) > MAX_JOB_MINUTES: raise CatalogError(f"{where}.timeout_sec exceeds the {MAX_JOB_MINUTES}-minute job limit") diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index 071dcf2aad..558be58abd 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -10,6 +10,7 @@ import subprocess import sys import tempfile +import time from collections.abc import Callable, Mapping, Sequence from pathlib import Path from typing import Any @@ -33,6 +34,16 @@ _SCRUBBED_ENV = frozenset({"CLAUDE_CODE_OAUTH_TOKEN", *SECRET_ENV}) _STDERR_TAIL_CHARS = 2000 +CLEAN_INSTALL_ALLOWED_WARNINGS: dict[tuple[str, str], str] = { + ( + "pytest_temp_capacity", + "/dev/shm: 67108864 bytes free of 67108864 total; 0 pytest generations under " + "/dev/shm/autoskillit-pytest-1000 (count unavailable: [Errno 2] No such file or " + "directory: '/dev/shm/autoskillit-pytest-1000') (below 2000000000-byte threshold); " + "run: task cleanup-shm", + ): "Docker's default shared-memory allocation is sufficient for this scenario, which " + "runs no pytest and creates no pytest generations.", +} Runner = Callable[..., subprocess.CompletedProcess[str]] CallResult = tuple[subprocess.CompletedProcess[str] | None, str | None] @@ -58,18 +69,42 @@ def run_command( ) -def _call(runner: Runner, argv: Sequence[str], *, what: str, **kwargs: Any) -> CallResult: +def _call( + runner: Runner, argv: Sequence[str], *, what: str, evidence: Path | None = None, **kwargs: Any +) -> CallResult: """Run *argv*, turning a nonzero exit, a timeout or a launch error into a failure string.""" + result = None + failure: str | None + stdout: str | bytes | None + stderr: str | bytes | None try: result = runner(argv, **kwargs) except subprocess.TimeoutExpired as exc: - return None, f"{what}: timed out after {exc.timeout}s" + failure = f"{what}: timed out after {exc.timeout}s" + stdout, stderr, outcome = exc.stdout, exc.stderr, "timeout" except OSError as exc: - return None, f"{what}: {exc}" - if result.returncode != 0: - stderr = (result.stderr or "").strip()[-_STDERR_TAIL_CHARS:] - return result, f"{what}: exit {result.returncode}: {stderr}" - return result, None + failure = f"{what}: {exc}" + stdout, stderr, outcome = "", str(exc), "launch_error" + else: + stdout, stderr, outcome = result.stdout, result.stderr, "completed" + failure = None + if result.returncode != 0: + detail = (result.stderr or "").strip()[-_STDERR_TAIL_CHARS:] + failure = f"{what}: exit {result.returncode}: {detail}" + if evidence is not None: + for stream, value in (("stdout", stdout), ("stderr", stderr)): + data = value if isinstance(value, bytes) else (value or "").encode("utf-8") + evidence.with_suffix(f".{stream}.txt").write_bytes(data) + record = { + "argv": list(argv), + "outcome": outcome, + "returncode": result.returncode if result is not None else None, + "failure": failure, + } + evidence.with_suffix(".command.json").write_text( + json.dumps(record, indent=2) + "\n", encoding="utf-8" + ) + return result, failure def child_env(env: Mapping[str, str]) -> dict[str, str]: @@ -339,6 +374,96 @@ def run_recipe( return failures +def check_clean_install_doctor( + stdout: str, expected_failures: Sequence[Mapping[str, str]] +) -> tuple[list[str], list[dict[str, str]]]: + try: + payload = json.loads(stdout) + except json.JSONDecodeError as exc: + return [f"doctor: invalid JSON: {exc}"], [] + if not isinstance(payload, dict): + return ["doctor: report must be an object"], [] + rows = payload.get("results") + if not isinstance(rows, list) or not rows: + return ["doctor: results must be a non-empty list"], [] + expected = {(row["severity"], row["check"], row["message"]): row for row in expected_failures} + failures: list[str] = [] + matched: dict[tuple[str, str, str], dict[str, str]] = {} + for index, row in enumerate(rows): + if ( + not isinstance(row, dict) + or not isinstance(row.get("severity"), str) + or row["severity"] not in ("ok", "info", "warning", "error") + or not isinstance(row.get("check"), str) + or not isinstance(row.get("message"), str) + ): + failures.append(f"doctor: invalid result at index {index}") + continue + severity, check, message = row["severity"], row["check"], row["message"] + if severity in ("ok", "info") or ( + severity == "warning" and CLEAN_INSTALL_ALLOWED_WARNINGS.get((check, message)) + ): + continue + identity = (severity, check, message) + if identity in expected: + matched[identity] = dict(expected[identity]) + else: + failures.append(f"doctor: unexpected {severity} {check}: {message}") + for identity, row in expected.items(): + if identity not in matched: + failures.append( + f"doctor: expected diagnostic absent {identity}; remove stale expected failure " + f"for fixed bug {row['issue']}" + ) + return failures, list(matched.values()) + + +def run_clean_install( + test: e2e_catalog.CatalogTest, out: Path, env: Mapping[str, str], runner: Runner +) -> tuple[list[str], list[dict[str, str]]]: + deadline = time.monotonic() + test.timeout_sec + scrubbed = { + name: value for name, value in child_env(env).items() if not name.startswith("OPENAI_") + } + scratch_root = out / ".autoskillit" / "temp" / "clean-install" + scratch_root.mkdir(parents=True, exist_ok=True) + scratch = Path(tempfile.mkdtemp(dir=scratch_root)) + commands = ( + ("install", ["autoskillit", "install"]), + ("git-init", ["git", "init"]), + ("init", ["autoskillit", "init", "--test-command", "git diff --check"]), + ("doctor", ["autoskillit", "doctor", "--output-json"]), + ) + try: + for stage, argv in commands: + if stage == "init": + (scratch / ".pre-commit-config.yaml").write_text( + "repos:\n - repo: https://github.com/gitleaks/gitleaks\n" + " rev: v8.30.0\n hooks:\n - id: gitleaks\n", + encoding="utf-8", + ) + remaining = deadline - time.monotonic() + if remaining <= 0: + return [f"clean-install: timeout budget exhausted before {stage}"], [] + result, failure = _call( + runner, + argv, + what=stage, + evidence=out / stage, + cwd=scratch, + env=scrubbed, + timeout=remaining, + ) + if stage == "doctor": + shutil.copyfile(out / "doctor.stdout.txt", out / "doctor.json") + if failure is not None: + return [failure], [] + assert result is not None + return check_clean_install_doctor(result.stdout, test.expected_failures) + finally: + shutil.rmtree(scratch) + + def _run_test( test: e2e_catalog.CatalogTest, catalog: e2e_catalog.Catalog, @@ -347,18 +472,20 @@ def _run_test( home: Path, env: Mapping[str, str], runner: Runner, -) -> list[str]: +) -> tuple[list[str], list[dict[str, str]]]: + if test.kind == "clean-install": + return run_clean_install(test, out, env, runner) required = SECRET_ENV if test.kind == "recipe" else ("MINIMAX_API_KEY",) missing = [f"{name} is not set" for name in required if not env.get(name)] if missing: - return missing + return missing, [] write_claude_settings(home, env["MINIMAX_API_KEY"]) install_autoskillit_config(home) scrubbed = child_env(env) if test.kind == "canary": - return run_canary(test, out, scrubbed, runner) + return run_canary(test, out, scrubbed, runner), [] token = env["E2E_SANDBOX_TOKEN"] - return run_recipe(test, catalog.sandbox_repository, token, out, scrubbed, runner) + return run_recipe(test, catalog.sandbox_repository, token, out, scrubbed, runner), [] def run_test( @@ -372,11 +499,19 @@ def run_test( ) -> list[str]: """Run *test* and always record ``out/result.json``; return the failures.""" out.mkdir(parents=True, exist_ok=True) + matched_findings: list[dict[str, str]] = [] try: - failures = _run_test(test, catalog, out=out, home=home, env=env, runner=runner) + failures, matched_findings = _run_test( + test, catalog, out=out, home=home, env=env, runner=runner + ) except Exception as exc: failures = [f"harness error: {exc}"] result = {"test": test.name, "passed": not failures, "failures": failures} + if test.kind == "clean-install": + outcome = "failed" if failures else "expected_failure" if matched_findings else "passed" + result.update( + outcome=outcome, passed=outcome == "passed", expected_findings=matched_findings + ) (out / "result.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") return failures @@ -447,7 +582,10 @@ def _run(test_name: str, out: Path, catalog_path: Path) -> int: ) for failure in failures: print(f"e2e_harness: FAIL {failure}", file=sys.stderr) - print(f"e2e_harness: {test.name} {'failed' if failures else 'passed'}") + outcome = "failed" if failures else "passed" + if test.kind == "clean-install": + outcome = json.loads((out / "result.json").read_text(encoding="utf-8"))["outcome"] + print(f"e2e_harness: {test.name} {outcome}") return 1 if failures else 0 @@ -462,7 +600,7 @@ def _redact_command(dest: Path, secret_env: Sequence[str], sources: Sequence[str def main(argv: Sequence[str]) -> int: - """``run`` exits 0 iff the test passed; ``redact`` exits 0 iff the redacted copy is clean.""" + """``run`` accepts passes and expected failures; ``redact`` requires a clean copy.""" parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest="command", required=True) run = commands.add_parser("run", help="Run one catalog test.") diff --git a/scripts/e2e/e2e_select.py b/scripts/e2e/e2e_select.py index a5192dd515..906e66a69b 100644 --- a/scripts/e2e/e2e_select.py +++ b/scripts/e2e/e2e_select.py @@ -167,8 +167,12 @@ def select_for_event( def matrix_json(selection: Selection, catalog: e2e_catalog.Catalog) -> str: include = [ - {"test": name, "timeout_minutes": e2e_catalog.job_timeout_minutes(catalog.get(name))} - for name in selection.tests + { + "test": test.name, + "kind": test.kind, + "timeout_minutes": e2e_catalog.job_timeout_minutes(test), + } + for test in (catalog.get(name) for name in selection.tests) ] return json.dumps({"include": include}, separators=(",", ":")) diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index d15179b334..1e800b9d3e 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import math import os import stat import subprocess @@ -41,6 +42,7 @@ {"type": "result", "is_error": False, "result": "AUTOSKILLIT-E2E-CANARY-OK"} ) _SKILL_NOTICE = "2 skills unavailable on this backend (codex): a, b" +_CLEAN_INSTALL_ISSUE = "https://github.com/TalonT-Org/AutoSkillit/issues/5231" @dataclass @@ -164,6 +166,80 @@ def _run_canary(tmp_path: Path, runner: FakeRunner, monkeypatch, env=None): return failures, json.loads((out / "result.json").read_text(encoding="utf-8")) +def _clean_install_catalog(expected_failures: list[dict[str, str]] | None = None): + test = { + "name": "clean-install", + "kind": "clean-install", + "peak_sessions": 0, + "timeout_sec": 90, + "trigger_paths": ["src/*"], + } + if expected_failures is not None: + test["expected_failures"] = expected_failures + return e2e_catalog.parse_catalog({"sandbox_repository": SANDBOX, "tests": [test]}) + + +def _clean_install_runner( + *, + doctor_stdout: str = ( + '{"results": [{"severity": "ok", "check": "install", "message": "ready"}, ' + '{"severity": "info", "check": "backend_onboarding", "message": "optional setup"}]}' + ), + responses: dict[str, subprocess.CompletedProcess[str] | BaseException] | None = None, +): + responses = responses or {} + observed: dict[str, object] = {} + runner: FakeRunner + + def handle(argv: list[str]): + if argv[:2] == ["autoskillit", "install"]: + stage = "install" + elif argv == ["git", "init"]: + stage = "git-init" + elif argv[:2] == ["autoskillit", "init"]: + stage = "init" + call = runner.calls[-1] + observed["cwd"] = call.cwd + if call.cwd is not None: + observed["scanner_config"] = (call.cwd / ".pre-commit-config.yaml").read_text() + elif argv == ["autoskillit", "doctor", "--output-json"]: + stage = "doctor" + else: + return None + if stage in responses: + return responses[stage] + return _completed(argv, doctor_stdout if stage == "doctor" else f"{stage} ok\n") + + runner = FakeRunner(handle) + return runner, observed + + +def _run_clean_install( + tmp_path: Path, + runner: FakeRunner, + monkeypatch: pytest.MonkeyPatch, + *, + expected_failures: list[dict[str, str]] | None = None, + env: dict[str, str] | None = None, +): + catalog = _clean_install_catalog(expected_failures) + out = tmp_path / "out" + failures = harness.run_test( + catalog.get("clean-install"), + catalog, + out=out, + home=tmp_path / "separate-home", + env=env if env is not None else {"HOME": str(tmp_path / "home"), "PATH": "/usr/bin"}, + runner=runner, + ) + result = json.loads((out / "result.json").read_text(encoding="utf-8")) + return failures, result, out, catalog + + +def _diagnostic(severity: str, check: str, message: str) -> dict[str, str]: + return {"severity": severity, "check": check, "message": message} + + class TestProviderSetup: def test_settings_env_is_the_minimax_map_and_keeps_other_keys(self, tmp_path: Path) -> None: settings_path = tmp_path / ".claude" / "settings.json" @@ -428,6 +504,314 @@ def test_missing_minimax_key_fails_before_any_launch( assert runner.calls == [] +class TestCleanInstallFlow: + @pytest.mark.parametrize("credentials", [False, True]) + def test_clean_flow_uses_one_scratch_directory_and_keeps_evidence( + self, tmp_path, monkeypatch, credentials + ): + runner, observed = _clean_install_runner() + env = { + "HOME": str(tmp_path / "home"), + "PATH": "/usr/bin", + "XDG_CONFIG_HOME": str(tmp_path / "xdg"), + } + if credentials: + env.update( + _env(HOME=env["HOME"], OPENAI_API_KEY="openai-secret", OPENAI_ORG_ID="org-test") + ) + failures, result, out, catalog = _run_clean_install(tmp_path, runner, monkeypatch, env=env) + + assert failures == [] + assert result["outcome"] == "passed" + assert result["expected_findings"] == [] + assert [call.argv for call in runner.calls] == [ + ["autoskillit", "install"], + ["git", "init"], + ["autoskillit", "init", "--test-command", "git diff --check"], + ["autoskillit", "doctor", "--output-json"], + ] + scratch = runner.calls[0].cwd + assert scratch is not None + assert all(call.cwd == scratch for call in runner.calls) + assert observed["cwd"] == scratch + assert observed["scanner_config"] == ( + "repos:\n - repo: https://github.com/gitleaks/gitleaks\n" + " rev: v8.30.0\n hooks:\n - id: gitleaks\n" + ) + assert all(call.env == runner.calls[0].env for call in runner.calls) + assert runner.calls[0].env["HOME"] == env["HOME"] + assert str(tmp_path / "separate-home") != runner.calls[0].env["HOME"] + assert not any(name.startswith("OPENAI_") for name in runner.calls[0].env) + assert not any( + name in runner.calls[0].env + for name in ( + "ANTHROPIC_API_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", + "MINIMAX_API_KEY", + "E2E_SANDBOX_TOKEN", + ) + ) + timeouts = [call.timeout for call in runner.calls] + assert all( + value is not None and math.isfinite(value) and 0 < value <= 90 for value in timeouts + ) + assert all(later <= earlier for earlier, later in zip(timeouts, timeouts[1:])) + for stage in ("install", "git-init", "init", "doctor"): + assert (out / f"{stage}.stdout.txt").is_file() + assert (out / f"{stage}.stderr.txt").is_file() + assert (out / f"{stage}.command.json").is_file() + assert (out / "doctor.json").read_text() == (out / "doctor.stdout.txt").read_text() + assert not (tmp_path / "separate-home" / ".claude" / "settings.json").exists() + assert not (tmp_path / "separate-home" / ".autoskillit" / "config.yaml").exists() + assert catalog.get("clean-install").peak_sessions == 0 + assert not scratch.exists() + + @pytest.mark.parametrize("additional_finding", [False, True]) + def test_expected_finding_admits_only_the_linked_diagnostic( + self, tmp_path, monkeypatch, additional_finding + ): + diagnostic = _diagnostic("error", "runtime", "known clean-install failure") + expected = {**diagnostic, "issue": _CLEAN_INSTALL_ISSUE} + rows = [diagnostic] + if additional_finding: + rows.append(_diagnostic("error", "another-check", "new defect")) + runner, _ = _clean_install_runner(doctor_stdout=json.dumps({"results": rows})) + failures, result, _out, _catalog = _run_clean_install( + tmp_path, runner, monkeypatch, expected_failures=[expected] + ) + + assert bool(failures) == additional_finding + assert result["outcome"] == ("failed" if additional_finding else "expected_failure") + assert result["passed"] is False + assert result["expected_findings"] == [expected] + + @pytest.mark.parametrize( + ("doctor_stdout", "expected_failure"), + [ + ("not json", "doctor: invalid JSON:"), + ("[]", "doctor: report must be an object"), + ("{}", "doctor: results must be a non-empty list"), + ('{"results": {}}', "doctor: results must be a non-empty list"), + ('{"results": []}', "doctor: results must be a non-empty list"), + ('{"results": [null]}', "doctor: invalid result at index 0"), + ( + '{"results": [{"severity": "fatal", "check": "x", "message": "y"}]}', + "doctor: invalid result at index 0", + ), + ( + '{"results": [{"severity": "warning", "check": "x", "message": 3}]}', + "doctor: invalid result at index 0", + ), + ], + ids=[ + "invalid-json", + "not-object", + "missing-results", + "results-not-list", + "empty-results", + "row-not-object", + "bad-severity", + "bad-message", + ], + ) + def test_invalid_doctor_output_fails( + self, tmp_path, monkeypatch, doctor_stdout, expected_failure + ): + runner, _ = _clean_install_runner(doctor_stdout=doctor_stdout) + failures, result, _out, _catalog = _run_clean_install(tmp_path, runner, monkeypatch) + + assert len(failures) == 1 + assert failures[0].startswith(expected_failure) + assert result["outcome"] == "failed" + + def test_unlisted_warning_and_zero_exit_error_fail(self, tmp_path, monkeypatch): + for name, severity in (("warning", "warning"), ("error", "error")): + case = tmp_path / name + diagnostic = _diagnostic(severity, "new-check", "new diagnostic") + runner, _ = _clean_install_runner(doctor_stdout=json.dumps({"results": [diagnostic]})) + failures, result, _out, _catalog = _run_clean_install(case, runner, monkeypatch) + assert failures + assert result["outcome"] == "failed" + + def test_warning_allowlist_requires_an_exact_check_and_message(self, tmp_path, monkeypatch): + assert all(reason.strip() for reason in harness.CLEAN_INSTALL_ALLOWED_WARNINGS.values()) + key = ("environment", "known warning") + monkeypatch.setattr(harness, "CLEAN_INSTALL_ALLOWED_WARNINGS", {key: "tracked reason"}) + for message, outcome in ((key[1], "passed"), ("changed warning", "failed")): + case = tmp_path / ("allowed" if outcome == "passed" else "changed") + diagnostic = _diagnostic("warning", key[0], message) + runner, _ = _clean_install_runner(doctor_stdout=json.dumps({"results": [diagnostic]})) + failures, result, _out, _catalog = _run_clean_install(case, runner, monkeypatch) + assert (not failures) == (outcome == "passed") + assert result["outcome"] == outcome + + @pytest.mark.parametrize( + "check", + [ + "mcp_server_registered", + "plugin_cache_exists", + "hook_registration", + "hook_registry_drift", + ], + ) + def test_installation_diagnostic_warning_fails_and_is_saved( + self, tmp_path, monkeypatch, check + ): + stdout = json.dumps({"results": [_diagnostic("warning", check, "installation defect")]}) + runner, _ = _clean_install_runner(doctor_stdout=stdout) + failures, result, out, _catalog = _run_clean_install(tmp_path, runner, monkeypatch) + + assert failures + assert result["outcome"] == "failed" + assert check in failures[0] + assert (out / "doctor.json").read_text() == stdout + + @pytest.mark.parametrize("stage", ["install", "init", "doctor"]) + def test_nonzero_command_exit_is_recorded_and_stops_the_flow( + self, tmp_path, monkeypatch, stage + ): + command = { + "install": ["autoskillit", "install"], + "init": ["autoskillit", "init", "--test-command", "git diff --check"], + "doctor": ["autoskillit", "doctor", "--output-json"], + }[stage] + stdout = '{"results": []}' if stage == "doctor" else "partial output" + runner, _ = _clean_install_runner( + responses={stage: _completed(command, stdout, returncode=2, stderr="command failed")} + ) + failures, result, out, _catalog = _run_clean_install(tmp_path, runner, monkeypatch) + + assert failures + assert result["outcome"] == "failed" + assert (out / f"{stage}.command.json").is_file() + assert runner.calls[-1].argv == command + if stage == "doctor": + assert (out / "doctor.json").read_text(encoding="utf-8") == stdout + + @pytest.mark.parametrize("stage", ["install", "init", "doctor"]) + def test_timeout_saves_partial_byte_and_text_streams(self, tmp_path, monkeypatch, stage): + command = { + "install": ["autoskillit", "install"], + "init": ["autoskillit", "init", "--test-command", "git diff --check"], + "doctor": ["autoskillit", "doctor", "--output-json"], + }[stage] + timeout = subprocess.TimeoutExpired( + command, 1.0, output=b"partial stdout", stderr="partial stderr" + ) + runner, _ = _clean_install_runner(responses={stage: timeout}) + failures, result, out, _catalog = _run_clean_install(tmp_path, runner, monkeypatch) + + assert failures + assert result["outcome"] == "failed" + assert (out / f"{stage}.stdout.txt").read_text(encoding="utf-8") == "partial stdout" + assert (out / f"{stage}.stderr.txt").read_text(encoding="utf-8") == "partial stderr" + if stage == "doctor": + assert (out / "doctor.json").read_bytes() == b"partial stdout" + + def test_launch_error_is_explicitly_recorded(self, tmp_path, monkeypatch): + runner, _ = _clean_install_runner( + responses={"install": FileNotFoundError(2, "No such file or directory", "autoskillit")} + ) + failures, result, out, _catalog = _run_clean_install(tmp_path, runner, monkeypatch) + + assert failures + assert result["outcome"] == "failed" + evidence = (out / "install.command.json").read_text(encoding="utf-8") + assert "No such file or directory" in evidence + + def test_exhausted_deadline_does_not_launch_another_command(self, tmp_path, monkeypatch): + ticks = iter([0, 1, 90]) + monkeypatch.setattr(harness.time, "monotonic", lambda: next(ticks)) + runner, _ = _clean_install_runner() + failures, result, _out, _catalog = _run_clean_install(tmp_path, runner, monkeypatch) + + assert "budget exhausted before git-init" in failures[0] + assert result["outcome"] == "failed" + assert len(runner.calls) == 1 + assert runner.calls[0].timeout == 89 + assert not runner.calls[0].cwd.exists() + + def test_stale_expectation_fails_and_unexpected_diagnostic_is_unlisted( + self, tmp_path, monkeypatch + ): + old = {**_diagnostic("error", "runtime", "old message"), "issue": _CLEAN_INSTALL_ISSUE} + current = _diagnostic("error", "runtime", "changed message") + runner, _ = _clean_install_runner(doctor_stdout=json.dumps({"results": [current]})) + failures, result, _out, _catalog = _run_clean_install( + tmp_path, runner, monkeypatch, expected_failures=[old] + ) + + assert failures + assert result["outcome"] == "failed" + assert result["expected_findings"] == [] + + def test_cli_names_expected_failure_and_run_test_handles_exceptions( + self, tmp_path, monkeypatch, capsys + ): + diagnostic = _diagnostic("error", "runtime", "known clean-install failure") + expected = {**diagnostic, "issue": _CLEAN_INSTALL_ISSUE} + catalog_path = tmp_path / "catalog.json" + catalog_path.write_text( + json.dumps( + { + "sandbox_repository": SANDBOX, + "tests": [ + { + "name": "clean-install", + "kind": "clean-install", + "peak_sessions": 0, + "timeout_sec": 90, + "trigger_paths": ["src/*"], + "expected_failures": [expected], + } + ], + } + ), + encoding="utf-8", + ) + scratch_parent = tmp_path / "cli-scratch" + scratch_parent.mkdir() + monkeypatch.setattr(tempfile, "tempdir", str(scratch_parent)) + runner, _ = _clean_install_runner(doctor_stdout=json.dumps({"results": [diagnostic]})) + monkeypatch.setattr(harness, "run_command", runner) + + assert ( + harness.main( + [ + "run", + "--test", + "clean-install", + "--out", + str(tmp_path / "cli-out"), + "--catalog", + str(catalog_path), + ] + ) + == 0 + ) + assert "e2e_harness: clean-install expected_failure" in capsys.readouterr().out + + def crash(*_args, **_kwargs): + raise RuntimeError("runner crashed") + + monkeypatch.setattr(harness, "run_command", crash) + assert ( + harness.main( + [ + "run", + "--test", + "clean-install", + "--out", + str(tmp_path / "crash-out"), + "--catalog", + str(catalog_path), + ] + ) + == 1 + ) + assert "harness error: runner crashed" in capsys.readouterr().err + + class TestRedaction: def test_secrets_are_replaced_in_nested_text_and_binary_files(self, tmp_path: Path) -> None: source = tmp_path / "out" diff --git a/tests/infra/test_e2e_select.py b/tests/infra/test_e2e_select.py index d21b5f7f25..72ef31376d 100644 --- a/tests/infra/test_e2e_select.py +++ b/tests/infra/test_e2e_select.py @@ -4,7 +4,9 @@ import hashlib import itertools +from collections.abc import Mapping, MutableMapping from pathlib import Path +from typing import cast import pytest @@ -56,6 +58,35 @@ def _entry(name: str, trigger_paths: tuple[str, ...] = ("src/*",), **overrides) return entry +_CLEAN_INSTALL_TRIGGER_PATHS = ( + "src/autoskillit/cli/*", + "src/autoskillit/workspace/*", + "scripts/docker/*", + "scripts/e2e/*", + ".github/workflows/e2e.yml", +) + + +def _clean_install_entry(**overrides) -> dict: + entry = _entry( + "clean-install", + _CLEAN_INSTALL_TRIGGER_PATHS, + kind="clean-install", + peak_sessions=0, + timeout_sec=300, + ) + entry.update(overrides) + return entry + + +_EXPECTED_FAILURE = { + "severity": "error", + "check": "install-check", + "message": "expected install failure", + "issue": "https://github.com/TalonT-Org/AutoSkillit/issues/123", +} + + def _raw_catalog(*entries: dict) -> dict: return {"sandbox_repository": "owner/sandbox", "tests": list(entries)} @@ -245,6 +276,16 @@ def test_matrix_for_an_empty_selection_is_parseable() -> None: assert select.matrix_json(selection, _catalog(_entry("a"))) == '{"include":[]}' +def test_matrix_includes_kind_from_each_catalog_entry() -> None: + catalog = _catalog(_clean_install_entry(), _entry("canary")) + selection = select.Selection(("clean-install", "canary"), "selected") + assert select.matrix_json(selection, catalog) == ( + '{"include":[{"test":"clean-install","kind":"clean-install",' + '"timeout_minutes":37},{"test":"canary","kind":"canary",' + '"timeout_minutes":37}]}' + ) + + _RECIPE_FIELDS = { "recipe": "implementation", "ingredients": {"task": "Add a greeting"}, @@ -265,7 +306,11 @@ def _recipe_entry_without(field: str) -> dict: "bad-name": _raw_catalog(_entry("Bad_Name")), "unknown-kind": _raw_catalog(_entry("a", kind="smoke")), "zero-peak-sessions": _raw_catalog(_entry("a", peak_sessions=0)), + "zero-recipe-peak-sessions": _raw_catalog( + _entry("impl", kind="recipe", peak_sessions=0, **_RECIPE_FIELDS) + ), "nine-peak-sessions": _raw_catalog(_entry("a", peak_sessions=9)), + "clean-install-nonzero-peak-sessions": _raw_catalog(_clean_install_entry(peak_sessions=1)), "timeout-over-job-limit": _raw_catalog(_entry("a", timeout_sec=_MAX_TIMEOUT_SEC + 1)), "recipe-without-recipe": _raw_catalog(_recipe_entry_without("recipe")), "recipe-without-ingredients": _raw_catalog(_recipe_entry_without("ingredients")), @@ -274,6 +319,64 @@ def _recipe_entry_without(field: str) -> dict: _entry("a", kind="recipe", **{**_RECIPE_FIELDS, "expected_pull_request_state": "draft"}) ), "canary-with-recipe-fields": _raw_catalog(_entry("a", **_RECIPE_FIELDS)), + "clean-install-with-recipe-fields": _raw_catalog(_clean_install_entry(**_RECIPE_FIELDS)), + "clean-install-unknown-key": _raw_catalog(_clean_install_entry(extra=1)), + "expected-failures-on-canary": _raw_catalog( + _entry("a", expected_failures=[_EXPECTED_FAILURE]) + ), + "expected-failures-on-recipe": _raw_catalog( + _entry( + "impl", + kind="recipe", + expected_failures=[_EXPECTED_FAILURE], + **_RECIPE_FIELDS, + ) + ), + "expected-failures-missing-field": _raw_catalog( + _clean_install_entry( + expected_failures=[ + {key: value for key, value in _EXPECTED_FAILURE.items() if key != "message"} + ] + ) + ), + "expected-failures-extra-field": _raw_catalog( + _clean_install_entry(expected_failures=[{**_EXPECTED_FAILURE, "extra": "value"}]) + ), + "expected-failures-empty-message": _raw_catalog( + _clean_install_entry(expected_failures=[{**_EXPECTED_FAILURE, "message": ""}]) + ), + "expected-failures-empty-check": _raw_catalog( + _clean_install_entry(expected_failures=[{**_EXPECTED_FAILURE, "check": ""}]) + ), + "expected-failures-unknown-severity": _raw_catalog( + _clean_install_entry(expected_failures=[{**_EXPECTED_FAILURE, "severity": "info"}]) + ), + "expected-failures-zero-issue": _raw_catalog( + _clean_install_entry( + expected_failures=[ + { + **_EXPECTED_FAILURE, + "issue": "https://github.com/TalonT-Org/AutoSkillit/issues/0", + } + ] + ) + ), + "expected-failures-bad-issue-url": _raw_catalog( + _clean_install_entry( + expected_failures=[{**_EXPECTED_FAILURE, "issue": "https://example.com/issues/123"}] + ) + ), + "expected-failures-duplicate-identity": _raw_catalog( + _clean_install_entry( + expected_failures=[ + _EXPECTED_FAILURE, + { + **_EXPECTED_FAILURE, + "issue": "https://github.com/TalonT-Org/AutoSkillit/issues/124", + }, + ] + ) + ), "empty-trigger-paths": _raw_catalog(_entry("a", ())), "unknown-key": _raw_catalog(_entry("a", extra=1)), "unknown-top-level-key": {**_raw_catalog(_entry("a")), "extra": 1}, @@ -289,6 +392,34 @@ def test_shipped_catalog_holds_the_canary(self) -> None: assert canary.recipe is None assert e2e_catalog.job_timeout_minutes(canary) == 37 + clean_install = catalog.get("clean-install") + assert clean_install.kind == "clean-install" + assert clean_install.peak_sessions == 0 + assert clean_install.timeout_sec == 300 + assert clean_install.trigger_paths == _CLEAN_INSTALL_TRIGGER_PATHS + assert clean_install.recipe is None + + def test_clean_install_expected_failures_are_immutable_mapping_rows(self) -> None: + rows = [{**_EXPECTED_FAILURE}, {**_EXPECTED_FAILURE, "severity": "warning"}] + test = _catalog(_clean_install_entry(expected_failures=rows)).get("clean-install") + + assert isinstance(test.expected_failures, tuple) + assert len(test.expected_failures) == 2 + for row in test.expected_failures: + assert isinstance(row, Mapping) + assert not isinstance(row, MutableMapping) + with pytest.raises(TypeError): + cast(MutableMapping[str, str], row)["message"] = "changed" + + def test_clean_install_expected_failures_allow_distinct_identities(self) -> None: + rows = [ + _EXPECTED_FAILURE, + {**_EXPECTED_FAILURE, "message": "another message"}, + {**_EXPECTED_FAILURE, "severity": "warning"}, + ] + test = _catalog(_clean_install_entry(expected_failures=rows)).get("clean-install") + assert len(test.expected_failures) == 3 + def test_recipe_entry_parses(self) -> None: catalog = _catalog(_entry("impl", kind="recipe", **_RECIPE_FIELDS)) test = catalog.get("impl") diff --git a/tests/infra/test_e2e_select_cli.py b/tests/infra/test_e2e_select_cli.py index a0ed198a84..0049a64170 100644 --- a/tests/infra/test_e2e_select_cli.py +++ b/tests/infra/test_e2e_select_cli.py @@ -16,7 +16,11 @@ REPO_ROOT = Path(__file__).resolve().parents[2] _SCRIPT = REPO_ROOT / "scripts" / "e2e" / "e2e_select.py" REPOSITORY = "TalonT-Org/AutoSkillit" -_CANARY_MATRIX = '{"include":[{"test":"canary","timeout_minutes":37}]}' +_CANARY_MATRIX = '{"include":[{"test":"canary","kind":"canary","timeout_minutes":37}]}' +_BOTH_MATRIX = ( + '{"include":[{"test":"canary","kind":"canary","timeout_minutes":37},' + '{"test":"clean-install","kind":"clean-install","timeout_minutes":37}]}' +) def _run( @@ -65,7 +69,9 @@ def _select(tmp_path: Path, changed_paths: Path, event_name: str, payload: objec ) -def test_labelled_pull_request_selects_the_touched_canary(tmp_path: Path) -> None: +def test_shared_harness_pull_request_selects_both_shipped_tests_deterministically( + tmp_path: Path, +) -> None: changed = tmp_path / "changed-paths.txt" changed.write_text("scripts/e2e/e2e_harness.py\n", encoding="utf-8") payload = _pull_request(changed_files=1, additions=5, labels=("e2e",)) @@ -73,7 +79,11 @@ def test_labelled_pull_request_selects_the_touched_canary(tmp_path: Path) -> Non assert result.returncode == 0, result.stderr outputs = _outputs(result.stdout) assert outputs["selected"] == "true" - assert outputs["matrix"] == _CANARY_MATRIX + assert outputs["matrix"] == _BOTH_MATRIX + + repeated = _select(tmp_path, changed, "pull_request", payload) + assert repeated.returncode == 0, repeated.stderr + assert _outputs(repeated.stdout)["matrix"] == outputs["matrix"] def test_small_unlabelled_pull_request_selects_nothing(tmp_path: Path) -> None: @@ -105,6 +115,19 @@ def test_dispatch_selects_the_named_test(tmp_path: Path) -> None: assert outputs["matrix"] == _CANARY_MATRIX +def test_dispatch_selects_both_named_tests_in_request_order(tmp_path: Path) -> None: + payload = {"inputs": {"tests": "clean-install, canary"}} + result = _select(tmp_path, tmp_path / "missing.txt", "workflow_dispatch", payload) + assert result.returncode == 0, result.stderr + outputs = _outputs(result.stdout) + assert outputs["selected"] == "true" + assert outputs["matrix"] == ( + '{"include":[{"test":"clean-install","kind":"clean-install",' + '"timeout_minutes":37},{"test":"canary","kind":"canary",' + '"timeout_minutes":37}]}' + ) + + def test_dispatch_of_an_unknown_test_fails(tmp_path: Path) -> None: payload = {"inputs": {"tests": "nope"}} result = _select(tmp_path, tmp_path / "missing.txt", "workflow_dispatch", payload) diff --git a/tests/infra/test_e2e_workflow.py b/tests/infra/test_e2e_workflow.py index a83f7b05f9..e719cfd8a9 100644 --- a/tests/infra/test_e2e_workflow.py +++ b/tests/infra/test_e2e_workflow.py @@ -47,6 +47,17 @@ def _all_steps(workflow: dict): yield from job["steps"] +def _model_branch(run: str) -> tuple[str, str]: + lines = run.splitlines() + start = next( + index + for index, line in enumerate(lines) + if line.strip() == "if [[ '${{ matrix.kind }}' != 'clean-install' ]]; then" + ) + end = next(index for index in range(start + 1, len(lines)) if lines[index].strip() == "fi") + return "\n".join(lines[start + 1 : end]), "\n".join(lines[:start] + lines[end + 1 :]) + + class TestTriggers: def test_pull_request_types_include_labeled(self, workflow: dict) -> None: types = _on(workflow)["pull_request"]["types"] @@ -76,6 +87,18 @@ def test_secrets_reach_only_the_test_and_redaction_steps(self, workflow: dict) - assert holders == {"e2e", "redact"} assert all("secrets." not in str(job.get("env", {})) for job in workflow["jobs"].values()) + @pytest.mark.parametrize("step_id", ["e2e", "redact"]) + def test_model_credentials_are_empty_for_clean_install( + self, workflow: dict, step_id: str + ) -> None: + env = _step(workflow, "run", step_id=step_id)["env"] + assert env["MINIMAX_API_KEY"] == ( + "${{ matrix.kind != 'clean-install' && secrets.MINIMAX_API_KEY || '' }}" + ) + assert env["E2E_SANDBOX_TOKEN"] == ( + "${{ matrix.kind != 'clean-install' && secrets.E2E_SANDBOX_TOKEN || '' }}" + ) + def test_report_step_uses_only_the_workflow_token(self, workflow: dict) -> None: env = _step(workflow, "run", name="Report failure")["env"] assert env["GH_TOKEN"] == "${{ github.token }}" @@ -84,9 +107,85 @@ def test_report_step_uses_only_the_workflow_token(self, workflow: dict) -> None: @pytest.mark.parametrize("step_id", ["e2e", "redact"]) def test_docker_run_forwards_secrets_by_name_only(self, workflow: dict, step_id: str) -> None: run = _step(workflow, "run", step_id=step_id)["run"] - forwarded = re.findall(r"--env (\S+)", run) + forwarded = re.findall(r"--env ([A-Z0-9_]+)", run) assert forwarded == ["MINIMAX_API_KEY", "E2E_SANDBOX_TOKEN"] + def test_redaction_keeps_required_secret_names(self, workflow: dict) -> None: + run = _step(workflow, "run", step_id="redact")["run"] + branch, outside = _model_branch(run) + assert re.findall(r"--secret-env (\S+)", outside) == [ + "MINIMAX_API_KEY", + "E2E_SANDBOX_TOKEN", + ] + assert "--secret-env" not in branch + + +@pytest.mark.parametrize( + ("step_id", "common_mounts"), + [ + ( + "e2e", + ( + '--volume "$GITHUB_WORKSPACE/scripts/e2e:/opt/e2e:ro"', + '--volume "$RUNNER_TEMP/e2e/out:/artifacts"', + ), + ), + ( + "redact", + ( + '--volume "$GITHUB_WORKSPACE/scripts/e2e:/opt/e2e:ro"', + '--volume "$RUNNER_TEMP/e2e:/e2e"', + ), + ), + ], +) +def test_clean_install_uses_the_common_user_image_and_artifacts( + workflow: dict, step_id: str, common_mounts: tuple[str, ...] +) -> None: + run = _step(workflow, "run", step_id=step_id)["run"] + branch, outside = _model_branch(run) + + assert run.count("docker_args=()") == 1 + assert '"${docker_args[@]}"' in outside + assert "docker run --rm" in outside + assert '"$IMAGE"' in outside + for mount in common_mounts: + assert mount in outside + assert not any(mount in branch for mount in common_mounts) + + credential_lines = [ + line.strip() for line in branch.splitlines() if "--env MINIMAX_API_KEY" in line + ] + assert len(credential_lines) == 1 + assert credential_lines[0].startswith("docker_args+=(") + assert re.findall(r"--env ([A-Z0-9_]+)", credential_lines[0]) == [ + "MINIMAX_API_KEY", + "E2E_SANDBOX_TOKEN", + ] + assert not re.search(r"--env (?:MINIMAX_API_KEY|E2E_SANDBOX_TOKEN)\b", outside) + + data_mounts = re.findall(r'--volume "\$RUNNER_TEMP/e2e/data:[^"]+"', run) + if step_id == "e2e": + assert len(data_mounts) == 1 + assert data_mounts[0] in branch + assert data_mounts[0] not in outside + assert any( + line.strip().startswith("docker_args+=(") and data_mounts[0] in line + for line in branch.splitlines() + ) + else: + assert not data_mounts + + +def test_user_image_is_built_for_the_matrix_run_and_workflow_does_not_install_or_init( + workflow: dict, +) -> None: + build = _step(workflow, "run", name="Build the user image") + assert build["with"]["target"] == "user" + for step in _all_steps(workflow): + run = step.get("run", "") + assert not re.search(r"\bautoskillit\s+(?:install|init)\b", run) + class TestRunJob: def test_one_repository_wide_queue(self, workflow: dict) -> None: