Add headless sandbox smoke coverage and verified teardown

.github/workflows/e2e.yml

@@ -106,16 +106,48 @@ jobs:
           E2E_SANDBOX_TOKEN: ${{ matrix.kind != 'clean-install' && secrets.E2E_SANDBOX_TOKEN || '' }}
           TEST: ${{ matrix.test }}
         run: |
+          set -euo pipefail
           docker_args=()
           if [[ '${{ matrix.kind }}' != 'clean-install' ]]; then
             docker_args+=(--env MINIMAX_API_KEY --env E2E_SANDBOX_TOKEN)
             docker_args+=(--volume "$RUNNER_TEMP/e2e/data:/home/autoskillit/.local/share/autoskillit")
           fi
-          docker run --rm "${docker_args[@]}" \
-            --volume "$GITHUB_WORKSPACE/scripts/e2e:/opt/e2e:ro" \
-            --volume "$RUNNER_TEMP/e2e/out:/artifacts" \
-            "$IMAGE" \
-            python3 /opt/e2e/e2e_harness.py run --test "$TEST" --out /artifacts
+          if [[ "$TEST" == 'headless-smoke' ]]; then
+            cid_file="$RUNNER_TEMP/e2e/out/${TEST}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.cid"
+            if [[ -e "$cid_file" ]]; then
+              echo "refusing to reuse existing container ID file: $cid_file" >&2
+              exit 1
+            fi
+
+            source "$GITHUB_WORKSPACE/scripts/e2e/smoke-cleanup.sh"
+            cleanup_smoke() {
+              model_status=$?
+              set +e
+              lifecycle_file="$RUNNER_TEMP/e2e/out/${TEST}-lifecycle.txt"
+              cleanup_smoke_resources "$cid_file" "$GITHUB_WORKSPACE/scripts/e2e" \
+                "$RUNNER_TEMP/e2e/out" "$IMAGE" "$TEST"
+
+              trap - EXIT
+              finish_smoke_lifecycle "$lifecycle_file" "$model_status"
+              exit $?
+            }
+            trap cleanup_smoke EXIT
+
+            # The harness budget is 900 seconds plus 120 seconds of grace. The job
+            # timeout reserves the catalog's remaining overhead for teardown and upload.
+            timeout --signal=TERM --kill-after=30s 1020s docker run --rm \
+              --cidfile "$cid_file" "${docker_args[@]}" \
+              --volume "$GITHUB_WORKSPACE/scripts/e2e:/opt/e2e:ro" \
+              --volume "$RUNNER_TEMP/e2e/out:/artifacts" \
+              "$IMAGE" \
+              python3 /opt/e2e/e2e_harness.py run --test "$TEST" --out /artifacts
+          else
+            docker run --rm "${docker_args[@]}" \
+              --volume "$GITHUB_WORKSPACE/scripts/e2e:/opt/e2e:ro" \
+              --volume "$RUNNER_TEMP/e2e/out:/artifacts" \
+              "$IMAGE" \
+              python3 /opt/e2e/e2e_harness.py run --test "$TEST" --out /artifacts
+          fi
 
       - name: Redact secrets from the artifacts
         id: redact

scripts/e2e/README.md

@@ -10,7 +10,7 @@ This directory holds everything the workflow runs:
 | `catalog.json` | Registered E2E tests (the only source of test names, budgets and trigger paths) |
 | `e2e_catalog.py` | Stdlib loader and validator for the catalog |
 | `e2e_select.py` | Picks the tests an event runs (`select`) and judges the required check (`gate`) |
-| `e2e_harness.py` | Runs one test inside the image (`run`) and redacts its artifacts (`redact`) |
+| `e2e_harness.py` | Runs one test (`run`), performs exact sandbox teardown (`cleanup`) and redacts artifacts (`redact`) |
 | `autoskillit-config.yaml` | AutoSkillit user-layer config the harness installs |
 | `post-e2e-failure.sh` | Appends a failure to the test's tracking issue |
 
@@ -35,7 +35,9 @@ model tests:
    | Actions: read | `wait_for_ci` |
    | Commit statuses: read | `wait_for_ci` |
 
-   The canary needs only `MINIMAX_API_KEY`; no Anthropic credential is used anywhere.
+   The canary needs only `MINIMAX_API_KEY`; no Anthropic API key or Claude OAuth credential
+   is supplied. The harness writes MiniMax's key into the Claude-compatible settings file
+   because MiniMax serves the Anthropic Messages API.
 3. Create the `e2e` label in this repository.
 4. In branch protection, mark only `e2e-gate` as required — never `select` or `run (…)`.
 
@@ -216,12 +218,48 @@ requests. The test container never sees the workflow token.
 
 ## Recipe test assertions
 
-A recipe test passes when `fleet run` exits 0, its envelope — the last non-empty stdout
+Regular recipe tests pass when `fleet run` exits 0, its envelope — the last non-empty stdout
 line, since earlier lines can be plain-text notices — reports `"success": true`, and exactly
 one sandbox pull request numbered above the pre-run baseline exists in the catalog's
 `expected_pull_request_state` with a non-empty diff. `gh` reports states in upper case; the
-harness lower-cases them. Afterwards the harness closes, with `--delete-branch`, every new
-pull request still open — but only once it captured the baseline.
+harness lower-cases them. Its existing cleanup closes newly opened PRs with their branches.
+
+`headless-smoke` exercises one orchestrator and one recipe worker (`peak_sessions: 2`). The
+harness stages `sandbox-smoke.yaml` from the read-only `/opt/e2e` mount into the disposable
+sandbox clone's `.autoskillit/recipes/` directory before invoking fleet by recipe name. This
+lets the clone's project name resolve the staged recipe; the host scripts remain read-only.
+Fleet resolves `base_branch` from the sandbox project's existing
+`branching.default_base_branch` configuration, matching the clone's base branch.
+The worker changes and commits only `sandbox/text.py` and `tests/test_smoke_canary.py`.
+The orchestrator runs the sandbox test gate, pushes the assigned per-run branch, opens a PR
+and closes it. Before cleanup, the harness verifies the successful
+envelope, that exactly one PR for the assigned branch is closed, the complete PR diff contains
+only those prescribed paths, and the PR head, remote ref and fetched commit agree. The source
+and test must contain the requested sentinel and an executable assertion. It saves the PR,
+commit, diff and branch evidence for redaction and upload.
+
+The harness writes a lifecycle descriptor before fleet starts, so cleanup does not depend on
+the model process or a fleet result. The workflow wraps this test's model container in a
+per-run Docker CID file. Its exit trap captures the model result, removes the recorded
+container by its exact ID, verifies that ID is absent, then launches a fresh user-image
+container for `e2e_harness.py cleanup`. That command uses only `E2E_SANDBOX_TOKEN`, closes any
+remaining PR for the assigned branch, deletes that exact remote branch and verifies no open
+PR remains and the ref is absent. Cleanup runs after model success, failure or timeout, and its result is part of the step
+status. The normal artifact redaction step then runs even when the test step failed. Recipes
+without this fixture keep their existing in-container cleanup path.
+
+Recipe `expected_failures` rows identify an individual, verified product defect by exact
+`severity: error`, `check` equal to the failed envelope's `error`, `message` equal to its
+`user_visible_message`, and an open bug issue URL. Only that exact failed envelope can be
+accepted; process timeouts, malformed results, other failures and cleanup failures remain
+fatal. An accepted defect is provisional until mandatory branch/PR cleanup succeeds. A clean
+run that observes a configured row makes it stale and fails with instructions to remove the
+row and close the bug. The smoke catalog currently records
+[#5250](https://github.com/TalonT-Org/AutoSkillit/issues/5250) for `open_kitchen_failed`
+during kitchen initialization, before the recipe executes. This row does not accept
+arbitrary failures from the recipe's `failed` stop step. Fleet translates the L3 result's
+`reason` into the failed command envelope's `error` and `user_visible_message` fields;
+the harness matches that envelope rather than the recipe payload's `failed_step`/`reason`.
 
 ## Catalog schema
 
@@ -251,10 +289,12 @@ pull request still open — but only once it captured the baseline.
 ```
 
 `recipe`, `ingredients` and `expected_pull_request_state` are required for `recipe` tests
-and forbidden for `canary` and `clean-install` tests. `expected_failures` is optional only
-for `clean-install`: each row contains nonempty `severity` (`error` or `warning`), `check`,
-`message`, and `issue` (`https://github.com/TalonT-Org/AutoSkillit/issues/<number>`).
-Duplicate diagnostic identities and unknown keys are rejected. `peak_sessions` must be
+and forbidden for `canary` and `clean-install` tests. `expected_failures` is optional for
+`clean-install` and recipe tests. Clean-install rows use the exact doctor `severity`, `check`
+and `message`; recipe rows require `severity: error`, `check` and `message` matching the failed
+envelope. Every row includes an open bug issue URL
+(`https://github.com/TalonT-Org/AutoSkillit/issues/<number>`). Duplicate diagnostic identities
+and unknown keys are rejected. `peak_sessions` must be
 1–8 for model tests and exactly zero for clean-install. `timeout_sec` must keep the
 derived job timeout within 360 minutes.
 
@@ -292,3 +332,59 @@ docker run --rm \
   autoskillit-e2e:local \
   python3 /opt/e2e/e2e_harness.py run --test clean-install --out /artifacts
 ```
+
+### Sandbox smoke with teardown
+
+Set `MINIMAX_API_KEY` and `E2E_SANDBOX_TOKEN` in the local environment first. Build the same
+user image and run this wrapper from the repository root. It creates the `out`, `data` and
+`upload` directories used by CI. The model and cleanup containers see E2E scripts read-only;
+the cleanup container receives the sandbox token without the MiniMax key. The EXIT trap
+keeps the model's status, removes and verifies the exact container, runs remote cleanup, then
+runs redaction even if an earlier stage failed. It retains the CID file if container absence
+cannot be verified.
+
+```bash
+git archive HEAD | docker build --file scripts/docker/Dockerfile --target user \
+  --tag autoskillit-e2e:local -
+
+set -euo pipefail
+IMAGE=autoskillit-e2e:local
+TEST=headless-smoke
+E2E_DIR="$PWD/.autoskillit/temp/e2e"
+mkdir -p "$E2E_DIR"/{out,data,upload}
+chmod 0777 "$E2E_DIR"/{out,data,upload}
+RUN_ID="$(date +%s)-$$"
+CID_FILE="$E2E_DIR/out/${TEST}-${RUN_ID}.cid"
+[[ ! -e "$CID_FILE" ]]
+
+source "$PWD/scripts/e2e/smoke-cleanup.sh"
+cleanup_smoke() {
+  model_status=$?
+  set +e
+  redact_status=0
+  lifecycle_file="$E2E_DIR/out/${TEST}-lifecycle.txt"
+  cleanup_smoke_resources "$CID_FILE" "$PWD/scripts/e2e" "$E2E_DIR/out" "$IMAGE" "$TEST"
+
+  docker run --rm --env MINIMAX_API_KEY --env E2E_SANDBOX_TOKEN \
+    --volume "$PWD/scripts/e2e:/opt/e2e:ro" \
+    --volume "$E2E_DIR:/e2e" \
+    "$IMAGE" python3 /opt/e2e/e2e_harness.py redact --dest /e2e/upload \
+      --secret-env MINIMAX_API_KEY --secret-env E2E_SANDBOX_TOKEN \
+      /e2e/out /e2e/data/logs
+  redact_status=$?
+
+  trap - EXIT
+  finish_smoke_lifecycle "$lifecycle_file" "$model_status" "$redact_status"
+  exit $?
+}
+trap cleanup_smoke EXIT
+
+timeout --signal=TERM --kill-after=30s 1020s docker run --rm --cidfile "$CID_FILE" \
+  --env MINIMAX_API_KEY --env E2E_SANDBOX_TOKEN \
+  --volume "$PWD/scripts/e2e:/opt/e2e:ro" \
+  --volume "$E2E_DIR/out:/artifacts" \
+  --volume "$E2E_DIR/data:/home/autoskillit/.local/share/autoskillit" \
+  "$IMAGE" python3 /opt/e2e/e2e_harness.py run --test "$TEST" --out /artifacts
+```
+
+Select it directly with workflow dispatch input `tests=headless-smoke`.

scripts/e2e/catalog.json

@@ -50,6 +50,34 @@
           "issue": "https://github.com/TalonT-Org/AutoSkillit/issues/5244"
         }
       ]
+    },
+    {
+      "name": "headless-smoke",
+      "kind": "recipe",
+      "peak_sessions": 2,
+      "timeout_sec": 900,
+      "trigger_paths": [
+        "src/autoskillit/execution/headless/*",
+        "src/autoskillit/execution/backends/*",
+        "src/autoskillit/cli/fleet/*",
+        "src/autoskillit/fleet/dispatch/*",
+        "src/autoskillit/server/*",
+        "src/autoskillit/hooks/*",
+        "scripts/e2e/*",
+        ".github/workflows/e2e.yml"
+      ],
+      "recipe": "sandbox-smoke",
+      "recipe_fixture": "sandbox-smoke.yaml",
+      "ingredients": {},
+      "expected_pull_request_state": "closed",
+      "expected_failures": [
+        {
+          "severity": "error",
+          "check": "open_kitchen_failed",
+          "message": "open_kitchen_failed",
+          "issue": "https://github.com/TalonT-Org/AutoSkillit/issues/5250"
+        }
+      ]
     }
   ]
 }

scripts/e2e/e2e_catalog.py

@@ -24,6 +24,7 @@
 PULL_REQUEST_STATES = ("open", "merged", "closed")
 
 _NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9-]*$")
+_RECIPE_FIXTURE_PATTERN = re.compile(r"^[a-z0-9][a-z0-9-]*\.yaml$")
 _CATALOG_KEYS = frozenset({"sandbox_repository", "tests"})
 _COMMON_KEYS = frozenset({"name", "kind", "peak_sessions", "timeout_sec", "trigger_paths"})
 _RECIPE_KEYS = frozenset({"recipe", "ingredients", "expected_pull_request_state"})
@@ -48,6 +49,7 @@ class CatalogTest:
     ingredients: tuple[tuple[str, str], ...]
     expected_pull_request_state: str | None
     expected_failures: tuple[Mapping[str, str], ...] = ()
+    recipe_fixture: str | None = None
 
 
 @dataclass(frozen=True, slots=True)
@@ -156,7 +158,9 @@ def _parse_test(raw: object, index: int) -> CatalogTest:
     if kind not in KINDS:
         raise CatalogError(f"{where}.kind must be one of {KINDS}")
     keys = _COMMON_KEYS | _RECIPE_KEYS if kind == "recipe" else _COMMON_KEYS
-    if kind == "clean-install" and "expected_failures" in raw:
+    if kind == "recipe" and "recipe_fixture" in raw:
+        keys |= {"recipe_fixture"}
+    if kind in ("recipe", "clean-install") and "expected_failures" in raw:
         keys |= {"expected_failures"}
     _require_keys(raw, keys, where)
     name = _require_str(raw["name"], f"{where}.name")
@@ -165,6 +169,14 @@ def _parse_test(raw: object, index: int) -> CatalogTest:
     recipe, ingredients, state = (
         _parse_recipe_fields(raw, where) if kind == "recipe" else (None, (), None)
     )
+    recipe_fixture = None
+    if kind == "recipe" and "recipe_fixture" in raw:
+        recipe_fixture = _require_str(raw["recipe_fixture"], f"{where}.recipe_fixture")
+        if not _RECIPE_FIXTURE_PATTERN.fullmatch(recipe_fixture):
+            raise CatalogError(f"{where}.recipe_fixture must be a .yaml basename")
+    expected_failures = _parse_expected_failures(raw.get("expected_failures", []), where)
+    if kind == "recipe" and any(row["severity"] != "error" for row in expected_failures):
+        raise CatalogError(f"{where}.expected_failures severity must be error for recipe tests")
     test = CatalogTest(
         name=name,
         kind=kind,
@@ -179,7 +191,8 @@ def _parse_test(raw: object, index: int) -> CatalogTest:
         recipe=recipe,
         ingredients=ingredients,
         expected_pull_request_state=state,
-        expected_failures=_parse_expected_failures(raw.get("expected_failures", []), where),
+        expected_failures=expected_failures,
+        recipe_fixture=recipe_fixture,
     )
     if job_timeout_minutes(test) > MAX_JOB_MINUTES:
         raise CatalogError(f"{where}.timeout_sec exceeds the {MAX_JOB_MINUTES}-minute job limit")