diff --git a/.github/workflows/egress-deny-guard.yml b/.github/workflows/egress-deny-guard.yml new file mode 100644 index 000000000..8e834046b --- /dev/null +++ b/.github/workflows/egress-deny-guard.yml @@ -0,0 +1,325 @@ +name: Egress Deny Guard + +# The only automated check that a sandbox declared `egressDeny` cannot reach the internet. +# Weekly rather than per-PR: it needs a real account and a MicroVM image build, and what it +# guards against is an AWS-side behaviour change rather than a change in this repository. + +on: + schedule: + - cron: "43 4 * * 2" + workflow_dispatch: + +permissions: + contents: read + id-token: write + +concurrency: + group: egress-deny-guard + cancel-in-progress: false + +env: + STACK_NAME: egress-deny-guard + # Docker base the bundle builds on. AL2023 matches the MicroVM base image the emitter names and + # ships the curl the probe runs. + SANDBOX_BASE_IMAGE: public.ecr.aws/amazonlinux/amazonlinux:2023 + AGENT_TARGET: aarch64-unknown-linux-musl + # Fall back to local compilation when the shared sccache backend returns a + # transient error (e.g. webdav 403) instead of failing the whole build. + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + +jobs: + egress-deny: + name: egressDeny is enforced + runs-on: depot-ubuntu-24.04-arm-8 + timeout-minutes: 90 + # Makes the OIDC `sub` `repo:OWNER/REPO:environment:egress-deny-guard` whatever branch a + # manual run starts from, so one trust policy covers the schedule and a deliberate break. + environment: egress-deny-guard + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Configure git credentials + run: git config --global url."https://x-access-token:${{ secrets.REPO_ACCESS_TOKEN }}@github.com/".insteadOf "https://github.com/" + + - uses: dtolnay/rust-toolchain@7c8d7d138f5c09cef361f8214cf96882cd029cdb # nightly + with: + toolchain: nightly + targets: aarch64-unknown-linux-musl + + - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + continue-on-error: true + with: + version: v0.16.0 + + - uses: taiki-e/install-action@43cb5d9d3c33252b8482ffa34f4e609859a530d8 # cargo-nextest + with: + tool: cargo-nextest + + - name: Install protoc + uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3 + with: + version: "27.x" + repo-token: ${{ github.token }} + + - name: Build the sandbox agent + run: | + set -euo pipefail + cargo build -p alien-sandbox-agent --release --target "$AGENT_TARGET" + agent="target/$AGENT_TARGET/release/alien-sandbox-agent" + # A MicroVM image accepts aarch64 only, and the wrong architecture surfaces minutes + # later as an image that never becomes active rather than as a build failure. + file "$agent" + file "$agent" | grep -q "ARM aarch64" + echo "AGENT_BINARY=$agent" >> "$GITHUB_ENV" + + - name: Write the image bundle + run: | + set -euo pipefail + mkdir -p .guard + cargo run -p alien-build --example sandbox-bundle -- \ + "$AGENT_BINARY" "$SANDBOX_BASE_IMAGE" .guard/sandbox-bundle.zip + # Per-run key, so nothing collides. A cancelled run cannot remove its own object and + # no later run can guess the key: expiry on the `egress-deny-guard/` prefix is what + # reclaims it, and that lives in the bucket's lifecycle configuration. + echo "BUNDLE_KEY=egress-deny-guard/${{ github.run_id }}-${{ github.run_attempt }}.zip" \ + >> "$GITHUB_ENV" + + - name: Render the sandbox stack + env: + ARTIFACT_BUCKET: ${{ secrets.EGRESS_GUARD_ARTIFACT_BUCKET }} + run: | + set -euo pipefail + cat > .guard/alien.json < .guard/stack-settings.yaml <<'YAML' + network: + type: create + availabilityZones: 2 + YAML + # Rendered by the shipped emitter rather than written out here: the deny is a + # loopback-only egress rule on the connector's security group, and a hand-written copy + # would guard the copy instead of what deployments get. + cargo run -p alien-cli --bin alien -- render \ + --format cloudformation \ + --stack .guard/alien.json \ + --stack-settings .guard/stack-settings.yaml \ + --registration-mode outputs \ + --output .guard + + - name: Build the guard test + # Built by nextest, and before credentials are assumed: nextest compiles its own test + # binaries, and doing that after the assume spends the session on compilation. + run: cargo nextest run -p alien-aws-clients --lib --no-run + + - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6 + with: + role-to-assume: ${{ secrets.EGRESS_GUARD_AWS_ROLE_ARN }} + aws-region: ${{ vars.EGRESS_GUARD_AWS_REGION }} + # The job budget is 90 minutes and a role's default ceiling is one hour, which would + # expire mid-test and fail on signing rather than on the property under test. + role-duration-seconds: 5400 + + - name: Resolve the target account + run: | + set -euo pipefail + account_id=$(aws sts get-caller-identity --query Account --output text) + echo "::add-mask::$account_id" + echo "ACCOUNT_ID=$account_id" >> "$GITHUB_ENV" + + - name: Remove anything an earlier run left behind + env: + AWS_TARGET_ACCOUNT_ID: ${{ env.ACCOUNT_ID }} + AWS_TARGET_REGION: ${{ vars.EGRESS_GUARD_AWS_REGION }} + AWS_TARGET_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} + AWS_TARGET_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} + AWS_TARGET_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} + run: ./scripts/egress-deny-guard-teardown.sh "$STACK_NAME" + + - name: Upload the image bundle + env: + ARTIFACT_BUCKET: ${{ secrets.EGRESS_GUARD_ARTIFACT_BUCKET }} + run: aws s3 cp .guard/sandbox-bundle.zip "s3://$ARTIFACT_BUCKET/$BUNDLE_KEY" + + - name: Deploy the sandbox stack + env: + MANAGING_ROLE_ARN: ${{ secrets.EGRESS_GUARD_AWS_ROLE_ARN }} + run: | + set -euo pipefail + aws cloudformation deploy \ + --template-file .guard/template.yaml \ + --stack-name "$STACK_NAME" \ + --no-fail-on-empty-changeset \ + --capabilities CAPABILITY_NAMED_IAM CAPABILITY_AUTO_EXPAND \ + --parameter-overrides \ + "ManagingRoleArn=$MANAGING_ROLE_ARN" \ + "ManagingAccountId=$ACCOUNT_ID" + + - name: Read the probe image and connector + run: | + set -euo pipefail + resources=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" \ + --query "Stacks[0].Outputs[?OutputKey=='DeploymentResources'].OutputValue" \ + --output text) + image_arn=$(printf '%s' "$resources" | + jq -r '.[] | select(.type == "sandbox") | .importData.imageArn // empty' | head -1) + image_version=$(printf '%s' "$resources" | + jq -r '.[] | select(.type == "sandbox") | .importData.imageVersion // empty' | head -1) + connector=$(printf '%s' "$resources" | + jq -r '.[] | select(.type == "sandbox") | .importData.egressConnectorArns[0] // empty' | head -1) + echo "::add-mask::$image_arn" + echo "::add-mask::$connector" + + # RunMicrovm refuses a bare name with "Malformed ARN", which reads like a broken image + # rather than the wrong identifier having been read out of the stack. + case "$image_arn" in + arn:*) ;; + *) echo "::error::the stack's sandbox output is not an image ARN"; exit 1 ;; + esac + case "$connector" in + arn:*) ;; + *) echo "::error::the stack produced no egress connector ARN"; exit 1 ;; + esac + if [ -z "$image_version" ]; then + echo "::error::the probe image reports no active version" + exit 1 + fi + + { + echo "PROBE_IMAGE_NAME=$image_arn" + echo "PROBE_IMAGE_VERSION=$image_version" + echo "PROBE_CONNECTOR_ARN=$connector" + } >> "$GITHUB_ENV" + + - name: A denied sandbox cannot reach the internet + env: + AWS_TARGET_ACCOUNT_ID: ${{ env.ACCOUNT_ID }} + AWS_TARGET_REGION: ${{ vars.EGRESS_GUARD_AWS_REGION }} + AWS_TARGET_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} + AWS_TARGET_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} + AWS_TARGET_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} + run: | + set -euo pipefail + # No --no-capture: nextest prints a failing test's output anyway, and streaming the + # passing run puts the session endpoint in a world-readable log. + if ! cargo nextest run -p alien-aws-clients --lib \ + --no-tests=fail --run-ignored=all \ + -E 'test(=aws::lambda_microvms::live_deny::a_denied_sandbox_cannot_reach_the_internet_and_an_open_one_can)' + then + echo "::error title=egressDeny regression::a sandbox declared egressDeny was not denied egress, or the guard could not prove it — the assertion above says which" + exit 1 + fi + + - name: Refresh credentials for teardown + if: always() + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6 + with: + role-to-assume: ${{ secrets.EGRESS_GUARD_AWS_ROLE_ARN }} + aws-region: ${{ vars.EGRESS_GUARD_AWS_REGION }} + # The session above may have expired inside the job; take the web-identity path again + # rather than inheriting it. + unset-current-credentials: true + + - name: Tear down + if: always() + env: + AWS_TARGET_ACCOUNT_ID: ${{ env.ACCOUNT_ID }} + AWS_TARGET_REGION: ${{ vars.EGRESS_GUARD_AWS_REGION }} + AWS_TARGET_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} + AWS_TARGET_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} + AWS_TARGET_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} + run: ./scripts/egress-deny-guard-teardown.sh "$STACK_NAME" + + - name: Remove the uploaded bundle + if: always() && env.BUNDLE_KEY != '' + env: + ARTIFACT_BUCKET: ${{ secrets.EGRESS_GUARD_ARTIFACT_BUCKET }} + run: aws s3 rm "s3://$ARTIFACT_BUCKET/$BUNDLE_KEY" + + # Cleanup safety net. The teardown above is a step in the job it cleans up after, so a timeout + # or a cancel gives it only whatever window the runner allows — which does not reliably cover + # deleting a VPC. This job has its own timeout and its own credentials. + # + # It checks with the CLI first and only builds when something actually survived: a cancelled run + # leaves a MicroVM and an image version holding the image open, no stock `aws` subcommand + # terminates either, and a delete on a held image is accepted while removing nothing — so + # `delete-stack` alone cannot be trusted to clear it. + cleanup: + needs: [egress-deny] + if: always() + runs-on: depot-ubuntu-24.04-arm-8 + timeout-minutes: 45 + environment: egress-deny-guard + steps: + - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6 + id: creds + with: + role-to-assume: ${{ secrets.EGRESS_GUARD_AWS_ROLE_ARN }} + aws-region: ${{ vars.EGRESS_GUARD_AWS_REGION }} + mask-aws-account-id: true + + - name: Did anything survive? + id: survived + run: | + set -uo pipefail + if ! err=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" 2>&1 >/dev/null); then + case "$err" in + *"does not exist"*) echo "nothing left behind"; echo "stack=absent" >> "$GITHUB_OUTPUT"; exit 0 ;; + *) echo "::error::describe-stacks was inconclusive: $err"; exit 1 ;; + esac + fi + echo "::warning::$STACK_NAME outlived its job; reclaiming it" + echo "stack=present" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + if: steps.survived.outputs.stack == 'present' + + - name: Configure git credentials + if: steps.survived.outputs.stack == 'present' + run: git config --global url."https://x-access-token:${{ secrets.REPO_ACCESS_TOKEN }}@github.com/".insteadOf "https://github.com/" + + - uses: dtolnay/rust-toolchain@7c8d7d138f5c09cef361f8214cf96882cd029cdb # nightly + if: steps.survived.outputs.stack == 'present' + with: + toolchain: nightly + + - uses: taiki-e/install-action@43cb5d9d3c33252b8482ffa34f4e609859a530d8 # cargo-nextest + if: steps.survived.outputs.stack == 'present' + with: + tool: cargo-nextest + + - name: Install protoc + if: steps.survived.outputs.stack == 'present' + uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3 + with: + version: "27.x" + repo-token: ${{ github.token }} + + - name: Reclaim and remove the stack + if: steps.survived.outputs.stack == 'present' + env: + # The account id is a plain step output; the key, secret and token are exported to the + # environment instead and are only outputs under `output-credentials`. + AWS_TARGET_ACCOUNT_ID: ${{ steps.creds.outputs.aws-account-id }} + AWS_TARGET_REGION: ${{ vars.EGRESS_GUARD_AWS_REGION }} + AWS_TARGET_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} + AWS_TARGET_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} + AWS_TARGET_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} + run: ./scripts/egress-deny-guard-teardown.sh "$STACK_NAME" diff --git a/crates/alien-aws-clients/src/aws/lambda_microvms.rs b/crates/alien-aws-clients/src/aws/lambda_microvms.rs index fac41e8f6..ba197f2c6 100644 --- a/crates/alien-aws-clients/src/aws/lambda_microvms.rs +++ b/crates/alien-aws-clients/src/aws/lambda_microvms.rs @@ -992,12 +992,21 @@ mod live_deny { fn client() -> LambdaMicrovmsClient { let root: StdPathBuf = workspace_root::get_workspace_root(); dotenvy::from_path(root.join(".env.test")).ok(); + // Empty is unset, not configured: a workflow step reading an expression that never + // resolved passes "" rather than nothing, and signing with it surfaces a 403 instead of + // saying the credentials were never there. + let required = |name: &str| { + std::env::var(name) + .ok() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| panic!("{name} must be set and non-empty")) + }; let config = crate::AwsClientConfig { - account_id: std::env::var("AWS_TARGET_ACCOUNT_ID").expect("AWS_TARGET_ACCOUNT_ID"), - region: std::env::var("AWS_TARGET_REGION").expect("AWS_TARGET_REGION"), + account_id: required("AWS_TARGET_ACCOUNT_ID"), + region: required("AWS_TARGET_REGION"), credentials: AwsCredentials::AccessKeys { - access_key_id: std::env::var("AWS_TARGET_ACCESS_KEY_ID").expect("key"), - secret_access_key: std::env::var("AWS_TARGET_SECRET_ACCESS_KEY").expect("secret"), + access_key_id: required("AWS_TARGET_ACCESS_KEY_ID"), + secret_access_key: required("AWS_TARGET_SECRET_ACCESS_KEY"), session_token: std::env::var("AWS_TARGET_SESSION_TOKEN") .ok() .filter(|token| !token.is_empty()), @@ -1123,6 +1132,11 @@ mod live_deny { } } + /// The only automated check that `egressDeny` denies, run weekly by `egress-deny-guard`. + /// + /// Left `#[ignore]`d and named exactly by that workflow rather than gated on the probe + /// environment: a gate that skipped when the image was missing would report a provisioning + /// failure as a pass, which is the hole this exists to close. #[tokio::test] #[ignore] async fn a_denied_sandbox_cannot_reach_the_internet_and_an_open_one_can() { @@ -1159,15 +1173,159 @@ mod live_deny { // the image, the agent or the probe was broken for both. assert!( open_output.contains("HTTP:200"), - "the control must reach the internet, or the deny result means nothing:\n{open_output}" + "the control must reach the internet, or the egressDeny result means \ + nothing:\n{open_output}" ); assert!( !denied_output.contains("HTTP:200"), - "a sandbox under deny reached the internet:\n{denied_output}" + "egressDeny regression: a sandbox under egressDeny reached the internet:\n\ + {denied_output}" ); assert!( denied_output.contains("HTTP:000"), - "deny should fail to connect rather than get some other status:\n{denied_output}" + "egressDeny regression: a denied sandbox should fail to connect rather than get some \ + other status:\n{denied_output}" + ); + } + + /// Reclaims everything a guard run leaves on the probe image: its sessions, then its versions. + /// + /// A MicroVM the run did not terminate bills for hours and no stack delete reaches it, and a + /// surviving version holds the image open so the delete that follows cannot remove it. + #[tokio::test] + #[ignore] + async fn reclaim_the_probe_image() { + let image = std::env::var("PROBE_IMAGE_NAME").expect( + "PROBE_IMAGE_NAME, the image ARN egress-deny-guard-teardown.sh reads from the stack", + ); + let client = client(); + + // Proves the identifier addresses something before the version list is read, so an empty + // list means reclaimed rather than misaddressed: every other caller puts PROBE_IMAGE_NAME + // in a request body, and this is the first to put it in a path. + if let Err(error) = client.get_microvm_image(&image).await { + assert_eq!( + error.code, "REMOTE_RESOURCE_NOT_FOUND", + "the probe image was unreadable for a reason other than being gone, so a reclaim \ + cannot prove anything: {error}" + ); + println!("probe image already reclaimed"); + return; + } + + let mut stranded = Vec::new(); + let mut terminating = Vec::new(); + let mut undeleted = Vec::new(); + for version in client + .list_microvm_image_versions(&image) + .await + .expect("ListMicrovmImageVersions") + { + let version = version.image_version.expect("a version identifier"); + for microvm in client + .list_microvms(&image, &version) + .await + .expect("ListMicrovms") + { + let id = microvm.microvm_id.expect("a MicroVM id"); + // Every MicroVM is attempted before anything fails: one that refuses to + // terminate would otherwise abandon the rest, which bill by the hour. + match client.terminate_microvm(&id).await { + Ok(()) => terminating.push(id), + Err(error) => { + println!("{id}: refused: {error}"); + stranded.push(id); + } + } + } + + // Terminate returns once AWS accepts it, the way suspend documents for itself, so the + // version delete below would race the sessions still holding it. Waiting here turns a + // refusal into a real failure rather than a retry that reads like one. + for id in terminating.drain(..) { + let mut gone = false; + for _ in 0..60 { + match client.get_microvm(&id).await { + Err(error) if error.code == "REMOTE_RESOURCE_NOT_FOUND" => { + gone = true; + break; + } + Ok(current) if current.state.as_deref() == Some("TERMINATED") => { + gone = true; + break; + } + _ => tokio::time::sleep(Duration::from_secs(5)).await, + } + } + if gone { + println!("terminated {id}"); + } else { + println!("{id}: still terminating"); + stranded.push(id); + } + } + + // Best-effort for the same reason termination is: a version that refuses to delete + // would otherwise abandon every version after it, and abort before the report below + // naming what is still billing. + match client + .send::( + Method::DELETE, + &format!("/{API_VERSION}/microvm-images/{image}/versions/{version}"), + &[], + None, + "DeleteMicrovmImageVersion", + ) + .await + { + Ok(_) => println!("deleted version {version}"), + Err(error) => { + println!("version {version}: delete refused: {error}"); + undeleted.push(version); + } + } + } + + assert!( + stranded.is_empty(), + "MicroVMs the reclaim could not terminate keep billing and hold the image open: \ + {stranded:?}" + ); + assert!( + undeleted.is_empty(), + "image versions the reclaim could not delete hold the image open, so the stack \ + delete cannot remove it: {undeleted:?}" + ); + let left = client + .list_microvm_image_versions(&image) + .await + .expect("ListMicrovmImageVersions"); + assert!( + left.is_empty(), + "the probe image still holds versions, so the stack delete cannot remove it: {left:?}" + ); + } + + /// The probe image is gone once its stack is. + /// + /// Asserted rather than assumed: a delete on a `CREATED` image is accepted and removes + /// nothing while its versions survive (`delete_images_versions_first`), so a teardown can + /// report success over an image that is still there. + #[tokio::test] + #[ignore] + async fn the_probe_image_is_gone() { + let image = std::env::var("PROBE_IMAGE_NAME").expect( + "PROBE_IMAGE_NAME, the image ARN egress-deny-guard-teardown.sh reads from the stack", + ); + + let error = client().get_microvm_image(&image).await.expect_err( + "the guard's probe image must not survive its own teardown; if AWS now answers a \ + deleted image with a terminal state instead of a 404, widen this to accept that \ + state rather than dropping the check", + ); + assert_eq!( + error.code, "REMOTE_RESOURCE_NOT_FOUND", + "the probe image is still readable after teardown: {error}" ); } diff --git a/crates/alien-build/examples/sandbox-bundle.rs b/crates/alien-build/examples/sandbox-bundle.rs new file mode 100644 index 000000000..b5f0c4421 --- /dev/null +++ b/crates/alien-build/examples/sandbox-bundle.rs @@ -0,0 +1,32 @@ +//! Writes the bundle a Lambda MicroVM image is built from. +//! +//! ```text +//! cargo run -p alien-build --example sandbox-bundle -- +//! ``` +//! +//! The agent must be an aarch64 Linux binary: MicroVM images accept no other architecture, and +//! an x86 one produces an image that never becomes active. + +use std::path::Path; +use std::process::ExitCode; + +use alien_build::sandbox_bundle::write_bundle; + +fn main() -> ExitCode { + let arguments: Vec = std::env::args().skip(1).collect(); + let [agent, base_image, destination] = arguments.as_slice() else { + eprintln!("usage: sandbox-bundle "); + return ExitCode::FAILURE; + }; + + match write_bundle(Path::new(destination), base_image, Path::new(agent)) { + Ok(()) => { + println!("{destination}"); + ExitCode::SUCCESS + } + Err(error) => { + eprintln!("{error}"); + ExitCode::FAILURE + } + } +} diff --git a/scripts/README.md b/scripts/README.md index 2d0937044..565a010aa 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -18,6 +18,12 @@ Small operational scripts used by root `package.json` commands and GitHub Action - **`write-gke-kubeconfig.sh`** — Used by `configure-e2e-provider-ingress.sh` and GKE Terraform distribution jobs to replace Terraform's static-client kubeconfig with one authenticated as the selected target service account. - **`cleanup-aws-e2e-resources.sh`** — Used by `.github/workflows/e2e-cloud.yml` before and after cloud E2E jobs to remove AWS resources for an E2E slot or explicit resource prefix. Run only with target-account AWS credentials and a scoped `ALIEN_E2E_SLOT` or `ALIEN_E2E_RESOURCE_PREFIX`. +## Sandbox egress guard + +- **`egress-deny-guard-teardown.sh`** — Used by `.github/workflows/egress-deny-guard.yml` before and after the weekly `egressDeny` check to remove the probe stack, and to fail the run if any of it survives. Takes the stack name and needs target-account AWS credentials in both the CLI's and the Rust client's environment (`AWS_TARGET_*`). + + Two preconditions live outside this repository: the OIDC role needs `MaxSessionDuration` of at least 5400 seconds, or every run dies at credential assumption; and a lifecycle rule expiring the artifact bucket's `egress-deny-guard/` prefix is what reclaims a bundle a cancelled run could not delete. + ## Example testing - **`test-examples-local.sh`** (`pnpm test:examples`) — Used by CI Fast and local development to test examples against local source by temporarily injecting `pnpm.overrides`. Always restores `examples/package.json` and `examples/pnpm-lock.yaml` via trap cleanup. diff --git a/scripts/egress-deny-guard-teardown.sh b/scripts/egress-deny-guard-teardown.sh new file mode 100755 index 000000000..a9130e40c --- /dev/null +++ b/scripts/egress-deny-guard-teardown.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# +# Removes everything the egress-deny guard creates, and fails if any of it survives — except a +# stack it deliberately keeps when a reclaim leaves resources behind, so the next run can find them. +# +# Runs before the guard as well as after it: a stack an earlier run left in a failed state would +# otherwise wedge every run that follows. +# +# Destructive, and assumes no guard run is in flight. The workflow serializes runs through a +# concurrency group; a hand-run invocation has nothing holding that lock. + +set -uo pipefail + +STACK_NAME=${1:?usage: egress-deny-guard-teardown.sh } + +cd "$(dirname "$0")/.." || exit 1 + +run_ignored_test() { + cargo nextest run -p alien-aws-clients --lib --no-tests=fail --run-ignored=all \ + -E "test(=aws::lambda_microvms::live_deny::$1)" +} + +# An absent stack is the success case; anything else — expired credentials, AccessDenied, a +# throttle — must not be read as one. Matching the message is how `cleanup-aws-e2e-resources.sh` +# draws the same distinction. +stack_state() { + local err + if err=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" 2>&1 >/dev/null); then + echo "present" + return 0 + fi + case "$err" in + *"does not exist"*) echo "absent" ;; + *) echo "::error::describe-stacks on $STACK_NAME was inconclusive: $err" >&2; return 1 ;; + esac +} + +state=$(stack_state) || exit 1 +if [ "$state" = "absent" ]; then + echo "no $STACK_NAME stack to remove" + exit 0 +fi + +# A stack that never reached its change set carries no Outputs, which is expected here — that +# state is exactly what this run exists to clean up. A call that fails outright is not: reading it +# as "no outputs" would skip the reclaim and the image check while still exiting 0. +# stderr goes to its own file rather than into the value, so one stray CLI warning cannot make a +# well-formed stack look like a malformed one. +outputs_err=$(mktemp) +trap 'rm -f "$outputs_err"' EXIT +if ! resources=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" \ + --query "Stacks[0].Outputs[?OutputKey=='DeploymentResources'].OutputValue" \ + --output text 2>"$outputs_err"); then + echo "::error::reading $STACK_NAME outputs was inconclusive: $(cat "$outputs_err")" + exit 1 +fi + +image_arn="" +if [ -n "$resources" ] && [ "$resources" != "None" ]; then + if ! image_arn=$(printf '%s' "$resources" | + jq -re '.[] | select(.type == "sandbox") | .importData.imageArn' 2>/dev/null | head -1); then + echo "::error::$STACK_NAME reported outputs that carry no sandbox image ARN" + exit 1 + fi +fi + +# Outputs are absent in the stack states this script exists to clear, so the image is resolved +# from the stack's own resources instead. Only a stack that genuinely declares no image may leave +# this empty: gating the reclaim and the is-it-gone check on a silently empty value would report a +# clean teardown over an image nobody looked at. +if [ -z "$image_arn" ]; then + if ! image_arn=$(aws cloudformation list-stack-resources --stack-name "$STACK_NAME" \ + --query "StackResourceSummaries[?ResourceType=='AWS::Lambda::MicrovmImage'].PhysicalResourceId" \ + --output text 2>"$outputs_err" | head -1); then + echo "::error::listing $STACK_NAME resources was inconclusive: $(cat "$outputs_err")" + exit 1 + fi + [ "$image_arn" = "None" ] && image_arn="" +fi + +failed=0 + +# Sessions and image versions outlive the stack: the delete below reaches no running MicroVM, and +# a delete on the image is accepted while removing nothing for as long as a version survives. +# So versions go first, and the stack stays standing if that fails: it is the only way back to the +# image, whether through its outputs or its resources, and deleting it would leave MicroVMs +# billing with nothing left pointing at them. +if [ -n "$image_arn" ]; then + echo "::add-mask::$image_arn" + export PROBE_IMAGE_NAME="$image_arn" + if ! run_ignored_test reclaim_the_probe_image; then + echo "::error title=egressDeny guard::the reclaim did not clear the probe image, so \ +$STACK_NAME stays up — it is what the next run needs to reach whatever is left. The failure \ +above says whether resources survived or the reclaim could not run." + exit 1 + fi +fi + +aws cloudformation delete-stack --stack-name "$STACK_NAME" || failed=1 +aws cloudformation wait stack-delete-complete --stack-name "$STACK_NAME" || failed=1 + +# Runs whatever the delete reported: a stack can reach DELETE_COMPLETE over an image the delete +# was accepted for and never removed, and this is the only thing that would notice. +if [ -n "$image_arn" ]; then + run_ignored_test the_probe_image_is_gone || failed=1 +fi + +state=$(stack_state) || exit 1 +if [ "$state" = "present" ]; then + echo "::error title=egressDeny guard::$STACK_NAME survived its own teardown" + exit 1 +fi + +exit "$failed"