From f6f2c5f5f0436a7413f7093809c9e298a4cc639f Mon Sep 17 00:00:00 2001 From: Toddy Mladenov Date: Tue, 27 Oct 2026 11:22:50 -0700 Subject: [PATCH 1/3] test(acquire): validate mirror history end to end Exercise the existing mirror and promotion workflows with guarded disposable registry fixtures, actual service HTTP calls, rendered dashboard assertions, and verified cleanup. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/_mirror-image.yml | 21 +- .github/workflows/validate-mirror-history.yml | 384 ++++++++++++++++++ docs/contributing/workflow-naming.md | 3 +- docs/guides/acquire/README.md | 3 + .../acquire/validating-mirror-history.md | 94 +++++ docs/reference/mirror-history.md | 3 + tests/mirror_history/mirror_history_e2e.py | 352 ++++++++++++++++ .../mirror_history/test_mirror_history_e2e.py | 216 ++++++++++ 8 files changed, 1074 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/validate-mirror-history.yml create mode 100644 docs/guides/acquire/validating-mirror-history.md create mode 100644 tests/mirror_history/mirror_history_e2e.py create mode 100644 tests/mirror_history/test_mirror_history_e2e.py diff --git a/.github/workflows/_mirror-image.yml b/.github/workflows/_mirror-image.yml index 448eae7..ce573e4 100644 --- a/.github/workflows/_mirror-image.yml +++ b/.github/workflows/_mirror-image.yml @@ -57,6 +57,21 @@ on: required: false default: true type: boolean + graph_events_artifact_name: + description: "Graph-event artifact name. Use distinct names when calling this workflow more than once in a run." + required: false + default: graph-events + type: string + outputs: + copied: + description: "Whether the mirror copied the image." + value: ${{ jobs.mirror.outputs.copied }} + skipped: + description: "Whether mirror history skipped the copy." + value: ${{ jobs.mirror.outputs.skipped }} + digest: + description: "Resolved image digest, including history-skipped runs." + value: ${{ jobs.mirror.outputs.digest }} secrets: source_registry_username: description: "Username for source_login_registry. Required only when source_login_registry is set." @@ -68,6 +83,10 @@ on: jobs: mirror: runs-on: ubuntu-latest + outputs: + copied: ${{ steps.mirror.outputs.copied || 'false' }} + skipped: ${{ inputs.record_history && steps.mirror.outputs.digest == '' }} + digest: ${{ steps.mirror.outputs.digest || steps.source.outputs.digest }} permissions: contents: read packages: write @@ -249,7 +268,7 @@ jobs: if: ${{ steps.mirror.outputs.digest != '' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: graph-events + name: ${{ inputs.graph_events_artifact_name }} path: ${{ runner.temp }}/graph-events if-no-files-found: ignore retention-days: 1 diff --git a/.github/workflows/validate-mirror-history.yml b/.github/workflows/validate-mirror-history.yml new file mode 100644 index 0000000..a78fb8c --- /dev/null +++ b/.github/workflows/validate-mirror-history.yml @@ -0,0 +1,384 @@ +name: validate / mirror-history + +on: + pull_request: + paths: + - ".github/workflows/validate-mirror-history.yml" + - ".github/workflows/_mirror-image.yml" + - ".github/workflows/_promote-from-quarantine.yml" + - ".github/actions/mirror-history/**" + - ".github/actions/mirror-image/**" + - ".github/actions/delete-image/**" + - ".github/actions/enumerate-tags/**" + - "apps/python-app/libs/cssc_common/**" + - "apps/python-app/services/packages-service/**" + - "apps/python-app/services/dashboard-web/**" + - "apps/python-app/services/issues-service/**" + - "tests/mirror_history/**" + workflow_dispatch: + +concurrency: + group: validate-mirror-history-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + checks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.12" + - name: Install declared test dependencies + run: | + python -m pip install \ + -e 'apps/python-app/libs/cssc_common[test]' \ + -e 'apps/python-app/libs/cssc_graph[test]' \ + -e 'apps/python-app/services/packages-service[test]' \ + -e 'apps/python-app/services/dashboard-web[test]' \ + -e 'apps/python-app/services/issues-service[test]' \ + -e 'apps/python-app/services/graph-service[test]' + - name: Run regression and per-service tests + run: | + set -euo pipefail + python -m pytest -q tests/mirror_history + for project in apps/python-app/libs/cssc_common apps/python-app/libs/cssc_graph apps/python-app/services/*; do + (cd "${project}" && python -m pytest -q) + done + - name: Lint workflows and changed shell blocks + run: | + set -euo pipefail + archive=actionlint_1.7.12_linux_amd64.tar.gz + curl --fail --location --silent --show-error \ + "https://github.com/rhysd/actionlint/releases/download/v1.7.12/${archive}" -o "${RUNNER_TEMP}/${archive}" + curl --fail --location --silent --show-error \ + https://github.com/rhysd/actionlint/releases/download/v1.7.12/actionlint_1.7.12_checksums.txt \ + -o "${RUNNER_TEMP}/actionlint-checksums.txt" + (cd "${RUNNER_TEMP}" && grep " ${archive}$" actionlint-checksums.txt | sha256sum -c -) + tar -xzf "${RUNNER_TEMP}/${archive}" -C "${RUNNER_TEMP}" actionlint + "${RUNNER_TEMP}/actionlint" .github/workflows/validate-mirror-history.yml .github/workflows/_mirror-image.yml + shellcheck --version + python - <<'PY' + import subprocess + import yaml + with open(".github/workflows/validate-mirror-history.yml") as source: + workflow = yaml.safe_load(source) + for job in workflow["jobs"].values(): + for step in job.get("steps", []): + if "run" in step: + subprocess.run( + ["shellcheck", "-s", "bash", "-S", "warning", "-"], + input=step["run"], text=True, check=True, + ) + PY + + prepare: + needs: checks + if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + outputs: + namespace: ${{ steps.paths.outputs.namespace }} + quarantine_image: ${{ steps.paths.outputs.quarantine_image }} + golden_image: ${{ steps.paths.outputs.golden_image }} + digest: ${{ steps.fixture.outputs.digest }} + owner_type: ${{ steps.identity.outputs.owner_type }} + username: ${{ steps.identity.outputs.username }} + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.12" + - run: python -m pip install -e apps/python-app/libs/cssc_common + - name: Verify the deletion credential before publishing + id: identity + env: + GH_TOKEN: ${{ secrets.GHCR_DELETE_TOKEN }} + run: | + set -euo pipefail + if [ -z "${GH_TOKEN}" ]; then + echo "::error::GHCR_DELETE_TOKEN is required for fixture deletion." + exit 1 + fi + username="$(gh api user --jq '.login')" + scopes="$(gh api --include user | tr -d '\r' | awk 'tolower($1) == "x-oauth-scopes:" {sub(/^[^:]*:[[:space:]]*/, ""); print}')" + if ! printf '%s' "${scopes}" | tr -d ' ' | tr ',' '\n' | grep -qx 'delete:packages'; then + echo "::error::GHCR_DELETE_TOKEN requires the delete:packages scope before any fixture is published." + exit 1 + fi + if ! printf '%s' "${scopes}" | tr -d ' ' | tr ',' '\n' | grep -qxE '(read|write):packages'; then + echo "::error::GHCR_DELETE_TOKEN also requires package read access." + exit 1 + fi + owner_type="$(gh api "users/${GITHUB_REPOSITORY_OWNER}" --jq '.type')" + case "${owner_type}" in + User) + if [ "${username,,}" != "${GITHUB_REPOSITORY_OWNER,,}" ]; then + echo "::error::The deletion token must belong to the fixture owner." + exit 1 + fi + owner_type=user + ;; + Organization) owner_type=org ;; + *) echo "::error::Unsupported owner type."; exit 1 ;; + esac + printf 'username=%s\nowner_type=%s\n' "${username}" "${owner_type}" >> "${GITHUB_OUTPUT}" + - name: Compute guarded fixture coordinates + id: paths + run: python tests/mirror_history/mirror_history_e2e.py coordinates + - uses: imjasonh/setup-crane@31b88efe9de28ae0ffa220711af4b60be9435f6e # v0.4 + - uses: ./.github/actions/registry-login + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - name: Publish a minimal scratch-image upstream fixture + id: fixture + env: + GOLDEN_IMAGE: ${{ steps.paths.outputs.golden_image }} + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/mirror-history-fixture" + printf 'Mirror-history acceptance fixture.\n' > "${RUNNER_TEMP}/mirror-history-fixture/fixture.txt" + tar -cf "${RUNNER_TEMP}/mirror-history-fixture.tar" -C "${RUNNER_TEMP}/mirror-history-fixture" fixture.txt + crane append --new_layer "${RUNNER_TEMP}/mirror-history-fixture.tar" --new_tag "${GOLDEN_IMAGE}:upstream" + crane mutate "${GOLDEN_IMAGE}:upstream" --set-platform linux/amd64 --tag "${GOLDEN_IMAGE}:upstream" + digest="$(crane digest "${GOLDEN_IMAGE}:upstream")" + echo "digest=${digest}" >> "${GITHUB_OUTPUT}" + + mirror-initial: + needs: prepare + permissions: + contents: read + packages: write + uses: ./.github/workflows/_mirror-image.yml + with: + source_image: ${{ needs.prepare.outputs.golden_image }} + source_tag: upstream + dest_image: ${{ needs.prepare.outputs.quarantine_image }} + dest_tag: image + graph_events_artifact_name: graph-events-initial-${{ github.run_attempt }} + + verify-initial: + needs: [prepare, mirror-initial] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.12" + - run: >- + python -m pip install -e apps/python-app/libs/cssc_common + -e apps/python-app/services/packages-service + -e apps/python-app/services/issues-service + -e apps/python-app/services/dashboard-web + - name: Verify the initial registry state and rendered dashboard + env: + GITHUB_TOKEN: ${{ secrets.GHCR_DELETE_TOKEN }} + GITHUB_OWNER_TYPE: ${{ needs.prepare.outputs.owner_type }} + GITHUB_USERNAME: ${{ needs.prepare.outputs.username }} + DIGEST: ${{ needs.mirror-initial.outputs.digest }} + COPIED: ${{ needs.mirror-initial.outputs.copied }} + SKIPPED: ${{ needs.mirror-initial.outputs.skipped }} + EXPECTED_DIGEST: ${{ needs.prepare.outputs.digest }} + run: | + set -euo pipefail + test "${DIGEST}" = "${EXPECTED_DIGEST}" + python tests/mirror_history/mirror_history_e2e.py initial \ + --digest "${DIGEST}" --copied "${COPIED}" --skipped "${SKIPPED}" \ + --output "${RUNNER_TEMP}/evidence" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: mirror-history-initial-${{ github.run_attempt }} + path: ${{ runner.temp }}/evidence + retention-days: 14 + if-no-files-found: error + + promote: + needs: [prepare, verify-initial] + permissions: + contents: read + packages: write + # Required by the reusable workflow even though override notifications are disabled. + issues: write + uses: ./.github/workflows/_promote-from-quarantine.yml + with: + source_repo: ${{ needs.prepare.outputs.quarantine_image }} + dest_repo: ${{ needs.prepare.outputs.golden_image }} + severity_threshold: HIGH + delete_source: true + enable_approval: false + secrets: + ghcr_delete_token: ${{ secrets.GHCR_DELETE_TOKEN }} + + verify-promoted: + needs: [prepare, promote] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.12" + - run: >- + python -m pip install -e apps/python-app/libs/cssc_common + -e apps/python-app/services/packages-service + -e apps/python-app/services/issues-service + -e apps/python-app/services/dashboard-web + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: mirror-history-initial-${{ github.run_attempt }} + path: ${{ runner.temp }}/initial + - name: Verify promotion, deletion and surviving history + env: + GITHUB_TOKEN: ${{ secrets.GHCR_DELETE_TOKEN }} + GITHUB_OWNER_TYPE: ${{ needs.prepare.outputs.owner_type }} + GITHUB_USERNAME: ${{ needs.prepare.outputs.username }} + DIGEST: ${{ needs.prepare.outputs.digest }} + run: | + python tests/mirror_history/mirror_history_e2e.py promoted \ + --digest "${DIGEST}" --expected "${RUNNER_TEMP}/initial/snapshot.json" \ + --output "${RUNNER_TEMP}/evidence" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: mirror-history-promoted-${{ github.run_attempt }} + path: ${{ runner.temp }}/evidence + retention-days: 14 + if-no-files-found: error + + mirror-again: + needs: [prepare, verify-promoted] + permissions: + contents: read + packages: write + uses: ./.github/workflows/_mirror-image.yml + with: + source_image: ${{ needs.prepare.outputs.golden_image }} + source_tag: upstream + dest_image: ${{ needs.prepare.outputs.quarantine_image }} + dest_tag: image + graph_events_artifact_name: graph-events-skipped-${{ github.run_attempt }} + + verify-skipped: + needs: [prepare, mirror-again] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.12" + - run: >- + python -m pip install -e apps/python-app/libs/cssc_common + -e apps/python-app/services/packages-service + -e apps/python-app/services/issues-service + -e apps/python-app/services/dashboard-web + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: mirror-history-initial-${{ github.run_attempt }} + path: ${{ runner.temp }}/initial + - name: Verify an actual history skip and unchanged dashboard history + env: + GITHUB_TOKEN: ${{ secrets.GHCR_DELETE_TOKEN }} + GITHUB_OWNER_TYPE: ${{ needs.prepare.outputs.owner_type }} + GITHUB_USERNAME: ${{ needs.prepare.outputs.username }} + DIGEST: ${{ needs.mirror-again.outputs.digest }} + COPIED: ${{ needs.mirror-again.outputs.copied }} + SKIPPED: ${{ needs.mirror-again.outputs.skipped }} + run: | + python tests/mirror_history/mirror_history_e2e.py skipped \ + --digest "${DIGEST}" --copied "${COPIED}" --skipped "${SKIPPED}" \ + --expected "${RUNNER_TEMP}/initial/snapshot.json" --output "${RUNNER_TEMP}/evidence" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: mirror-history-skipped-${{ github.run_attempt }} + path: ${{ runner.temp }}/evidence + retention-days: 14 + if-no-files-found: error + + mirror-force: + needs: [prepare, verify-skipped] + permissions: + contents: read + packages: write + uses: ./.github/workflows/_mirror-image.yml + with: + source_image: ${{ needs.prepare.outputs.golden_image }} + source_tag: upstream + dest_image: ${{ needs.prepare.outputs.quarantine_image }} + dest_tag: image + force: true + graph_events_artifact_name: graph-events-forced-${{ github.run_attempt }} + + verify-forced: + needs: [prepare, mirror-force] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.12" + - run: >- + python -m pip install -e apps/python-app/libs/cssc_common + -e apps/python-app/services/packages-service + -e apps/python-app/services/issues-service + -e apps/python-app/services/dashboard-web + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: mirror-history-initial-${{ github.run_attempt }} + path: ${{ runner.temp }}/initial + - name: Verify forced copying and append-only history + env: + GITHUB_TOKEN: ${{ secrets.GHCR_DELETE_TOKEN }} + GITHUB_OWNER_TYPE: ${{ needs.prepare.outputs.owner_type }} + GITHUB_USERNAME: ${{ needs.prepare.outputs.username }} + DIGEST: ${{ needs.mirror-force.outputs.digest }} + COPIED: ${{ needs.mirror-force.outputs.copied }} + SKIPPED: ${{ needs.mirror-force.outputs.skipped }} + run: | + python tests/mirror_history/mirror_history_e2e.py forced \ + --digest "${DIGEST}" --copied "${COPIED}" --skipped "${SKIPPED}" \ + --expected "${RUNNER_TEMP}/initial/snapshot.json" --output "${RUNNER_TEMP}/evidence" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: mirror-history-forced-${{ github.run_attempt }} + path: ${{ runner.temp }}/evidence + retention-days: 14 + if-no-files-found: error + + cleanup: + needs: [prepare, mirror-initial, verify-initial, promote, verify-promoted, mirror-again, verify-skipped, mirror-force, verify-forced] + if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.12" + - run: python -m pip install -e apps/python-app/libs/cssc_common + - name: Delete only this run's disposable packages and verify absence + env: + GITHUB_TOKEN: ${{ secrets.GHCR_DELETE_TOKEN }} + run: python tests/mirror_history/mirror_history_e2e.py cleanup + + acceptance: + needs: [checks, prepare, mirror-initial, verify-initial, promote, verify-promoted, mirror-again, verify-skipped, mirror-force, verify-forced, cleanup] + if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository) }} + runs-on: ubuntu-latest + steps: + - name: Require every lifecycle stage and cleanup to pass + env: + RESULTS: ${{ toJSON(needs) }} + run: | + python3 - <<'PY' + import json + import os + results = json.loads(os.environ["RESULTS"]) + failures = {name: result["result"] for name, result in results.items() if result["result"] != "success"} + if failures: + raise SystemExit(f"Mirror-history acceptance failed: {failures}") + print("PASS: mirror, promote/delete, history skip, force, dashboard, and fixture cleanup.") + PY diff --git a/docs/contributing/workflow-naming.md b/docs/contributing/workflow-naming.md index c9f8897..7a3062e 100644 --- a/docs/contributing/workflow-naming.md +++ b/docs/contributing/workflow-naming.md @@ -15,6 +15,7 @@ both in the file system and in the Actions UI. | **Build** | Build an application image on top of a mirrored base | `build-.yml` | `build / ` | `build-` | | **Promote override** | Act on a maintainer's approve/deny of a blocked image (issue-comment or dispatch) | `promote-override.yml` | `promote override` | `promote-override-` | | **Report** | Watch other workflows and report CI failures as tracking issues | `report-.yml` | `report / ` | `report-` | +| **Validate** | Exercise a capability with tests and disposable registry fixtures | `validate-.yml` | `validate / ` | `validate--` | | **Reusable** | Shared logic invoked by other workflows; never triggered directly | `_.yml` (leading underscore) | `_reusable / ` | n/a | | **Composite action** | A single reusable step shared across workflows | `.github/actions//action.yml` | `name: ` | n/a | @@ -22,7 +23,7 @@ both in the file system and in the Actions UI. 1. **Verb prefix.** Every workflow filename starts with a category verb: `mirror-`, `promote-from-quarantine-`, `promote-override`, `build-`, - `report-`, or a leading underscore + `report-`, `validate-`, or a leading underscore (`_`) for reusable workflows. This groups related workflows together alphabetically and makes intent obvious at a glance. diff --git a/docs/guides/acquire/README.md b/docs/guides/acquire/README.md index 367850e..679e1c8 100644 --- a/docs/guides/acquire/README.md +++ b/docs/guides/acquire/README.md @@ -7,3 +7,6 @@ How-to guides for the **Acquire** stage. - [Mirroring base images from Docker Hub to GHCR](mirroring-base-images.md) — how the mirror actions work and how to add a new mirror for another image or tag. +- [Validating mirror history end to end](validating-mirror-history.md) — + exercise promotion, deletion, history skips, forced synchronization, and the + dashboard using disposable registry fixtures. diff --git a/docs/guides/acquire/validating-mirror-history.md b/docs/guides/acquire/validating-mirror-history.md new file mode 100644 index 0000000..3f0f7aa --- /dev/null +++ b/docs/guides/acquire/validating-mirror-history.md @@ -0,0 +1,94 @@ +# Validate mirror history end to end + +Use the [mirror-history validation workflow](../../../.github/workflows/validate-mirror-history.yml) +to prove that a synchronized image is not acquired again after promotion removes +it from quarantine. This exercises the real mirror and promotion workflows, +GitHub Container Registry (GHCR), the package and issue services, and the +dashboard's rendered Acquisition view. Service checks use real local HTTP +servers on the runner, not mocked registry or service responses. + +## Prerequisites and safety + +- The repository must have a `GHCR_DELETE_TOKEN` secret with package read and + deletion permissions. The same secret is already used by promotion. +- The workflow's built-in token needs `packages: write` for fixture publication. +- Live tests run only for same-repository pull requests or manual dispatch. + Fork pull requests run the read-only tests without receiving secrets. +- Registry writes and deletes are restricted to two computed package paths: + `ghcr.io//quarantine/mirror-history-e2e--/image` and + `ghcr.io//golden/mirror-history-e2e--/image`. + Normal quarantine and golden packages are never modified. + +The fixture is a tiny scratch image containing a text file. It is first +published as `:upstream` in the disposable golden package, then treated as the +mirror's upstream. Using a controlled image avoids moving public tags and +changing vulnerability databases making the lifecycle test nondeterministic. +The normal Trivy scan and HIGH-severity promotion gate still run; neither +exceptions nor approval overrides are used. + +## Run the validation + +Changes to relevant actions, workflows, services, or the test harness trigger +the workflow on a pull request. After the workflow is on the default branch, +you can also run it from Actions, or with GitHub CLI: + +```bash +gh workflow run validate-mirror-history.yml --ref main +gh run list --workflow validate-mirror-history.yml --limit 5 +gh run watch --exit-status +``` + +The run must pass all these checks: + +| Phase | Required result | +| --- | --- | +| Initial mirror | The image digest is preserved, history has exactly one non-forced entry, and the workflow reports copied rather than skipped. | +| Promote and delete | The image reaches golden, its quarantine tag and referrers are deleted, and the quarantine package survives with only `mirror-history`. | +| Mirror again | The workflow reports a history skip; the quarantine image remains absent and history is unchanged. | +| Force | The image is copied again and exactly one forced entry is appended without changing earlier entries. | +| Dashboard | The fixture card shows the digests, timestamps and run links; a history-only package has zero image tags and is marked not in quarantine. | +| Cleanup | Both disposable packages are deleted and verified absent. | + +The workflow also runs regression tests, each service's tests, `actionlint`, and +`shellcheck -S warning` on its shell blocks. + +The reusable [mirror workflow](../../../.github/workflows/_mirror-image.yml) +exposes `copied`, `skipped`, and `digest` outputs for these assertions. Its +optional `graph_events_artifact_name` defaults to the existing `graph-events` +name; callers invoking it repeatedly in one run must choose distinct names. + +## Inspect evidence and failures + +Each successful observation phase publishes a `mirror-history--` +artifact containing `snapshot.json` and the rendered fixture card +`acquisition.html`. Snapshots include the source digest and complete history, +but no credentials. These artifacts are retained for 14 days. + +For success, the final `acceptance` job and `cleanup` must both pass. A green +mirror or promotion job alone is insufficient: deletion can otherwise be +skipped, and a history-only package could still be incorrectly shown in the +dashboard. + +```bash +gh run download --pattern 'mirror-history-*' --dir mirror-history-evidence +gh run view --log-failed +``` + +The checks before publishing reject missing credentials. Subsequent failures +still invoke cleanup, including partial setup failures. Runs are not +automatically cancelled when a newer commit is pushed. + +If a runner is forcibly terminated before cleanup, obtain the original run ID +and attempt from that run, install the declared `cssc_common` dependencies, and +run the same guarded cleanup helper. Provide the deletion token through the +process environment as `GITHUB_TOKEN`; never place its value in shell history, +source files, or uploaded artifacts. + +```bash +python tests/mirror_history/mirror_history_e2e.py cleanup \ + --owner --run-id --attempt +``` + +The helper only computes the two fixture package names; it does not accept an +arbitrary repository to delete. Authentication and deletion failures are +errors, not successful cleanup. diff --git a/docs/reference/mirror-history.md b/docs/reference/mirror-history.md index ebd3e84..943ff35 100644 --- a/docs/reference/mirror-history.md +++ b/docs/reference/mirror-history.md @@ -86,3 +86,6 @@ cat out/mirror-history.json | jq . The CSSC Dashboard's Acquisition view surfaces this history per repository via the `packages-service` `GET /packages/{name}/history` endpoint. + +For the repeatable mirror, promote, delete, skip, force and dashboard acceptance +test, see [Validate mirror history end to end](../guides/acquire/validating-mirror-history.md). diff --git a/tests/mirror_history/mirror_history_e2e.py b/tests/mirror_history/mirror_history_e2e.py new file mode 100644 index 0000000..ebc4bae --- /dev/null +++ b/tests/mirror_history/mirror_history_e2e.py @@ -0,0 +1,352 @@ +"""Live mirror-history assertions and cleanup for disposable GHCR fixtures.""" + +from __future__ import annotations + +import argparse +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass +from datetime import datetime +import os +from pathlib import Path +import re +import socket +import subprocess +import sys +import tempfile +import time +from typing import Iterator +from urllib.parse import quote + +import httpx +from pydantic import BaseModel, Field, TypeAdapter + +from cssc_common import MirrorHistoryEntry, MirroredImage, Tag + +IMAGE_TAG = "image" +SOURCE_TAG = "upstream" +HISTORY_TAG = "mirror-history" + + +@dataclass(frozen=True) +class Fixture: + owner: str + run_id: str + attempt: str + + def __post_init__(self) -> None: + if not re.fullmatch(r"[a-z0-9](?:[a-z0-9-]{0,37}[a-z0-9])?", self.owner): + raise ValueError("A lowercase GitHub owner is required") + if not all(re.fullmatch(r"[1-9][0-9]*", value) for value in (self.run_id, self.attempt)): + raise ValueError("Run ID and attempt must be positive integers") + + @property + def namespace(self) -> str: + return f"quarantine/mirror-history-e2e-{self.run_id}-{self.attempt}" + + @property + def quarantine(self) -> str: + return f"{self.namespace}/image" + + @property + def golden(self) -> str: + return f"golden/mirror-history-e2e-{self.run_id}-{self.attempt}/image" + + def coordinates(self) -> dict[str, str]: + return { + "namespace": self.namespace, + "quarantine": self.quarantine, + "golden": self.golden, + "quarantine_image": f"ghcr.io/{self.owner}/{self.quarantine}", + "golden_image": f"ghcr.io/{self.owner}/{self.golden}", + } + + +class Snapshot(BaseModel): + owner: str + run_id: str + attempt: str + digest: str = Field(pattern=r"^sha256:[0-9a-f]{64}$") + entries: list[MirrorHistoryEntry] + + +@contextmanager +def serve(module: str, environment: dict[str, str]) -> Iterator[str]: + with socket.socket() as listener, tempfile.TemporaryFile(mode="w+") as log: + listener.bind(("127.0.0.1", 0)) + listener.listen() + url = f"http://127.0.0.1:{listener.getsockname()[1]}" + # Passing the bound socket avoids a free-port race between processes. + process = subprocess.Popen( + [sys.executable, "-m", "uvicorn", module, "--fd", str(listener.fileno())], + env={**os.environ, **environment}, + pass_fds=(listener.fileno(),), + stdout=log, + stderr=subprocess.STDOUT, + ) + try: + deadline = time.monotonic() + 30 + with httpx.Client(timeout=1) as client: + while True: + if process.poll() is not None: + raise RuntimeError(f"{module} exited during startup ({process.returncode})") + try: + response = client.get(f"{url}/healthz") + response.raise_for_status() + break + except httpx.TransportError: + if time.monotonic() >= deadline: + raise TimeoutError(f"{module} did not become ready at {url}") + time.sleep(0.1) + yield url + finally: + failed = sys.exc_info()[0] is not None + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + if failed: + log.seek(0) + detail = log.read() + for key in ("GITHUB_TOKEN", "GHCR_DELETE_TOKEN"): + token = environment.get(key) or os.environ.get(key) + if token: + detail = detail.replace(token, "[REDACTED]") + if detail: + print(f"{module} log:\n{detail}", file=sys.stderr) + + +def validate_state( + phase: str, + fixture: Fixture, + quarantine_tags: list[Tag], + golden_tags: list[Tag], + history: list[MirrorHistoryEntry], + digest: str, + expected: Snapshot | None, + copied: str | None, + skipped: str | None, +) -> Snapshot: + assert history, "History must exist after the initial mirror" + assert re.fullmatch(r"sha256:[0-9a-f]{64}", digest), "Missing or invalid workflow digest" + quarantine = {tag.tag: tag.digest for tag in quarantine_tags} + golden = {tag.tag: tag.digest for tag in golden_tags} + assert HISTORY_TAG in quarantine, "The standalone history tag must survive" + assert golden.get(SOURCE_TAG) == digest, "The controlled upstream image must remain unchanged" + times = [] + for entry in history: + assert entry.source_tag == SOURCE_TAG + assert entry.dest_tag == IMAGE_TAG + assert entry.source_digest == digest + assert entry.run_id == fixture.run_id + assert entry.run_attempt == fixture.attempt + assert entry.synced_at, "Every entry needs a timestamp" + timestamp = datetime.fromisoformat(entry.synced_at.replace("Z", "+00:00")) + assert timestamp.utcoffset() is not None, "History timestamps must include a timezone" + times.append(timestamp) + assert entry.run_url == ( + f"https://github.com/{fixture.owner}/cssc-framework/actions/runs/{fixture.run_id}" + ) + assert times == sorted(times), "History must remain chronological" + + if phase == "initial": + assert expected is None + assert copied == "true" and skipped == "false" + assert quarantine.get(IMAGE_TAG) == digest + assert IMAGE_TAG not in golden, "Promotion must not have happened yet" + assert len(history) == 1 and history[0].force is False + elif phase in {"promoted", "skipped", "forced"}: + assert expected is not None, "The initial snapshot is required" + assert (expected.owner, expected.run_id, expected.attempt) == ( + fixture.owner, fixture.run_id, fixture.attempt + ), "The snapshot belongs to a different fixture" + assert digest == expected.digest + assert golden.get(IMAGE_TAG) == digest, "Promotion must preserve the digest" + if phase == "forced": + assert copied == "true" and skipped == "false" + assert quarantine.get(IMAGE_TAG) == digest + assert history[:-1] == expected.entries, "Force must not rewrite earlier history" + assert len(history) == len(expected.entries) + 1 and history[-1].force is True + else: + assert set(quarantine) == {HISTORY_TAG}, "Only history may remain after deletion" + assert history == expected.entries, "Promotion and history skips must not append entries" + if phase == "skipped": + assert copied == "false" and skipped == "true", "The actual workflow must skip" + else: + raise ValueError(f"Unknown validation phase: {phase}") + + return Snapshot( + owner=fixture.owner, + run_id=fixture.run_id, + attempt=fixture.attempt, + digest=digest, + entries=history, + ) + + +def validate_dashboard(html: str, fixture: Fixture, snapshot: Snapshot, history_only: bool) -> str: + cards = [ + part.split("", 1)[0] + for part in html.split('
')[1:] + if f"{fixture.quarantine}" in part.split("
", 1)[0] + ] + assert len(cards) == 1, "The dashboard must contain exactly one fixture card" + card = cards[0] + assert f"Synchronized ({len(snapshot.entries)})" in card + assert snapshot.digest[:19] in card + for entry in snapshot.entries: + assert entry.synced_at and entry.synced_at in card + assert entry.run_url and entry.run_url in card + assert (">not in quarantine<" in card) == history_only + if history_only: + assert "0 tag(s)" in card + if any(entry.force for entry in snapshot.entries): + assert ">forced" in card + return f'
{card}
\n' + + +def check_live( + phase: str, + fixture: Fixture, + digest: str, + expected: Snapshot | None, + copied: str | None, + skipped: str | None, + output: Path, +) -> None: + token = os.environ["GITHUB_TOKEN"] + if not token: + raise ValueError("GITHUB_TOKEN is required for live validation") + environment = { + "GITHUB_TOKEN": token, + "GITHUB_OWNER": fixture.owner, + "GITHUB_REPO": "cssc-framework", + "GITHUB_OWNER_TYPE": os.environ.get("GITHUB_OWNER_TYPE", "user"), + "GITHUB_USERNAME": os.environ.get("GITHUB_USERNAME", fixture.owner), + "CACHE_TTL_SECONDS": "0", + } + with ExitStack() as stack: + packages = stack.enter_context(serve("packages_service.app:app", environment)) + issues = stack.enter_context(serve("issues_service.app:app", environment)) + dashboard = stack.enter_context(serve("dashboard_web.app:app", { + **environment, + "PACKAGES_SERVICE_URL": packages, + "ISSUES_SERVICE_URL": issues, + "QUARANTINE_NAMESPACE": fixture.namespace, + })) + with httpx.Client(timeout=60) as client: + deadline = time.monotonic() + 90 + while True: + try: + response = client.get(f"{packages}/packages", params={"namespace": fixture.namespace}) + response.raise_for_status() + images = TypeAdapter(list[MirroredImage]).validate_json(response.content) + assert [image.name for image in images] == [fixture.quarantine] + response = client.get(f"{packages}/packages/{fixture.quarantine}/history") + response.raise_for_status() + history = TypeAdapter(list[MirrorHistoryEntry]).validate_json(response.content) + tags = [] + for repository in (fixture.quarantine, fixture.golden): + response = client.get(f"{packages}/packages/{repository}/tags") + response.raise_for_status() + tags.append(TypeAdapter(list[Tag]).validate_json(response.content)) + snapshot = validate_state( + phase, fixture, tags[0], tags[1], history, + digest, expected, copied, skipped, + ) + response = client.get(f"{dashboard}/stages/acquisition/fragment") + response.raise_for_status() + card = validate_dashboard(response.text, fixture, snapshot, phase in {"promoted", "skipped"}) + break + except AssertionError as error: + if time.monotonic() >= deadline: + raise + print(f"Waiting for registry/package visibility: {error}", flush=True) + time.sleep(3) + output.mkdir(parents=True, exist_ok=True) + (output / "snapshot.json").write_text(snapshot.model_dump_json(indent=2) + "\n") + (output / "acquisition.html").write_text(card) + print(f"PASS: {phase}; {len(snapshot.entries)} history entries; registry, workflow and dashboard verified") + + +def cleanup(fixture: Fixture, client: httpx.Client) -> None: + response = client.get(f"/users/{fixture.owner}") + response.raise_for_status() + owner_type = response.json()["type"] + if owner_type == "Organization": + root = f"/orgs/{fixture.owner}" + elif owner_type == "User": + response = client.get("/user") + response.raise_for_status() + if response.json()["login"].lower() != fixture.owner: + raise ValueError("The deletion token must belong to the fixture owner") + root = "/user" + else: + raise ValueError(f"Unsupported fixture owner type: {owner_type}") + for repository in (fixture.quarantine, fixture.golden): + path = f"{root}/packages/container/{quote(repository, safe='')}" + response = client.get(path) + if response.status_code == 404: + print(f"Fixture package already absent: {repository}") + continue + response.raise_for_status() + response = client.delete(path) + response.raise_for_status() + deadline = time.monotonic() + 30 + while True: + response = client.get(path) + if response.status_code == 404: + break + response.raise_for_status() + if time.monotonic() >= deadline: + raise RuntimeError(f"Fixture package still exists after deletion: {repository}") + time.sleep(1) + print(f"Deleted and verified absent: {repository}") + + +def main() -> None: + if not __debug__: + raise RuntimeError("Run validation without -O or PYTHONOPTIMIZE; assertions are required") + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("phase", choices=["coordinates", "initial", "promoted", "skipped", "forced", "cleanup"]) + parser.add_argument("--owner", default=os.environ.get("GITHUB_REPOSITORY_OWNER", "").lower()) + parser.add_argument("--run-id", default=os.environ.get("GITHUB_RUN_ID", "")) + parser.add_argument("--attempt", default=os.environ.get("GITHUB_RUN_ATTEMPT", "")) + parser.add_argument("--digest", default="") + parser.add_argument("--copied", choices=["true", "false"]) + parser.add_argument("--skipped", choices=["true", "false"]) + parser.add_argument("--expected", type=Path) + parser.add_argument("--output", type=Path, default=Path("mirror-history-evidence")) + args = parser.parse_args() + fixture = Fixture(args.owner, args.run_id, args.attempt) + if args.phase == "coordinates": + with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: + for name, value in fixture.coordinates().items(): + output.write(f"{name}={value}\n") + return + if args.phase == "cleanup": + token = os.environ["GITHUB_TOKEN"] + if not token: + raise ValueError("GITHUB_TOKEN is required for fixture cleanup") + with httpx.Client( + base_url="https://api.github.com", + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }, + timeout=30, + ) as client: + cleanup(fixture, client) + return + expected = Snapshot.model_validate_json(args.expected.read_text()) if args.expected else None + check_live( + args.phase, fixture, args.digest, expected, + args.copied, args.skipped, args.output, + ) + + +if __name__ == "__main__": + main() diff --git a/tests/mirror_history/test_mirror_history_e2e.py b/tests/mirror_history/test_mirror_history_e2e.py new file mode 100644 index 0000000..c14de7b --- /dev/null +++ b/tests/mirror_history/test_mirror_history_e2e.py @@ -0,0 +1,216 @@ +from datetime import datetime, timezone +from pathlib import Path +import subprocess +import sys +from unittest.mock import patch + +import httpx +import pytest +import yaml + +from cssc_common import MirrorHistoryEntry, Tag +from mirror_history_e2e import Fixture, cleanup, serve, validate_dashboard, validate_state + +DIGEST = "sha256:" + "a" * 64 +FIXTURE = Fixture("toddysm", "123", "1") + + +def entry(force: bool = False) -> MirrorHistoryEntry: + return MirrorHistoryEntry( + source_tag="upstream", + source_digest=DIGEST, + dest_tag="image", + synced_at=datetime(2026, 10, 27, tzinfo=timezone.utc).isoformat(), + run_url="https://github.com/toddysm/cssc-framework/actions/runs/123", + run_id="123", + run_attempt="1", + force=force, + ) + + +def initial(): + return validate_state( + "initial", FIXTURE, + [Tag(tag="image", digest=DIGEST), Tag(tag="mirror-history")], + [Tag(tag="upstream", digest=DIGEST)], + [entry()], DIGEST, None, "true", "false", + ) + + +@pytest.mark.parametrize("owner,run_id,attempt", [ + ("toddysm/normal-image", "123", "1"), + ("toddysm", "../python", "1"), + ("toddysm", "123", "0"), + ("toddysm", "123", "1/../../python"), + ("", "123", "1"), +]) +def test_fixture_rejects_unsafe_coordinates(owner, run_id, attempt): + with pytest.raises(ValueError): + Fixture(owner, run_id, attempt) + + +def test_complete_lifecycle(): + snapshot = initial() + for phase in ("promoted", "skipped"): + validate_state( + phase, FIXTURE, [Tag(tag="mirror-history")], + [Tag(tag="upstream", digest=DIGEST), Tag(tag="image", digest=DIGEST)], + snapshot.entries, DIGEST, snapshot, + "false" if phase == "skipped" else None, + "true" if phase == "skipped" else None, + ) + forced = validate_state( + "forced", FIXTURE, + [Tag(tag="image", digest=DIGEST), Tag(tag="mirror-history")], + [Tag(tag="upstream", digest=DIGEST), Tag(tag="image", digest=DIGEST)], + [entry(), entry(force=True)], DIGEST, snapshot, "true", "false", + ) + assert len(forced.entries) == 2 + + +@pytest.mark.parametrize("tags,history,copied,skipped", [ + ([Tag(tag="mirror-history"), Tag(tag="image", digest=DIGEST)], [entry()], "false", "true"), + ([Tag(tag="mirror-history")], [entry(), entry()], "false", "true"), + ([Tag(tag="mirror-history")], [entry()], "true", "false"), + ([], [entry()], "false", "true"), +]) +def test_skip_requires_no_copy_no_new_history_and_surviving_package(tags, history, copied, skipped): + with pytest.raises(AssertionError): + validate_state( + "skipped", FIXTURE, tags, + [Tag(tag="upstream", digest=DIGEST), Tag(tag="image", digest=DIGEST)], + history, DIGEST, initial(), copied, skipped, + ) + + +def test_force_cannot_rewrite_history(): + with pytest.raises(AssertionError): + validate_state( + "forced", FIXTURE, + [Tag(tag="mirror-history"), Tag(tag="image", digest=DIGEST)], + [Tag(tag="upstream", digest=DIGEST), Tag(tag="image", digest=DIGEST)], + [entry(force=True)], DIGEST, initial(), "true", "false", + ) + + +def test_dashboard_checks_the_fixture_card_not_another_image(): + snapshot = initial() + card = ( + f'
{FIXTURE.quarantine}' + f'0 tag(s)not in quarantine' + f'Synchronized (1) {DIGEST[:19]} {snapshot.entries[0].synced_at}' + f' run
' + ) + assert FIXTURE.quarantine in validate_dashboard(card, FIXTURE, snapshot, True) + with pytest.raises(AssertionError): + validate_dashboard(card.replace(FIXTURE.quarantine, "quarantine/python"), FIXTURE, snapshot, True) + with pytest.raises(AssertionError): + validate_dashboard(card.replace(">not in quarantine<", ">waiting<"), FIXTURE, snapshot, True) + + +def test_cleanup_only_deletes_the_two_computed_fixture_packages(): + deleted = [] + + def handler(request): + path = request.url.path + if path == "/users/toddysm": + return httpx.Response(200, json={"type": "User"}) + if path == "/user": + return httpx.Response(200, json={"login": "toddysm"}) + assert "mirror-history-e2e-123-1" in path + if request.method == "DELETE": + deleted.append(path) + return httpx.Response(204) + return httpx.Response(404 if path in deleted else 200, json={}) + + with httpx.Client(base_url="https://api.github.com", transport=httpx.MockTransport(handler)) as client: + cleanup(FIXTURE, client) + assert len(deleted) == 2 + assert all("quarantine/python" not in path for path in deleted) + + +def test_cleanup_does_not_hide_authentication_failure(): + with httpx.Client( + base_url="https://api.github.com", + transport=httpx.MockTransport(lambda request: httpx.Response(403)), + ) as client, pytest.raises(httpx.HTTPStatusError): + cleanup(FIXTURE, client) + + +def test_cleanup_rejects_another_users_token(): + def handler(request): + body = {"type": "User"} if request.url.path == "/users/toddysm" else {"login": "someone-else"} + assert request.method == "GET" + return httpx.Response(200, json=body) + + with httpx.Client( + base_url="https://api.github.com", transport=httpx.MockTransport(handler) + ) as client, pytest.raises(ValueError, match="fixture owner"): + cleanup(FIXTURE, client) + + +def test_cleanup_fails_when_a_deleted_package_remains_visible(): + def handler(request): + if request.url.path == "/users/toddysm": + return httpx.Response(200, json={"type": "User"}) + if request.url.path == "/user": + return httpx.Response(200, json={"login": "toddysm"}) + return httpx.Response(204 if request.method == "DELETE" else 200, json={}) + + with httpx.Client( + base_url="https://api.github.com", transport=httpx.MockTransport(handler) + ) as client, patch("mirror_history_e2e.time") as clock: + clock.monotonic.side_effect = [0, 31] + with pytest.raises(RuntimeError, match="still exists after deletion"): + cleanup(FIXTURE, client) + clock.sleep.assert_not_called() + + +def test_startup_errors_redact_server_credentials(tmp_path, capsys): + (tmp_path / "broken_server.py").write_text( + "import os\nraise RuntimeError(os.environ['GITHUB_TOKEN'])\n" + ) + with pytest.raises(RuntimeError, match="exited during startup"): + with serve("broken_server:app", { + "PYTHONPATH": str(tmp_path), + "GITHUB_TOKEN": "test-only-value", + }): + pytest.fail("A failed service must not become ready") + output = capsys.readouterr().err + assert "[REDACTED]" in output + assert "test-only-value" not in output + + +def test_cli_cannot_silently_disable_acceptance_assertions(): + result = subprocess.run( + [sys.executable, "-O", str(Path(__file__).with_name("mirror_history_e2e.py")), "coordinates"], + text=True, + capture_output=True, + ) + assert result.returncode != 0 + assert "assertions are required" in result.stderr + + +def test_actual_http_service_becomes_ready_and_stops(): + with serve("packages_service.app:app", {}) as url: + assert httpx.get(f"{url}/healthz").json() == {"status": "ok"} + with pytest.raises(httpx.ConnectError): + httpx.get(f"{url}/healthz", timeout=1) + + +def test_workflow_observability_preserves_existing_artifact_default(): + root = Path(__file__).resolve().parents[2] + workflow = yaml.load( + (root / ".github/workflows/_mirror-image.yml").read_text(), + Loader=yaml.BaseLoader, + ) + call = workflow["on"]["workflow_call"] + assert call["inputs"]["graph_events_artifact_name"]["default"] == "graph-events" + assert set(call["outputs"]) == {"copied", "skipped", "digest"} + live = yaml.load( + (root / ".github/workflows/validate-mirror-history.yml").read_text(), + Loader=yaml.BaseLoader, + ) + assert live["concurrency"]["cancel-in-progress"] == "false" + assert live["jobs"]["cleanup"]["if"].startswith("${{ always()") + assert live["jobs"]["promote"]["with"]["enable_approval"] == "false" From 746ebcce405fb06a72c3173460f44b13b61bcc6d Mon Sep 17 00:00:00 2001 From: Toddy Mladenov Date: Tue, 27 Oct 2026 11:33:35 -0700 Subject: [PATCH 2/3] fix(acquire): handle live registry history validation Follow GHCR blob redirects without forwarding registry authorization, redact signed download diagnostics, check untagged referrer removal, retry bounded visibility failures, and preserve fixture identity and independent cleanup on reruns. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/_mirror-image.yml | 4 + .github/workflows/validate-mirror-history.yml | 38 +++-- .../libs/cssc_common/cssc_common/registry.py | 11 +- .../libs/cssc_common/tests/test_registry.py | 62 +++++++ .../acquire/validating-mirror-history.md | 25 ++- tests/mirror_history/mirror_history_e2e.py | 152 ++++++++++++------ .../mirror_history/test_mirror_history_e2e.py | 151 +++++++++++++++-- 7 files changed, 365 insertions(+), 78 deletions(-) create mode 100644 apps/python-app/libs/cssc_common/tests/test_registry.py diff --git a/.github/workflows/_mirror-image.yml b/.github/workflows/_mirror-image.yml index ce573e4..f895c91 100644 --- a/.github/workflows/_mirror-image.yml +++ b/.github/workflows/_mirror-image.yml @@ -72,6 +72,9 @@ on: digest: description: "Resolved image digest, including history-skipped runs." value: ${{ jobs.mirror.outputs.digest }} + run_attempt: + description: "Execution attempt that produced these mirror results." + value: ${{ jobs.mirror.outputs.run_attempt }} secrets: source_registry_username: description: "Username for source_login_registry. Required only when source_login_registry is set." @@ -87,6 +90,7 @@ jobs: copied: ${{ steps.mirror.outputs.copied || 'false' }} skipped: ${{ inputs.record_history && steps.mirror.outputs.digest == '' }} digest: ${{ steps.mirror.outputs.digest || steps.source.outputs.digest }} + run_attempt: ${{ github.run_attempt }} permissions: contents: read packages: write diff --git a/.github/workflows/validate-mirror-history.yml b/.github/workflows/validate-mirror-history.yml index a78fb8c..d0f39ce 100644 --- a/.github/workflows/validate-mirror-history.yml +++ b/.github/workflows/validate-mirror-history.yml @@ -83,6 +83,7 @@ jobs: contents: read packages: write outputs: + fixture_attempt: ${{ steps.paths.outputs.fixture_attempt }} namespace: ${{ steps.paths.outputs.namespace }} quarantine_image: ${{ steps.paths.outputs.quarantine_image }} golden_image: ${{ steps.paths.outputs.golden_image }} @@ -167,6 +168,8 @@ jobs: verify-initial: needs: [prepare, mirror-initial] runs-on: ubuntu-latest + outputs: + evidence_artifact: mirror-history-initial-${{ needs.prepare.outputs.fixture_attempt }}-${{ github.run_attempt }} steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 @@ -186,15 +189,19 @@ jobs: COPIED: ${{ needs.mirror-initial.outputs.copied }} SKIPPED: ${{ needs.mirror-initial.outputs.skipped }} EXPECTED_DIGEST: ${{ needs.prepare.outputs.digest }} + FIXTURE_ATTEMPT: ${{ needs.prepare.outputs.fixture_attempt }} + MIRROR_ATTEMPT: ${{ needs.mirror-initial.outputs.run_attempt }} run: | set -euo pipefail test "${DIGEST}" = "${EXPECTED_DIGEST}" python tests/mirror_history/mirror_history_e2e.py initial \ + --attempt "${FIXTURE_ATTEMPT}" \ + --execution-attempt "${MIRROR_ATTEMPT}" \ --digest "${DIGEST}" --copied "${COPIED}" --skipped "${SKIPPED}" \ --output "${RUNNER_TEMP}/evidence" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: mirror-history-initial-${{ github.run_attempt }} + name: mirror-history-initial-${{ needs.prepare.outputs.fixture_attempt }}-${{ github.run_attempt }} path: ${{ runner.temp }}/evidence retention-days: 14 if-no-files-found: error @@ -217,7 +224,7 @@ jobs: ghcr_delete_token: ${{ secrets.GHCR_DELETE_TOKEN }} verify-promoted: - needs: [prepare, promote] + needs: [prepare, verify-initial, promote] runs-on: ubuntu-latest steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -231,7 +238,7 @@ jobs: -e apps/python-app/services/dashboard-web - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: mirror-history-initial-${{ github.run_attempt }} + name: ${{ needs.verify-initial.outputs.evidence_artifact }} path: ${{ runner.temp }}/initial - name: Verify promotion, deletion and surviving history env: @@ -239,13 +246,15 @@ jobs: GITHUB_OWNER_TYPE: ${{ needs.prepare.outputs.owner_type }} GITHUB_USERNAME: ${{ needs.prepare.outputs.username }} DIGEST: ${{ needs.prepare.outputs.digest }} + FIXTURE_ATTEMPT: ${{ needs.prepare.outputs.fixture_attempt }} run: | python tests/mirror_history/mirror_history_e2e.py promoted \ + --attempt "${FIXTURE_ATTEMPT}" \ --digest "${DIGEST}" --expected "${RUNNER_TEMP}/initial/snapshot.json" \ --output "${RUNNER_TEMP}/evidence" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: mirror-history-promoted-${{ github.run_attempt }} + name: mirror-history-promoted-${{ needs.prepare.outputs.fixture_attempt }}-${{ github.run_attempt }} path: ${{ runner.temp }}/evidence retention-days: 14 if-no-files-found: error @@ -264,7 +273,7 @@ jobs: graph_events_artifact_name: graph-events-skipped-${{ github.run_attempt }} verify-skipped: - needs: [prepare, mirror-again] + needs: [prepare, verify-initial, mirror-again] runs-on: ubuntu-latest steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -278,7 +287,7 @@ jobs: -e apps/python-app/services/dashboard-web - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: mirror-history-initial-${{ github.run_attempt }} + name: ${{ needs.verify-initial.outputs.evidence_artifact }} path: ${{ runner.temp }}/initial - name: Verify an actual history skip and unchanged dashboard history env: @@ -288,13 +297,15 @@ jobs: DIGEST: ${{ needs.mirror-again.outputs.digest }} COPIED: ${{ needs.mirror-again.outputs.copied }} SKIPPED: ${{ needs.mirror-again.outputs.skipped }} + FIXTURE_ATTEMPT: ${{ needs.prepare.outputs.fixture_attempt }} run: | python tests/mirror_history/mirror_history_e2e.py skipped \ + --attempt "${FIXTURE_ATTEMPT}" \ --digest "${DIGEST}" --copied "${COPIED}" --skipped "${SKIPPED}" \ --expected "${RUNNER_TEMP}/initial/snapshot.json" --output "${RUNNER_TEMP}/evidence" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: mirror-history-skipped-${{ github.run_attempt }} + name: mirror-history-skipped-${{ needs.prepare.outputs.fixture_attempt }}-${{ github.run_attempt }} path: ${{ runner.temp }}/evidence retention-days: 14 if-no-files-found: error @@ -314,7 +325,7 @@ jobs: graph_events_artifact_name: graph-events-forced-${{ github.run_attempt }} verify-forced: - needs: [prepare, mirror-force] + needs: [prepare, verify-initial, mirror-force] runs-on: ubuntu-latest steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -328,7 +339,7 @@ jobs: -e apps/python-app/services/dashboard-web - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: mirror-history-initial-${{ github.run_attempt }} + name: ${{ needs.verify-initial.outputs.evidence_artifact }} path: ${{ runner.temp }}/initial - name: Verify forced copying and append-only history env: @@ -338,13 +349,17 @@ jobs: DIGEST: ${{ needs.mirror-force.outputs.digest }} COPIED: ${{ needs.mirror-force.outputs.copied }} SKIPPED: ${{ needs.mirror-force.outputs.skipped }} + FIXTURE_ATTEMPT: ${{ needs.prepare.outputs.fixture_attempt }} + MIRROR_ATTEMPT: ${{ needs.mirror-force.outputs.run_attempt }} run: | python tests/mirror_history/mirror_history_e2e.py forced \ + --attempt "${FIXTURE_ATTEMPT}" \ + --execution-attempt "${MIRROR_ATTEMPT}" \ --digest "${DIGEST}" --copied "${COPIED}" --skipped "${SKIPPED}" \ --expected "${RUNNER_TEMP}/initial/snapshot.json" --output "${RUNNER_TEMP}/evidence" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: mirror-history-forced-${{ github.run_attempt }} + name: mirror-history-forced-${{ needs.prepare.outputs.fixture_attempt }}-${{ github.run_attempt }} path: ${{ runner.temp }}/evidence retention-days: 14 if-no-files-found: error @@ -362,7 +377,8 @@ jobs: - name: Delete only this run's disposable packages and verify absence env: GITHUB_TOKEN: ${{ secrets.GHCR_DELETE_TOKEN }} - run: python tests/mirror_history/mirror_history_e2e.py cleanup + FIXTURE_ATTEMPT: ${{ needs.prepare.outputs.fixture_attempt || github.run_attempt }} + run: python tests/mirror_history/mirror_history_e2e.py cleanup --attempt "${FIXTURE_ATTEMPT}" acceptance: needs: [checks, prepare, mirror-initial, verify-initial, promote, verify-promoted, mirror-again, verify-skipped, mirror-force, verify-forced, cleanup] diff --git a/apps/python-app/libs/cssc_common/cssc_common/registry.py b/apps/python-app/libs/cssc_common/cssc_common/registry.py index 38a5824..6de9ef0 100644 --- a/apps/python-app/libs/cssc_common/cssc_common/registry.py +++ b/apps/python-app/libs/cssc_common/cssc_common/registry.py @@ -112,8 +112,17 @@ def fetch_json_artifact( blob_response = self._client.get( f"/v2/{repository}/blobs/{layers[0]['digest']}", headers=auth, + follow_redirects=True, ) - blob_response.raise_for_status() + try: + blob_response.raise_for_status() + except httpx.HTTPStatusError: + url = blob_response.request.url.copy_with(query=None) + raise httpx.HTTPStatusError( + f"Registry blob fetch failed: HTTP {blob_response.status_code} from {url}", + request=blob_response.request, + response=blob_response, + ) from None return blob_response.json() def close(self) -> None: diff --git a/apps/python-app/libs/cssc_common/tests/test_registry.py b/apps/python-app/libs/cssc_common/tests/test_registry.py new file mode 100644 index 0000000..7f15389 --- /dev/null +++ b/apps/python-app/libs/cssc_common/tests/test_registry.py @@ -0,0 +1,62 @@ +import httpx +import pytest + +from cssc_common import OciRegistryClient + + +@pytest.mark.parametrize("injected_client", [False, True]) +def test_json_blob_redirect_is_followed_without_forwarding_registry_credentials(injected_client): + seen = [] + + def handler(request): + seen.append(request) + if request.url.path == "/token": + return httpx.Response(200, json={"token": "test-registry-pull-token"}) + if "/manifests/" in request.url.path: + return httpx.Response(200, json={"layers": [{"digest": "sha256:blob"}]}) + if "/blobs/" in request.url.path: + assert request.headers["Authorization"] == "Bearer test-registry-pull-token" + return httpx.Response(307, headers={"Location": "https://cdn.example.test/blob?sig=test-only"}) + assert request.url.host == "cdn.example.test" + assert "Authorization" not in request.headers + return httpx.Response(200, json={"schemaVersion": 1, "entries": []}) + + transport = httpx.MockTransport(handler) + if injected_client: + http = httpx.Client(base_url="https://ghcr.io", transport=transport) + registry = OciRegistryClient(owner="toddysm", client=http) + else: + http = None + registry = OciRegistryClient(owner="toddysm", transport=transport) + try: + assert registry.fetch_json_artifact("quarantine/test", "mirror-history") == { + "schemaVersion": 1, "entries": [], + } + assert len(seen) == 4 + finally: + registry.close() + if http is not None: + http.close() + + +@pytest.mark.parametrize("status", [404, 503]) +def test_redirected_blob_errors_preserve_status_without_exposing_signed_queries(status): + def handler(request): + if request.url.path == "/token": + return httpx.Response(200, json={"token": "test-registry-pull-token"}) + if "/manifests/" in request.url.path: + return httpx.Response(200, json={"layers": [{"digest": "sha256:blob"}]}) + if "/blobs/" in request.url.path: + return httpx.Response(307, headers={"Location": "https://cdn.example.test/blob?sig=test-only"}) + return httpx.Response(status) + + registry = OciRegistryClient(owner="toddysm", transport=httpx.MockTransport(handler)) + try: + with pytest.raises(httpx.HTTPStatusError) as failure: + registry.fetch_json_artifact("quarantine/test", "mirror-history") + assert failure.value.response.status_code == status + assert "https://cdn.example.test/blob" in str(failure.value) + assert "test-only" not in str(failure.value) + assert "?sig=" not in str(failure.value) + finally: + registry.close() diff --git a/docs/guides/acquire/validating-mirror-history.md b/docs/guides/acquire/validating-mirror-history.md index 3f0f7aa..3543453 100644 --- a/docs/guides/acquire/validating-mirror-history.md +++ b/docs/guides/acquire/validating-mirror-history.md @@ -43,7 +43,7 @@ The run must pass all these checks: | Phase | Required result | | --- | --- | | Initial mirror | The image digest is preserved, history has exactly one non-forced entry, and the workflow reports copied rather than skipped. | -| Promote and delete | The image reaches golden, its quarantine tag and referrers are deleted, and the quarantine package survives with only `mirror-history`. | +| Promote and delete | The image reaches golden; the quarantine package's complete version inventory contains only the unchanged `mirror-history` artifact, with no tagged or untagged image/referrer versions left. | | Mirror again | The workflow reports a history skip; the quarantine image remains absent and history is unchanged. | | Force | The image is copied again and exactly one forced entry is appended without changing earlier entries. | | Dashboard | The fixture card shows the digests, timestamps and run links; a history-only package has zero image tags and is marked not in quarantine. | @@ -53,16 +53,26 @@ The workflow also runs regression tests, each service's tests, `actionlint`, and `shellcheck -S warning` on its shell blocks. The reusable [mirror workflow](../../../.github/workflows/_mirror-image.yml) -exposes `copied`, `skipped`, and `digest` outputs for these assertions. Its +exposes `copied`, `skipped`, `digest`, and `run_attempt` outputs for these assertions. Its optional `graph_events_artifact_name` defaults to the existing `graph-events` name; callers invoking it repeatedly in one run must choose distinct names. ## Inspect evidence and failures -Each successful observation phase publishes a `mirror-history--` +Each successful observation phase publishes a +`mirror-history---` artifact containing `snapshot.json` and the rendered fixture card `acquisition.html`. Snapshots include the source digest and complete history, -but no credentials. These artifacts are retained for 14 days. +the history artifact digest, and the quarantine version digests, but no +credentials. These artifacts are retained for 14 days. The initial observation +requires an actual referrer so deletion coverage cannot pass vacuously. + +Visibility checks retry transient 404, 429 and server errors within a 90-second +deadline; authentication/authorization failures are immediate errors. Diagnostic +logs redact credentials and download-URL queries, including signed blob links. +The registry reader follows blob redirects without forwarding registry +authorization to another host; redirected-fetch errors retain the HTTP status +but omit signed query strings. For success, the final `acceptance` job and `cleanup` must both pass. A green mirror or promotion job alone is insufficient: deletion can otherwise be @@ -78,6 +88,13 @@ The checks before publishing reject missing credentials. Subsequent failures still invoke cleanup, including partial setup failures. Runs are not automatically cancelled when a newer commit is pushed. +Partial job reruns reuse the fixture-creation attempt emitted by `prepare`, +rather than accidentally targeting the new execution attempt's package names. +The initial evidence artifact and each mirror's producing attempt are also +carried explicitly between jobs. Cleanup attempts both packages independently +and fails if either cannot be removed. For a fresh acceptance lifecycle, +dispatch a new run instead of replaying already-completed mutations. + If a runner is forcibly terminated before cleanup, obtain the original run ID and attempt from that run, install the declared `cssc_common` dependencies, and run the same guarded cleanup helper. Provide the deletion token through the diff --git a/tests/mirror_history/mirror_history_e2e.py b/tests/mirror_history/mirror_history_e2e.py index ebc4bae..b5df747 100644 --- a/tests/mirror_history/mirror_history_e2e.py +++ b/tests/mirror_history/mirror_history_e2e.py @@ -14,13 +14,13 @@ import sys import tempfile import time -from typing import Iterator +from typing import Callable, Iterator from urllib.parse import quote import httpx from pydantic import BaseModel, Field, TypeAdapter -from cssc_common import MirrorHistoryEntry, MirroredImage, Tag +from cssc_common import GitHubClient, MirrorHistoryEntry, MirroredImage, Tag IMAGE_TAG = "image" SOURCE_TAG = "upstream" @@ -53,6 +53,7 @@ def golden(self) -> str: def coordinates(self) -> dict[str, str]: return { + "fixture_attempt": self.attempt, "namespace": self.namespace, "quarantine": self.quarantine, "golden": self.golden, @@ -66,9 +67,23 @@ class Snapshot(BaseModel): run_id: str attempt: str digest: str = Field(pattern=r"^sha256:[0-9a-f]{64}$") + history_digest: str = Field(pattern=r"^sha256:[0-9a-f]{64}$") + quarantine_versions: list[str] entries: list[MirrorHistoryEntry] +class PackageVersion(BaseModel): + name: str + + +def redact_diagnostics(text: str, environment: dict[str, str]) -> str: + for key in ("GITHUB_TOKEN", "GHCR_DELETE_TOKEN"): + token = environment.get(key) or os.environ.get(key) + if token: + text = text.replace(token, "[REDACTED]") + return re.sub(r"(https?://[^\s'\"?]+)\?[^\s'\"]+", r"\1?[REDACTED]", text) + + @contextmanager def serve(module: str, environment: dict[str, str]) -> Iterator[str]: with socket.socket() as listener, tempfile.TemporaryFile(mode="w+") as log: @@ -109,11 +124,7 @@ def serve(module: str, environment: dict[str, str]) -> Iterator[str]: process.wait() if failed: log.seek(0) - detail = log.read() - for key in ("GITHUB_TOKEN", "GHCR_DELETE_TOKEN"): - token = environment.get(key) or os.environ.get(key) - if token: - detail = detail.replace(token, "[REDACTED]") + detail = redact_diagnostics(log.read(), environment) if detail: print(f"{module} log:\n{detail}", file=sys.stderr) @@ -128,12 +139,15 @@ def validate_state( expected: Snapshot | None, copied: str | None, skipped: str | None, + quarantine_versions: set[str], ) -> Snapshot: assert history, "History must exist after the initial mirror" assert re.fullmatch(r"sha256:[0-9a-f]{64}", digest), "Missing or invalid workflow digest" quarantine = {tag.tag: tag.digest for tag in quarantine_tags} golden = {tag.tag: tag.digest for tag in golden_tags} assert HISTORY_TAG in quarantine, "The standalone history tag must survive" + history_digest = quarantine[HISTORY_TAG] + assert isinstance(history_digest, str) and re.fullmatch(r"sha256:[0-9a-f]{64}", history_digest) assert golden.get(SOURCE_TAG) == digest, "The controlled upstream image must remain unchanged" times = [] for entry in history: @@ -141,7 +155,8 @@ def validate_state( assert entry.dest_tag == IMAGE_TAG assert entry.source_digest == digest assert entry.run_id == fixture.run_id - assert entry.run_attempt == fixture.attempt + assert entry.run_attempt and re.fullmatch(r"[1-9][0-9]*", entry.run_attempt) + assert int(entry.run_attempt) >= int(fixture.attempt) assert entry.synced_at, "Every entry needs a timestamp" timestamp = datetime.fromisoformat(entry.synced_at.replace("Z", "+00:00")) assert timestamp.utcoffset() is not None, "History timestamps must include a timezone" @@ -157,6 +172,8 @@ def validate_state( assert quarantine.get(IMAGE_TAG) == digest assert IMAGE_TAG not in golden, "Promotion must not have happened yet" assert len(history) == 1 and history[0].force is False + assert {digest, history_digest}.issubset(quarantine_versions) + assert quarantine_versions - {digest, history_digest}, "An initial referrer must exist to test its deletion" elif phase in {"promoted", "skipped", "forced"}: assert expected is not None, "The initial snapshot is required" assert (expected.owner, expected.run_id, expected.attempt) == ( @@ -169,8 +186,11 @@ def validate_state( assert quarantine.get(IMAGE_TAG) == digest assert history[:-1] == expected.entries, "Force must not rewrite earlier history" assert len(history) == len(expected.entries) + 1 and history[-1].force is True + assert history_digest != expected.history_digest, "Appended history must be persisted as a new artifact" else: assert set(quarantine) == {HISTORY_TAG}, "Only history may remain after deletion" + assert quarantine_versions == {history_digest}, "Untagged image/referrer versions must also be deleted" + assert history_digest == expected.history_digest assert history == expected.entries, "Promotion and history skips must not append entries" if phase == "skipped": assert copied == "false" and skipped == "true", "The actual workflow must skip" @@ -182,10 +202,27 @@ def validate_state( run_id=fixture.run_id, attempt=fixture.attempt, digest=digest, + history_digest=history_digest, + quarantine_versions=sorted(quarantine_versions), entries=history, ) +def wait_for_visibility(check: Callable[[], tuple[Snapshot, str]]) -> tuple[Snapshot, str]: + deadline = time.monotonic() + 90 + while True: + try: + return check() + except (AssertionError, httpx.HTTPStatusError) as error: + if isinstance(error, httpx.HTTPStatusError): + if error.response.status_code not in {404, 429, 500, 502, 503, 504}: + raise + if time.monotonic() >= deadline: + raise + print(f"Waiting for registry/package visibility: {redact_diagnostics(str(error), {})}", flush=True) + time.sleep(3) + + def validate_dashboard(html: str, fixture: Fixture, snapshot: Snapshot, history_only: bool) -> str: cards = [ part.split("", 1)[0] @@ -215,6 +252,7 @@ def check_live( copied: str | None, skipped: str | None, output: Path, + execution_attempt: str | None, ) -> None: token = os.environ["GITHUB_TOKEN"] if not token: @@ -228,6 +266,8 @@ def check_live( "CACHE_TTL_SECONDS": "0", } with ExitStack() as stack: + github = GitHubClient(token=token, cache_ttl=0) + stack.callback(github.close) packages = stack.enter_context(serve("packages_service.app:app", environment)) issues = stack.enter_context(serve("issues_service.app:app", environment)) dashboard = stack.enter_context(serve("dashboard_web.app:app", { @@ -237,34 +277,37 @@ def check_live( "QUARANTINE_NAMESPACE": fixture.namespace, })) with httpx.Client(timeout=60) as client: - deadline = time.monotonic() + 90 - while True: - try: - response = client.get(f"{packages}/packages", params={"namespace": fixture.namespace}) - response.raise_for_status() - images = TypeAdapter(list[MirroredImage]).validate_json(response.content) - assert [image.name for image in images] == [fixture.quarantine] - response = client.get(f"{packages}/packages/{fixture.quarantine}/history") + def observe() -> tuple[Snapshot, str]: + response = client.get(f"{packages}/packages", params={"namespace": fixture.namespace}) + response.raise_for_status() + images = TypeAdapter(list[MirroredImage]).validate_json(response.content) + assert [image.name for image in images] == [fixture.quarantine] + response = client.get(f"{packages}/packages/{fixture.quarantine}/history") + response.raise_for_status() + history = TypeAdapter(list[MirrorHistoryEntry]).validate_json(response.content) + if phase in {"initial", "forced"}: + assert execution_attempt and re.fullmatch(r"[1-9][0-9]*", execution_attempt) + assert history and history[-1].run_attempt == execution_attempt + tags = [] + for repository in (fixture.quarantine, fixture.golden): + response = client.get(f"{packages}/packages/{repository}/tags") response.raise_for_status() - history = TypeAdapter(list[MirrorHistoryEntry]).validate_json(response.content) - tags = [] - for repository in (fixture.quarantine, fixture.golden): - response = client.get(f"{packages}/packages/{repository}/tags") - response.raise_for_status() - tags.append(TypeAdapter(list[Tag]).validate_json(response.content)) - snapshot = validate_state( - phase, fixture, tags[0], tags[1], history, - digest, expected, copied, skipped, - ) - response = client.get(f"{dashboard}/stages/acquisition/fragment") - response.raise_for_status() - card = validate_dashboard(response.text, fixture, snapshot, phase in {"promoted", "skipped"}) - break - except AssertionError as error: - if time.monotonic() >= deadline: - raise - print(f"Waiting for registry/package visibility: {error}", flush=True) - time.sleep(3) + tags.append(TypeAdapter(list[Tag]).validate_json(response.content)) + owner_root = {"user": "users", "org": "orgs"}[environment["GITHUB_OWNER_TYPE"]] + versions = TypeAdapter(list[PackageVersion]).validate_python(github.get_all( + f"/{owner_root}/{fixture.owner}/packages/container/{quote(fixture.quarantine, safe='')}/versions", + params={"per_page": 100}, + )) + snapshot = validate_state( + phase, fixture, tags[0], tags[1], history, + digest, expected, copied, skipped, {version.name for version in versions}, + ) + response = client.get(f"{dashboard}/stages/acquisition/fragment") + response.raise_for_status() + card = validate_dashboard(response.text, fixture, snapshot, phase in {"promoted", "skipped"}) + return snapshot, card + + snapshot, card = wait_for_visibility(observe) output.mkdir(parents=True, exist_ok=True) (output / "snapshot.json").write_text(snapshot.model_dump_json(indent=2) + "\n") (output / "acquisition.html").write_text(card) @@ -285,25 +328,33 @@ def cleanup(fixture: Fixture, client: httpx.Client) -> None: root = "/user" else: raise ValueError(f"Unsupported fixture owner type: {owner_type}") + failures: list[str] = [] for repository in (fixture.quarantine, fixture.golden): path = f"{root}/packages/container/{quote(repository, safe='')}" - response = client.get(path) - if response.status_code == 404: - print(f"Fixture package already absent: {repository}") - continue - response.raise_for_status() - response = client.delete(path) - response.raise_for_status() - deadline = time.monotonic() + 30 - while True: + try: response = client.get(path) if response.status_code == 404: - break + print(f"Fixture package already absent: {repository}") + continue response.raise_for_status() - if time.monotonic() >= deadline: - raise RuntimeError(f"Fixture package still exists after deletion: {repository}") - time.sleep(1) - print(f"Deleted and verified absent: {repository}") + response = client.delete(path) + response.raise_for_status() + deadline = time.monotonic() + 30 + while True: + response = client.get(path) + if response.status_code == 404: + break + response.raise_for_status() + if time.monotonic() >= deadline: + raise RuntimeError(f"Fixture package still exists after deletion: {repository}") + time.sleep(1) + print(f"Deleted and verified absent: {repository}") + except (httpx.HTTPError, RuntimeError) as error: + detail = f"{repository}: {redact_diagnostics(str(error), {})}" + print(f"Fixture cleanup failed: {detail}", file=sys.stderr) + failures.append(detail) + if failures: + raise RuntimeError("Fixture cleanup incomplete:\n" + "\n".join(failures)) def main() -> None: @@ -314,6 +365,7 @@ def main() -> None: parser.add_argument("--owner", default=os.environ.get("GITHUB_REPOSITORY_OWNER", "").lower()) parser.add_argument("--run-id", default=os.environ.get("GITHUB_RUN_ID", "")) parser.add_argument("--attempt", default=os.environ.get("GITHUB_RUN_ATTEMPT", "")) + parser.add_argument("--execution-attempt") parser.add_argument("--digest", default="") parser.add_argument("--copied", choices=["true", "false"]) parser.add_argument("--skipped", choices=["true", "false"]) @@ -344,7 +396,7 @@ def main() -> None: expected = Snapshot.model_validate_json(args.expected.read_text()) if args.expected else None check_live( args.phase, fixture, args.digest, expected, - args.copied, args.skipped, args.output, + args.copied, args.skipped, args.output, args.execution_attempt, ) diff --git a/tests/mirror_history/test_mirror_history_e2e.py b/tests/mirror_history/test_mirror_history_e2e.py index c14de7b..d95abeb 100644 --- a/tests/mirror_history/test_mirror_history_e2e.py +++ b/tests/mirror_history/test_mirror_history_e2e.py @@ -1,5 +1,6 @@ from datetime import datetime, timezone from pathlib import Path +import os import subprocess import sys from unittest.mock import patch @@ -9,9 +10,14 @@ import yaml from cssc_common import MirrorHistoryEntry, Tag -from mirror_history_e2e import Fixture, cleanup, serve, validate_dashboard, validate_state +from mirror_history_e2e import ( + Fixture, cleanup, redact_diagnostics, serve, validate_dashboard, validate_state, wait_for_visibility, +) DIGEST = "sha256:" + "a" * 64 +HISTORY_DIGEST = "sha256:" + "b" * 64 +REFERRER_DIGEST = "sha256:" + "c" * 64 +FORCED_HISTORY_DIGEST = "sha256:" + "d" * 64 FIXTURE = Fixture("toddysm", "123", "1") @@ -31,9 +37,9 @@ def entry(force: bool = False) -> MirrorHistoryEntry: def initial(): return validate_state( "initial", FIXTURE, - [Tag(tag="image", digest=DIGEST), Tag(tag="mirror-history")], + [Tag(tag="image", digest=DIGEST), Tag(tag="mirror-history", digest=HISTORY_DIGEST)], [Tag(tag="upstream", digest=DIGEST)], - [entry()], DIGEST, None, "true", "false", + [entry()], DIGEST, None, "true", "false", {DIGEST, HISTORY_DIGEST, REFERRER_DIGEST}, ) @@ -53,25 +59,27 @@ def test_complete_lifecycle(): snapshot = initial() for phase in ("promoted", "skipped"): validate_state( - phase, FIXTURE, [Tag(tag="mirror-history")], + phase, FIXTURE, [Tag(tag="mirror-history", digest=HISTORY_DIGEST)], [Tag(tag="upstream", digest=DIGEST), Tag(tag="image", digest=DIGEST)], snapshot.entries, DIGEST, snapshot, "false" if phase == "skipped" else None, "true" if phase == "skipped" else None, + {HISTORY_DIGEST}, ) forced = validate_state( "forced", FIXTURE, - [Tag(tag="image", digest=DIGEST), Tag(tag="mirror-history")], + [Tag(tag="image", digest=DIGEST), Tag(tag="mirror-history", digest=FORCED_HISTORY_DIGEST)], [Tag(tag="upstream", digest=DIGEST), Tag(tag="image", digest=DIGEST)], [entry(), entry(force=True)], DIGEST, snapshot, "true", "false", + {DIGEST, HISTORY_DIGEST, FORCED_HISTORY_DIGEST, REFERRER_DIGEST}, ) assert len(forced.entries) == 2 @pytest.mark.parametrize("tags,history,copied,skipped", [ - ([Tag(tag="mirror-history"), Tag(tag="image", digest=DIGEST)], [entry()], "false", "true"), - ([Tag(tag="mirror-history")], [entry(), entry()], "false", "true"), - ([Tag(tag="mirror-history")], [entry()], "true", "false"), + ([Tag(tag="mirror-history", digest=HISTORY_DIGEST), Tag(tag="image", digest=DIGEST)], [entry()], "false", "true"), + ([Tag(tag="mirror-history", digest=HISTORY_DIGEST)], [entry(), entry()], "false", "true"), + ([Tag(tag="mirror-history", digest=HISTORY_DIGEST)], [entry()], "true", "false"), ([], [entry()], "false", "true"), ]) def test_skip_requires_no_copy_no_new_history_and_surviving_package(tags, history, copied, skipped): @@ -79,7 +87,7 @@ def test_skip_requires_no_copy_no_new_history_and_surviving_package(tags, histor validate_state( "skipped", FIXTURE, tags, [Tag(tag="upstream", digest=DIGEST), Tag(tag="image", digest=DIGEST)], - history, DIGEST, initial(), copied, skipped, + history, DIGEST, initial(), copied, skipped, {HISTORY_DIGEST}, ) @@ -87,12 +95,96 @@ def test_force_cannot_rewrite_history(): with pytest.raises(AssertionError): validate_state( "forced", FIXTURE, - [Tag(tag="mirror-history"), Tag(tag="image", digest=DIGEST)], + [Tag(tag="mirror-history", digest=FORCED_HISTORY_DIGEST), Tag(tag="image", digest=DIGEST)], [Tag(tag="upstream", digest=DIGEST), Tag(tag="image", digest=DIGEST)], [entry(force=True)], DIGEST, initial(), "true", "false", + {DIGEST, HISTORY_DIGEST, FORCED_HISTORY_DIGEST, REFERRER_DIGEST}, ) +def test_force_rerun_preserves_creation_identity_and_records_execution_attempt(): + forced = entry(force=True).model_copy(update={"run_attempt": "2"}) + snapshot = validate_state( + "forced", FIXTURE, + [Tag(tag="image", digest=DIGEST), Tag(tag="mirror-history", digest=FORCED_HISTORY_DIGEST)], + [Tag(tag="upstream", digest=DIGEST), Tag(tag="image", digest=DIGEST)], + [entry(), forced], DIGEST, initial(), "true", "false", + {DIGEST, HISTORY_DIGEST, FORCED_HISTORY_DIGEST, REFERRER_DIGEST}, + ) + assert snapshot.attempt == "1" and snapshot.entries[-1].run_attempt == "2" + + +def test_explicit_creation_attempt_survives_a_partial_rerun(tmp_path): + output = tmp_path / "outputs" + subprocess.run( + [ + sys.executable, str(Path(__file__).with_name("mirror_history_e2e.py")), + "coordinates", "--owner", "toddysm", "--run-id", "123", "--attempt", "1", + ], + env={**os.environ, "GITHUB_RUN_ATTEMPT": "2", "GITHUB_OUTPUT": str(output)}, + check=True, + ) + text = output.read_text() + assert "fixture_attempt=1\n" in text + assert "mirror-history-e2e-123-1/image" in text + assert "mirror-history-e2e-123-2" not in text + + +def test_untagged_referrers_cannot_survive_promotion(): + with pytest.raises(AssertionError, match="Untagged image/referrer"): + validate_state( + "promoted", FIXTURE, [Tag(tag="mirror-history", digest=HISTORY_DIGEST)], + [Tag(tag="upstream", digest=DIGEST), Tag(tag="image", digest=DIGEST)], + [entry()], DIGEST, initial(), None, None, {HISTORY_DIGEST, REFERRER_DIGEST}, + ) + + +@pytest.mark.parametrize("status", [404, 429, 500, 502, 503, 504]) +def test_visibility_poll_retries_transient_http_errors(status): + attempts = 0 + + def observe(): + nonlocal attempts + attempts += 1 + if attempts == 1: + httpx.Response(status, request=httpx.Request("GET", "https://api.github.com/fixture")).raise_for_status() + return initial(), "card" + + with patch("mirror_history_e2e.time.sleep"): + snapshot, card = wait_for_visibility(observe) + assert attempts == 2 and card == "card" and snapshot.digest == DIGEST + + +@pytest.mark.parametrize("status", [401, 403]) +def test_visibility_poll_does_not_hide_authentication_failure(status): + def observe(): + httpx.Response(status, request=httpx.Request("GET", "https://api.github.com/fixture")).raise_for_status() + return initial(), "card" + + with patch("mirror_history_e2e.time.sleep") as sleep, pytest.raises(httpx.HTTPStatusError): + wait_for_visibility(observe) + sleep.assert_not_called() + + +def test_visibility_poll_has_a_real_deadline(): + def observe(): + raise AssertionError("not visible") + + with patch("mirror_history_e2e.time") as clock: + clock.monotonic.side_effect = [0, 91] + with pytest.raises(AssertionError, match="not visible"): + wait_for_visibility(observe) + clock.sleep.assert_not_called() + + +def test_diagnostics_remove_signed_download_queries_and_tokens(): + text = "token=test-only-value redirect='https://cdn.example.test/blob?sig=example-signature&hmac=example-hmac'" + redacted = redact_diagnostics(text, {"GITHUB_TOKEN": "test-only-value"}) + assert "test-only-value" not in redacted + assert "example-signature" not in redacted and "example-hmac" not in redacted + assert "https://cdn.example.test/blob?[REDACTED]" in redacted + + def test_dashboard_checks_the_fixture_card_not_another_image(): snapshot = initial() card = ( @@ -160,12 +252,34 @@ def handler(request): with httpx.Client( base_url="https://api.github.com", transport=httpx.MockTransport(handler) ) as client, patch("mirror_history_e2e.time") as clock: - clock.monotonic.side_effect = [0, 31] + clock.monotonic.side_effect = [0, 31, 31, 62] with pytest.raises(RuntimeError, match="still exists after deletion"): cleanup(FIXTURE, client) clock.sleep.assert_not_called() +def test_cleanup_still_deletes_golden_after_quarantine_deletion_fails(): + attempted = [] + + def handler(request): + path = request.url.path + if path == "/users/toddysm": + return httpx.Response(200, json={"type": "User"}) + if path == "/user": + return httpx.Response(200, json={"login": "toddysm"}) + if request.method == "DELETE": + attempted.append(path) + return httpx.Response(403 if "/quarantine/" in path else 204) + return httpx.Response(404 if path in attempted and "/golden/" in path else 200, json={}) + + with httpx.Client( + base_url="https://api.github.com", transport=httpx.MockTransport(handler) + ) as client, pytest.raises(RuntimeError, match="cleanup incomplete"): + cleanup(FIXTURE, client) + assert len(attempted) == 2 + assert "/quarantine/" in attempted[0] and "/golden/" in attempted[1] + + def test_startup_errors_redact_server_credentials(tmp_path, capsys): (tmp_path / "broken_server.py").write_text( "import os\nraise RuntimeError(os.environ['GITHUB_TOKEN'])\n" @@ -206,7 +320,7 @@ def test_workflow_observability_preserves_existing_artifact_default(): ) call = workflow["on"]["workflow_call"] assert call["inputs"]["graph_events_artifact_name"]["default"] == "graph-events" - assert set(call["outputs"]) == {"copied", "skipped", "digest"} + assert set(call["outputs"]) == {"copied", "skipped", "digest", "run_attempt"} live = yaml.load( (root / ".github/workflows/validate-mirror-history.yml").read_text(), Loader=yaml.BaseLoader, @@ -214,3 +328,16 @@ def test_workflow_observability_preserves_existing_artifact_default(): assert live["concurrency"]["cancel-in-progress"] == "false" assert live["jobs"]["cleanup"]["if"].startswith("${{ always()") assert live["jobs"]["promote"]["with"]["enable_approval"] == "false" + assert live["jobs"]["prepare"]["outputs"]["fixture_attempt"] == "${{ steps.paths.outputs.fixture_attempt }}" + cleanup_step = live["jobs"]["cleanup"]["steps"][-1] + assert cleanup_step["env"]["FIXTURE_ATTEMPT"] == "${{ needs.prepare.outputs.fixture_attempt || github.run_attempt }}" + for name in ("verify-initial", "verify-promoted", "verify-skipped", "verify-forced"): + steps = live["jobs"][name]["steps"] + assertion = next(step for step in steps if "FIXTURE_ATTEMPT" in step.get("env", {})) + assert '--attempt "${FIXTURE_ATTEMPT}"' in assertion["run"] + for step in steps: + if "download-artifact@" in step.get("uses", ""): + assert step["with"]["name"] == "${{ needs.verify-initial.outputs.evidence_artifact }}" + for name in ("verify-initial", "verify-forced"): + assertion = next(step for step in live["jobs"][name]["steps"] if "MIRROR_ATTEMPT" in step.get("env", {})) + assert '--execution-attempt "${MIRROR_ATTEMPT}"' in assertion["run"] From 364e225446e831788bec30d34dff749469f28a6d Mon Sep 17 00:00:00 2001 From: Toddy Mladenov Date: Tue, 27 Oct 2026 11:44:00 -0700 Subject: [PATCH 3/3] fix(catalog): exclude referrer indexes from promotion Match the existing SBOM promotion path: only image tags enter the scan matrix. This prevents a concurrent fallback-index promotion from deleting discovery metadata before image-owned referrer cleanup, which the live version-inventory assertion exposed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../workflows/_promote-from-quarantine.yml | 1 + .github/workflows/validate-mirror-history.yml | 5 ++- .../acquire/validating-mirror-history.md | 5 +++ tests/mirror_history/mirror_history_e2e.py | 5 ++- .../mirror_history/test_mirror_history_e2e.py | 41 +++++++++++++++++++ 5 files changed, 55 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_promote-from-quarantine.yml b/.github/workflows/_promote-from-quarantine.yml index 3e40acc..38d29a5 100644 --- a/.github/workflows/_promote-from-quarantine.yml +++ b/.github/workflows/_promote-from-quarantine.yml @@ -118,6 +118,7 @@ jobs: uses: ./.github/actions/enumerate-tags with: repository: ${{ inputs.source_repo }} + exclude-referrer-tags: "true" - name: Note empty repository if: steps.enumerate.outputs.has-tags != 'true' diff --git a/.github/workflows/validate-mirror-history.yml b/.github/workflows/validate-mirror-history.yml index d0f39ce..3266408 100644 --- a/.github/workflows/validate-mirror-history.yml +++ b/.github/workflows/validate-mirror-history.yml @@ -49,6 +49,8 @@ jobs: (cd "${project}" && python -m pytest -q) done - name: Lint workflows and changed shell blocks + env: + SHELLCHECK_OPTS: "--severity=warning" run: | set -euo pipefail archive=actionlint_1.7.12_linux_amd64.tar.gz @@ -59,7 +61,8 @@ jobs: -o "${RUNNER_TEMP}/actionlint-checksums.txt" (cd "${RUNNER_TEMP}" && grep " ${archive}$" actionlint-checksums.txt | sha256sum -c -) tar -xzf "${RUNNER_TEMP}/${archive}" -C "${RUNNER_TEMP}" actionlint - "${RUNNER_TEMP}/actionlint" .github/workflows/validate-mirror-history.yml .github/workflows/_mirror-image.yml + "${RUNNER_TEMP}/actionlint" .github/workflows/validate-mirror-history.yml \ + .github/workflows/_mirror-image.yml .github/workflows/_promote-from-quarantine.yml shellcheck --version python - <<'PY' import subprocess diff --git a/docs/guides/acquire/validating-mirror-history.md b/docs/guides/acquire/validating-mirror-history.md index 3543453..5cfb404 100644 --- a/docs/guides/acquire/validating-mirror-history.md +++ b/docs/guides/acquire/validating-mirror-history.md @@ -26,6 +26,11 @@ changing vulnerability databases making the lifecycle test nondeterministic. The normal Trivy scan and HIGH-severity promotion gate still run; neither exceptions nor approval overrides are used. +Promotion excludes both `mirror-history` and the `sha256-*` fallback-referrer +tags from its image matrix. A fallback index must remain available until the +owning image's cleanup discovers and deletes its referrers; promoting that +metadata tag concurrently would race with cleanup and leave untagged versions. + ## Run the validation Changes to relevant actions, workflows, services, or the test harness trigger diff --git a/tests/mirror_history/mirror_history_e2e.py b/tests/mirror_history/mirror_history_e2e.py index b5df747..583e82c 100644 --- a/tests/mirror_history/mirror_history_e2e.py +++ b/tests/mirror_history/mirror_history_e2e.py @@ -189,7 +189,10 @@ def validate_state( assert history_digest != expected.history_digest, "Appended history must be persisted as a new artifact" else: assert set(quarantine) == {HISTORY_TAG}, "Only history may remain after deletion" - assert quarantine_versions == {history_digest}, "Untagged image/referrer versions must also be deleted" + assert quarantine_versions == {history_digest}, ( + "Untagged image/referrer versions must also be deleted; remaining: " + f"{sorted(quarantine_versions - {history_digest})}" + ) assert history_digest == expected.history_digest assert history == expected.entries, "Promotion and history skips must not append entries" if phase == "skipped": diff --git a/tests/mirror_history/test_mirror_history_e2e.py b/tests/mirror_history/test_mirror_history_e2e.py index d95abeb..3d8457c 100644 --- a/tests/mirror_history/test_mirror_history_e2e.py +++ b/tests/mirror_history/test_mirror_history_e2e.py @@ -1,4 +1,5 @@ from datetime import datetime, timezone +import json from pathlib import Path import os import subprocess @@ -312,6 +313,37 @@ def test_actual_http_service_becomes_ready_and_stops(): httpx.get(f"{url}/healthz", timeout=1) +@pytest.mark.parametrize("exclude,expected", [ + ("true", ["image"]), + ("false", ["image", "sha256-subject"]), +]) +def test_tag_enumeration_filters_metadata_without_losing_image_tags(tmp_path, exclude, expected): + root = Path(__file__).resolve().parents[2] + action = yaml.load( + (root / ".github/actions/enumerate-tags/action.yml").read_text(), + Loader=yaml.BaseLoader, + ) + crane = tmp_path / "crane" + crane.write_text("#!/bin/sh\nprintf '%s\\n' image mirror-history sha256-subject\n") + crane.chmod(0o755) + output = tmp_path / "outputs" + subprocess.run( + ["bash", "-c", action["runs"]["steps"][0]["run"]], + env={ + **os.environ, + "PATH": f"{tmp_path}{os.pathsep}{os.environ['PATH']}", + "TMPDIR": str(tmp_path), + "GITHUB_OUTPUT": str(output), + "REPOSITORY": "ghcr.io/toddysm/quarantine/test-fixture", + "EXCLUDE_REFERRER_TAGS": exclude, + }, + check=True, + ) + results = dict(line.split("=", 1) for line in output.read_text().splitlines()) + assert json.loads(results["tags-json"]) == expected + assert results["has-tags"] == "true" + + def test_workflow_observability_preserves_existing_artifact_default(): root = Path(__file__).resolve().parents[2] workflow = yaml.load( @@ -328,6 +360,15 @@ def test_workflow_observability_preserves_existing_artifact_default(): assert live["concurrency"]["cancel-in-progress"] == "false" assert live["jobs"]["cleanup"]["if"].startswith("${{ always()") assert live["jobs"]["promote"]["with"]["enable_approval"] == "false" + for name in ("_promote-from-quarantine.yml", "_promote-from-quarantine-sbom.yml"): + promotion = yaml.load( + (root / ".github/workflows" / name).read_text(), Loader=yaml.BaseLoader, + ) + enumeration = next( + step for step in promotion["jobs"]["enumerate"]["steps"] + if step.get("uses") == "./.github/actions/enumerate-tags" + ) + assert enumeration["with"]["exclude-referrer-tags"] == "true" assert live["jobs"]["prepare"]["outputs"]["fixture_attempt"] == "${{ steps.paths.outputs.fixture_attempt }}" cleanup_step = live["jobs"]["cleanup"]["steps"][-1] assert cleanup_step["env"]["FIXTURE_ATTEMPT"] == "${{ needs.prepare.outputs.fixture_attempt || github.run_attempt }}"