From 5438b6901845fdf4604a9c89d41b7534ecbfa15c Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 14:22:38 -0700 Subject: [PATCH 1/2] ci: alarm on red or missing automation, and harden the release The nightly build was red for nine nights with nobody told, and release 0.1.76 left a tag with no image after a Docker Hub 429. This adds a watchdog that files an issue per finding and closes it once the finding clears, and makes a release unable to cancel another or to stop half way without a retry. hack/ci_alarm keeps one issue per alarm key (label ci-alarm, assigned to the owner with an @mention); a changed body comments, an unchanged one comments again only after 24 h. When the repository has issues disabled it prints a ::error annotation and the step summary and exits 3. hack/watchdog (.github/workflows/watchdog.yml: hourly at :47 and after CI, Docs and every release workflow completes on main) raises stale: and red: for scheduled workflows (failed jobs, the failed log up to its last error, a signature hint, one re-dispatch per failed run), disabled:, ci-missing and ci-main for main (dispatching ci.yml once when HEAD has no run), release: when the newest tag older than 45 min lacks its Docker Hub or GHCR image, its .deb, .rpm and SHA256SUMS, or its AUR update (one release re-dispatch when the Hub image is missing), bot-pr-stuck for autobump PRs over 72 h without green CI, and simulate on demand. Its own errors become watchdog-error. The tests replay the October red nights and the 0.1.76 tag from the real run list, jobs and failed log. build-and-push.yml queues each release version on its own, builds each variant in up to three attempts, and pushes only the versioned tags; the floating tags move afterwards, from hack/release-tags, only for the highest aur-v* tag, and aur-deploy.yml never moves AUR back. hack/release-paths is the one list of shipped paths (now including console/, images.lock and the shipped bench and oracle files): auto-release.yml repeats it as its push paths and release_cas.sh reads it, and a mode=changed dispatch releases only when one of them changed. Every job has a timeout, every workflow a permissions block, and no cron fires at minute 0 or 30. --- .github/workflows/aur-deploy.yml | 21 + .github/workflows/auto-release.yml | 45 +- .github/workflows/build-and-push.yml | 177 ++++- .github/workflows/ci.yml | 51 ++ .github/workflows/dependabot-auto-merge.yml | 1 + .github/workflows/docs.yml | 3 + .github/workflows/packages.yml | 3 + .github/workflows/watchdog.yml | 61 ++ hack/ci_alarm | 146 ++++ hack/ci_alarm_signatures.tsv | 8 + hack/release-paths | 29 + hack/release-tags | 38 + hack/release_cas.sh | 15 +- hack/watchdog | 669 ++++++++++++++++++ tests/auto_release_cas_test.sh | 34 +- tests/ci_alarm_test.sh | 174 +++++ tests/ci_wiring_test.sh | 10 + tests/image_test.sh | 31 +- tests/ops/fake_gh.py | 187 +++++ .../build-and-push_schedule_runs.json | 1 + tests/ops/fixtures/run_39412124435_jobs.json | 73 ++ .../fixtures/run_39412124435_log_failed.txt | 69 ++ tests/ops/fixtures/workflows.json | 1 + tests/ops/stub_registry.py | 24 + tests/ops/workflow_checks.py | 161 +++++ tests/release_tags_test.sh | 101 +++ tests/watchdog_test.sh | 422 +++++++++++ 27 files changed, 2510 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/watchdog.yml create mode 100755 hack/ci_alarm create mode 100644 hack/ci_alarm_signatures.tsv create mode 100644 hack/release-paths create mode 100755 hack/release-tags create mode 100755 hack/watchdog create mode 100755 tests/ci_alarm_test.sh create mode 100755 tests/ops/fake_gh.py create mode 100644 tests/ops/fixtures/build-and-push_schedule_runs.json create mode 100644 tests/ops/fixtures/run_39412124435_jobs.json create mode 100644 tests/ops/fixtures/run_39412124435_log_failed.txt create mode 100644 tests/ops/fixtures/workflows.json create mode 100644 tests/ops/stub_registry.py create mode 100755 tests/ops/workflow_checks.py create mode 100755 tests/release_tags_test.sh create mode 100755 tests/watchdog_test.sh diff --git a/.github/workflows/aur-deploy.yml b/.github/workflows/aur-deploy.yml index 873b7f42..a486055d 100644 --- a/.github/workflows/aur-deploy.yml +++ b/.github/workflows/aur-deploy.yml @@ -11,10 +11,14 @@ on: required: true type: string +permissions: + contents: read + jobs: deploy: name: Publish k3d-gpu to AUR runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -30,12 +34,29 @@ jobs: echo "version=$REF" >> "$GITHUB_OUTPUT" fi + # Releases build in parallel (one build-and-push queue per version), so an + # older release can reach this deploy after a newer one: AUR never moves back. + # When the AUR RPC cannot be read the deploy goes ahead. + - name: AUR already newer? + id: aur + env: + VER: ${{ steps.ver.outputs.version }} + run: | + cur=$(curl -fsSg --retry 3 --max-time 30 'https://aur.archlinux.org/rpc/v5/info?arg[]=k3d-gpu' \ + | jq -r '.results[0].Version // empty' | cut -d- -f1) + if [ -n "$cur" ] && [ "$cur" != "$VER" ] && [ "$(printf '%s\n%s\n' "$cur" "$VER" | sort -V | tail -1)" = "$cur" ]; then + echo "AUR has k3d-gpu $cur, newer than $VER; leaving it" + echo "skip=true" >> "$GITHUB_OUTPUT" + fi + - name: Update PKGBUILD + if: steps.aur.outputs.skip != 'true' run: | sed -i "s/^pkgver=.*/pkgver=${{ steps.ver.outputs.version }}/" packaging/aur/PKGBUILD sed -i "s/^pkgrel=.*/pkgrel=1/" packaging/aur/PKGBUILD - name: Publish to AUR + if: steps.aur.outputs.skip != 'true' uses: KSXGitHub/github-actions-deploy-aur@084b0d9b15415bf9cdb65d44dad1efe37a354050 # v4.2.0 with: pkgname: k3d-gpu diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index 82cd1cd6..d9ebe3c9 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -1,12 +1,21 @@ name: Auto-release AUR -# Auto-tag a release on changes to packaging/, scripts/, share/ or the Dockerfile, -# and when build-and-push.yml's nightly check bumps upstream pins (dispatch). +# Auto-tag a release when a commit on main changes a path hack/release-paths lists +# (what the packages, the formula and the image ship: packaging/, scripts/, +# share/, the Dockerfile, console/, images.lock and the shipped bench/ and oracle/ +# files), and when build-and-push.yml's nightly check bumps upstream pins +# (dispatch). The push paths: below repeat hack/release-paths (GitHub reads no +# file there); tests/ci_wiring_test.sh checks they are equal. # Tag format: aur-vX.Y.Z. The launcher at that tag defaults to node image # cryptoandcoffee/k3d-gpu:X.Y.Z, so after tagging this dispatches build-and-push.yml # with release=X.Y.Z; that run publishes the image and then dispatches aur-deploy.yml. # Dispatches bypass the GITHUB_TOKEN downstream-trigger limit. # +# A dispatch's mode: force (the default) always releases; changed releases only +# when a commit above the last tag changed a shipped path (release_cas.sh runs as +# for a push), so a merge that a GITHUB_TOKEN push did not announce can still cut +# its release, and a dispatch with nothing shipped cuts none. +# # hack/release_cas.sh releases from a fresh origin/main and pushes main and the # tag in one atomic push (--atomic), retrying when another writer moved main or # took the tag. On those lock and non-fast-forward races GitHub rejected both @@ -14,9 +23,9 @@ name: Auto-release AUR # other rejection classes a release commit can land # without its tag, and the next run tags it in place. A push run whose commits # an earlier run already released exits with no tag. The -# concurrency group only queues runs; correctness does not depend on it. Push and -# dispatch runs queue separately, so a pending forced dispatch is never replaced -# by a push run that finds nothing to release. +# concurrency group only queues runs; correctness does not depend on it. Push +# runs and each dispatch mode queue separately, so a pending forced dispatch is +# never replaced by a run that finds nothing to release. on: push: @@ -26,7 +35,28 @@ on: - 'scripts/**' - 'share/**' - 'Dockerfile' + - 'console/**' + - 'images.lock' + - 'bench/vs.py' + - 'bench/llm_load.py' + - 'bench/energy.sh' + - 'bench/agent_trace.py' + - 'bench/agent_trace_gen.py' + - 'bench/apb/apb.py' + - 'bench/apb/capacity.py' + - 'bench/competitors.lock' + - 'oracle/bundle.py' + - 'oracle/project.py' + - 'oracle/validate.py' + - 'oracle/share.py' + - 'oracle/schema/bundle-v1.json' workflow_dispatch: + inputs: + mode: + description: 'force: always release; changed: release only when a shipped path changed since the last tag' + type: choice + options: [force, changed] + default: force permissions: contents: write @@ -36,8 +66,9 @@ jobs: release: if: "github.actor != '88plug-bot' && !contains(github.event.head_commit.message, '[skip release]')" runs-on: ubuntu-latest + timeout-minutes: 15 concurrency: - group: auto-release-${{ github.event_name }} + group: auto-release-${{ github.event_name }}-${{ inputs.mode || 'push' }} cancel-in-progress: false steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -48,7 +79,7 @@ jobs: - name: Release (atomic push of main and the tag) id: rel env: - EVENT: ${{ github.event_name }} + EVENT: ${{ github.event_name == 'workflow_dispatch' && inputs.mode == 'changed' && 'push' || github.event_name }} run: | git config user.name "88plug-bot" git config user.email "andrew@88plug.com" diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 46c75bd8..accb56f8 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -5,19 +5,23 @@ name: Auto-update, Build & Release # check (nightly, or manual without `release`): resolve upstream versions; when # one moved (or `force`), commit the bumped pins to main and dispatch # auto-release.yml, which cuts a launcher release for them. -# release (dispatched by auto-release.yml with `release=X.Y.Z`): build the image -# from tag aur-vX.Y.Z, publish :X.Y.Z (the launcher's default -# image) next to the floating tags, then dispatch aur-deploy.yml and -# packages.yml (deb, rpm, Homebrew) so the launcher reaches every package -# channel only once its image exists. +# release (dispatched by auto-release.yml with `release=X.Y.Z`, or by +# hack/watchdog when a release tag has no image): build the image from +# tag aur-vX.Y.Z, publish :X.Y.Z (the launcher's default image), +# move the floating tags when X.Y.Z is the newest release, then dispatch +# aur-deploy.yml and packages.yml (deb, rpm, Homebrew) so the launcher +# reaches every package channel only once its image exists. # # Every image is built from a release tag, so every published image has a # launcher version. Pushes made with GITHUB_TOKEN start no workflow runs, but # workflow_dispatch does, hence the explicit dispatches. +# +# Each release version queues on its own (one release never replaces another +# pending release); the nightly check queues in `nightly`. on: schedule: - - cron: '0 6 * * *' + - cron: '17 6 * * *' workflow_dispatch: inputs: force: @@ -30,7 +34,7 @@ on: default: '' concurrency: - group: build-and-push + group: build-and-push-${{ inputs.release && format('release-{0}', inputs.release) || 'nightly' }} cancel-in-progress: false permissions: @@ -42,6 +46,7 @@ jobs: check: if: ${{ !inputs.release }} runs-on: ubuntu-latest + timeout-minutes: 15 steps: - name: Checkout main uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -292,6 +297,7 @@ jobs: release: if: ${{ inputs.release }} runs-on: ubuntu-latest + timeout-minutes: 60 env: RELEASE: ${{ inputs.release }} steps: @@ -347,30 +353,128 @@ jobs: exit 1 fi + # hack/release-tags comes from main: the release tag checked out above can + # predate it (the watchdog re-dispatches an old release whose image is missing). + - name: Checkout main's release tooling + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + path: .ci-main + sparse-checkout: hack + persist-credentials: false + + # The builds push the versioned tags (:X.Y.Z, -ubuntu26.04, -ubuntu24.04) + # only; the floating tags move after both builds, below. + - name: Tags to push + id: tags + env: + K3S: ${{ steps.current.outputs.k3s }} + RELEASE_REPOS: cryptoandcoffee/k3d-gpu ghcr.io/${{ github.repository_owner }}/k3d-gpu + run: | + tags=$(.ci-main/hack/release-tags "$RELEASE" "$K3S") + printf '%s\n' "$tags" + for v in 26.04 24.04; do + echo "v${v/./}<> "$GITHUB_OUTPUT" + # Ubuntu 26.04 is the default (:latest and :X.Y.Z); 24.04 is also published. # Each carries the baked device plugins, so clusters expose GPUs with no apply. - - name: Build & push — Ubuntu 26.04 (default / latest) + # Each variant builds in up to 3 attempts with the same inputs (a Docker Hub + # 429 stopped release 0.1.76 with its tag cut and no image): attempts 1 and 2 + # continue on error and are followed by a 2 and a 4 min wait; attempt 3 fails + # the job. + - name: Build & push — Ubuntu 26.04 (attempt 1) + id: b2604_1 + continue-on-error: true + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + push: true + platforms: linux/amd64 + labels: | + org.opencontainers.image.version=${{ inputs.release }} + org.opencontainers.image.revision=aur-v${{ inputs.release }} + tags: ${{ steps.tags.outputs.v2604 }} + + - name: Wait 2 min before Ubuntu 26.04 attempt 2 + if: steps.b2604_1.outcome == 'failure' + run: sleep 120 + + - name: Build & push — Ubuntu 26.04 (attempt 2) + id: b2604_2 + if: steps.b2604_1.outcome == 'failure' + continue-on-error: true + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + push: true + platforms: linux/amd64 + labels: | + org.opencontainers.image.version=${{ inputs.release }} + org.opencontainers.image.revision=aur-v${{ inputs.release }} + tags: ${{ steps.tags.outputs.v2604 }} + + - name: Wait 4 min before Ubuntu 26.04 attempt 3 + if: steps.b2604_2.outcome == 'failure' + run: sleep 240 + + - name: Build & push — Ubuntu 26.04 (attempt 3) + id: b2604_3 + if: steps.b2604_2.outcome == 'failure' + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + push: true + platforms: linux/amd64 + labels: | + org.opencontainers.image.version=${{ inputs.release }} + org.opencontainers.image.revision=aur-v${{ inputs.release }} + tags: ${{ steps.tags.outputs.v2604 }} + + - name: Build & push — Ubuntu 24.04 (attempt 1) + id: b2404_1 + continue-on-error: true + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + push: true + platforms: linux/amd64 + build-args: | + UBUNTU_IMAGE=${{ steps.current.outputs.ubuntu2404 }} + labels: | + org.opencontainers.image.version=${{ inputs.release }} + org.opencontainers.image.revision=aur-v${{ inputs.release }} + tags: ${{ steps.tags.outputs.v2404 }} + + - name: Wait 2 min before Ubuntu 24.04 attempt 2 + if: steps.b2404_1.outcome == 'failure' + run: sleep 120 + + - name: Build & push — Ubuntu 24.04 (attempt 2) + id: b2404_2 + if: steps.b2404_1.outcome == 'failure' + continue-on-error: true uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . push: true platforms: linux/amd64 + build-args: | + UBUNTU_IMAGE=${{ steps.current.outputs.ubuntu2404 }} labels: | org.opencontainers.image.version=${{ inputs.release }} org.opencontainers.image.revision=aur-v${{ inputs.release }} - tags: | - cryptoandcoffee/k3d-gpu:${{ inputs.release }} - cryptoandcoffee/k3d-gpu:${{ inputs.release }}-ubuntu26.04 - cryptoandcoffee/k3d-gpu:latest - cryptoandcoffee/k3d-gpu:ubuntu26.04 - cryptoandcoffee/k3d-gpu:${{ steps.current.outputs.k3s }}-ubuntu26.04 - ghcr.io/${{ github.repository_owner }}/k3d-gpu:${{ inputs.release }} - ghcr.io/${{ github.repository_owner }}/k3d-gpu:${{ inputs.release }}-ubuntu26.04 - ghcr.io/${{ github.repository_owner }}/k3d-gpu:latest - ghcr.io/${{ github.repository_owner }}/k3d-gpu:ubuntu26.04 - ghcr.io/${{ github.repository_owner }}/k3d-gpu:${{ steps.current.outputs.k3s }}-ubuntu26.04 - - - name: Build & push — Ubuntu 24.04 (variant) + tags: ${{ steps.tags.outputs.v2404 }} + + - name: Wait 4 min before Ubuntu 24.04 attempt 3 + if: steps.b2404_2.outcome == 'failure' + run: sleep 240 + + - name: Build & push — Ubuntu 24.04 (attempt 3) + id: b2404_3 + if: steps.b2404_2.outcome == 'failure' uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . @@ -381,13 +485,26 @@ jobs: labels: | org.opencontainers.image.version=${{ inputs.release }} org.opencontainers.image.revision=aur-v${{ inputs.release }} - tags: | - cryptoandcoffee/k3d-gpu:${{ inputs.release }}-ubuntu24.04 - cryptoandcoffee/k3d-gpu:ubuntu24.04 - cryptoandcoffee/k3d-gpu:${{ steps.current.outputs.k3s }}-ubuntu24.04 - ghcr.io/${{ github.repository_owner }}/k3d-gpu:${{ inputs.release }}-ubuntu24.04 - ghcr.io/${{ github.repository_owner }}/k3d-gpu:ubuntu24.04 - ghcr.io/${{ github.repository_owner }}/k3d-gpu:${{ steps.current.outputs.k3s }}-ubuntu24.04 + tags: ${{ steps.tags.outputs.v2404 }} + + # The floating tags (latest, ubuntu26.04, ubuntu24.04, -ubuntu26.04, + # -ubuntu24.04) move only when this release is the highest aur-v* tag on + # origin now, after both variants pushed: releases build in parallel (one + # queue per version), so an older one that finishes late leaves them alone. + # Each is a registry-side copy of the versioned tag pushed above. + - name: Move the floating tags (newest release only) + env: + K3S: ${{ steps.current.outputs.k3s }} + RELEASE_REPOS: cryptoandcoffee/k3d-gpu ghcr.io/${{ github.repository_owner }}/k3d-gpu + run: | + tags=$(.ci-main/hack/release-tags "$RELEASE" "$K3S") + n=0 + while read -r _ kind ref src; do + [ "$kind" = floating ] || continue + docker buildx imagetools create --tag "$ref" "$src" + n=$((n + 1)) + done <<< "$tags" + echo "moved $n floating tags" - name: Create GitHub Release uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 @@ -445,6 +562,7 @@ jobs: list-components: if: ${{ !inputs.release }} runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: read outputs: @@ -463,6 +581,7 @@ jobs: needs: list-components if: needs.list-components.outputs.names != '[]' runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: write pull-requests: write @@ -540,6 +659,7 @@ jobs: repin-images: if: ${{ !inputs.release }} runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: write pull-requests: write @@ -634,6 +754,7 @@ jobs: bump-convert-pins: if: ${{ !inputs.release }} runs-on: ubuntu-latest + timeout-minutes: 15 permissions: contents: write pull-requests: write diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54bacd6d..6c63ca54 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,9 @@ on: branches: [main] pull_request: branches: [main] + # hack/watchdog dispatches it on main when main's HEAD has no CI run (a push made + # with GITHUB_TOKEN starts no workflow); it tests the dispatched ref's head. + workflow_dispatch: permissions: contents: read @@ -18,6 +21,7 @@ jobs: shellcheck: name: Shellcheck runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: ludeeus/action-shellcheck@00cae500b08a931fb5698e11e79bfbd38e612a38 # 2.0.0 @@ -29,6 +33,7 @@ jobs: actionlint: name: Workflow lint runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # rhysd/actionlint, pinned by digest in hack/ci-images.env (the nightly repin moves it). @@ -40,6 +45,7 @@ jobs: detect-tests: name: Detection tests runs-on: ubuntu-latest + timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: tests/detect_test.sh @@ -60,6 +66,7 @@ jobs: quickstart-tests: name: Quickstart, uninstall and installer runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: tests/quickstart_test.sh @@ -68,6 +75,7 @@ jobs: fuzz-props: name: Launcher input properties (fuzz) runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # A fresh seed each run; a failure prints its FUZZ_SEED replay line. @@ -76,6 +84,7 @@ jobs: remote-tests: name: Remote OpenAI endpoints (resolver, Secret, sandbox egress, redaction) runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: python3 tests/remote_test.py @@ -84,6 +93,7 @@ jobs: telemetry-tests: name: Telemetry manifests runs-on: ubuntu-latest + timeout-minutes: 10 env: PROMETHEUS_VERSION: 3.15.1 PROMETHEUS_SHA256: 6a726885328ad9efebbf36e876ce2fb6225468a71fd1e7ce3ba666d5f4d97a84 @@ -108,6 +118,7 @@ jobs: agent-manifests: name: Agent manifests (kubeconform) runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: tests/agent_kubeconform.sh @@ -115,6 +126,7 @@ jobs: intel-gpu-dra-manifests: name: Intel GPU DRA manifests (kubeconform) runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: tests/kubeconform.sh @@ -122,6 +134,7 @@ jobs: share-manifests: name: share/ manifests (kubeconform, CRD schemas) runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends python3-yaml @@ -130,6 +143,7 @@ jobs: console-tests: name: Console spine (facts/v1, plan, audit) runs-on: ubuntu-latest + timeout-minutes: 40 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install jsonschema for the schema cross-check (pinned) @@ -152,6 +166,7 @@ jobs: images-lock: name: images.lock complete, every image pinned by digest runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: scripts/images-lock check @@ -160,6 +175,7 @@ jobs: report-html: name: report.html (goldens, offline load in headless Chrome) runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - id: chrome @@ -177,6 +193,7 @@ jobs: oracle: name: Capacity oracle (schema, privacy, validator, fuzz, matrix, leaderboard, share) runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -200,6 +217,7 @@ jobs: claims-check: name: Docs claims guard runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: tests/claims_check_test.sh @@ -213,6 +231,7 @@ jobs: bench-tests: name: Bench harness tests runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: tests/bench_test.sh @@ -222,6 +241,7 @@ jobs: fit-tests: name: k3d-gpu fit and guard tests runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # tests/fit_test.py (the fit arithmetic on fixture hosts and models, the guard @@ -234,6 +254,7 @@ jobs: tune-tests: name: k3d-gpu tune tests runs-on: ubuntu-latest + timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # TUNE_E2E_FULL: two whole tunes against tests/fake_ovms.py, one where no @@ -245,6 +266,7 @@ jobs: capacity-tests: name: k3d-gpu capacity tests (stub server) runs-on: ubuntu-latest + timeout-minutes: 25 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: python3 bench/apb/test_capacity.py @@ -276,6 +298,7 @@ jobs: mcp-tests: name: MCP server tests runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: tests/mcp_test.sh @@ -283,6 +306,7 @@ jobs: kuberay-example: name: KubeRay example (static) runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: sudo apt-get update && sudo apt-get install -y python3-yaml @@ -291,6 +315,7 @@ jobs: chat-ui-manifest: name: Chat UI manifest (static + kubeconform) runs-on: ubuntu-latest + timeout-minutes: 10 env: KUBECONFORM_VERSION: v0.8.0 KUBECONFORM_SHA256: 9bc2bffbf71f261128533edaf912153948b7ff238f9a531ae6d34466ec287883 @@ -310,6 +335,7 @@ jobs: sim-fixtures: name: Simulated hardware list runs-on: ubuntu-latest + timeout-minutes: 10 outputs: platforms: ${{ steps.list.outputs.platforms }} steps: @@ -321,6 +347,7 @@ jobs: name: Simulated hardware (${{ matrix.platform }}) needs: sim-fixtures runs-on: ubuntu-latest + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -334,6 +361,7 @@ jobs: agent-examples: name: Agent framework examples (${{ matrix.example }}) runs-on: ubuntu-latest + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -345,6 +373,7 @@ jobs: register-model-test: name: LiteLLM register-model.sh runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: tests/register_model_test.sh @@ -352,6 +381,7 @@ jobs: render: name: Rendered components runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install pinned helm @@ -370,6 +400,7 @@ jobs: packaging-tests: name: Packaging consistency tests runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: tests/packaging_test.sh @@ -377,6 +408,7 @@ jobs: deb-rpm: name: Build and test .deb + .rpm (nfpm) runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: packaging/check.sh nfpm @@ -384,6 +416,7 @@ jobs: homebrew: name: Homebrew formula style, audit, install, test runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: packaging/check.sh brew @@ -391,6 +424,7 @@ jobs: nix-flake: name: Nix flake check runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: packaging/check.sh nix @@ -398,6 +432,7 @@ jobs: hadolint: name: Dockerfile lint runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: hadolint/hadolint-action@06be81baf89a55ffd0e24b8f04a4185738dd3387 # v3.5.0 @@ -410,6 +445,7 @@ jobs: arch-image: name: Arch image pin runs-on: ubuntu-latest + timeout-minutes: 10 outputs: ref: ${{ steps.pin.outputs.ref }} steps: @@ -427,6 +463,7 @@ jobs: name: Lint PKGBUILD in Arch container needs: arch-image runs-on: ubuntu-latest + timeout-minutes: 10 container: ${{ needs.arch-image.outputs.ref }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -447,6 +484,7 @@ jobs: kserve-example: name: KServe example (stub install + kubeconform 1.34/1.35) runs-on: ubuntu-latest + timeout-minutes: 10 env: KUBECONFORM_VERSION: v0.8.0 KUBECONFORM_SHA256: 9bc2bffbf71f261128533edaf912153948b7ff238f9a531ae6d34466ec287883 @@ -463,3 +501,16 @@ jobs: sudo tar -xzf kubeconform.tar.gz -C /usr/local/bin kubeconform sudo apt-get update && sudo apt-get install -y --no-install-recommends python3-yaml - run: tests/kserve_test.sh + + # ─── Alarm, watchdog and release hardening ────────────────────────────── + ops-tests: + name: Alarm, watchdog and release tags + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: tests/ci_alarm_test.sh + - run: tests/watchdog_test.sh + - run: tests/release_tags_test.sh diff --git a/.github/workflows/dependabot-auto-merge.yml b/.github/workflows/dependabot-auto-merge.yml index cd7a4ecf..8750fdc6 100644 --- a/.github/workflows/dependabot-auto-merge.yml +++ b/.github/workflows/dependabot-auto-merge.yml @@ -10,6 +10,7 @@ jobs: automerge: if: github.actor == 'dependabot[bot]' runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Enable auto-merge (squash) run: gh pr merge --auto --squash "$PR_URL" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d333cad2..50665fd6 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -41,6 +41,7 @@ jobs: image: name: Docs interpreter pin runs-on: ubuntu-latest + timeout-minutes: 10 outputs: ref: ${{ steps.pin.outputs.ref }} steps: @@ -58,6 +59,7 @@ jobs: name: Build (mkdocs --strict) needs: image runs-on: ubuntu-latest + timeout-minutes: 15 outputs: pages: ${{ steps.pages.outputs.enabled }} container: @@ -89,6 +91,7 @@ jobs: if: needs.build.outputs.pages == 'true' needs: build runs-on: ubuntu-latest + timeout-minutes: 15 permissions: pages: write id-token: write diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 419040bc..868bd818 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -45,6 +45,7 @@ jobs: deb-rpm: name: Build and attach .deb + .rpm runs-on: ubuntu-latest + timeout-minutes: 15 steps: - name: Determine version id: ver @@ -85,6 +86,7 @@ jobs: homebrew: name: Update Homebrew formula runs-on: ubuntu-latest + timeout-minutes: 15 steps: - name: Determine version id: ver @@ -147,6 +149,7 @@ jobs: needs: deb-rpm if: ${{ !inputs.dry_run }} runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Determine version id: ver diff --git a/.github/workflows/watchdog.yml b/.github/workflows/watchdog.yml new file mode 100644 index 00000000..87beefbf --- /dev/null +++ b/.github/workflows/watchdog.yml @@ -0,0 +1,61 @@ +name: Watchdog + +# hack/watchdog checks, hourly and after each listed workflow completes on main: +# scheduled workflows that went red or stopped succeeding, disabled workflows, +# main without CI or with red CI, release tags without their images, packages or +# AUR update, and autobump PRs stuck without green CI. Each finding is one issue +# labelled ci-alarm (hack/ci_alarm), assigned to the owner with an @mention; it +# closes itself once the finding clears. Dispatch with simulate_red=true to file +# the `simulate` alarm; the next run without it closes it. +# +# It reads main, never the triggering run's code. workflow_run passes only runs on +# main (branches: and the job's if:); the hourly cron covers a missed event. + +on: + schedule: + - cron: '47 * * * *' + workflow_run: + workflows: + - 'CI' + - 'Auto-update, Build & Release' + - 'Auto-release AUR' + - 'Release packages (deb, rpm, Homebrew)' + - 'AUR Deploy' + - 'Docs' + types: [completed] + branches: [main] + workflow_dispatch: + inputs: + simulate_red: + description: 'File the `simulate` alarm, to check that alarms reach the owner' + type: boolean + default: false + +permissions: + contents: read + issues: write + actions: write + pull-requests: read + +concurrency: + group: watchdog + cancel-in-progress: false + +jobs: + watchdog: + name: Watchdog + if: github.event_name != 'workflow_run' || github.event.workflow_run.head_branch == 'main' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + fetch-depth: 0 # the aur-v* tags and their dates + persist-credentials: false + - name: Check and alarm + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + SIMULATE_RED: ${{ inputs.simulate_red }} + run: hack/watchdog diff --git a/hack/ci_alarm b/hack/ci_alarm new file mode 100755 index 00000000..37e20c69 --- /dev/null +++ b/hack/ci_alarm @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# ci_alarm: one GitHub issue per alarm key, labelled ci-alarm. +# +# hack/ci_alarm open <body-file> +# Keeps one open issue titled "[ci-alarm] <key>: <title>", labelled ci-alarm +# (created with `gh label create --force` when the issue is filed), assigned +# to $ALARM_OWNER with an @mention in its body. When the key's issue is +# already open: a new body replaces the issue body and is posted as a +# comment; the same body comments again only once 24 h have passed since +# the last comment (or since the issue was opened). +# hack/ci_alarm close <key> <comment> +# Comments and closes the key's open issue; no open issue is a no-op. +# hack/ci_alarm list +# Prints the key of every open ci-alarm issue, one per line. +# +# A key is [A-Za-z0-9._:/@+-]+ (no spaces), e.g. red:build-and-push.yml, +# release:0.1.76, watchdog-error. gh picks the repository from GH_REPO, else the +# checkout's remote. +# +# When the repository has issues disabled, `open` prints a ::error workflow +# annotation with the title and appends the body to $GITHUB_STEP_SUMMARY, and +# every command exits 3. Exit 2 is a usage error, 1 any other gh failure. +# +# Env: ALARM_OWNER (default 88plug), CI_ALARM_NOW (epoch seconds, for tests), +# GITHUB_STEP_SUMMARY. +set -euo pipefail + +OWNER=${ALARM_OWNER:-88plug} +LABEL=ci-alarm +NOW=${CI_ALARM_NOW:-$(date -u +%s)} +DAY=86400 + +usage() { sed -n '2,24p' "$0" >&2; exit 2; } +TMP=$(mktemp -d "${TMPDIR:-/tmp}/ci_alarm.XXXXXX") +trap 'rm -rf "$TMP"' EXIT + +# gh_run ARGS...: run gh; on failure print its stderr and exit 3 (issues +# disabled) or 1 (anything else). +gh_run() { + if gh "$@" > "$TMP/gh.out" 2> "$TMP/gh.err"; then + cat "$TMP/gh.out" + return 0 + fi + cat "$TMP/gh.err" >&2 + if grep -qi 'disabled issues' "$TMP/gh.err"; then exit 3; fi + exit 1 +} + +# open_issues: JSON array of the open ci-alarm issues. +open_issues() { + gh_run issue list --label "$LABEL" --state open --limit 200 --json number,title,body,comments,createdAt +} + +# issue_for KEY: the key's open issue (lowest number) as JSON, or nothing. +issue_for() { + open_issues > "$TMP/issues.json" + jq -c --arg p "[ci-alarm] $1: " \ + 'map(select(.title | startswith($p))) | sort_by(.number) | .[0] // empty' "$TMP/issues.json" +} + +check_key() { + [[ "$1" =~ ^[A-Za-z0-9._:/@+-]+$ ]] || { echo "ci_alarm: bad key '$1' (want [A-Za-z0-9._:/@+-]+)" >&2; exit 2; } +} + +cmd_open() { + [ $# -eq 3 ] || usage + local key=$1 title=$2 file=$3 full issue num old new last + check_key "$key" + [ -n "$title" ] || { echo "ci_alarm: empty title" >&2; exit 2; } + [ -r "$file" ] || { echo "ci_alarm: cannot read body file '$file'" >&2; exit 2; } + full="[ci-alarm] $key: $title" + # shellcheck disable=SC2016 # markdown backticks, not a command substitution + { printf '@%s ci-alarm `%s`\n\n' "$OWNER" "$key"; cat "$file"; } > "$TMP/body" + + issue=$(issue_for "$key") || exit $? + if [ -z "$issue" ]; then + gh_run label create "$LABEL" --force --color B60205 \ + --description "Automation alarm filed by hack/ci_alarm" >/dev/null + gh_run issue create --title "$full" --body-file "$TMP/body" --label "$LABEL" --assignee "$OWNER" + return 0 + fi + + num=$(jq -r .number <<< "$issue") + # GitHub may store the body with CRLF and without the trailing newline. + old=$(jq -r '.body | gsub("\r\n"; "\n") | sub("\\s+$"; "")' <<< "$issue") + new=$(jq -Rrs 'gsub("\r\n"; "\n") | sub("\\s+$"; "")' "$TMP/body") + if [ "$old" != "$new" ] || [ "$(jq -r .title <<< "$issue")" != "$full" ]; then + gh_run issue edit "$num" --title "$full" --body-file "$TMP/body" >/dev/null + gh_run issue comment "$num" --body-file "$TMP/body" >/dev/null + echo "ci_alarm: $key updated (#$num)" + return 0 + fi + last=$(jq -r '[.createdAt] + [.comments[].createdAt] | max' <<< "$issue") + if [ $(( NOW - $(date -u -d "$last" +%s) )) -ge "$DAY" ]; then + printf '@%s still open: %s\n' "$OWNER" "$full" > "$TMP/reminder" + gh_run issue comment "$num" --body-file "$TMP/reminder" >/dev/null + echo "ci_alarm: $key reminded (#$num)" + else + echo "ci_alarm: $key unchanged (#$num)" + fi +} + +cmd_close() { + [ $# -eq 2 ] || usage + local key=$1 comment=$2 issue num + check_key "$key" + issue=$(issue_for "$key") || exit $? + if [ -z "$issue" ]; then + echo "ci_alarm: no open alarm $key" + return 0 + fi + num=$(jq -r .number <<< "$issue") + gh_run issue close "$num" --comment "$comment" >/dev/null + echo "ci_alarm: $key closed (#$num)" +} + +cmd_list() { + [ $# -eq 0 ] || usage + open_issues > "$TMP/issues.json" + jq -r '.[].title | capture("^\\[ci-alarm\\] (?<k>[^ ]+): ") | .k' "$TMP/issues.json" +} + +# Issues disabled: `open` still reaches the run's log and summary. +annotate() { + local key=$1 title=$2 file=$3 + echo "::error title=ci-alarm ${key}::${title} (not filed: this repository has issues disabled)" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + # shellcheck disable=SC2016 # markdown backticks + { printf '### ci-alarm `%s`: %s\n\n' "$key" "$title"; cat "$file"; echo; } >> "$GITHUB_STEP_SUMMARY" + fi +} + +[ $# -ge 1 ] || usage +cmd=$1; shift +case "$cmd" in + open) + set +e + (cmd_open "$@") + rc=$? + set -e + if [ "$rc" -eq 3 ] && [ $# -eq 3 ]; then annotate "$@"; fi + exit "$rc" ;; + close) cmd_close "$@" ;; + list) cmd_list "$@" ;; + *) usage ;; +esac diff --git a/hack/ci_alarm_signatures.tsv b/hack/ci_alarm_signatures.tsv new file mode 100644 index 00000000..b9c4d373 --- /dev/null +++ b/hack/ci_alarm_signatures.tsv @@ -0,0 +1,8 @@ +# Known failure signatures for hack/watchdog: <Python regex> TAB <hint>. +# The watchdog scans a failed run's log and puts the hint of the first matching +# line (top to bottom in this file) in the alarm body. Lines starting with # are comments. +registry serves sha256 An upstream tag was re-pushed (the registry serves a new digest for a pinned tag): repin with scripts/images-lock repin, then update and check. +without .workflows. permission A bot push touched .github/workflows/: GITHUB_TOKEN cannot create or update workflow files. Delete the stale autobump branch (its workflows differ from main) or move the pin out of the workflow file. +refusing to allow a GitHub App A bot push touched .github/workflows/: GITHUB_TOKEN cannot create or update workflow files. Delete the stale autobump branch (its workflows differ from main) or move the pin out of the workflow file. +429 Too Many Requests Docker Hub rate limit: retry later; the release build retries 3 times with backoff. +non-fast-forward Release race: another writer moved main between fetch and push; the next run releases from the new tip. diff --git a/hack/release-paths b/hack/release-paths new file mode 100644 index 00000000..a3bc94e3 --- /dev/null +++ b/hack/release-paths @@ -0,0 +1,29 @@ +# The paths whose change ships to users: a commit on main that touches one of them +# (and carries no [skip release]) cuts a launcher release. One entry per line: a +# file, or dir/** for everything under dir. # starts a comment. +# +# Read by hack/release_cas.sh (its git log path filter for push runs and +# mode=changed dispatches). .github/workflows/auto-release.yml repeats the list as +# its literal push paths: (GitHub reads no file there); tests/ci_wiring_test.sh +# checks the two are equal, and that this list covers every file +# packaging/nfpm/nfpm.yaml and Formula/k3d-gpu.rb install (README.md and +# LICENSE.md excepted: documentation rides the next release). +packaging/** +scripts/** +share/** +Dockerfile +console/** +images.lock +bench/vs.py +bench/llm_load.py +bench/energy.sh +bench/agent_trace.py +bench/agent_trace_gen.py +bench/apb/apb.py +bench/apb/capacity.py +bench/competitors.lock +oracle/bundle.py +oracle/project.py +oracle/validate.py +oracle/share.py +oracle/schema/bundle-v1.json diff --git a/hack/release-tags b/hack/release-tags new file mode 100755 index 00000000..4be75439 --- /dev/null +++ b/hack/release-tags @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# release-tags: the image tags a release build pushes, for build-and-push.yml. +# +# hack/release-tags <version X.Y.Z> <k3s tag vX.Y.Z-k3sN> +# +# Prints one line per tag: +# <variant> versioned <ref> pushed by the build of every release +# <variant> floating <ref> <source> copied from <source> after both builds +# variant is 26.04 (the default image) or 24.04. Versioned: <repo>:<ver>, +# <repo>:<ver>-ubuntu26.04, <repo>:<ver>-ubuntu24.04. Floating: <repo>:latest, +# <repo>:ubuntu26.04, <repo>:<k3s>-ubuntu26.04, <repo>:ubuntu24.04, +# <repo>:<k3s>-ubuntu24.04. Floating tags are printed only when <version> is the +# highest aur-v* tag on $REMOTE now (git ls-remote, sort -V), so an older release +# that builds late never moves them back. An unreadable remote is an error. +# +# Env: RELEASE_REPOS (default "cryptoandcoffee/k3d-gpu ghcr.io/88plug/k3d-gpu"), +# REMOTE (default origin). +set -euo pipefail + +[ $# -eq 2 ] || { echo "usage: hack/release-tags <version> <k3s tag>" >&2; exit 2; } +VER=$1 K3S=$2 +[[ "$VER" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo "release-tags: version '$VER' is not X.Y.Z" >&2; exit 2; } +[[ "$K3S" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-k3s[0-9]+$ ]] || { echo "release-tags: k3s tag '$K3S' is not vX.Y.Z-k3sN" >&2; exit 2; } +read -r -a REPOS <<< "${RELEASE_REPOS:-cryptoandcoffee/k3d-gpu ghcr.io/88plug/k3d-gpu}" + +refs=$(git ls-remote --tags "${REMOTE:-origin}" 'aur-v*') +highest=$(sed -n 's|^[0-9a-f]*[[:space:]]*refs/tags/aur-v\([0-9]*\.[0-9]*\.[0-9]*\)$|\1|p' <<< "$refs" | sort -V | tail -n 1) + +for r in "${REPOS[@]}"; do + echo "26.04 versioned $r:$VER" + echo "26.04 versioned $r:$VER-ubuntu26.04" + echo "24.04 versioned $r:$VER-ubuntu24.04" +done +[ "$VER" = "$highest" ] || exit 0 +for r in "${REPOS[@]}"; do + for t in latest ubuntu26.04 "$K3S-ubuntu26.04"; do echo "26.04 floating $r:$t $r:$VER"; done + for t in ubuntu24.04 "$K3S-ubuntu24.04"; do echo "24.04 floating $r:$t $r:$VER-ubuntu24.04"; done +done diff --git a/hack/release_cas.sh b/hack/release_cas.sh index 97b4c1f6..261ca4f8 100755 --- a/hack/release_cas.sh +++ b/hack/release_cas.sh @@ -18,9 +18,12 @@ # release commit whose tag did not land), else one patch above the last tag. # # EVENT=push releases when main's VERSION is above the last tag, or when a -# commit above the last tag touches packaging/, scripts/, share/ or Dockerfile -# and carries no [skip release] in its message; otherwise it exits 0 with no -# tag. Any other EVENT (workflow_dispatch) always releases. +# commit above the last tag touches a path main's hack/release-paths lists (what +# nfpm, the AUR package, the formula and the image ship) and carries no +# [skip release] in its message; otherwise it prints "nothing to release" and +# exits 0 with no tag. auto-release.yml passes EVENT=push for pushes and for +# mode=changed dispatches. Any other EVENT (a forced workflow_dispatch) always +# releases. # # Env: EVENT, REMOTE (default origin), RELEASE_CAS_ATTEMPTS, GITHUB_OUTPUT # (receives tag= and ver= when a release is cut). @@ -43,8 +46,12 @@ for attempt in $(seq 1 "${RELEASE_CAS_ATTEMPTS:-5}"); do VER=0.1.0 else if [ "$EVENT" = push ]; then + # hack/release-paths entries: a file, or dir/** (the pathspec dir). + [ -f hack/release-paths ] || { echo "no hack/release-paths on $REMOTE/main"; exit 1; } + mapfile -t paths < <(sed -e 's/#.*//' -e 's/[[:space:]]*$//' -e '/^$/d' -e 's|/\*\*$||' hack/release-paths) + [ "${#paths[@]}" -gt 0 ] || { echo "hack/release-paths lists no path"; exit 1; } n=$(git log -F --invert-grep --grep='[skip release]' --format=%H "$LATEST..HEAD" \ - -- packaging scripts share Dockerfile | wc -l) + -- "${paths[@]}" | wc -l) if [ "$n" -eq 0 ]; then echo "nothing to release above $LATEST"; exit 0; fi fi VER=${LAST%.*}.$(( ${LAST##*.} + 1 )) diff --git a/hack/watchdog b/hack/watchdog new file mode 100755 index 00000000..2e1e76ed --- /dev/null +++ b/hack/watchdog @@ -0,0 +1,669 @@ +#!/usr/bin/env python3 +"""watchdog: notice red, missing or stuck automation within hours, not days. + +Each run (.github/workflows/watchdog.yml: hourly, and after every listed workflow +completes on main) checks the repository and opens or closes one hack/ci_alarm +issue per finding: + + a. scheduled workflows Every .github/workflows/*.yml with a cron. stale:<file> + when its last successful scheduled run is older than the + cron period + 6 h (none at all counts, unless the workflow + was added within that time). red:<file> when the latest + completed scheduled run concluded failure, startup_failure + or timed_out: the run, its failed jobs, the last 60 lines + of its failed log up to the last ##[error], and the hint of + the first matching hack/ci_alarm_signatures.tsv line. Once + per failed run id (kept in the alarm body) the workflow is + re-dispatched on main when it has workflow_dispatch. A + green latest scheduled run closes red:<file>. + b. workflow state A workflow file whose Actions state is not active: + disabled:<file>. + c. main CI main's HEAD older than 30 min with no ci.yml run (push or + workflow_dispatch) for its SHA: dispatch ci.yml on main + (the dispatched run is for that SHA, so it happens once); + still none 2 h after HEAD: ci-missing. The latest completed + ci.yml run on main red: ci-main; green closes it. + d. release integrity The newest aur-v* tag older than 45 min (tag time = its + commit's committer date in --root's git) must have + cryptoandcoffee/k3d-gpu:<ver> on Docker Hub, + ghcr.io/<owner>/k3d-gpu:<ver>, the GitHub release + aur-v<ver> with its .deb, .rpm and SHA256SUMS, and AUR at + <ver> or later. A missing Docker Hub image dispatches + build-and-push.yml -f release=<ver> once (kept in the alarm + body). Any gap: release:<ver>, listing older tags from the + last 14 days with gaps as information. Other release:* + alarms close as superseded. Skipped while a release + workflow is queued or running. + e. stuck bot PRs Open autobump/* PRs older than 72 h whose head SHA has no + successful ci.yml run: bot-pr-stuck. + f. simulate_red SIMULATE_RED=true opens simulate; a run without it closes it. + +A failed gh, git, registry or AUR call is collected and filed as watchdog-error +with its text (the other checks still run); a run with no error closes it. Exit 0 +unless an alarm could not be filed (the repository has issues disabled, or +ci_alarm failed for watchdog-error itself): then ::error annotations and exit 1. + +Usage: hack/watchdog [--root DIR] [--list-scheduled] + --root DIR the checkout to read workflows and aur-v* tags from (default: + this repo). The workflow needs fetch-depth: 0 for the tags. + --list-scheduled print "<file> <cron period in hours>" per scheduled workflow + +Env: GH_REPO or GITHUB_REPOSITORY (owner/repo), SIMULATE_RED, ALARM_OWNER (for +ci_alarm), GITHUB_STEP_SUMMARY; for tests: WATCHDOG_NOW (epoch seconds), +WATCHDOG_ALARM (ci_alarm path), WATCHDOG_IMAGES_LOCK (the file registry_digest is +loaded from; default scripts/images-lock), WATCHDOG_AUR_URL. + +Python standard library; gh on PATH. +""" + +import argparse +import datetime +import importlib.machinery +import importlib.util +import json +import os +import re +import subprocess +import sys +import tempfile +import traceback +import urllib.error +import urllib.request + +HERE = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT = os.path.dirname(HERE) +MINUTE, HOUR, DAY = 60, 3600, 86400 +RED = ("failure", "startup_failure", "timed_out") +DECIDED = RED + ("success",) +RELEASE_WORKFLOWS = ("Auto-update, Build & Release", "Auto-release AUR", + "Release packages (deb, rpm, Homebrew)", "AUR Deploy") +HUB_REPO = "cryptoandcoffee/k3d-gpu" +AUR_URL = "https://aur.archlinux.org/rpc/v5/info?arg[]=k3d-gpu" +TAG_RE = re.compile(r"^aur-v(\d+)\.(\d+)\.(\d+)$") +ANSI_RE = re.compile(r"\x1b\[[0-9;]*[A-Za-z]") +CRON_LINE_RE = re.compile(r"""^\s*-\s*cron:\s*(['"]?)([^'"#\n]+?)\1\s*(?:#.*)?$""", re.M) +DISPATCH_RE = re.compile(r"^\s+workflow_dispatch:|^on:.*\bworkflow_dispatch\b", re.M) +LOG_LINES, LOG_WIDTH = 60, 300 + + +class ToolError(Exception): + """A gh, git, registry or AUR call failed.""" + + +def ts(s): + return datetime.datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp() + + +def iso(t): + return datetime.datetime.fromtimestamp(t, datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def hours(seconds): + return "%g" % round(seconds / HOUR, 2) + + +def vkey(ver): + return tuple(int(x) for x in ver.split(".")) + + +# ── cron ───────────────────────────────────────────────────────────────────── + +def cron_values(field, lo, hi): + """The set a cron field allows: *, n, a-b, with /step, comma lists.""" + out = set() + for part in field.split(","): + step = 1 + if "/" in part: + part, s = part.split("/", 1) + step = int(s) + if part == "*": + a, b = lo, hi + elif "-" in part: + a, b = (int(x) for x in part.split("-", 1)) + else: + a = int(part) + b = hi if step > 1 else a + if a < lo or b > hi or a > b or step < 1: + raise ValueError("cron field %r out of range %d-%d" % (field, lo, hi)) + out.update(range(a, b + 1, step)) + return out + + +def cron_period(crons): + """The longest gap, in seconds, between consecutive firings of the crons (UTC).""" + specs = [] + for c in crons: + f = c.split() + if len(f) != 5: + raise ValueError("cron %r: want 5 fields" % c) + specs.append((cron_values(f[0], 0, 59), cron_values(f[1], 0, 23), cron_values(f[2], 1, 31), + cron_values(f[3], 1, 12), {d % 7 for d in cron_values(f[4], 0, 7)}, + f[2] != "*", f[4] != "*")) + start = datetime.datetime(2026, 1, 5, tzinfo=datetime.timezone.utc) # a Monday + fires = [] + for day in range(800): + d = start + datetime.timedelta(days=day) + dow = (d.weekday() + 1) % 7 # cron: Sunday is 0 + for mins, hrs, doms, mons, dows, dom_set, dow_set in specs: + if d.month not in mons: + continue + dom_ok, dow_ok = d.day in doms, dow in dows + if not ((dom_ok or dow_ok) if (dom_set and dow_set) else (dom_ok and dow_ok)): + continue + base = d.timestamp() + fires.extend(base + h * HOUR + m * MINUTE for h in hrs for m in mins) + fires = sorted(set(fires)) + if len(fires) < 2: + return 366 * DAY + return max(b - a for a, b in zip(fires, fires[1:])) + + +def scheduled_workflows(root): + d = os.path.join(root, ".github", "workflows") + out = [] + for f in sorted(os.listdir(d)): + if not f.endswith((".yml", ".yaml")): + continue + with open(os.path.join(d, f)) as fh: + text = fh.read() + crons = [m.group(2).strip() for m in CRON_LINE_RE.finditer(text)] + if crons: + out.append({"file": f, "crons": crons, "period": cron_period(crons), + "dispatch": bool(DISPATCH_RE.search(text))}) + return out + + +def has_dispatch(root, f): + try: + with open(os.path.join(root, ".github", "workflows", f)) as fh: + return bool(DISPATCH_RE.search(fh.read())) + except FileNotFoundError: + return False + + +# ── tools ──────────────────────────────────────────────────────────────────── + +class Gh: + def __init__(self, repo): + self.repo = repo + + def run(self, *args): + p = subprocess.run(["gh", *args], capture_output=True, text=True) + if p.returncode != 0: + raise ToolError("gh %s: exit %d: %s" % (" ".join(args), p.returncode, p.stderr.strip()[-800:])) + return p.stdout + + def json(self, *args): + out = self.run(*args) + try: + return json.loads(out) if out.strip() else None + except ValueError: + raise ToolError("gh %s: not JSON: %s" % (" ".join(args), out[:200])) from None + + +def load_registry_digest(): + path = os.environ.get("WATCHDOG_IMAGES_LOCK") or os.path.join(REPO_ROOT, "scripts", "images-lock") + loader = importlib.machinery.SourceFileLoader("watchdog_images_lock", path) + spec = importlib.util.spec_from_loader("watchdog_images_lock", loader) + mod = importlib.util.module_from_spec(spec) + loader.exec_module(mod) + return mod.registry_digest + + +def marker(body, name): + m = re.search(r"<!-- watchdog:%s=(\S+) -->" % re.escape(name), body or "") + return m.group(1) if m else None + + +class Alarms: + """hack/ci_alarm, plus the open alarms (key -> issue body) read once per run.""" + + def __init__(self, gh, script): + self.script = script + self.open = None # None: unknown (the listing failed) + self.disabled = False + self.undelivered = [] + self.opened, self.closed = [], [] + try: + issues = gh.json("issue", "list", "--label", "ci-alarm", "--state", "open", "--limit", "200", + "--json", "number,title,body") or [] + self.open = {} + for it in issues: + m = re.match(r"^\[ci-alarm\] (\S+): ", it["title"]) + if m: + self.open.setdefault(m.group(1), it.get("body") or "") + except ToolError as e: + if "disabled issues" in str(e).lower(): + self.disabled = True + self.open = {} + self.list_error = e + else: + self.list_error = None + + def body(self, key): + return (self.open or {}).get(key) + + def keys(self, prefix): + return [k for k in (self.open or {}) if k.startswith(prefix)] + + def _call(self, *args): + p = subprocess.run([self.script, *args], capture_output=True, text=True) + sys.stdout.write(p.stdout) + sys.stderr.write(p.stderr) + return p + + def raise_(self, key, title, body): + with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) as f: + f.write(body.rstrip() + "\n") + try: + p = self._call("open", key, title, f.name) + finally: + os.unlink(f.name) + if p.returncode == 3: + self.undelivered.append(key) + return + if p.returncode != 0: + raise ToolError("ci_alarm open %s: exit %d: %s" % (key, p.returncode, p.stderr.strip()[-400:])) + print("::warning title=ci-alarm %s::%s" % (key, title)) + self.opened.append((key, title)) + + def clear(self, key, comment): + if self.disabled or (self.open is not None and key not in self.open): + return + p = self._call("close", key, comment) + if p.returncode not in (0, 3): + raise ToolError("ci_alarm close %s: exit %d: %s" % (key, p.returncode, p.stderr.strip()[-400:])) + if self.open and key in self.open: + self.closed.append(key) + + +class Watchdog: + def __init__(self, root, repo, now): + self.root, self.repo, self.now = root, repo, now + self.gh = Gh(repo) + self.errors = [] + self.alarms = Alarms(self.gh, os.environ.get("WATCHDOG_ALARM") or os.path.join(HERE, "ci_alarm")) + if self.alarms.list_error: + self.errors.append("listing open alarms: %s" % self.alarms.list_error) + self._registry = None + self.signatures = [] + with open(os.path.join(HERE, "ci_alarm_signatures.tsv")) as f: + for ln in f: + if ln.strip() and not ln.startswith("#"): + rx, hint = ln.rstrip("\n").split("\t", 1) + self.signatures.append((re.compile(rx), hint)) + + def guard(self, name, fn, *args): + try: + return fn(*args) + except ToolError as e: + self.errors.append("%s: %s" % (name, e)) + except Exception: # a bug in a check must not silence the others + self.errors.append("%s: %s" % (name, traceback.format_exc(limit=3).strip())) + return None + + # ── failed-run description ─────────────────────────────────────────── + + def describe_failed_run(self, run): + rid = str(run["databaseId"]) + jobs = (self.gh.json("run", "view", rid, "-R", self.repo, "--json", "jobs") or {}).get("jobs", []) + failed = [j["name"] for j in jobs if j.get("conclusion") in ("failure", "timed_out")] + lines = ["Run: %s" % run.get("url", rid), + "Conclusion: **%s**, created %s, commit %s" % (run.get("conclusion"), run.get("createdAt", "?"), + run.get("headSha", "?")), + "Failed jobs: %s" % (", ".join(failed) or "(none reported)")] + try: + log = ANSI_RE.sub("", self.gh.run("run", "view", rid, "-R", self.repo, "--log-failed")) + except ToolError as e: + lines.append("Failed log unavailable: %s" % str(e)[-300:]) + return lines + rows = log.splitlines() + hint = next((h for rx, h in self.signatures if any(rx.search(r) for r in rows)), None) + lines.append("Hint: %s" % (hint or "no known signature matched")) + errs = [i for i, r in enumerate(rows) if "##[error]" in r] + end = errs[-1] + 1 if errs else len(rows) + tail = [r.replace("\t", " ")[:LOG_WIDTH].replace("```", "'''") for r in rows[max(0, end - LOG_LINES):end]] + lines += ["", "<details><summary>Last %d lines of the failed log</summary>" % len(tail), "", + "```text", *tail, "```", "</details>"] + return lines + + # ── a + b ──────────────────────────────────────────────────────────── + + def check_workflow_state(self): + doc = self.gh.json("api", "repos/%s/actions/workflows?per_page=100" % self.repo) or {} + meta = {} + for w in doc.get("workflows", []): + path = w.get("path", "") + if not path.startswith(".github/workflows/"): + continue + f = path.rsplit("/", 1)[1] + meta[f] = w + if not os.path.exists(os.path.join(self.root, path)): + continue + key = "disabled:" + f + if w.get("state") != "active": + self.alarms.raise_(key, "%s is %s" % (f, w.get("state")), "\n".join([ + "Workflow `%s` (%s) has Actions state `%s`, so it runs no more." % (f, w.get("name"), w.get("state")), + "", "Enable it: Actions, %s, Enable workflow (or `gh workflow enable %s`)." % (w.get("name"), f)])) + else: + self.alarms.clear(key, "%s is active again." % f) + return meta + + def check_scheduled(self, wf, meta): + f = wf["file"] + fields = "databaseId,conclusion,status,createdAt,url,headSha,workflowName" + runs = self.gh.json("run", "list", "-R", self.repo, "--workflow", f, "--event", "schedule", + "--limit", "30", "--json", fields) or [] + success = next((r for r in runs if r.get("conclusion") == "success"), None) + if success is None: + more = self.gh.json("run", "list", "-R", self.repo, "--workflow", f, "--event", "schedule", + "--status", "success", "--limit", "1", "--json", fields) or [] + success = more[0] if more else None + threshold = wf["period"] + 6 * HOUR + created = (meta or {}).get(f, {}).get("created_at") + fresh = success is not None and self.now - ts(success["createdAt"]) <= threshold + new = success is None and created is not None and self.now - ts(created) <= threshold + key = "stale:" + f + if fresh or new: + self.alarms.clear(key, "A scheduled run of %s succeeded within %s h." % (f, hours(threshold))) + else: + last = ("%s at %s" % (success["url"], success["createdAt"])) if success else "none found" + self.alarms.raise_(key, "no successful scheduled run within %s h" % hours(threshold), "\n".join([ + "`%s` (cron %s, every %s h) has no successful scheduled run within %s h (period + 6 h)." + % (f, ", ".join("`%s`" % c for c in wf["crons"]), hours(wf["period"]), hours(threshold)), + "", "Last successful scheduled run: %s" % last, + "", "A scheduled run that never starts leaves no red run: check the workflow is enabled " + "and that its last runs were not skipped."])) + + latest = next((r for r in runs if r.get("status") == "completed" and r.get("conclusion") in DECIDED), None) + key = "red:" + f + if latest is None: + return + if latest["conclusion"] == "success": + self.alarms.clear(key, "The latest scheduled run is green: %s" % latest.get("url")) + return + rid = str(latest["databaseId"]) + body = ["The latest scheduled run of `%s` failed." % f, ""] + self.describe_failed_run(latest) + done = marker(self.alarms.body(key), "dispatched") + if wf["dispatch"]: + if done != rid: + try: + self.gh.run("workflow", "run", f, "-R", self.repo, "--ref", "main") + done = rid + print("re-dispatched %s on main after run %s" % (f, rid)) + except ToolError as e: + self.errors.append("re-dispatching %s: %s" % (f, e)) + body.append("\nRe-dispatch failed: %s" % e) + if done == rid: + body += ["", "Re-dispatched `%s` on main once after this run; a further re-dispatch " + "happens only after a new failed scheduled run." % f, + "<!-- watchdog:dispatched=%s -->" % rid] + self.alarms.raise_(key, "scheduled run %s" % latest["conclusion"], "\n".join(body)) + + # ── c ──────────────────────────────────────────────────────────────── + + def check_main_ci(self): + head = self.gh.json("api", "repos/%s/commits/main" % self.repo) or {} + sha = head["sha"] + age = self.now - ts(head["commit"]["committer"]["date"]) + runs = self.gh.json("run", "list", "-R", self.repo, "--workflow", "ci.yml", "--commit", sha, + "--limit", "20", "--json", "databaseId,event,status,conclusion") or [] + if any(r.get("event") in ("push", "workflow_dispatch") for r in runs): + self.alarms.clear("ci-missing", "ci.yml runs for main's HEAD %s." % sha) + elif age >= 2 * HOUR: + self.alarms.raise_("ci-missing", "main's HEAD has no ci.yml run", "\n".join([ + "main's HEAD `%s` (committed %s) has no ci.yml run from a push or workflow_dispatch, " + "2 h after the commit." % (sha, head["commit"]["committer"]["date"]), + "", "A push made with GITHUB_TOKEN starts no workflow; the watchdog dispatched ci.yml on main " + "once when HEAD was 30 min old. Dispatch it: `gh workflow run ci.yml --ref main`."])) + else: + self.alarms.clear("ci-missing", "main moved to %s." % sha) + if age >= 30 * MINUTE and has_dispatch(self.root, "ci.yml"): + self.gh.run("workflow", "run", "ci.yml", "-R", self.repo, "--ref", "main") + print("dispatched ci.yml on main for HEAD %s (no ci.yml run %s min after the commit)" + % (sha, int(age // MINUTE))) + + runs = self.gh.json("run", "list", "-R", self.repo, "--workflow", "ci.yml", "--branch", "main", + "--limit", "20", "--json", "databaseId,event,status,conclusion,url,headSha,createdAt") or [] + latest = next((r for r in runs if r.get("status") == "completed" + and r.get("event") in ("push", "workflow_dispatch") + and r.get("conclusion") in DECIDED), None) + if latest is None: + return + if latest["conclusion"] == "success": + self.alarms.clear("ci-main", "ci.yml on main is green: %s" % latest.get("url")) + else: + self.alarms.raise_("ci-main", "ci.yml on main %s" % latest["conclusion"], + "\n".join(["The latest completed ci.yml run on main failed.", ""] + + self.describe_failed_run(latest))) + + # ── d ──────────────────────────────────────────────────────────────── + + def registry_status(self, name, tag): + if self._registry is None: + self._registry = load_registry_digest() + try: + self._registry(name, tag) + return "ok" + except ValueError as e: + if re.search(r"HTTP 404\b", str(e)): + return "missing" + raise ToolError(str(e)) from None + except (urllib.error.URLError, OSError) as e: + raise ToolError("%s:%s: %s" % (name, tag, e)) from None + + def aur_version(self): + url = os.environ.get("WATCHDOG_AUR_URL") or AUR_URL + try: + with urllib.request.urlopen(url, timeout=30) as r: + doc = json.load(r) + except (urllib.error.URLError, OSError, ValueError) as e: + raise ToolError("AUR RPC %s: %s" % (url, e)) from None + res = doc.get("results") or [] + return res[0]["Version"].split("-", 1)[0] if res else None + + def release_gaps(self, ver, newest): + owner = self.repo.split("/", 1)[0].lower() + ghcr = "ghcr.io/%s/k3d-gpu" % owner + gaps = [] + hub_missing = self.registry_status("docker.io/" + HUB_REPO, ver) == "missing" + if hub_missing: + gaps.append("Docker Hub: %s:%s has no manifest (HTTP 404)" % (HUB_REPO, ver)) + if self.registry_status(ghcr, ver) == "missing": + gaps.append("GHCR: %s:%s has no manifest (HTTP 404)" % (ghcr, ver)) + want = ["k3d-gpu_%s-1_all.deb" % ver, "k3d-gpu-%s-1.noarch.rpm" % ver, "SHA256SUMS"] + try: + rel = self.gh.json("release", "view", "aur-v" + ver, "-R", self.repo, "--json", "assets") or {} + have = {a["name"] for a in rel.get("assets", []) if a.get("state", "uploaded") == "uploaded"} + missing = [w for w in want if w not in have] + if missing: + gaps.append("GitHub release aur-v%s lacks %s" % (ver, ", ".join(missing))) + except ToolError as e: + if "release not found" not in str(e): + raise + gaps.append("GitHub release aur-v%s does not exist (no %s)" % (ver, ", ".join(want))) + if newest: + aur = self.aur_version() + if aur is None or not re.match(r"^\d+\.\d+\.\d+$", aur) or vkey(aur) < vkey(ver): + gaps.append("AUR: k3d-gpu is at %s, not %s" % (aur or "nothing", ver)) + return gaps, hub_missing + + def release_tags(self): + p = subprocess.run(["git", "-C", self.root, "for-each-ref", "--format=%(refname:strip=2) %(creatordate:unix)", + "refs/tags/aur-v*"], capture_output=True, text=True) + if p.returncode != 0: + raise ToolError("git for-each-ref: %s" % p.stderr.strip()) + out = [] + for ln in p.stdout.splitlines(): + name, _, when = ln.partition(" ") + m = TAG_RE.match(name) + if m and when.isdigit(): + out.append((".".join(m.groups()), int(when))) + return sorted(out, key=lambda t: vkey(t[0]), reverse=True) + + def check_release(self): + recent = self.gh.json("run", "list", "-R", self.repo, "--limit", "100", + "--json", "workflowName,status,databaseId") or [] + busy = sorted({r["workflowName"] for r in recent + if r.get("workflowName") in RELEASE_WORKFLOWS and r.get("status") != "completed"}) + if busy: + print("release check deferred: %s is queued or running" % ", ".join(busy)) + return + cands = [(v, t) for v, t in self.release_tags() if self.now - t >= 45 * MINUTE] + if not cands: + return + ver = cands[0][0] + key = "release:" + ver + gaps, hub_missing = self.release_gaps(ver, newest=True) + for k in self.alarms.keys("release:"): + if k != key: + self.alarms.clear(k, "Superseded: the newest release older than 45 min is %s%s." + % (ver, "" if gaps else ", and it is complete")) + if not gaps: + self.alarms.clear(key, "Release %s is complete: images, packages and AUR." % ver) + return + body = ["Release %s (tag aur-v%s) is incomplete:" % (ver, ver), ""] + ["- " + g for g in gaps] + done = marker(self.alarms.body(key), "dispatched") + if hub_missing: + if done != ver: + try: + self.gh.run("workflow", "run", "build-and-push.yml", "-R", self.repo, "--ref", "main", + "-f", "release=" + ver) + done = ver + print("dispatched build-and-push.yml release=%s (no Docker Hub image)" % ver) + except ToolError as e: + self.errors.append("re-dispatching release %s: %s" % (ver, e)) + body += ["", "Re-dispatch failed: %s" % e] + if done == ver: + body += ["", "Dispatched `build-and-push.yml -f release=%s` once; it builds the image from the tag " + "and then dispatches the AUR and package releases. It is not dispatched again." % ver, + "<!-- watchdog:dispatched=%s -->" % ver] + elif any(g.startswith(("Docker Hub", "GHCR")) for g in gaps): + body += ["", "Docker Hub has %s:%s, so a release build would stop at its overwrite guard; " + "push the GHCR tag by hand or cut a new release." % (HUB_REPO, ver)] + info = [] + for v, t in cands[1:21]: + if self.now - t > 14 * DAY: + break + try: + g, _ = self.release_gaps(v, newest=False) + except ToolError as e: + g = ["not checked: %s" % str(e)[:200]] + if g: + info.append("- %s: %s" % (v, "; ".join(g))) + if info: + body += ["", "Older releases from the last 14 days with gaps (information only; a newer release " + "supersedes them):", ""] + info + self.alarms.raise_(key, "release %s is incomplete" % ver, "\n".join(body)) + + # ── e + f ──────────────────────────────────────────────────────────── + + def check_bot_prs(self): + prs = self.gh.json("pr", "list", "-R", self.repo, "--state", "open", "--limit", "100", + "--json", "number,headRefName,headRefOid,createdAt,url,title") or [] + stuck = [] + for pr in sorted(prs, key=lambda p: p["number"]): + if not pr["headRefName"].startswith("autobump/") or self.now - ts(pr["createdAt"]) <= 72 * HOUR: + continue + runs = self.gh.json("run", "list", "-R", self.repo, "--workflow", "ci.yml", "--commit", pr["headRefOid"], + "--limit", "20", "--json", "databaseId,event,status,conclusion") or [] + if any(r.get("conclusion") == "success" for r in runs): + continue + seen = sorted({"%s %s" % (r.get("event"), r.get("conclusion") or r.get("status")) for r in runs}) + stuck.append("- %s %s (`%s`, open since %s): ci.yml at %s: %s" % ( + pr["url"], pr["title"], pr["headRefName"], pr["createdAt"], pr["headRefOid"][:12], + ", ".join(seen) or "no run")) + if stuck: + self.alarms.raise_("bot-pr-stuck", "bot PRs open over 72 h without green CI", "\n".join( + ["These autobump PRs are open over 72 h and their head SHA has no successful ci.yml run:", ""] + + stuck + ["", "A PR that GITHUB_TOKEN opened or updated gets approval-held CI runs (action_required): " + "approve them, or dispatch ci.yml on the branch."])) + else: + self.alarms.clear("bot-pr-stuck", "No autobump PR is stuck any more.") + + def check_simulate(self, simulate): + if simulate: + self.alarms.raise_("simulate", "simulated red", "\n".join([ + "The watchdog was dispatched with simulate_red=true: this checks that an alarm reaches its owner.", + "", "The next watchdog run without simulate_red closes it."])) + else: + self.alarms.clear("simulate", "The next watchdog run without simulate_red closed it.") + + # ── run ────────────────────────────────────────────────────────────── + + def run(self, simulate): + meta = self.guard("workflow state", self.check_workflow_state) + try: + scheduled = scheduled_workflows(self.root) + except (OSError, ValueError) as e: + scheduled = [] + self.errors.append("reading scheduled workflows: %s" % e) + for wf in scheduled: + self.guard("scheduled %s" % wf["file"], self.check_scheduled, wf, meta) + self.guard("main CI", self.check_main_ci) + self.guard("release integrity", self.check_release) + self.guard("bot PRs", self.check_bot_prs) + self.guard("simulate_red", self.check_simulate, simulate) + + fatal = False + if self.errors: + for e in self.errors: + print("::error title=watchdog::%s" % e.splitlines()[0][:400], file=sys.stderr) + body = "\n".join(["The watchdog hit %d error(s); the checks they stopped did not run:" % len(self.errors), + "", "```text", *[e.replace("```", "'''") for e in self.errors], "```"]) + try: + self.alarms.raise_("watchdog-error", "the watchdog could not complete its checks", body) + except ToolError as e: + print("::error title=watchdog::cannot file watchdog-error: %s" % e, file=sys.stderr) + fatal = True + else: + try: + self.alarms.clear("watchdog-error", "A watchdog run completed every check.") + except ToolError as e: + print("::error title=watchdog::cannot close watchdog-error: %s" % e, file=sys.stderr) + fatal = True + + if self.alarms.undelivered: + print("::error title=watchdog::%d alarm(s) not filed (%s): this repository has issues disabled; " + "enable Issues in Settings > General > Features" % (len(self.alarms.undelivered), + ", ".join(self.alarms.undelivered)), + file=sys.stderr) + fatal = True + self.summary() + return 1 if fatal else 0 + + def summary(self): + lines = ["## Watchdog %s" % iso(self.now), ""] + lines += ["- open: `%s` %s" % (k, t) for k, t in self.alarms.opened] + lines += ["- closed: `%s`" % k for k in self.alarms.closed] + lines += ["- not filed: `%s`" % k for k in self.alarms.undelivered] + lines += ["- error: %s" % e.splitlines()[0][:300] for e in self.errors] + if len(lines) == 2: + lines.append("No alarm.") + text = "\n".join(lines) + "\n" + print(text) + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: + f.write(text) + + +def main(argv): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--root", default=REPO_ROOT) + ap.add_argument("--list-scheduled", action="store_true") + a = ap.parse_args(argv) + if a.list_scheduled: + for wf in scheduled_workflows(a.root): + print("%s %s" % (wf["file"], hours(wf["period"]))) + return 0 + repo = os.environ.get("GH_REPO") or os.environ.get("GITHUB_REPOSITORY") + if not repo: + print("watchdog: set GH_REPO or GITHUB_REPOSITORY (owner/repo)", file=sys.stderr) + return 2 + now = float(os.environ.get("WATCHDOG_NOW") or datetime.datetime.now(datetime.timezone.utc).timestamp()) + simulate = os.environ.get("SIMULATE_RED", "").strip().lower() == "true" + return Watchdog(a.root, repo, now).run(simulate) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/auto_release_cas_test.sh b/tests/auto_release_cas_test.sh index 567995ee..ff9e5d85 100755 --- a/tests/auto_release_cas_test.sh +++ b/tests/auto_release_cas_test.sh @@ -36,7 +36,9 @@ setup() { git clone -q "file://$D/origin.git" "$D/seed" 2>/dev/null ( cd "$D/seed" || exit 1 git checkout -q -b main - mkdir -p scripts packaging/aur Formula share console + mkdir -p scripts packaging/aur Formula share console hack + # The shipped paths a push run releases for: the repo's own list. + cp "$ROOT/hack/release-paths" hack/release-paths # Stand-in for console/model_opt.py ledger-rules: the JSON names the ledger's line count. cat > console/model_opt.py <<'PY' import sys @@ -57,7 +59,7 @@ PY write() { ( cd "$D/seed" || exit 1 git fetch -q origin; git reset -q --hard origin/main - echo "x $RANDOM" >> "$1"; git add "$1"; git commit -q -m "$2"; git push -q origin HEAD:main ) + mkdir -p "$(dirname "$1")"; echo "x $RANDOM" >> "$1"; git add "$1"; git commit -q -m "$2"; git push -q origin HEAD:main ) } runner() { git clone -q "file://$D/origin.git" "$D/$1" 2>/dev/null; } # release DIR EVENT [HOOK]: run the release body in a runner checkout. @@ -205,6 +207,34 @@ got=$(o show "$m:console/ledger-rules.json" 2>&1) c=1; [ "$rc" -eq 0 ] && [ "$(tagsha aur-v0.1.6)" = "$m" ] && [ "$got" = "$want" ] && c=0 check ledger_rules_regenerated $c "rc=$rc got=$got want=$want" +# shipped_paths: a push run (and a mode=changed dispatch, which runs as a push) +# releases for a change to any path hack/release-paths lists: console/, a shipped +# bench or oracle file; and not for the ledger or a bench file nothing ships. +for c in "console/plan.py:release" "bench/vs.py:release" "oracle/schema/bundle-v1.json:release" \ + "images.lock:release" "EXPERIMENTS.md:none" "bench/h54/run.sh:none" "docs/x.md:none"; do + f=${c%%:*}; want=${c#*:} + setup "shipped_${f//\//_}"; write "$f" "change: $f"; before=$(main); runner r; release r push; rc=$? + if [ "$want" = release ]; then + c=1; [ "$rc" -eq 0 ] && [ "$(tagsha aur-v0.1.6)" = "$(main)" ] && o merge-base --is-ancestor "$before" "$(main)" && c=0 + else + c=1; [ "$rc" -eq 0 ] && [ "$(main)" = "$before" ] && [ "$(tags)" = "aur-v0.1.5 " ] && grep -q 'nothing to release' "$D/r.log" && c=0 + fi + check "shipped_paths: $f -> $want" $c "rc=$rc tags=$(tags) $(tail -1 "$D/r.log")" +done + +# release_paths_read: the path filter is read from main's hack/release-paths, not +# built in: listing docs/** makes a docs-only change releasable. +setup release_paths_read +( cd "$D/seed" && echo 'docs/**' >> hack/release-paths && git commit -qam "ci: ship docs [skip release]" && git push -q origin HEAD:main ) +write docs/x.md "docs: x"; runner r; release r push; rc=$? +c=1; [ "$rc" -eq 0 ] && [ "$(tagsha aur-v0.1.6)" = "$(main)" ] && c=0 +check release_paths_read $c "rc=$rc tags=$(tags) $(tail -1 "$D/r.log")" +if grep -qF 'hack/release-paths' "$REL" && ! grep -qE -- '-- packaging scripts share Dockerfile' "$REL"; then + ok "release_cas.sh takes its path filter from hack/release-paths" +else + fail "release_cas.sh does not read hack/release-paths (or keeps a built-in path list)" +fi + # no_env_command_hook: the release script runs no command named by the environment. if grep -n 'BEFORE_PUSH' "$REL"; then fail "no_env_command_hook: env-named command in $REL" else ok "no_env_command_hook"; fi diff --git a/tests/ci_alarm_test.sh b/tests/ci_alarm_test.sh new file mode 100755 index 00000000..be2c3c2d --- /dev/null +++ b/tests/ci_alarm_test.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +# hack/ci_alarm against tests/ops/fake_gh.py on PATH (an issue store in a temp +# directory, no network). Checks: open files one labelled, assigned issue with an +# @mention per key; the same body within 24 h adds nothing; a changed body +# comments and replaces the body; the same body after 24 h comments once more; +# close comments and closes; list prints the open keys; a repository with issues +# disabled gets a workflow error annotation and exit 3 instead. +# +# Usage: tests/ci_alarm_test.sh + +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +ALARM=${ALARM:-${ROOT}/hack/ci_alarm} +T=$(mktemp -d "${TMPDIR:-/tmp}/ci_alarm_test.XXXXXX") +trap 'rm -rf "$T"' EXIT +PASS=0 FAILS=0 +ok() { PASS=$((PASS + 1)); echo "ok $1"; } +fail() { FAILS=$((FAILS + 1)); echo "FAIL $1"; } + +mkdir -p "$T/bin" +printf '#!/bin/sh\nexec python3 %q "$@"\n' "${ROOT}/tests/ops/fake_gh.py" > "$T/bin/gh" +chmod +x "$T/bin/gh" +export PATH="$T/bin:$PATH" GH_REPO=88plug/k3d-gpu +unset GITHUB_STEP_SUMMARY ALARM_OWNER + +# fresh NAME: a new empty issue store; the clock at 2026-10-29T13:00:00Z. +fresh() { + export FAKE_GH="$T/$1" + mkdir -p "$FAKE_GH" + clock 2026-10-29T13:00:00Z +} +clock() { export FAKE_GH_NOW=$1; CI_ALARM_NOW=$(date -u -d "$1" +%s); export CI_ALARM_NOW; } +body() { printf '%s\n' "$@" > "$T/body"; echo "$T/body"; } +alarm() { bash "$ALARM" "$@" > "$FAKE_GH/out" 2> "$FAKE_GH/err"; } +issues() { jq "$@" "$FAKE_GH/issues.json"; } +calls() { grep -c "^issue $1" "$FAKE_GH/calls.log" 2>/dev/null || true; } + +# 1. open: one issue, titled, labelled ci-alarm, assigned to 88plug, @mention in the body. +fresh open +alarm open red:build-and-push.yml "nightly is red" "$(body "run 39412124435 failed")"; rc=$? +if [ "$rc" -eq 0 ] \ + && [ "$(issues length)" = 1 ] \ + && [ "$(issues -r '.[0].title')" = "[ci-alarm] red:build-and-push.yml: nightly is red" ] \ + && [ "$(issues -r '.[0].labels[0].name')" = ci-alarm ] \ + && [ "$(issues -r '.[0].assignees[0].login')" = 88plug ] \ + && issues -r '.[0].body' | grep -q '@88plug' \ + && issues -r '.[0].body' | grep -q 'run 39412124435 failed' \ + && jq -e 'index("ci-alarm")' "$FAKE_GH/labels.json" >/dev/null; then + ok "open files one labelled, assigned issue with an @mention" +else + fail "open: rc=$rc issues=$(cat "$FAKE_GH/issues.json" 2>/dev/null) err=$(cat "$FAKE_GH/err")" +fi + +# 2. the same body within 24 h: no comment, no edit, no second issue. +clock 2026-10-29T20:00:00Z +before=$(calls comment) +alarm open red:build-and-push.yml "nightly is red" "$(body "run 39412124435 failed")"; rc=$? +if [ "$rc" -eq 0 ] && [ "$(issues length)" = 1 ] && [ "$(issues '.[0].comments | length')" = 0 ] \ + && [ "$(calls comment)" = "$before" ] && [ "$(calls edit)" = 0 ]; then + ok "the same body within 24 h adds nothing" +else + fail "same body within 24 h: rc=$rc comments=$(issues '.[0].comments | length') edits=$(calls edit)" +fi + +# 3. a changed body: one comment carrying it, and the issue body replaced. +alarm open red:build-and-push.yml "nightly is red" "$(body "run 39591066580 failed")"; rc=$? +if [ "$rc" -eq 0 ] && [ "$(issues length)" = 1 ] && [ "$(issues '.[0].comments | length')" = 1 ] \ + && issues -r '.[0].comments[0].body' | grep -q 'run 39591066580 failed' \ + && issues -r '.[0].body' | grep -q 'run 39591066580 failed' \ + && ! issues -r '.[0].body' | grep -q 'run 39412124435'; then + ok "a changed body comments and replaces the issue body" +else + fail "changed body: rc=$rc issues=$(cat "$FAKE_GH/issues.json")" +fi + +# 4. the same body 24 h after the last comment: one reminder comment, then quiet again. +clock 2026-10-30T20:00:01Z +alarm open red:build-and-push.yml "nightly is red" "$(body "run 39591066580 failed")"; rc1=$? +alarm open red:build-and-push.yml "nightly is red" "$(body "run 39591066580 failed")"; rc2=$? +if [ "$rc1" -eq 0 ] && [ "$rc2" -eq 0 ] && [ "$(issues '.[0].comments | length')" = 2 ] \ + && issues -r '.[0].comments[1].body' | grep -q '@88plug'; then + ok "the same body after 24 h comments once, with an @mention" +else + fail "24 h reminder: rc=$rc1/$rc2 comments=$(issues '.[0].comments | length')" +fi + +# 5. a second key is its own issue; a key that extends another is not confused with it. +alarm open red:build-and-push.yml.bak "other" "$(body x)"; rc1=$? +alarm open release:0.1.76 "no image" "$(body y)"; rc2=$? +if [ "$rc1" -eq 0 ] && [ "$rc2" -eq 0 ] && [ "$(issues length)" = 3 ]; then + ok "each key gets its own issue" +else + fail "keys: rc=$rc1/$rc2 issues=$(issues length)" +fi + +# 6. list prints the open keys, one per line. +alarm list; rc=$? +if [ "$rc" -eq 0 ] && [ "$(sort "$FAKE_GH/out" | tr '\n' ' ')" = "red:build-and-push.yml red:build-and-push.yml.bak release:0.1.76 " ]; then + ok "list prints the open keys" +else + fail "list: rc=$rc out=$(tr '\n' ' ' < "$FAKE_GH/out")" +fi + +# 7. close comments and closes; list no longer prints the key; closing again is a no-op. +alarm close red:build-and-push.yml "the nightly is green again"; rc1=$? +n=$(issues -r '.[] | select(.title | startswith("[ci-alarm] red:build-and-push.yml:")) | .number') +# shellcheck disable=SC2016 # $n is a jq variable +st=$(issues -r --argjson n "$n" '.[] | select(.number == $n) | .state') +# shellcheck disable=SC2016 +last=$(issues -r --argjson n "$n" '.[] | select(.number == $n) | .comments[-1].body') +alarm close red:build-and-push.yml "again"; rc2=$? +alarm list +if [ "$rc1" -eq 0 ] && [ "$rc2" -eq 0 ] && [ "$st" = CLOSED ] && [ "$last" = "the nightly is green again" ] \ + && ! grep -qx 'red:build-and-push.yml' "$FAKE_GH/out" && grep -qx 'release:0.1.76' "$FAKE_GH/out" \ + && [ "$(issues '[.[] | select(.state == "OPEN")] | length')" = 2 ]; then + ok "close comments and closes; a second close is a no-op" +else + fail "close: rc=$rc1/$rc2 state=$st last=$last" +fi + +# 8. reopening a closed key files a new issue. +alarm open red:build-and-push.yml "red again" "$(body z)"; rc=$? +if [ "$rc" -eq 0 ] && [ "$(issues '[.[] | select(.title | startswith("[ci-alarm] red:build-and-push.yml:"))] | length')" = 2 ]; then + ok "a key opened after close files a new issue" +else + fail "reopen: rc=$rc" +fi + +# 9. ALARM_OWNER moves the assignee and the mention. +fresh owner +ALARM_OWNER=someone-else alarm open simulate "simulated red" "$(body s)"; rc=$? +if [ "$rc" -eq 0 ] && [ "$(issues -r '.[0].assignees[0].login')" = someone-else ] \ + && issues -r '.[0].body' | grep -q '@someone-else'; then + ok "ALARM_OWNER sets the assignee and the mention" +else + fail "ALARM_OWNER: rc=$rc issues=$(cat "$FAKE_GH/issues.json")" +fi + +# 10. a key with a space or an empty title is refused (exit 2), nothing filed. +fresh bad +alarm open "bad key" "t" "$(body b)"; rc1=$? +alarm open good "" "$(body b)"; rc2=$? +if [ "$rc1" -eq 2 ] && [ "$rc2" -eq 2 ] && [ ! -s "$FAKE_GH/issues.json" ]; then + ok "a bad key or an empty title is refused" +else + fail "bad input: rc=$rc1/$rc2" +fi + +# 11. issues disabled: open prints a ::error annotation and the summary, exits 3. +fresh disabled +printf '^(issue|label) \t-\t1\tthe %s repository has disabled issues\n' "'88plug/k3d-gpu'" > "$FAKE_GH/routes.tsv" +GITHUB_STEP_SUMMARY="$FAKE_GH/summary" alarm open simulate "simulated red" "$(body "the simulated body")"; rc=$? +if [ "$rc" -eq 3 ] && grep -q '^::error title=ci-alarm simulate::' "$FAKE_GH/out" \ + && grep -q 'the simulated body' "$FAKE_GH/summary" && grep -qi 'disabled issues' "$FAKE_GH/err"; then + ok "issues disabled: ::error annotation, step summary, exit 3" +else + fail "issues disabled: rc=$rc out=$(cat "$FAKE_GH/out") err=$(cat "$FAKE_GH/err")" +fi +alarm list; rc=$? +if [ "$rc" -eq 3 ]; then ok "issues disabled: list exits 3"; else fail "issues disabled: list rc=$rc"; fi + +# 12. any other gh failure is exit 1, with gh's error on stderr. +fresh broken +printf '^issue list\t-\t1\tHTTP 502: Bad Gateway\n' > "$FAKE_GH/routes.tsv" +alarm open simulate "simulated red" "$(body b)"; rc=$? +if [ "$rc" -eq 1 ] && grep -q 'HTTP 502' "$FAKE_GH/err"; then + ok "a gh error exits 1 with the error text" +else + fail "gh error: rc=$rc err=$(cat "$FAKE_GH/err")" +fi + +echo "${PASS} passed, ${FAILS} failed" +[ "${FAILS}" -eq 0 ] diff --git a/tests/ci_wiring_test.sh b/tests/ci_wiring_test.sh index 6426f9b7..f6ad4927 100755 --- a/tests/ci_wiring_test.sh +++ b/tests/ci_wiring_test.sh @@ -156,5 +156,15 @@ else fail "release does not read UBUNTU_2404_IMAGE from the Dockerfile for the 24.04 variant" fi +# Alarm, watchdog and release hardening: tests/ops/workflow_checks.py reads every +# workflow with PyYAML (a second parser: hack/watchdog reads crons with its own +# regex) and prints one "ok <what>" or "FAIL <what>" line per check. +while IFS= read -r line; do + case "${line}" in + "ok "*) ok "${line#ok }" ;; + *) fail "${line#FAIL }" ;; + esac +done < <(python3 "${ROOT}/tests/ops/workflow_checks.py" "${ROOT}" 2>&1 || echo "FAIL tests/ops/workflow_checks.py exited non-zero") + echo "${PASS} passed, ${FAILS} failed" [ "${FAILS}" -eq 0 ] diff --git a/tests/image_test.sh b/tests/image_test.sh index ee487f3d..84879229 100755 --- a/tests/image_test.sh +++ b/tests/image_test.sh @@ -167,10 +167,33 @@ contract() { local name=$1 file=$2 re=$3 if matches "${re}" "$(cat "${WF}/${file}")"; then pass "${name}"; else fail "${name} (/${re}/ in ${file})"; fi } -contract "release build publishes <repo>:<version> on Docker Hub" build-and-push.yml \ - '^ +cryptoandcoffee/k3d-gpu:\$\{\{ inputs\.release \}\}$' -contract "release build publishes <repo>:<version> on GHCR" build-and-push.yml \ - '^ +ghcr\.io/.*/k3d-gpu:\$\{\{ inputs\.release \}\}$' +# The build pushes the versioned tags hack/release-tags prints; every release gets +# <repo>:<version> on Docker Hub and GHCR, the newest one also the floating tags. +contract "release build pushes the versioned tags hack/release-tags prints" build-and-push.yml \ + 'tags=\$\(\.ci-main/hack/release-tags "\$RELEASE" "\$K3S"\)' +contract "release build pushes to Docker Hub and GHCR" build-and-push.yml \ + 'RELEASE_REPOS: cryptoandcoffee/k3d-gpu ghcr\.io/\$\{\{ github\.repository_owner \}\}/k3d-gpu$' +contract "release build retries the 26.04 build (attempt 3)" build-and-push.yml \ + '^ +- name: Build & push — Ubuntu 26\.04 \(attempt 3\)$' +contract "release build retries the 24.04 build (attempt 3)" build-and-push.yml \ + '^ +- name: Build & push — Ubuntu 24\.04 \(attempt 3\)$' +contract "release build moves the floating tags only through hack/release-tags" build-and-push.yml \ + 'docker buildx imagetools create --tag "\$ref" "\$src"' +contract "release build pushes no literal :latest tag" build-and-push.yml \ + '!^ +(cryptoandcoffee|ghcr\.io/.*)/k3d-gpu:latest$' +RT_ORIGIN="${WORK}/rt-origin.git" +( export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 + git init -q --bare "${RT_ORIGIN}" && cd "${WORK}" && git init -q rt && cd rt \ + && git -c user.name=t -c user.email=t@t commit -q --allow-empty -m r && git tag "aur-v${VERSION}" \ + && git push -q "${RT_ORIGIN}" "aur-v${VERSION}" ) +rt_out=$(cd "${WORK}/rt" && GIT_CONFIG_GLOBAL=/dev/null REMOTE="${RT_ORIGIN}" bash "${ROOT}/hack/release-tags" "${VERSION}" v1.37.1-k3s1 2>&1) +if matches "^26\.04 versioned $(ere "${PINNED}")$" "${rt_out}" \ + && matches "^26\.04 versioned ghcr\.io/88plug/k3d-gpu:$(ere "${VERSION}")$" "${rt_out}" \ + && matches "^26\.04 floating $(ere "${REPO}:latest ${PINNED}")$" "${rt_out}"; then + pass "hack/release-tags gives the launcher's <repo>:<version> and moves latest for the newest release" +else + fail "hack/release-tags does not print ${PINNED} (versioned) and ${REPO}:latest (floating): ${rt_out}" +fi contract "release build checks out the release tag" build-and-push.yml \ "ref: .*aur-v" contract "release build dispatches the AUR deploy after pushing" build-and-push.yml \ diff --git a/tests/ops/fake_gh.py b/tests/ops/fake_gh.py new file mode 100755 index 00000000..27fa9f08 --- /dev/null +++ b/tests/ops/fake_gh.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""A stand-in for the gh CLI, for tests/ci_alarm_test.sh and tests/watchdog_test.sh. + +Install it on PATH as `gh` (a symlink or a one-line wrapper). State lives in the +directory $FAKE_GH: + + routes.tsv <regex> TAB <response file or -> TAB <exit code> [TAB <stderr text>] + The first line whose regex matches the arguments joined with single + spaces (re.search) answers: the file's bytes on stdout, the exit code, + the stderr text. Routes come before the built-in commands below, so a + route can make `issue list` fail the way a repository with issues + disabled does. Blank lines and lines starting with # are skipped. + issues.json the issue store the built-in issue commands read and write + labels.json label names `label create` added + calls.log every invocation, the arguments joined with single spaces + dispatches.log every `workflow run`: "<file> ref=<ref> [k=v ...]" + +Built in (shapes as gh 2.88 prints them for --json): + issue list [--label L] [--state open|closed|all] [--limit N] [--json F] + issue create --title T (--body B | --body-file F) [--label L]... [--assignee A]... + issue comment N (--body B | --body-file F) + issue edit N [--title T] [--body B | --body-file F] + issue close N [--comment C] [--reason R] + label create NAME [--force] [--color C] [--description D] + workflow run FILE [--ref R] [-f k=v]... + +FAKE_GH_NOW (ISO 8601, default 2026-10-29T13:00:00Z) stamps createdAt. +Anything else exits 1 with "fake gh: no route for: <args>" on stderr. +""" + +import json +import os +import re +import sys + +STATE = os.environ["FAKE_GH"] +NOW = os.environ.get("FAKE_GH_NOW", "2026-10-29T13:00:00Z") +REPO = os.environ.get("GH_REPO", "88plug/k3d-gpu") + + +def path(name): + return os.path.join(STATE, name) + + +def load(name, default): + try: + with open(path(name)) as f: + return json.load(f) + except FileNotFoundError: + return default + + +def save(name, obj): + with open(path(name), "w") as f: + json.dump(obj, f, indent=1) + + +def opts(args, multi=()): + """Parse --flag value pairs (and -R/-f/-F short forms) into a dict and positionals.""" + alias = {"-R": "--repo", "-f": "--raw-field", "-F": "--field", "-b": "--body", "-t": "--title", + "-l": "--label", "-a": "--assignee", "-L": "--limit", "-s": "--state", "-c": "--comment", + "-r": "--reason", "-d": "--description"} + flags = {"--force"} + out, pos, i = {}, [], 0 + while i < len(args): + a = alias.get(args[i], args[i]) + if a in flags: + out[a] = True + i += 1 + elif a.startswith("--") and "=" in a: + k, v = a.split("=", 1) + out.setdefault(k, []).append(v) if k in multi else out.__setitem__(k, v) + i += 1 + elif a.startswith("-") and i + 1 < len(args): + out.setdefault(a, []).append(args[i + 1]) if a in multi else out.__setitem__(a, args[i + 1]) + i += 2 + else: + pos.append(args[i]) + i += 1 + return out, pos + + +def body_of(o): + if "--body-file" in o: + with open(o["--body-file"]) as f: + return f.read() + return o.get("--body", "") + + +def find(issues, num): + for it in issues: + if it["number"] == int(num): + return it + sys.stderr.write("GraphQL: Could not resolve to an issue or pull request with the number of %s.\n" % num) + sys.exit(1) + + +def comment(it, body): + it["comments"].append({"author": {"login": "github-actions"}, "authorAssociation": "NONE", + "body": body, "createdAt": NOW}) + + +def builtin(args): + issues = load("issues.json", []) + if args[:2] == ["issue", "list"]: + o, _ = opts(args[2:]) + state = o.get("--state", "open") + out = [it for it in issues + if (state == "all" or it["state"] == state.upper()) + and ("--label" not in o or any(lb["name"] == o["--label"] for lb in it["labels"]))] + out.sort(key=lambda it: -it["number"]) + print(json.dumps(out[:int(o.get("--limit", 30))])) + return 0 + if args[:2] == ["issue", "create"]: + o, _ = opts(args[2:], multi=("--label", "--assignee")) + num = max([it["number"] for it in issues] + [0]) + 1 + issues.append({"number": num, "title": o["--title"], "body": body_of(o), "state": "OPEN", + "labels": [{"name": n} for n in o.get("--label", [])], + "assignees": [{"login": n} for n in o.get("--assignee", [])], + "comments": [], "createdAt": NOW}) + save("issues.json", issues) + print("https://github.com/%s/issues/%d" % (REPO, num)) + return 0 + if args[:2] == ["issue", "comment"]: + o, pos = opts(args[2:]) + comment(find(issues, pos[0]), body_of(o)) + save("issues.json", issues) + return 0 + if args[:2] == ["issue", "edit"]: + o, pos = opts(args[2:]) + it = find(issues, pos[0]) + if "--title" in o: + it["title"] = o["--title"] + if "--body" in o or "--body-file" in o: + it["body"] = body_of(o) + save("issues.json", issues) + return 0 + if args[:2] == ["issue", "close"]: + o, pos = opts(args[2:]) + it = find(issues, pos[0]) + if "--comment" in o: + comment(it, o["--comment"]) + it["state"] = "CLOSED" + save("issues.json", issues) + return 0 + if args[:2] == ["label", "create"]: + _, pos = opts(args[2:]) + labels = load("labels.json", []) + if pos[0] not in labels: + labels.append(pos[0]) + save("labels.json", labels) + return 0 + if args[:2] == ["workflow", "run"]: + o, pos = opts(args[2:], multi=("--raw-field", "--field")) + fields = o.get("--raw-field", []) + o.get("--field", []) + with open(path("dispatches.log"), "a") as f: + f.write(" ".join([pos[0], "ref=%s" % o.get("--ref", "")] + fields) + "\n") + return 0 + return None + + +def main(args): + line = " ".join(args) + with open(path("calls.log"), "a") as f: + f.write(line + "\n") + try: + with open(path("routes.tsv")) as f: + routes = [ln.rstrip("\n").split("\t") for ln in f if ln.strip() and not ln.startswith("#")] + except FileNotFoundError: + routes = [] + for r in routes: + if re.search(r[0], line): + if len(r) > 1 and r[1] not in ("", "-"): + with open(r[1] if os.path.isabs(r[1]) else path(r[1])) as f: + sys.stdout.write(f.read()) + if len(r) > 3: + sys.stderr.write(r[3] + "\n") + return int(r[2]) if len(r) > 2 and r[2] else 0 + rc = builtin(args) + if rc is None: + sys.stderr.write("fake gh: no route for: %s\n" % line) + return 1 + return rc + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/ops/fixtures/build-and-push_schedule_runs.json b/tests/ops/fixtures/build-and-push_schedule_runs.json new file mode 100644 index 00000000..bdf6d8ce --- /dev/null +++ b/tests/ops/fixtures/build-and-push_schedule_runs.json @@ -0,0 +1 @@ +[{"attempt":1,"conclusion":"success","createdAt":"2026-10-29T12:25:59Z","databaseId":40873203081,"displayTitle":"Auto-update, Build & Release","event":"schedule","headBranch":"main","headSha":"c2b649615dfc697c68236c847d5e93cff1928bff","name":"Auto-update, Build & Release","number":637,"startedAt":"2026-10-29T12:25:59Z","status":"completed","updatedAt":"2026-10-29T12:38:08Z","url":"https://github.com/88plug/k3d-gpu/actions/runs/40873203081","workflowName":"Auto-update, Build & Release"},{"attempt":1,"conclusion":"failure","createdAt":"2026-10-28T12:16:17Z","databaseId":40686436167,"displayTitle":"Auto-update, Build & Release","event":"schedule","headBranch":"main","headSha":"91e958f6ecfb2416a5dedf91eae516ca4cc5a52a","name":"Auto-update, Build & Release","number":632,"startedAt":"2026-10-28T12:16:17Z","status":"completed","updatedAt":"2026-10-28T12:28:45Z","url":"https://github.com/88plug/k3d-gpu/actions/runs/40686436167","workflowName":"Auto-update, Build & Release"},{"attempt":1,"conclusion":"failure","createdAt":"2026-10-27T11:43:34Z","databaseId":40498289109,"displayTitle":"Auto-update, Build & Release","event":"schedule","headBranch":"main","headSha":"91e958f6ecfb2416a5dedf91eae516ca4cc5a52a","name":"Auto-update, Build & Release","number":631,"startedAt":"2026-10-27T11:43:34Z","status":"completed","updatedAt":"2026-10-27T11:54:34Z","url":"https://github.com/88plug/k3d-gpu/actions/runs/40498289109","workflowName":"Auto-update, Build & Release"},{"attempt":1,"conclusion":"failure","createdAt":"2026-10-26T12:24:58Z","databaseId":40322356297,"displayTitle":"Auto-update, Build & Release","event":"schedule","headBranch":"main","headSha":"91e958f6ecfb2416a5dedf91eae516ca4cc5a52a","name":"Auto-update, Build & Release","number":630,"startedAt":"2026-10-26T12:24:58Z","status":"completed","updatedAt":"2026-10-26T12:37:33Z","url":"https://github.com/88plug/k3d-gpu/actions/runs/40322356297","workflowName":"Auto-update, Build & Release"},{"attempt":1,"conclusion":"failure","createdAt":"2026-10-25T10:54:11Z","databaseId":40186662031,"displayTitle":"Auto-update, Build & Release","event":"schedule","headBranch":"main","headSha":"91e958f6ecfb2416a5dedf91eae516ca4cc5a52a","name":"Auto-update, Build & Release","number":629,"startedAt":"2026-10-25T10:54:11Z","status":"completed","updatedAt":"2026-10-25T11:06:29Z","url":"https://github.com/88plug/k3d-gpu/actions/runs/40186662031","workflowName":"Auto-update, Build & Release"},{"attempt":1,"conclusion":"failure","createdAt":"2026-10-24T10:26:50Z","databaseId":40085941186,"displayTitle":"Auto-update, Build & Release","event":"schedule","headBranch":"main","headSha":"91e958f6ecfb2416a5dedf91eae516ca4cc5a52a","name":"Auto-update, Build & Release","number":628,"startedAt":"2026-10-24T10:26:50Z","status":"completed","updatedAt":"2026-10-24T10:37:26Z","url":"https://github.com/88plug/k3d-gpu/actions/runs/40085941186","workflowName":"Auto-update, Build & Release"},{"attempt":1,"conclusion":"failure","createdAt":"2026-10-23T11:09:08Z","databaseId":39944071889,"displayTitle":"Auto-update, Build & Release","event":"schedule","headBranch":"main","headSha":"91e958f6ecfb2416a5dedf91eae516ca4cc5a52a","name":"Auto-update, Build & Release","number":627,"startedAt":"2026-10-23T11:09:08Z","status":"completed","updatedAt":"2026-10-23T11:19:30Z","url":"https://github.com/88plug/k3d-gpu/actions/runs/39944071889","workflowName":"Auto-update, Build & Release"},{"attempt":1,"conclusion":"failure","createdAt":"2026-10-22T11:18:24Z","databaseId":39769577054,"displayTitle":"Auto-update, Build & Release","event":"schedule","headBranch":"main","headSha":"91e958f6ecfb2416a5dedf91eae516ca4cc5a52a","name":"Auto-update, Build & Release","number":626,"startedAt":"2026-10-22T11:18:24Z","status":"completed","updatedAt":"2026-10-22T11:29:23Z","url":"https://github.com/88plug/k3d-gpu/actions/runs/39769577054","workflowName":"Auto-update, Build & Release"},{"attempt":1,"conclusion":"failure","createdAt":"2026-10-21T11:19:14Z","databaseId":39591066580,"displayTitle":"Auto-update, Build & Release","event":"schedule","headBranch":"main","headSha":"ced8cd8729632d5d4fe8aeb3a89ae3b9232e7e1d","name":"Auto-update, Build & Release","number":620,"startedAt":"2026-10-21T11:19:14Z","status":"completed","updatedAt":"2026-10-21T11:29:11Z","url":"https://github.com/88plug/k3d-gpu/actions/runs/39591066580","workflowName":"Auto-update, Build & Release"},{"attempt":1,"conclusion":"failure","createdAt":"2026-10-20T11:02:22Z","databaseId":39412124435,"displayTitle":"Auto-update, Build & Release","event":"schedule","headBranch":"main","headSha":"a32a9701bf9bc139c1b38a5c46673462b26f66b2","name":"Auto-update, Build & Release","number":611,"startedAt":"2026-10-20T11:02:22Z","status":"completed","updatedAt":"2026-10-20T11:11:43Z","url":"https://github.com/88plug/k3d-gpu/actions/runs/39412124435","workflowName":"Auto-update, Build & Release"},{"attempt":1,"conclusion":"success","createdAt":"2026-10-19T12:24:18Z","databaseId":39251789909,"displayTitle":"Auto-update, Build & Release","event":"schedule","headBranch":"main","headSha":"3745468bc1be296deecd0f89203971cdb3bee5cb","name":"Auto-update, Build & Release","number":609,"startedAt":"2026-10-19T12:24:18Z","status":"completed","updatedAt":"2026-10-19T12:34:00Z","url":"https://github.com/88plug/k3d-gpu/actions/runs/39251789909","workflowName":"Auto-update, Build & Release"},{"attempt":1,"conclusion":"success","createdAt":"2026-10-18T10:31:50Z","databaseId":39119616480,"displayTitle":"Auto-update, Build & Release","event":"schedule","headBranch":"main","headSha":"5dae99b16c8e88d3f8c2bb8e0db552efa3c3b296","name":"Auto-update, Build & Release","number":605,"startedAt":"2026-10-18T10:31:50Z","status":"completed","updatedAt":"2026-10-18T10:40:21Z","url":"https://github.com/88plug/k3d-gpu/actions/runs/39119616480","workflowName":"Auto-update, Build & Release"},{"attempt":1,"conclusion":"success","createdAt":"2026-10-17T10:16:15Z","databaseId":39024257395,"displayTitle":"Auto-update, Build & Release","event":"schedule","headBranch":"main","headSha":"8b0526ef8ab52d247242fddb54c5f03a8c42c28d","name":"Auto-update, Build & Release","number":575,"startedAt":"2026-10-17T10:16:15Z","status":"completed","updatedAt":"2026-10-17T10:20:25Z","url":"https://github.com/88plug/k3d-gpu/actions/runs/39024257395","workflowName":"Auto-update, Build & Release"},{"attempt":1,"conclusion":"success","createdAt":"2026-10-16T10:53:26Z","databaseId":38890188335,"displayTitle":"Auto-update, Build & Release","event":"schedule","headBranch":"main","headSha":"83d9ce0686b71dcc49f4e43b55f4e859f2227f18","name":"Auto-update, Build & Release","number":560,"startedAt":"2026-10-16T10:53:26Z","status":"completed","updatedAt":"2026-10-16T10:53:41Z","url":"https://github.com/88plug/k3d-gpu/actions/runs/38890188335","workflowName":"Auto-update, Build & Release"}] diff --git a/tests/ops/fixtures/run_39412124435_jobs.json b/tests/ops/fixtures/run_39412124435_jobs.json new file mode 100644 index 00000000..499ec81b --- /dev/null +++ b/tests/ops/fixtures/run_39412124435_jobs.json @@ -0,0 +1,73 @@ +{ + "jobs": [ + { + "completedAt": "2026-10-20T11:11:42Z", + "conclusion": "failure", + "databaseId": 118592194165, + "name": "repin-images", + "startedAt": "2026-10-20T11:02:24Z", + "status": "completed", + "steps": [ + { + "completedAt": "2026-10-20T11:11:41Z", + "conclusion": "failure", + "name": "Open or update the digest PR", + "number": 7, + "startedAt": "2026-10-20T11:11:39Z", + "status": "completed" + } + ], + "url": "https://github.com/88plug/k3d-gpu/actions/runs/39412124435/job/118592194165" + }, + { + "completedAt": "2026-10-20T11:02:36Z", + "conclusion": "success", + "databaseId": 118592194452, + "name": "check", + "startedAt": "2026-10-20T11:02:25Z", + "status": "completed", + "steps": [], + "url": "https://github.com/88plug/k3d-gpu/actions/runs/39412124435/job/118592194452" + }, + { + "completedAt": "2026-10-20T11:02:31Z", + "conclusion": "success", + "databaseId": 118592194594, + "name": "list-components", + "startedAt": "2026-10-20T11:02:25Z", + "status": "completed", + "steps": [], + "url": "https://github.com/88plug/k3d-gpu/actions/runs/39412124435/job/118592194594" + }, + { + "completedAt": "2026-10-20T11:02:22Z", + "conclusion": "skipped", + "databaseId": 118592195801, + "name": "release", + "startedAt": "2026-10-20T11:02:23Z", + "status": "completed", + "steps": [], + "url": "https://github.com/88plug/k3d-gpu/actions/runs/39412124435/job/118592195801" + }, + { + "completedAt": "2026-10-20T11:11:24Z", + "conclusion": "success", + "databaseId": 118592253980, + "name": "bump-components (nri-balloons)", + "startedAt": "2026-10-20T11:02:33Z", + "status": "completed", + "steps": [], + "url": "https://github.com/88plug/k3d-gpu/actions/runs/39412124435/job/118592253980" + }, + { + "completedAt": "2026-10-20T11:11:26Z", + "conclusion": "success", + "databaseId": 118592254119, + "name": "bump-components (nri-balloons-guard)", + "startedAt": "2026-10-20T11:02:33Z", + "status": "completed", + "steps": [], + "url": "https://github.com/88plug/k3d-gpu/actions/runs/39412124435/job/118592254119" + } + ] +} diff --git a/tests/ops/fixtures/run_39412124435_log_failed.txt b/tests/ops/fixtures/run_39412124435_log_failed.txt new file mode 100644 index 00000000..0023334f --- /dev/null +++ b/tests/ops/fixtures/run_39412124435_log_failed.txt @@ -0,0 +1,69 @@ +repin-images UNKNOWN STEP 2026-10-20T11:11:40.3921272Z [command]/usr/bin/git checkout --progress -B autobump/image-digests 3dafeaa7-ed44-406a-adab-6b3079798909 -- +repin-images UNKNOWN STEP 2026-10-20T11:11:40.4155261Z Switched to a new branch 'autobump/image-digests' +repin-images UNKNOWN STEP 2026-10-20T11:11:40.4170852Z [command]/usr/bin/git rev-list --right-only --count main...autobump/image-digests +repin-images UNKNOWN STEP 2026-10-20T11:11:40.4213474Z 1 +repin-images UNKNOWN STEP 2026-10-20T11:11:40.4218208Z Created branch 'autobump/image-digests' +repin-images UNKNOWN STEP 2026-10-20T11:11:40.4223267Z [command]/usr/bin/git rev-parse main +repin-images UNKNOWN STEP 2026-10-20T11:11:40.4257673Z a32a9701bf9bc139c1b38a5c46673462b26f66b2 +repin-images UNKNOWN STEP 2026-10-20T11:11:40.4279469Z [command]/usr/bin/git -c core.quotePath=false show --raw --cc --no-renames --no-abbrev --format=%H%n%T%n%P%n%G?%n%s%n%b%n###EOB### a32a9701bf9bc139c1b38a5c46673462b26f66b2 +repin-images UNKNOWN STEP 2026-10-20T11:11:40.4736792Z [command]/usr/bin/git rev-parse autobump/image-digests +repin-images UNKNOWN STEP 2026-10-20T11:11:40.4824065Z 44df43843575c34e4445f74c19275b0a8c368127 +repin-images UNKNOWN STEP 2026-10-20T11:11:40.4828650Z [command]/usr/bin/git log --format=%H main..autobump/image-digests +repin-images UNKNOWN STEP 2026-10-20T11:11:40.4890744Z 44df43843575c34e4445f74c19275b0a8c368127 +repin-images UNKNOWN STEP 2026-10-20T11:11:40.4900873Z [command]/usr/bin/git -c core.quotePath=false show --raw --cc --no-renames --no-abbrev --format=%H%n%T%n%P%n%G?%n%s%n%b%n###EOB### 44df43843575c34e4445f74c19275b0a8c368127 +repin-images UNKNOWN STEP 2026-10-20T11:11:40.4945493Z [command]/usr/bin/git branch --delete --force 3dafeaa7-ed44-406a-adab-6b3079798909 +repin-images UNKNOWN STEP 2026-10-20T11:11:40.4985736Z Deleted branch 3dafeaa7-ed44-406a-adab-6b3079798909 (was 44df438). +repin-images UNKNOWN STEP 2026-10-20T11:11:40.4993338Z [command]/usr/bin/git checkout --progress main -- +repin-images UNKNOWN STEP 2026-10-20T11:11:40.5185041Z Switched to branch 'main' +repin-images UNKNOWN STEP 2026-10-20T11:11:40.5186587Z Your branch is up to date with 'origin/main'. +repin-images UNKNOWN STEP 2026-10-20T11:11:40.5195480Z ##[endgroup] +repin-images UNKNOWN STEP 2026-10-20T11:11:40.5196354Z ##[group]Pushing pull request branch to 'origin/autobump/image-digests' +repin-images UNKNOWN STEP 2026-10-20T11:11:40.5201405Z [command]/usr/bin/git push --force-with-lease origin autobump/image-digests:refs/heads/autobump/image-digests +repin-images UNKNOWN STEP 2026-10-20T11:11:41.5322777Z To https://github.com/88plug/k3d-gpu +repin-images UNKNOWN STEP 2026-10-20T11:11:41.5323814Z ! [remote rejected] autobump/image-digests -> autobump/image-digests (refusing to allow a GitHub App to create or update workflow `.github/workflows/ci.yml` without `workflows` permission) +repin-images UNKNOWN STEP 2026-10-20T11:11:41.5324928Z error: failed to push some refs to 'https://github.com/88plug/k3d-gpu' +repin-images UNKNOWN STEP 2026-10-20T11:11:41.5362780Z ##[error]The process '/usr/bin/git' failed with exit code 1 +repin-images UNKNOWN STEP 2026-10-20T11:11:41.5371492Z ##[group]Restore git configuration +repin-images UNKNOWN STEP 2026-10-20T11:11:41.5372069Z [command]/usr/bin/git config --local --name-only --get-regexp http.https://github.com/.extraheader ^AUTHORIZATION: +repin-images UNKNOWN STEP 2026-10-20T11:11:41.5391040Z http.https://github.com/.extraheader +repin-images UNKNOWN STEP 2026-10-20T11:11:41.5401977Z [command]/usr/bin/git config --local --get-regexp http.https://github.com/.extraheader ^AUTHORIZATION: +repin-images UNKNOWN STEP 2026-10-20T11:11:41.5444012Z http.https://github.com/.extraheader AUTHORIZATION: basic *** +repin-images UNKNOWN STEP 2026-10-20T11:11:41.5449266Z [command]/usr/bin/git config --local --unset http.https://github.com/.extraheader ^AUTHORIZATION: +repin-images UNKNOWN STEP 2026-10-20T11:11:41.5489680Z Unset config key 'http.https://github.com/.extraheader' +repin-images UNKNOWN STEP 2026-10-20T11:11:41.5493641Z Restored checkout credential file: git-credentials-0855f328-330d-4402-b0df-e8084e6fb813.config +repin-images UNKNOWN STEP 2026-10-20T11:11:41.5497799Z [command]/usr/bin/git config --global --unset safe.directory /home/runner/work/k3d-gpu/k3d-gpu +repin-images UNKNOWN STEP 2026-10-20T11:11:41.5542247Z ##[endgroup] +repin-images UNKNOWN STEP 2026-10-20T11:11:41.5777709Z Post job cleanup. +repin-images UNKNOWN STEP 2026-10-20T11:11:41.6720620Z [command]/usr/bin/git version +repin-images UNKNOWN STEP 2026-10-20T11:11:41.6770976Z git version 2.55.0 +repin-images UNKNOWN STEP 2026-10-20T11:11:41.6810419Z Copying '/home/runner/.gitconfig' to '/home/runner/work/_temp/61145c41-06b6-42b8-b3ad-72172d8eeb20/.gitconfig' +repin-images UNKNOWN STEP 2026-10-20T11:11:41.6821283Z Temporarily overriding HOME='/home/runner/work/_temp/61145c41-06b6-42b8-b3ad-72172d8eeb20' before making global git config changes +repin-images UNKNOWN STEP 2026-10-20T11:11:41.6822668Z Adding repository directory to the temporary git global config as a safe directory +repin-images UNKNOWN STEP 2026-10-20T11:11:41.6827043Z [command]/usr/bin/git config --global --add safe.directory /home/runner/work/k3d-gpu/k3d-gpu +repin-images UNKNOWN STEP 2026-10-20T11:11:41.6868727Z Removing SSH command configuration +repin-images UNKNOWN STEP 2026-10-20T11:11:41.6876124Z [command]/usr/bin/git config --local --name-only --get-regexp core\.sshCommand +repin-images UNKNOWN STEP 2026-10-20T11:11:41.6918380Z [command]/usr/bin/git submodule foreach --recursive sh -c "git config --local --name-only --get-regexp 'core\.sshCommand' && git config --local --unset-all 'core.sshCommand' || :" +repin-images UNKNOWN STEP 2026-10-20T11:11:41.7214198Z Removing HTTP extra header +repin-images UNKNOWN STEP 2026-10-20T11:11:41.7221090Z [command]/usr/bin/git config --local --name-only --get-regexp http\.https\:\/\/github\.com\/\.extraheader +repin-images UNKNOWN STEP 2026-10-20T11:11:41.7263494Z [command]/usr/bin/git submodule foreach --recursive sh -c "git config --local --name-only --get-regexp 'http\.https\:\/\/github\.com\/\.extraheader' && git config --local --unset-all 'http.https://github.com/.extraheader' || :" +repin-images UNKNOWN STEP 2026-10-20T11:11:41.7561114Z Removing includeIf entries pointing to credentials config files +repin-images UNKNOWN STEP 2026-10-20T11:11:41.7573853Z [command]/usr/bin/git config --local --name-only --get-regexp ^includeIf\.gitdir: +repin-images UNKNOWN STEP 2026-10-20T11:11:41.7609505Z includeif.gitdir:/home/runner/work/k3d-gpu/k3d-gpu/.git.path +repin-images UNKNOWN STEP 2026-10-20T11:11:41.7610347Z includeif.gitdir:/home/runner/work/k3d-gpu/k3d-gpu/.git/worktrees/*.path +repin-images UNKNOWN STEP 2026-10-20T11:11:41.7611064Z includeif.gitdir:/github/workspace/.git.path +repin-images UNKNOWN STEP 2026-10-20T11:11:41.7611707Z includeif.gitdir:/github/workspace/.git/worktrees/*.path +repin-images UNKNOWN STEP 2026-10-20T11:11:41.7620469Z [command]/usr/bin/git config --local --get-all includeif.gitdir:/home/runner/work/k3d-gpu/k3d-gpu/.git.path +repin-images UNKNOWN STEP 2026-10-20T11:11:41.7652746Z /home/runner/work/_temp/git-credentials-0855f328-330d-4402-b0df-e8084e6fb813.config +repin-images UNKNOWN STEP 2026-10-20T11:11:41.7665336Z [command]/usr/bin/git config --local --unset includeif.gitdir:/home/runner/work/k3d-gpu/k3d-gpu/.git.path /home/runner/work/_temp/git-credentials-0855f328-330d-4402-b0df-e8084e6fb813.config +repin-images UNKNOWN STEP 2026-10-20T11:11:41.7719090Z [command]/usr/bin/git config --local --get-all includeif.gitdir:/home/runner/work/k3d-gpu/k3d-gpu/.git/worktrees/*.path +repin-images UNKNOWN STEP 2026-10-20T11:11:41.7754639Z /home/runner/work/_temp/git-credentials-0855f328-330d-4402-b0df-e8084e6fb813.config +repin-images UNKNOWN STEP 2026-10-20T11:11:41.7767248Z [command]/usr/bin/git config --local --unset includeif.gitdir:/home/runner/work/k3d-gpu/k3d-gpu/.git/worktrees/*.path /home/runner/work/_temp/git-credentials-0855f328-330d-4402-b0df-e8084e6fb813.config +repin-images UNKNOWN STEP 2026-10-20T11:11:41.7817451Z [command]/usr/bin/git config --local --get-all includeif.gitdir:/github/workspace/.git.path +repin-images UNKNOWN STEP 2026-10-20T11:11:41.7854260Z /github/runner_temp/git-credentials-0855f328-330d-4402-b0df-e8084e6fb813.config +repin-images UNKNOWN STEP 2026-10-20T11:11:41.7864827Z [command]/usr/bin/git config --local --unset includeif.gitdir:/github/workspace/.git.path /github/runner_temp/git-credentials-0855f328-330d-4402-b0df-e8084e6fb813.config +repin-images UNKNOWN STEP 2026-10-20T11:11:41.7914916Z [command]/usr/bin/git config --local --get-all includeif.gitdir:/github/workspace/.git/worktrees/*.path +repin-images UNKNOWN STEP 2026-10-20T11:11:41.7949826Z /github/runner_temp/git-credentials-0855f328-330d-4402-b0df-e8084e6fb813.config +repin-images UNKNOWN STEP 2026-10-20T11:11:41.7962282Z [command]/usr/bin/git config --local --unset includeif.gitdir:/github/workspace/.git/worktrees/*.path /github/runner_temp/git-credentials-0855f328-330d-4402-b0df-e8084e6fb813.config +repin-images UNKNOWN STEP 2026-10-20T11:11:41.8011711Z [command]/usr/bin/git submodule foreach --recursive git config --local --show-origin --name-only --get-regexp remote.origin.url +repin-images UNKNOWN STEP 2026-10-20T11:11:41.8271962Z Removing credentials config '/home/runner/work/_temp/git-credentials-0855f328-330d-4402-b0df-e8084e6fb813.config' +repin-images UNKNOWN STEP 2026-10-20T11:11:41.8452430Z Cleaning up orphan processes diff --git a/tests/ops/fixtures/workflows.json b/tests/ops/fixtures/workflows.json new file mode 100644 index 00000000..d527afcf --- /dev/null +++ b/tests/ops/fixtures/workflows.json @@ -0,0 +1 @@ +{"total_count":9,"workflows":[{"id":284520033,"node_id":"W_kwDOOd6rPM4Q9W5h","name":"AUR Deploy","path":".github/workflows/aur-deploy.yml","state":"active","created_at":"2026-05-27T15:37:24.000-07:00","updated_at":"2026-05-27T15:37:24.000-07:00","url":"https://api.github.com/repos/88plug/k3d-gpu/actions/workflows/284520033","html_url":"https://github.com/88plug/k3d-gpu/blob/main/.github/workflows/aur-deploy.yml","badge_url":"https://github.com/88plug/k3d-gpu/workflows/AUR%20Deploy/badge.svg"},{"id":284520036,"node_id":"W_kwDOOd6rPM4Q9W5k","name":"Auto-release AUR","path":".github/workflows/auto-release.yml","state":"active","created_at":"2026-05-27T15:37:24.000-07:00","updated_at":"2026-05-27T15:37:24.000-07:00","url":"https://api.github.com/repos/88plug/k3d-gpu/actions/workflows/284520036","html_url":"https://github.com/88plug/k3d-gpu/blob/main/.github/workflows/auto-release.yml","badge_url":"https://github.com/88plug/k3d-gpu/workflows/Auto-release%20AUR/badge.svg"},{"id":157632137,"node_id":"W_kwDOOd6rPM4JZUaJ","name":"Auto-update, Build & Release","path":".github/workflows/build-and-push.yml","state":"active","created_at":"2025-04-22T13:32:16.000-07:00","updated_at":"2025-04-22T13:43:51.000-07:00","url":"https://api.github.com/repos/88plug/k3d-gpu/actions/workflows/157632137","html_url":"https://github.com/88plug/k3d-gpu/blob/main/.github/workflows/build-and-push.yml","badge_url":"https://github.com/88plug/k3d-gpu/workflows/Auto-update,%20Build%20&%20Release/badge.svg"},{"id":284520038,"node_id":"W_kwDOOd6rPM4Q9W5m","name":"CI","path":".github/workflows/ci.yml","state":"active","created_at":"2026-05-27T15:37:24.000-07:00","updated_at":"2026-05-27T15:37:24.000-07:00","url":"https://api.github.com/repos/88plug/k3d-gpu/actions/workflows/284520038","html_url":"https://github.com/88plug/k3d-gpu/blob/main/.github/workflows/ci.yml","badge_url":"https://github.com/88plug/k3d-gpu/workflows/CI/badge.svg"},{"id":284520040,"node_id":"W_kwDOOd6rPM4Q9W5o","name":"Auto-merge Dependabot PRs","path":".github/workflows/dependabot-auto-merge.yml","state":"active","created_at":"2026-05-27T15:37:24.000-07:00","updated_at":"2026-05-27T15:37:24.000-07:00","url":"https://api.github.com/repos/88plug/k3d-gpu/actions/workflows/284520040","html_url":"https://github.com/88plug/k3d-gpu/blob/main/.github/workflows/dependabot-auto-merge.yml","badge_url":"https://github.com/88plug/k3d-gpu/workflows/Auto-merge%20Dependabot%20PRs/badge.svg"},{"id":387399486,"node_id":"W_kwDOOd6rPM4XFz8-","name":"Docs","path":".github/workflows/docs.yml","state":"active","created_at":"2026-10-16T22:44:34.000-07:00","updated_at":"2026-10-16T22:48:54.000-07:00","url":"https://api.github.com/repos/88plug/k3d-gpu/actions/workflows/387399486","html_url":"https://github.com/88plug/k3d-gpu/blob/main/.github/workflows/docs.yml","badge_url":"https://github.com/88plug/k3d-gpu/workflows/Docs/badge.svg"},{"id":387412638,"node_id":"W_kwDOOd6rPM4XF3Ke","name":"Release packages (deb, rpm, Homebrew)","path":".github/workflows/packages.yml","state":"active","created_at":"2026-10-16T23:12:09.000-07:00","updated_at":"2026-10-16T23:12:09.000-07:00","url":"https://api.github.com/repos/88plug/k3d-gpu/actions/workflows/387412638","html_url":"https://github.com/88plug/k3d-gpu/blob/main/.github/workflows/packages.yml","badge_url":"https://github.com/88plug/k3d-gpu/workflows/Release%20packages%20(deb,%20rpm,%20Homebrew)/badge.svg"},{"id":231869215,"node_id":"W_kwDOOd6rPM4N0gsf","name":"Copilot coding agent","path":"dynamic/copilot-swe-agent/copilot","state":"active","created_at":"2026-02-08T09:15:33.000-08:00","updated_at":"2026-02-08T09:15:33.000-08:00","url":"https://api.github.com/repos/88plug/k3d-gpu/actions/workflows/231869215","html_url":"https://github.com/88plug/k3d-gpu/actions/workflows/copilot-swe-agent/copilot","badge_url":"https://github.com/88plug/k3d-gpu/actions/workflows/copilot-swe-agent/copilot/badge.svg"},{"id":284520057,"node_id":"W_kwDOOd6rPM4Q9W55","name":"CodeQL","path":"dynamic/github-code-scanning/codeql","state":"active","created_at":"2026-05-27T15:37:28.000-07:00","updated_at":"2026-05-27T15:37:28.000-07:00","url":"https://api.github.com/repos/88plug/k3d-gpu/actions/workflows/284520057","html_url":"https://github.com/88plug/k3d-gpu/actions/workflows/github-code-scanning/codeql","badge_url":"https://github.com/88plug/k3d-gpu/actions/workflows/github-code-scanning/codeql/badge.svg"}]} \ No newline at end of file diff --git a/tests/ops/stub_registry.py b/tests/ops/stub_registry.py new file mode 100644 index 00000000..4a9f5e8d --- /dev/null +++ b/tests/ops/stub_registry.py @@ -0,0 +1,24 @@ +"""A stand-in for scripts/images-lock's registry_digest, for tests/watchdog_test.sh. + +hack/watchdog loads registry_digest from the file WATCHDOG_IMAGES_LOCK names; the +test points that at this file. $STUB_REGISTRY names a file of lines +"<name>:<tag> [status]": a listed reference without a status resolves to a fixed +digest, one with a status (e.g. 503) raises the way images-lock does for that HTTP +status, and an unlisted one raises HTTP 404. Every lookup is appended to +$STUB_REGISTRY.log. +""" + +import os + + +def registry_digest(name, tag): + ref = "%s:%s" % (name, tag) + with open(os.environ["STUB_REGISTRY"] + ".log", "a") as f: + f.write(ref + "\n") + with open(os.environ["STUB_REGISTRY"]) as f: + rows = dict((ln.split() + [""])[:2] for ln in f if ln.strip()) + if ref not in rows: + raise ValueError("%s: HTTP 404 from https://stub/v2/%s/manifests/%s" % (ref, name, tag)) + if rows[ref]: + raise ValueError("%s: HTTP %s from https://stub/v2/%s/manifests/%s" % (ref, rows[ref], name, tag)) + return "sha256:" + "0" * 64 diff --git a/tests/ops/workflow_checks.py b/tests/ops/workflow_checks.py new file mode 100755 index 00000000..4e3cb497 --- /dev/null +++ b/tests/ops/workflow_checks.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Workflow checks for tests/ci_wiring_test.sh (alarm, watchdog, release hardening). + +Reads every .github/workflows/*.yml with PyYAML and prints one line per check: +"ok <what>" or "FAIL <what>: <why>". Exit 0 when the checks ran (failures are +lines, not the exit code). + +Usage: tests/ops/workflow_checks.py REPO_ROOT +""" + +import glob +import os +import re +import subprocess +import sys + +import yaml + +ROOT = sys.argv[1] +WF = os.path.join(ROOT, ".github", "workflows") +DOCS = {os.path.basename(p): yaml.safe_load(open(p)) for p in sorted(glob.glob(WF + "/*.yml"))} + + +def say(cond, what, why=""): + print(("ok %s" % what) if cond else ("FAIL %s%s" % (what, (": " + str(why)) if why else ""))) + + +def on(d): + """A workflow's on: block (PyYAML reads the bare key `on` as True).""" + v = (d or {}).get("on", (d or {}).get(True)) + return v if isinstance(v, dict) else {} + + +# Every job has timeout-minutes; every workflow has a top-level permissions block. +for f, d in DOCS.items(): + missing = [j for j, v in (d.get("jobs") or {}).items() if not isinstance(v.get("timeout-minutes"), int)] + say(not missing, "%s: every job has timeout-minutes" % f, "missing in %s" % ", ".join(missing)) + say("permissions" in d, "%s: has a top-level permissions block" % f) +say(DOCS.get("aur-deploy.yml", {}).get("permissions") == {"contents": "read"}, "aur-deploy.yml: permissions contents: read") +# Releases now build in parallel, so the AUR deploy of an older one can come last: +# it must leave a newer AUR version alone. +ad = ((DOCS.get("aur-deploy.yml") or {}).get("jobs") or {}).get("deploy", {}).get("steps") or [] +guard = [s for s in ad if s.get("id") == "aur"] +pub = [s for s in ad if str(s.get("uses", "")).startswith("KSXGitHub/github-actions-deploy-aur@")] +say(guard and "sort -V" in guard[0].get("run", "") and "rpc/v5/info" in guard[0].get("run", "") + and pub and pub[0].get("if") == "steps.aur.outputs.skip != 'true'" + and ad.index(guard[0]) < ad.index(pub[0]), + "aur-deploy.yml never moves AUR back to an older version") + +# No cron fires at minute 0 or 30. +scheduled = {} +for f, d in DOCS.items(): + for c in on(d).get("schedule") or []: + scheduled.setdefault(f, []).append(c["cron"]) + minute = c["cron"].split()[0] + bad = not re.fullmatch(r"\d+(,\d+)*", minute) or bool({0, 30} & {int(x) for x in minute.split(",")}) + say(not bad, "%s: cron '%s' avoids minutes 0 and 30" % (f, c["cron"])) + +# hack/watchdog discovers every scheduled workflow. +p = subprocess.run([sys.executable, os.path.join(ROOT, "hack", "watchdog"), "--list-scheduled", "--root", ROOT], + capture_output=True, text=True) +found = {ln.split()[0] for ln in p.stdout.splitlines() if ln.strip()} +say(p.returncode == 0 and scheduled and found == set(scheduled), + "hack/watchdog --list-scheduled covers every scheduled workflow", + "watchdog %s, yaml %s %s" % (sorted(found), sorted(scheduled), p.stderr[-200:])) + +# watchdog.yml listens to CI, Docs and the release chain by their exact names, and +# to Canary, Freshness and Bot sweep once those workflows exist. +names = {d.get("name") for d in DOCS.values()} +wd = DOCS.get("watchdog.yml") or {} +listed = (on(wd).get("workflow_run") or {}).get("workflows") or [] +want = {"CI", "Auto-update, Build & Release", "Auto-release AUR", "Release packages (deb, rpm, Homebrew)", + "AUR Deploy", "Docs"} | ({"Canary", "Freshness", "Bot sweep"} & names) +say(set(listed) == want and len(listed) == len(want) and set(listed) <= names, + "watchdog.yml workflow_run names equal the name: fields of the workflows it watches", + "listed %s, want %s, no such workflow %s" % (sorted(listed), sorted(want), sorted(set(listed) - names))) +job = (wd.get("jobs") or {}).get("watchdog") or {} +perms = wd.get("permissions") or {} +say(perms.get("issues") == "write" and perms.get("actions") == "write" and perms.get("contents") == "read" + and wd.get("concurrency") == {"group": "watchdog", "cancel-in-progress": False} + and "head_branch == 'main'" in str(job.get("if", "")) + and (on(wd).get("workflow_run") or {}).get("branches") == ["main"] + and on(wd).get("schedule") == [{"cron": "47 * * * *"}] + and ((on(wd).get("workflow_dispatch") or {}).get("inputs") or {}).get("simulate_red", {}).get("type") == "boolean", + "watchdog.yml: hourly at :47, simulate_red, issues+actions write, one queue, workflow_run on main only") +co = [s for s in job.get("steps", []) if str(s.get("uses", "")).startswith("actions/checkout@")] +say(co and co[0].get("with", {}).get("ref") == "main" and co[0]["with"].get("fetch-depth") == 0 + and any(str(s.get("run", "")).strip() == "hack/watchdog" for s in job.get("steps", [])), + "watchdog.yml checks out main with every tag and runs hack/watchdog") + +# build-and-push.yml: one queue per release version, the nightly in its own, so a +# pending release never replaces another. +bp = DOCS.get("build-and-push.yml") or {} +say(bp.get("concurrency") == {"group": "build-and-push-${{ inputs.release && format('release-{0}', inputs.release) || 'nightly' }}", + "cancel-in-progress": False}, + "build-and-push.yml concurrency is keyed by the release version", bp.get("concurrency")) + +# The release job builds each variant in up to 3 identical attempts (1 and 2 +# continue on error and are followed by a 2 and a 4 min wait; 3 fails the job), +# pushing the versioned tags only; the floating tags are copied after both builds +# from hack/release-tags run at that moment. +steps = ((bp.get("jobs") or {}).get("release") or {}).get("steps") or [] +by_id = {s.get("id"): s for s in steps if s.get("id")} +order = [s.get("id") or s.get("name") for s in steps] +for v in ("2604", "2404"): + a = [by_id.get("b%s_%d" % (v, i)) for i in (1, 2, 3)] + good = all(a) and all(str(s.get("uses", "")).startswith("docker/build-push-action@") for s in a) + good = good and [s.get("continue-on-error") for s in a] == [True, True, None] + good = good and a[0].get("if") is None and a[1].get("if") == "steps.b%s_1.outcome == 'failure'" % v \ + and a[2].get("if") == "steps.b%s_2.outcome == 'failure'" % v + good = good and all(s["with"].get("tags") == "${{ steps.tags.outputs.v%s }}" % v for s in a) + good = good and len({yaml.safe_dump(s["with"]) for s in a}) == 1 and len({s.get("uses") for s in a}) == 1 + waits = [s for s in steps if "sleep" in str(s.get("run", "")) and s.get("if") in + ("steps.b%s_1.outcome == 'failure'" % v, "steps.b%s_2.outcome == 'failure'" % v)] + good = good and [w["run"].split()[-1] for w in waits] == ["120", "240"] + good = good and order.index("b%s_1" % v) < order.index(waits[0].get("id") or waits[0].get("name")) \ + < order.index("b%s_2" % v) < order.index(waits[1].get("id") or waits[1].get("name")) < order.index("b%s_3" % v) + say(good, "release builds the %s variant in 3 identical attempts, 2 and 4 min apart, versioned tags only" % v) +tags = by_id.get("tags") or {} +flo = [s for s in steps if "imagetools create" in str(s.get("run", ""))] +say("release-tags" in str(tags.get("run", "")) and len(flo) == 1 and "release-tags" in flo[0]["run"] + and "floating" in flo[0]["run"] and "b2404_3" in order and "b2604_3" in order + and order.index(flo[0].get("id") or flo[0].get("name")) > max(order.index("b2404_3"), order.index("b2604_3")), + "floating tags come from hack/release-tags at push time, after both builds") +say(any("already published" in str(s.get("run", "")) for s in steps), "release keeps its refusal to overwrite a published version") + +# auto-release.yml: its push paths equal hack/release-paths, which covers every file +# nfpm and the Homebrew formula ship (README and license excepted); mode=changed +# runs release_cas.sh as a push, so the path filter applies. +ar = DOCS.get("auto-release.yml") or {} +rp = [ln.split("#")[0].strip() for ln in open(os.path.join(ROOT, "hack", "release-paths"))] +rp = [x for x in rp if x] +say(rp and all(re.fullmatch(r"[A-Za-z0-9_.-]+(/[A-Za-z0-9_.-]+)*(/\*\*)?", x) for x in rp), + "hack/release-paths holds only paths and dir/** entries") +say((on(ar).get("push") or {}).get("paths") == rp, "auto-release.yml paths: equals hack/release-paths", + "yml %s, file %s" % ((on(ar).get("push") or {}).get("paths"), rp)) + + +def covered(src): + return any(src == r or (r.endswith("/**") and src.startswith(r[:-2])) for r in rp) + + +nfpm = yaml.safe_load(open(os.path.join(ROOT, "packaging", "nfpm", "nfpm.yaml"))) +srcs = [c["src"] for c in nfpm["contents"] if c.get("src")] +formula = open(os.path.join(ROOT, "Formula", "k3d-gpu.rb")).read() +inst = re.sub(r",\s*\n\s*", ", ", formula.split("def install", 1)[1].split("\n end", 1)[0]) +fsrcs = [q for ln in inst.splitlines() if ".install" in ln for q in re.findall(r'"([^"]+)"', ln.split(".install", 1)[1])] +docs_only = {"README.md", "LICENSE.md"} +miss = [x for x in srcs + fsrcs if x not in docs_only and not covered(x)] +say(bool(srcs) and bool(fsrcs) and not miss, "hack/release-paths covers every nfpm src and Formula source", + "not covered: %s" % miss) +mode = ((on(ar).get("workflow_dispatch") or {}).get("inputs") or {}).get("mode") or {} +say(mode.get("type") == "choice" and mode.get("options") == ["force", "changed"] and mode.get("default") == "force", + "auto-release.yml workflow_dispatch input mode: force (default) or changed", mode) +rel = (ar.get("jobs") or {}).get("release") or {} +ev = [(s.get("env") or {}).get("EVENT") for s in rel.get("steps", []) if "release_cas.sh" in str(s.get("run", ""))] +say(ev == ["${{ github.event_name == 'workflow_dispatch' && inputs.mode == 'changed' && 'push' || github.event_name }}"], + "mode=changed runs release_cas.sh as a push (the path filter applies)", "EVENT=%s" % ev) +grp = ((rel.get("concurrency") or {}).get("group") or "") +say("inputs.mode" in grp and rel.get("concurrency", {}).get("cancel-in-progress") is False, + "auto-release.yml queues each mode separately (a pending forced release is never replaced)", grp) diff --git a/tests/release_tags_test.sh b/tests/release_tags_test.sh new file mode 100755 index 00000000..f2b4967d --- /dev/null +++ b/tests/release_tags_test.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# hack/release-tags against a local bare origin over file:// (no network): the +# release build pushes the versioned tags of every release, and copies the +# floating tags (latest, ubuntu26.04, <k3s>-ubuntu26.04 and the 24.04 ones) only +# for the highest aur-v* tag on the remote at push time. +# +# Usage: tests/release_tags_test.sh + +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +RT=${RT:-${ROOT}/hack/release-tags} +T=$(mktemp -d "${TMPDIR:-/tmp}/release_tags_test.XXXXXX") +trap 'rm -rf "$T"' EXIT +PASS=0 FAILS=0 +ok() { PASS=$((PASS + 1)); echo "ok $1"; } +fail() { FAILS=$((FAILS + 1)); echo "FAIL $1"; } +export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 +export GIT_AUTHOR_NAME=t GIT_AUTHOR_EMAIL=t@t GIT_COMMITTER_NAME=t GIT_COMMITTER_EMAIL=t@t +unset RELEASE_REPOS + +# origin.git with tags aur-v0.1.9 and aur-v0.1.10 (sort -V, not sort), plus an +# annotated tag (its peeled ^{} line must not count) and a non-release tag. +git init -q --bare -b main "$T/origin.git" +git init -q -b main "$T/w" +( cd "$T/w" && git commit -q --allow-empty -m one && git tag aur-v0.1.9 \ + && git commit -q --allow-empty -m two && git tag aur-v0.1.10 \ + && git tag -a -m note aur-v0.1.8 HEAD~1 && git tag v9.9.9 \ + && git push -q "$T/origin.git" main --tags ) +K3S=v1.37.1-k3s1 +run() { ( cd "$T/w" && REMOTE="file://$T/origin.git" bash "$RT" "$@" ) > "$T/out" 2> "$T/err"; } + +want_versioned="26.04 versioned cryptoandcoffee/k3d-gpu:0.1.10 +26.04 versioned cryptoandcoffee/k3d-gpu:0.1.10-ubuntu26.04 +24.04 versioned cryptoandcoffee/k3d-gpu:0.1.10-ubuntu24.04 +26.04 versioned ghcr.io/88plug/k3d-gpu:0.1.10 +26.04 versioned ghcr.io/88plug/k3d-gpu:0.1.10-ubuntu26.04 +24.04 versioned ghcr.io/88plug/k3d-gpu:0.1.10-ubuntu24.04" +want_floating="26.04 floating cryptoandcoffee/k3d-gpu:latest cryptoandcoffee/k3d-gpu:0.1.10 +26.04 floating cryptoandcoffee/k3d-gpu:ubuntu26.04 cryptoandcoffee/k3d-gpu:0.1.10 +26.04 floating cryptoandcoffee/k3d-gpu:${K3S}-ubuntu26.04 cryptoandcoffee/k3d-gpu:0.1.10 +24.04 floating cryptoandcoffee/k3d-gpu:ubuntu24.04 cryptoandcoffee/k3d-gpu:0.1.10-ubuntu24.04 +24.04 floating cryptoandcoffee/k3d-gpu:${K3S}-ubuntu24.04 cryptoandcoffee/k3d-gpu:0.1.10-ubuntu24.04 +26.04 floating ghcr.io/88plug/k3d-gpu:latest ghcr.io/88plug/k3d-gpu:0.1.10 +26.04 floating ghcr.io/88plug/k3d-gpu:ubuntu26.04 ghcr.io/88plug/k3d-gpu:0.1.10 +26.04 floating ghcr.io/88plug/k3d-gpu:${K3S}-ubuntu26.04 ghcr.io/88plug/k3d-gpu:0.1.10 +24.04 floating ghcr.io/88plug/k3d-gpu:ubuntu24.04 ghcr.io/88plug/k3d-gpu:0.1.10-ubuntu24.04 +24.04 floating ghcr.io/88plug/k3d-gpu:${K3S}-ubuntu24.04 ghcr.io/88plug/k3d-gpu:0.1.10-ubuntu24.04" + +# 1. The highest release: versioned and floating tags. +run 0.1.10 "$K3S"; rc=$? +if [ "$rc" -eq 0 ] && diff <(sort "$T/out") <(printf '%s\n%s\n' "$want_versioned" "$want_floating" | sort) >/dev/null; then + ok "the highest aur-v* tag gets its versioned and floating tags" +else + fail "highest: rc=$rc $(diff <(sort "$T/out") <(printf '%s\n%s\n' "$want_versioned" "$want_floating" | sort) | head -5 | tr '\n' ' ') $(cat "$T/err")" +fi + +# 2. An older release (a newer tag exists on the remote): versioned tags only. +run 0.1.9 "$K3S"; rc=$? +if [ "$rc" -eq 0 ] && diff "$T/out" <(printf '%s\n' "${want_versioned//0.1.10/0.1.9}") >/dev/null; then + ok "an older release gets versioned tags only" +else + fail "older: rc=$rc out=$(tr '\n' ' ' < "$T/out")" +fi + +# 3. A version above every tag on the remote is not the highest tag: versioned only. +run 0.1.11 "$K3S"; rc=$? +if [ "$rc" -eq 0 ] && ! grep -q floating "$T/out" && [ "$(wc -l < "$T/out")" -eq 6 ]; then + ok "a version that is not a tag on the remote moves no floating tag" +else + fail "untagged: rc=$rc out=$(tr '\n' ' ' < "$T/out")" +fi + +# 4. RELEASE_REPOS picks the repositories. +( cd "$T/w" && RELEASE_REPOS="example.org/a/b" REMOTE="file://$T/origin.git" bash "$RT" 0.1.10 "$K3S" ) > "$T/out" 2>&1; rc=$? +if [ "$rc" -eq 0 ] && [ "$(grep -c . "$T/out")" -eq 8 ] && ! grep -qv ' example.org/a/b:' "$T/out"; then + ok "RELEASE_REPOS sets the repositories" +else + fail "RELEASE_REPOS: rc=$rc out=$(tr '\n' ' ' < "$T/out")" +fi + +# 5. The remote cannot be read: fail, print no tag (never fall back to floating). +( cd "$T/w" && REMOTE="file://$T/missing.git" bash "$RT" 0.1.10 "$K3S" ) > "$T/out" 2> "$T/err"; rc=$? +if [ "$rc" -ne 0 ] && [ ! -s "$T/out" ]; then + ok "an unreadable remote fails with no tag printed" +else + fail "unreadable remote: rc=$rc out=$(tr '\n' ' ' < "$T/out")" +fi + +# 6. Bad arguments exit 2. +run 0.1 "$K3S"; rc1=$? +run 0.1.10 "1.37.1"; rc2=$? +run 0.1.10; rc3=$? +if [ "$rc1" -eq 2 ] && [ "$rc2" -eq 2 ] && [ "$rc3" -eq 2 ]; then + ok "a bad version, k3s tag or argument count exits 2" +else + fail "bad args: rc=$rc1/$rc2/$rc3" +fi + +echo "${PASS} passed, ${FAILS} failed" +[ "${FAILS}" -eq 0 ] diff --git a/tests/watchdog_test.sh b/tests/watchdog_test.sh new file mode 100755 index 00000000..157de5dd --- /dev/null +++ b/tests/watchdog_test.sh @@ -0,0 +1,422 @@ +#!/usr/bin/env bash +# hack/watchdog against tests/ops/fake_gh.py (routes plus an issue store), a stub +# registry (tests/ops/stub_registry.py), a file:// AUR answer and a local git +# repository holding the aur-v* tags. No network. The real hack/ci_alarm files the +# alarms into the fake issue store. Run lists, jobs, the failed log and the +# workflow list are the real gh shapes (tests/ops/fixtures/, from the October red +# nights of build-and-push.yml); each case moves the clock (WATCHDOG_NOW) and the +# fixtures the fake gh answers with. +# +# Usage: tests/watchdog_test.sh + +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +WATCHDOG=${WATCHDOG:-${ROOT}/hack/watchdog} +FIX="${ROOT}/tests/ops/fixtures" +T=$(mktemp -d "${TMPDIR:-/tmp}/watchdog_test.XXXXXX") +trap '[ -n "${KEEP:-}" ] || rm -rf "$T"' EXIT +PASS=0 FAILS=0 +ok() { PASS=$((PASS + 1)); echo "ok $1"; } +fail() { FAILS=$((FAILS + 1)); echo "FAIL $1"; } + +mkdir -p "$T/bin" +printf '#!/bin/sh\nexec python3 %q "$@"\n' "${ROOT}/tests/ops/fake_gh.py" > "$T/bin/gh" +chmod +x "$T/bin/gh" +export PATH="$T/bin:$PATH" GH_REPO=88plug/k3d-gpu +export WATCHDOG_IMAGES_LOCK="${ROOT}/tests/ops/stub_registry.py" +export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 +unset GITHUB_STEP_SUMMARY SIMULATE_RED ALARM_OWNER +HEAD_SHA=890d1d16790ed656ae5aa86a8319e578310b0f80 +PR_SHA=1bf784c727bc3c2a051d5022681b0fa51b7d7ccf + +# iso OFFSET: $NOW moved by OFFSET (a GNU date relative item, e.g. "-73 hours"). +iso() { date -u -d "$NOW ${1:-}" +%Y-%m-%dT%H:%M:%SZ; } +at() { # at ISO: set the clock for the watchdog, ci_alarm and the fake gh + NOW=$1 + export FAKE_GH_NOW=$NOW + WATCHDOG_NOW=$(date -u -d "$NOW" +%s); CI_ALARM_NOW=$WATCHDOG_NOW + export WATCHDOG_NOW CI_ALARM_NOW +} +# route REGEX FILE [RC] [STDERR]: answered before every earlier route. +route() { + { printf '%s\t%s\t%s' "$1" "$2" "${3:-0}"; [ -n "${4:-}" ] && printf '\t%s' "$4"; printf '\n' + cat "$C/routes.tsv" 2>/dev/null; } > "$C/routes.tmp" + mv "$C/routes.tmp" "$C/routes.tsv" +} +put() { cat > "$C/$1"; } # put FILE: write a fixture file from stdin +# tags "aur-vX@OFFSET ...": the case root's git history, one commit per tag. +tags() { + local spec t + rm -rf "$C/root/.git" + git -C "$C/root" init -q -b main + for spec in $1; do + t=${spec%%@*} + GIT_AUTHOR_DATE=$(iso "${spec#*@}") GIT_COMMITTER_DATE=$(iso "${spec#*@}") \ + git -C "$C/root" -c user.name=t -c user.email=t@t commit -q --allow-empty -m "release: $t" + git -C "$C/root" tag "$t" + done +} +registry() { printf '%s\n' "$@" > "$C/registry"; } +aur() { printf '{"resultcount":1,"results":[{"Name":"k3d-gpu","Version":"%s"}],"type":"multiinfo","version":5}\n' "$1" > "$C/aur.json"; } +runs() { # runs FILE: a ci.yml run list; each argument "event:status:conclusion[:sha[:createdAt]]" + local f=$1; shift + printf '%s\n' "$@" | jq -R 'split(":") | {databaseId: 4100, event: .[0], status: .[1], conclusion: .[2], + headSha: (.[3] // ""), createdAt: (.[4] // ""), url: "https://github.com/88plug/k3d-gpu/actions/runs/4100"}' \ + | jq -s . > "$C/$f" +} + +# setup NAME [ISO]: a healthy repository at ISO (default 2026-10-29T13:00:00Z): +# every scheduled workflow green, main's HEAD an hour old with green CI, release +# 0.1.79 complete everywhere, no bot PR, no open alarm. +setup() { + C=$T/$1 + export FAKE_GH=$C + mkdir -p "$C/root/.github" + cp -r "${ROOT}/.github/workflows" "$C/root/.github/" + at "${2:-2026-10-29T13:00:00Z}" + # The workflow list: the real one, plus every workflow file the repo has now + # (watchdog.yml registered an hour ago). + python3 - "$FIX/workflows.json" "$C/root/.github/workflows" "$(iso '-1 hour')" > "$C/workflows.json" <<'PY' +import json, os, re, sys +doc = json.load(open(sys.argv[1])) +have = {w["path"] for w in doc["workflows"]} +for f in sorted(os.listdir(sys.argv[2])): + p = ".github/workflows/" + f + if p not in have: + name = re.search(r"^name:\s*(.+)$", open(os.path.join(sys.argv[2], f)).read(), re.M).group(1).strip("'\"") + doc["workflows"].append({"id": 900000 + len(doc["workflows"]), "name": name, "path": p, "state": "active", + "created_at": sys.argv[3], "updated_at": sys.argv[3]}) +doc["total_count"] = len(doc["workflows"]) +print(json.dumps(doc)) +PY + jq --arg t "$(iso '-1 hour')" '[.[0] | .createdAt = $t | .startedAt = $t | .updatedAt = $t]' \ + "$FIX/build-and-push_schedule_runs.json" > "$C/bp_sched.json" + jq -n --arg t "$(iso '-13 minutes')" '[{databaseId: 4200, conclusion: "success", status: "completed", + createdAt: $t, url: "https://github.com/88plug/k3d-gpu/actions/runs/4200", headSha: "x", + workflowName: "Watchdog", event: "schedule"}]' > "$C/wd_sched.json" + jq -n --arg s "$HEAD_SHA" --arg t "$(iso '-1 hour')" \ + '{sha: $s, commit: {committer: {name: "88plug", date: $t}, message: "bench(wildct): smoke3"}}' > "$C/head.json" + runs ci_head.json "push:completed:success:$HEAD_SHA:$(iso '-58 minutes')" + runs ci_main.json "push:completed:success:$HEAD_SHA:$(iso '-58 minutes')" + echo '[]' > "$C/busy.json" + echo '[]' > "$C/prs.json" + jq -n '{assets: [{name: "k3d-gpu-0.1.79-1.noarch.rpm", state: "uploaded"}, + {name: "k3d-gpu_0.1.79-1_all.deb", state: "uploaded"}, + {name: "SHA256SUMS", state: "uploaded"}]}' > "$C/rel79.json" + for v in 75 77 78; do sed "s/0\.1\.79/0.1.$v/g" "$C/rel79.json" > "$C/rel$v.json"; done + cp "$FIX/run_39412124435_jobs.json" "$C/jobs.json" + cp "$FIX/run_39412124435_log_failed.txt" "$C/log.txt" + tags "aur-v0.1.78@-6hours aur-v0.1.79@-2hours" + registry docker.io/cryptoandcoffee/k3d-gpu:0.1.78 docker.io/cryptoandcoffee/k3d-gpu:0.1.79 \ + ghcr.io/88plug/k3d-gpu:0.1.78 ghcr.io/88plug/k3d-gpu:0.1.79 + aur 0.1.79-1 + export STUB_REGISTRY="$C/registry" WATCHDOG_AUR_URL="file://$C/aur.json" + rm -f "$C/routes.tsv" + route '^api repos/88plug/k3d-gpu/actions/workflows' workflows.json + route '^run list .*--workflow build-and-push\.yml .*--event schedule' bp_sched.json + route '^run list .*--workflow watchdog\.yml .*--event schedule' wd_sched.json + route '^api repos/88plug/k3d-gpu/commits/main' head.json + route '^run list .*--workflow ci\.yml .*--commit ' ci_head.json + route '^run list .*--workflow ci\.yml .*--branch main' ci_main.json + route '^run list -R \S+ --limit \d+ --json workflowName' busy.json + route '^release view ' - 1 'release not found' + route '^release view aur-v0\.1\.79 ' rel79.json + route '^release view aur-v0\.1\.78 ' rel78.json + route '^pr list ' prs.json + route '^run view \d+ .*--json jobs' jobs.json + route '^run view \d+ .*--log-failed' log.txt +} +watch() { python3 "$WATCHDOG" --root "$C/root" "$@" > "$C/out" 2> "$C/err"; } +issue() { # issue KEY: the newest issue for the key, as JSON + jq -c --arg p "[ci-alarm] $1: " '[.[] | select(.title | startswith($p))] | max_by(.number) // empty' "$C/issues.json" 2>/dev/null +} +state() { local i; i=$(issue "$1"); if [ -z "$i" ]; then echo none; else jq -r .state <<< "$i"; fi; } +nopen() { jq '[.[] | select(.state == "OPEN")] | length' "$C/issues.json" 2>/dev/null || echo 0; } +dispatched() { grep -cxF "$1" "$C/dispatches.log" 2>/dev/null || true; } +ndispatch() { if [ -f "$C/dispatches.log" ]; then wc -l < "$C/dispatches.log"; else echo 0; fi; } +why() { echo "rc=$rc open=$(nopen) dispatches=$(ndispatch) err=$(tail -3 "$C/err" | tr '\n' ' ') keys=$(jq -r '[.[] | select(.state == "OPEN") | .title] | join(" | ")' "$C/issues.json" 2>/dev/null)"; } + +# 1. A healthy repository: no alarm, no dispatch, exit 0. +setup healthy +watch; rc=$? +if [ "$rc" -eq 0 ] && [ "$(nopen)" = 0 ] && [ "$(ndispatch)" = 0 ] && ! grep -q 'no route' "$C/err"; then + ok "healthy: no alarm, no dispatch" +else + fail "healthy: $(why)" +fi + +# 2. The October red nights of build-and-push.yml, replayed from the real run list. +# Night 1 (run 39412124435 red at 11:02, the watchdog at 11:47): red alarm and one +# re-dispatch; the hourly run after it adds neither. +setup october 2026-10-20T11:47:00Z +night() { # night ISO: the scheduled runs created up to ISO, newest first + at "$1" + jq --arg t "$1" '[.[] | select(.createdAt <= $t)]' "$FIX/build-and-push_schedule_runs.json" > "$C/bp_sched.json" + jq -n --arg s "$HEAD_SHA" --arg t "$(iso '-1 hour')" '{sha: $s, commit: {committer: {date: $t}}}' > "$C/head.json" + jq -n --arg t "$(iso '-13 minutes')" '[{databaseId: 4200, conclusion: "success", status: "completed", createdAt: $t, + url: "https://github.com/88plug/k3d-gpu/actions/runs/4200", headSha: "x"}]' > "$C/wd_sched.json" +} +night 2026-10-20T11:47:00Z +watch; rc=$? +red=$(issue red:build-and-push.yml) +if [ "$rc" -eq 0 ] && [ "$(state red:build-and-push.yml)" = OPEN ] \ + && [ "$(dispatched 'build-and-push.yml ref=main')" = 1 ] && [ "$(ndispatch)" = 1 ] \ + && jq -r .body <<< "$red" | grep -q 'actions/runs/39412124435' \ + && jq -r .body <<< "$red" | grep -q 'repin-images' \ + && jq -r .body <<< "$red" | grep -q 'refusing to allow a GitHub App' \ + && jq -r .body <<< "$red" | grep -q 'bot push touched .github/workflows' \ + && [ "$(jq -r '.assignees[0].login' <<< "$red")" = 88plug ] \ + && [ "$(state stale:build-and-push.yml)" = none ]; then + ok "October night 1: red:build-and-push.yml (run, failed job, log, hint) and one re-dispatch" +else + fail "October night 1: $(why) body=$(jq -r .body <<< "$red" | head -c 600)" +fi +night 2026-10-20T12:47:00Z +watch; rc=$? +if [ "$rc" -eq 0 ] && [ "$(ndispatch)" = 1 ] && [ "$(jq '.comments | length' <<< "$(issue red:build-and-push.yml)")" = 0 ] \ + && [ "$(jq '[.[] | select(.title | startswith("[ci-alarm] red:"))] | length' "$C/issues.json")" = 1 ] \ + && [ "$(state stale:build-and-push.yml)" = none ]; then + ok "October night 1, next hour: no second re-dispatch for the same run, no new comment, not stale at 24.4 h" +else + fail "October night 1, next hour: $(why)" +fi +# Night 2: a new red run; a new re-dispatch and a comment; 47 h since the last +# green scheduled run (threshold 24 h + 6 h): stale too. +night 2026-10-21T11:47:00Z +watch; rc=$? +if [ "$rc" -eq 0 ] && [ "$(ndispatch)" = 2 ] \ + && jq -r '.comments[-1].body' <<< "$(issue red:build-and-push.yml)" | grep -q 'actions/runs/39591066580' \ + && [ "$(state stale:build-and-push.yml)" = OPEN ]; then + ok "October night 2: the new red run re-dispatches once more and comments; stale opens" +else + fail "October night 2: $(why)" +fi +# 2026-10-29: the scheduled run is green again: red and stale close. +night 2026-10-29T13:00:00Z +watch; rc=$? +if [ "$rc" -eq 0 ] && [ "$(state red:build-and-push.yml)" = CLOSED ] && [ "$(state stale:build-and-push.yml)" = CLOSED ] \ + && [ "$(ndispatch)" = 2 ]; then + ok "green scheduled run closes red and stale" +else + fail "green closes: $(why)" +fi + +# 3. Stale: the last green scheduled run is 31 h old and nothing ran since. +setup stale +jq --arg t "$(iso '-31 hours')" '[.[0] | .createdAt = $t]' "$FIX/build-and-push_schedule_runs.json" > "$C/bp_sched.json" +watch; rc=$? +if [ "$rc" -eq 0 ] && [ "$(state stale:build-and-push.yml)" = OPEN ] && [ "$(state red:build-and-push.yml)" = none ] \ + && [ "$(ndispatch)" = 0 ]; then + ok "stale: a last success older than period + 6 h opens stale:build-and-push.yml" +else + fail "stale: $(why)" +fi +# No green scheduled run at all, and the workflow is old: stale. +echo '[]' > "$C/bp_sched.json" +watch; rc=$? +if [ "$rc" -eq 0 ] && [ "$(state stale:build-and-push.yml)" = OPEN ] && [ "$(nopen)" = 1 ]; then + ok "stale: no green scheduled run in 30 days keeps stale open" +else + fail "stale, none: $(why)" +fi + +# 4. A disabled workflow; a dynamic workflow or a deleted file is not alarmed. +setup disabled +jq '.workflows |= map(if .path == ".github/workflows/docs.yml" then .state = "disabled_inactivity" else . end) + | .workflows |= map(if .path | startswith("dynamic/") then .state = "disabled_manually" else . end) + | .workflows += [{id: 1, name: "Gone", path: ".github/workflows/gone.yml", state: "deleted", + created_at: "2025-01-01T00:00:00Z"}]' "$C/workflows.json" > "$C/w.json" +mv "$C/w.json" "$C/workflows.json" +watch; rc=$? +if [ "$rc" -eq 0 ] && [ "$(state disabled:docs.yml)" = OPEN ] && [ "$(nopen)" = 1 ] \ + && jq -r .body <<< "$(issue disabled:docs.yml)" | grep -q 'disabled_inactivity'; then + ok "disabled workflow: disabled:docs.yml (dynamic and deleted workflows ignored)" +else + fail "disabled: $(why)" +fi + +# 5. main's HEAD has no ci.yml run: at 10 min nothing yet (a push run may still +# appear); at 40 min one dispatch; at 2 h 10 min ci-missing. +setup cimissing +jq -n --arg s "$HEAD_SHA" --arg t "$(iso '-10 minutes')" '{sha: $s, commit: {committer: {date: $t}}}' > "$C/head.json" +runs ci_head.json "pull_request:completed:success:$HEAD_SHA" +watch; rc=$? +if [ "$rc" -eq 0 ] && [ "$(ndispatch)" = 0 ] && [ "$(nopen)" = 0 ]; then + ok "main HEAD 10 min old without CI: no dispatch yet, no alarm" +else + fail "young HEAD: $(why)" +fi +at "$(iso '+30 minutes')" +watch; rc1=$? +d1=$(dispatched 'ci.yml ref=main'); m1=$(state ci-missing) +at "$(iso '+90 minutes')" +watch; rc2=$? +if [ "$rc1" -eq 0 ] && [ "$rc2" -eq 0 ] && [ "$d1" = 1 ] && [ "$m1" = none ] \ + && [ "$(dispatched 'ci.yml ref=main')" = 1 ] && [ "$(state ci-missing)" = OPEN ] \ + && jq -r .body <<< "$(issue ci-missing)" | grep -q "$HEAD_SHA"; then + ok "main HEAD without CI: one ci.yml dispatch, then ci-missing" +else + rc=$rc2; fail "ci-missing: d1=$d1 m1=$m1 $(why)" +fi +runs ci_head.json "workflow_dispatch:in_progress::$HEAD_SHA" +watch; rc=$? +if [ "$rc" -eq 0 ] && [ "$(state ci-missing)" = CLOSED ]; then ok "a ci.yml run for HEAD closes ci-missing"; else fail "ci-missing close: $(why)"; fi + +# 6. main CI red: ci-main; green closes it. +setup cimain +runs ci_main.json "push:completed:failure:$HEAD_SHA:$(iso '-30 minutes')" "push:completed:success:abc:$(iso '-3 hours')" +watch; rc=$? +if [ "$rc" -eq 0 ] && [ "$(state ci-main)" = OPEN ] && jq -r .body <<< "$(issue ci-main)" | grep -q 'repin-images'; then + ok "red main CI opens ci-main with the failed jobs" +else + fail "ci-main: $(why)" +fi +runs ci_main.json "workflow_dispatch:completed:cancelled:$HEAD_SHA" "push:completed:success:$HEAD_SHA:$(iso '-10 minutes')" +watch; rc=$? +if [ "$rc" -eq 0 ] && [ "$(state ci-main)" = CLOSED ]; then ok "green main CI closes ci-main (a cancelled run is skipped)"; else fail "ci-main close: $(why)"; fi + +# 7. Release 0.1.76's shape: the tag exists, Docker Hub and GHCR have no image, no +# GitHub release, AUR at 0.1.75. One release re-dispatch, then the alarm; the next +# run neither re-dispatches nor comments. +setup rel76 +tags "aur-v0.1.75@-5hours aur-v0.1.76@-1hours" +registry docker.io/cryptoandcoffee/k3d-gpu:0.1.75 ghcr.io/88plug/k3d-gpu:0.1.75 +aur 0.1.75-1 +route '^release view aur-v0\.1\.75 ' rel75.json +watch; rc1=$? +at "$(iso '+1 hour')" +watch; rc2=$? +rel=$(issue release:0.1.76) +if [ "$rc1" -eq 0 ] && [ "$rc2" -eq 0 ] && [ "$(dispatched 'build-and-push.yml ref=main release=0.1.76')" = 1 ] \ + && [ "$(ndispatch)" = 1 ] && [ "$(state release:0.1.76)" = OPEN ] && [ "$(jq '.comments | length' <<< "$rel")" = 0 ] \ + && jq -r .body <<< "$rel" | grep -q 'cryptoandcoffee/k3d-gpu:0.1.76' \ + && jq -r .body <<< "$rel" | grep -q 'ghcr.io/88plug/k3d-gpu:0.1.76' \ + && jq -r .body <<< "$rel" | grep -qi 'SHA256SUMS' \ + && jq -r .body <<< "$rel" | grep -q 'AUR'; then + ok "0.1.76 shape: exactly one release re-dispatch, then release:0.1.76" +else + rc=$rc2; fail "0.1.76: $(why) body=$(jq -r .body <<< "$rel" | head -c 400)" +fi +# 0.1.77 then ships complete: release:0.1.76 closes as superseded; no dispatch. +tags "aur-v0.1.75@-6hours aur-v0.1.76@-2hours aur-v0.1.77@-1hours" +registry docker.io/cryptoandcoffee/k3d-gpu:0.1.75 ghcr.io/88plug/k3d-gpu:0.1.75 \ + docker.io/cryptoandcoffee/k3d-gpu:0.1.77 ghcr.io/88plug/k3d-gpu:0.1.77 +aur 0.1.77-1 +route '^release view aur-v0\.1\.77 ' rel77.json +watch; rc=$? +if [ "$rc" -eq 0 ] && [ "$(state release:0.1.76)" = CLOSED ] && [ "$(state release:0.1.77)" = none ] && [ "$(ndispatch)" = 1 ]; then + ok "a complete newer release closes the superseded release:0.1.76" +else + fail "superseded: $(why)" +fi + +# 8. A newer broken release lists the older broken one as information only, and a +# tag younger than 45 min is left alone. Only Docker Hub missing triggers the +# re-dispatch; GHCR alone does not (the build refuses to overwrite a published tag). +setup relinfo +tags "aur-v0.1.75@-9hours aur-v0.1.76@-8hours aur-v0.1.77@-2hours aur-v0.1.78@-20minutes" +registry docker.io/cryptoandcoffee/k3d-gpu:0.1.75 ghcr.io/88plug/k3d-gpu:0.1.75 docker.io/cryptoandcoffee/k3d-gpu:0.1.77 +aur 0.1.77-1 +route '^release view aur-v0\.1\.77 ' rel77.json +route '^release view aur-v0\.1\.75 ' rel75.json +watch; rc=$? +rel=$(issue release:0.1.77) +if [ "$rc" -eq 0 ] && [ "$(state release:0.1.77)" = OPEN ] && [ "$(ndispatch)" = 0 ] && [ "$(state release:0.1.78)" = none ] \ + && jq -r .body <<< "$rel" | grep -q 'ghcr.io/88plug/k3d-gpu:0.1.77' \ + && jq -r .body <<< "$rel" | grep -q '0\.1\.76' \ + && ! jq -r .body <<< "$rel" | grep -q 'cryptoandcoffee/k3d-gpu:0.1.77 '; then + ok "GHCR-only gap: alarm, no re-dispatch; older 0.1.76 listed; a 20 min old tag skipped" +else + fail "relinfo: $(why) body=$(jq -r .body <<< "$rel" | head -c 600)" +fi + +# 9. A release workflow still running: the release check waits (no dispatch, no alarm). +setup busy +tags "aur-v0.1.75@-5hours aur-v0.1.76@-1hours" +registry docker.io/cryptoandcoffee/k3d-gpu:0.1.75 ghcr.io/88plug/k3d-gpu:0.1.75 +jq -n '[{databaseId: 1, workflowName: "Auto-update, Build & Release", status: "in_progress"}]' > "$C/busy.json" +watch; rc=$? +if [ "$rc" -eq 0 ] && [ "$(ndispatch)" = 0 ] && [ "$(state release:0.1.76)" = none ] && grep -q 'release check deferred' "$C/out"; then + ok "a running release workflow defers the release check" +else + fail "busy: $(why)" +fi + +# 10. Bot PRs: an autobump PR open 73 h whose head has no green ci.yml run. +setup botpr +jq -n --arg old "$(iso '-73 hours')" --arg young "$(iso '-10 hours')" --arg s "$PR_SHA" '[ + {number: 240, headRefName: "autobump/convert-pins", headRefOid: $s, createdAt: $old, + url: "https://github.com/88plug/k3d-gpu/pull/240", title: "chore(convert-pins): bump", + author: {is_bot: true, login: "app/github-actions"}}, + {number: 241, headRefName: "autobump/image-digests", headRefOid: "feed", createdAt: $young, + url: "https://github.com/88plug/k3d-gpu/pull/241", title: "young", author: {login: "app/github-actions"}}, + {number: 242, headRefName: "autobump/nri-balloons", headRefOid: "beef", createdAt: $old, + url: "https://github.com/88plug/k3d-gpu/pull/242", title: "green", author: {login: "app/github-actions"}}, + {number: 243, headRefName: "claude/h54q-run", headRefOid: "cafe", createdAt: $old, + url: "https://github.com/88plug/k3d-gpu/pull/243", title: "human", author: {login: "88plug"}}]' > "$C/prs.json" +runs ci_pr.json "pull_request:completed:action_required:$PR_SHA" +runs ci_green.json "workflow_dispatch:completed:success:beef" +route "^run list .*--workflow ci\\.yml .*--commit $PR_SHA" ci_pr.json +route '^run list .*--workflow ci\.yml .*--commit beef' ci_green.json +route '^run list .*--workflow ci\.yml .*--commit cafe' - 1 'must not be asked' +watch; rc=$? +b=$(issue bot-pr-stuck) +if [ "$rc" -eq 0 ] && [ "$(state bot-pr-stuck)" = OPEN ] && jq -r .body <<< "$b" | grep -q 'pull/240' \ + && ! jq -r .body <<< "$b" | grep -qE 'pull/24[123]' && ! grep -q 'must not be asked' "$C/err"; then + ok "a 73 h autobump PR without green CI opens bot-pr-stuck (young, green and human PRs left out)" +else + fail "bot-pr-stuck: $(why) body=$(jq -r .body <<< "$b" | head -c 400)" +fi +echo '[]' > "$C/prs.json" +watch; rc=$? +if [ "$rc" -eq 0 ] && [ "$(state bot-pr-stuck)" = CLOSED ]; then ok "no stuck bot PR closes bot-pr-stuck"; else fail "bot-pr-stuck close: $(why)"; fi + +# 11. simulate_red: opens simulate (assigned to 88plug); the next run without it closes it. +setup simulate +SIMULATE_RED=true watch; rc1=$? +s1=$(state simulate); who=$(jq -r '.assignees[0].login' <<< "$(issue simulate)") +SIMULATE_RED=false watch; rc2=$? +if [ "$rc1" -eq 0 ] && [ "$rc2" -eq 0 ] && [ "$s1" = OPEN ] && [ "$who" = 88plug ] && [ "$(state simulate)" = CLOSED ]; then + ok "simulate_red opens [ci-alarm] simulate; the next run closes it" +else + rc=$rc2; fail "simulate: s1=$s1 who=$who $(why)" +fi + +# 12. A gh error: watchdog-error carries the error text, the other checks still run, +# exit 0; the next clean run closes it. +setup gherror +route '^api repos/88plug/k3d-gpu/actions/workflows' - 1 'HTTP 502: Bad Gateway (https://api.github.com/repos/88plug/k3d-gpu/actions/workflows)' +runs ci_main.json "push:completed:failure:$HEAD_SHA:$(iso '-30 minutes')" +watch; rc=$? +if [ "$rc" -eq 0 ] && [ "$(state watchdog-error)" = OPEN ] && jq -r .body <<< "$(issue watchdog-error)" | grep -q 'HTTP 502' \ + && [ "$(state ci-main)" = OPEN ]; then + ok "a gh error opens watchdog-error with its text; the other checks still run" +else + fail "watchdog-error: $(why)" +fi +route '^api repos/88plug/k3d-gpu/actions/workflows' workflows.json +watch; rc=$? +if [ "$rc" -eq 0 ] && [ "$(state watchdog-error)" = CLOSED ]; then ok "a clean run closes watchdog-error"; else fail "watchdog-error close: $(why)"; fi + +# 13. Issues disabled: no alarm can be filed; the run prints ::error and exits 1. +setup noissues +route '^(issue|label) ' - 1 "the '88plug/k3d-gpu' repository has disabled issues" +SIMULATE_RED=true watch; rc=$? +if [ "$rc" -eq 1 ] && grep -q '^::error title=ci-alarm simulate::' "$C/out" && grep -qi 'issues' "$C/err"; then + ok "issues disabled: ::error annotation and exit 1" +else + fail "issues disabled: rc=$rc out=$(tail -3 "$C/out" | tr '\n' ' ') err=$(tail -3 "$C/err" | tr '\n' ' ')" +fi + +# 14. --list-scheduled: every workflow with on.schedule, with its cron period in hours. +setup listsched +watch --list-scheduled; rc=$? +if [ "$rc" -eq 0 ] && grep -qx 'build-and-push.yml 24' "$C/out" && grep -qx 'watchdog.yml 1' "$C/out"; then + ok "--list-scheduled: build-and-push.yml 24, watchdog.yml 1" +else + fail "--list-scheduled: rc=$rc out=$(tr '\n' ' ' < "$C/out")" +fi + +echo "${PASS} passed, ${FAILS} failed" +[ "${FAILS}" -eq 0 ] From 2eb4fd15c131ac082cd21badcdb98fdf1f459764 Mon Sep 17 00:00:00 2001 From: Andrew Mello <andrew@88plug.com> Date: Thu, 29 Oct 2026 17:38:08 -0700 Subject: [PATCH 2/2] watchdog: record a re-dispatch before it runs, and ignore forks The watchdog re-dispatched a red scheduled workflow, or a release with no Docker Hub image, and only then wrote the record that stops the next re-dispatch into the alarm. With Issues disabled (this repository today) the record can never be written and every watchdog run exits 1, so each run re-dispatched build-and-push.yml, the missing release and watchdog.yml itself, and each dispatched watchdog run did the same, without end. A failed alarm write or a failed alarm listing lost the record the same way. The record now goes into the alarm before the dispatch. A run that cannot read or file the alarm dispatches nothing: with Issues disabled the alarm is only annotated, after a failed listing the alarm is left as it is so its record survives, and a failed write stops the dispatch. A failed dispatch rewrites the alarm without the record, so the next run tries again. workflow_run's branches filter matches the triggering run's head branch name, which a fork's pull request can also call main, so watchdog.yml now also requires the run to come from this repository. bot-pr-stuck skips cross-repository PRs, so a fork's autobump/* branch cannot hold it open. The AUR guard in aur-deploy.yml failed the deploy when the AUR RPC could not be read (pipefail made the command substitution fail the step), the opposite of its comment; an unreadable RPC now lets the deploy go ahead. --- .github/workflows/aur-deploy.yml | 2 +- .github/workflows/watchdog.yml | 9 ++- hack/watchdog | 108 ++++++++++++++++++++----------- tests/ops/workflow_checks.py | 30 +++++++++ tests/watchdog_test.sh | 93 ++++++++++++++++++++++++++ 5 files changed, 200 insertions(+), 42 deletions(-) diff --git a/.github/workflows/aur-deploy.yml b/.github/workflows/aur-deploy.yml index a486055d..48835f32 100644 --- a/.github/workflows/aur-deploy.yml +++ b/.github/workflows/aur-deploy.yml @@ -43,7 +43,7 @@ jobs: VER: ${{ steps.ver.outputs.version }} run: | cur=$(curl -fsSg --retry 3 --max-time 30 'https://aur.archlinux.org/rpc/v5/info?arg[]=k3d-gpu' \ - | jq -r '.results[0].Version // empty' | cut -d- -f1) + | jq -r '.results[0].Version // empty' | cut -d- -f1) || cur= if [ -n "$cur" ] && [ "$cur" != "$VER" ] && [ "$(printf '%s\n%s\n' "$cur" "$VER" | sort -V | tail -1)" = "$cur" ]; then echo "AUR has k3d-gpu $cur, newer than $VER; leaving it" echo "skip=true" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/watchdog.yml b/.github/workflows/watchdog.yml index 87beefbf..9b87f6c3 100644 --- a/.github/workflows/watchdog.yml +++ b/.github/workflows/watchdog.yml @@ -9,7 +9,9 @@ name: Watchdog # the `simulate` alarm; the next run without it closes it. # # It reads main, never the triggering run's code. workflow_run passes only runs on -# main (branches: and the job's if:); the hourly cron covers a missed event. +# this repository's main: branches: matches the triggering run's head branch name, +# which a fork's pull request can also call main, so the job's if: also requires +# the run to come from this repository. The hourly cron covers a missed event. on: schedule: @@ -44,7 +46,10 @@ concurrency: jobs: watchdog: name: Watchdog - if: github.event_name != 'workflow_run' || github.event.workflow_run.head_branch == 'main' + if: >- + github.event_name != 'workflow_run' + || (github.event.workflow_run.head_branch == 'main' + && github.event.workflow_run.head_repository.full_name == github.repository) runs-on: ubuntu-latest timeout-minutes: 15 steps: diff --git a/hack/watchdog b/hack/watchdog index 2e1e76ed..d1989b3d 100755 --- a/hack/watchdog +++ b/hack/watchdog @@ -34,10 +34,18 @@ issue per finding: last 14 days with gaps as information. Other release:* alarms close as superseded. Skipped while a release workflow is queued or running. - e. stuck bot PRs Open autobump/* PRs older than 72 h whose head SHA has no - successful ci.yml run: bot-pr-stuck. + e. stuck bot PRs Open autobump/* PRs from this repository (not a fork's) + older than 72 h whose head SHA has no successful ci.yml + run: bot-pr-stuck. f. simulate_red SIMULATE_RED=true opens simulate; a run without it closes it. +A re-dispatch (a and d) is recorded in its alarm before it runs, and the record is +what stops the next one. A run that cannot read or file the alarm (issues +disabled, the open alarms could not be listed, the alarm write failed) therefore +dispatches nothing; a dispatch that fails is not recorded, and the next run tries +again. Recording after dispatching let every watchdog run that could not file +alarms re-dispatch the red workflows, watchdog.yml among them, without end. + A failed gh, git, registry or AUR call is collected and filed as watchdog-error with its text (the other checks still run); a run with no error closes it. Exit 0 unless an alarm could not be filed (the repository has issues disabled, or @@ -260,12 +268,15 @@ class Alarms: finally: os.unlink(f.name) if p.returncode == 3: - self.undelivered.append(key) - return + if key not in self.undelivered: + self.undelivered.append(key) + return False if p.returncode != 0: raise ToolError("ci_alarm open %s: exit %d: %s" % (key, p.returncode, p.stderr.strip()[-400:])) print("::warning title=ci-alarm %s::%s" % (key, title)) - self.opened.append((key, title)) + if (key, title) not in self.opened: + self.opened.append((key, title)) + return True def clear(self, key, comment): if self.disabled or (self.open is not None and key not in self.open): @@ -302,6 +313,37 @@ class Watchdog: self.errors.append("%s: %s" % (name, traceback.format_exc(limit=3).strip())) return None + # ── one dispatch per token ─────────────────────────────────────────── + + def raise_dispatching(self, key, title, body, token, argv, done_text): + """Raise alarm KEY and run `gh ARGV` once per TOKEN. The record of the + dispatch (<!-- watchdog:dispatched=TOKEN -->) is written into the alarm + before the dispatch runs, so a run that cannot read or write the alarm + dispatches nothing: + - issues disabled: the alarm is annotated only, nothing is dispatched; + - the open alarms could not be listed: the alarm is left as it is (its + record stays), nothing is dispatched; watchdog-error says why; + - the alarm write fails: nothing is dispatched (the error is collected); + - the dispatch fails: the alarm is rewritten without the record and + says so, and the next run tries again.""" + claimed = "\n".join(body + ["", done_text, "<!-- watchdog:dispatched=%s -->" % token]) + if self.alarms.disabled: + self.alarms.raise_(key, title, "\n".join(body)) + return + if self.alarms.open is None: + return + if marker(self.alarms.body(key), "dispatched") == token: + self.alarms.raise_(key, title, claimed) + return + if not self.alarms.raise_(key, title, claimed): + return + try: + self.gh.run(*argv) + print("dispatched: gh %s" % " ".join(argv)) + except ToolError as e: + self.errors.append("dispatching for %s: %s" % (key, e)) + self.alarms.raise_(key, title, "\n".join(body + ["", "Re-dispatch failed: %s" % e])) + # ── failed-run description ─────────────────────────────────────────── def describe_failed_run(self, run): @@ -384,21 +426,13 @@ class Watchdog: return rid = str(latest["databaseId"]) body = ["The latest scheduled run of `%s` failed." % f, ""] + self.describe_failed_run(latest) - done = marker(self.alarms.body(key), "dispatched") - if wf["dispatch"]: - if done != rid: - try: - self.gh.run("workflow", "run", f, "-R", self.repo, "--ref", "main") - done = rid - print("re-dispatched %s on main after run %s" % (f, rid)) - except ToolError as e: - self.errors.append("re-dispatching %s: %s" % (f, e)) - body.append("\nRe-dispatch failed: %s" % e) - if done == rid: - body += ["", "Re-dispatched `%s` on main once after this run; a further re-dispatch " - "happens only after a new failed scheduled run." % f, - "<!-- watchdog:dispatched=%s -->" % rid] - self.alarms.raise_(key, "scheduled run %s" % latest["conclusion"], "\n".join(body)) + title = "scheduled run %s" % latest["conclusion"] + if not wf["dispatch"]: + self.alarms.raise_(key, title, "\n".join(body)) + return + self.raise_dispatching(key, title, body, rid, ("workflow", "run", f, "-R", self.repo, "--ref", "main"), + "Re-dispatched `%s` on main once after this run; a further re-dispatch " + "happens only after a new failed scheduled run." % f) # ── c ──────────────────────────────────────────────────────────────── @@ -523,22 +557,7 @@ class Watchdog: self.alarms.clear(key, "Release %s is complete: images, packages and AUR." % ver) return body = ["Release %s (tag aur-v%s) is incomplete:" % (ver, ver), ""] + ["- " + g for g in gaps] - done = marker(self.alarms.body(key), "dispatched") - if hub_missing: - if done != ver: - try: - self.gh.run("workflow", "run", "build-and-push.yml", "-R", self.repo, "--ref", "main", - "-f", "release=" + ver) - done = ver - print("dispatched build-and-push.yml release=%s (no Docker Hub image)" % ver) - except ToolError as e: - self.errors.append("re-dispatching release %s: %s" % (ver, e)) - body += ["", "Re-dispatch failed: %s" % e] - if done == ver: - body += ["", "Dispatched `build-and-push.yml -f release=%s` once; it builds the image from the tag " - "and then dispatches the AUR and package releases. It is not dispatched again." % ver, - "<!-- watchdog:dispatched=%s -->" % ver] - elif any(g.startswith(("Docker Hub", "GHCR")) for g in gaps): + if not hub_missing and any(g.startswith(("Docker Hub", "GHCR")) for g in gaps): body += ["", "Docker Hub has %s:%s, so a release build would stop at its overwrite guard; " "push the GHCR tag by hand or cut a new release." % (HUB_REPO, ver)] info = [] @@ -554,16 +573,27 @@ class Watchdog: if info: body += ["", "Older releases from the last 14 days with gaps (information only; a newer release " "supersedes them):", ""] + info - self.alarms.raise_(key, "release %s is incomplete" % ver, "\n".join(body)) + title = "release %s is incomplete" % ver + if not hub_missing: + self.alarms.raise_(key, title, "\n".join(body)) + return + self.raise_dispatching(key, title, body, ver, + ("workflow", "run", "build-and-push.yml", "-R", self.repo, "--ref", "main", + "-f", "release=" + ver), + "Dispatched `build-and-push.yml -f release=%s` once; it builds the image from the " + "tag and then dispatches the AUR and package releases. It is not dispatched again." + % ver) # ── e + f ──────────────────────────────────────────────────────────── def check_bot_prs(self): prs = self.gh.json("pr", "list", "-R", self.repo, "--state", "open", "--limit", "100", - "--json", "number,headRefName,headRefOid,createdAt,url,title") or [] + "--json", "number,headRefName,headRefOid,createdAt,url,title,isCrossRepository") or [] stuck = [] for pr in sorted(prs, key=lambda p: p["number"]): - if not pr["headRefName"].startswith("autobump/") or self.now - ts(pr["createdAt"]) <= 72 * HOUR: + # A fork can name its branch autobump/* too; only this repository's branches are bot PRs. + if (pr.get("isCrossRepository") or not pr["headRefName"].startswith("autobump/") + or self.now - ts(pr["createdAt"]) <= 72 * HOUR): continue runs = self.gh.json("run", "list", "-R", self.repo, "--workflow", "ci.yml", "--commit", pr["headRefOid"], "--limit", "20", "--json", "databaseId,event,status,conclusion") or [] diff --git a/tests/ops/workflow_checks.py b/tests/ops/workflow_checks.py index 4e3cb497..da0a2ef0 100755 --- a/tests/ops/workflow_checks.py +++ b/tests/ops/workflow_checks.py @@ -13,6 +13,7 @@ import re import subprocess import sys +import tempfile import yaml @@ -47,6 +48,31 @@ def on(d): and ad.index(guard[0]) < ad.index(pub[0]), "aur-deploy.yml never moves AUR back to an older version") + +def aur_guard(answer, ver): + """Run the guard step as Actions does (bash -eo pipefail) against a fake curl: + answer is the AUR version (a JSON reply) or None (curl fails). Returns + (exit code, GITHUB_OUTPUT text).""" + with tempfile.TemporaryDirectory() as d: + reply = '{"resultcount":1,"results":[{"Version":"%s"}]}' % answer if answer else "" + with open(os.path.join(d, "curl"), "w") as f: + f.write("#!/bin/sh\n" + ("printf '%%s\\n' '%s'\n" % reply if answer else "exit 22\n")) + os.chmod(os.path.join(d, "curl"), 0o755) + out = os.path.join(d, "out") + open(out, "w").close() + env = dict(os.environ, PATH=d + os.pathsep + os.environ["PATH"], VER=ver, GITHUB_OUTPUT=out) + p = subprocess.run(["bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", guard[0]["run"]], + env=env, capture_output=True, text=True) + return p.returncode, open(out).read() + + +if guard: + got = {k: aur_guard(a, "0.1.80") for k, a in + (("newer", "0.1.81-1"), ("same", "0.1.80-1"), ("older", "0.1.79-1"), ("unreadable", None))} + say(got["newer"] == (0, "skip=true\n") and got["same"] == (0, "") and got["older"] == (0, "") + and got["unreadable"] == (0, ""), + "aur-deploy.yml AUR guard: skips only for a newer AUR version; an unreadable RPC deploys", got) + # No cron fires at minute 0 or 30. scheduled = {} for f, d in DOCS.items(): @@ -83,6 +109,10 @@ def on(d): and on(wd).get("schedule") == [{"cron": "47 * * * *"}] and ((on(wd).get("workflow_dispatch") or {}).get("inputs") or {}).get("simulate_red", {}).get("type") == "boolean", "watchdog.yml: hourly at :47, simulate_red, issues+actions write, one queue, workflow_run on main only") +# workflow_run's branches: filter matches a fork's branch named main too (a fork PR +# from <fork>:main), so the job also requires the run to come from this repository. +say("github.event.workflow_run.head_repository.full_name == github.repository" in str(job.get("if", "")), + "watchdog.yml: a workflow_run from a fork's main does not start the watchdog", job.get("if")) co = [s for s in job.get("steps", []) if str(s.get("uses", "")).startswith("actions/checkout@")] say(co and co[0].get("with", {}).get("ref") == "main" and co[0]["with"].get("fetch-depth") == 0 and any(str(s.get("run", "")).strip() == "hack/watchdog" for s in job.get("steps", [])), diff --git a/tests/watchdog_test.sh b/tests/watchdog_test.sh index 157de5dd..c0ea6889 100755 --- a/tests/watchdog_test.sh +++ b/tests/watchdog_test.sh @@ -409,6 +409,99 @@ else fail "issues disabled: rc=$rc out=$(tail -3 "$C/out" | tr '\n' ' ') err=$(tail -3 "$C/err" | tr '\n' ' ')" fi +# 13b. Re-dispatches happen only once the alarm records them. Two runs in a row +# (the second stands for the run a re-dispatch of watchdog.yml starts) with red +# scheduled runs of watchdog.yml and build-and-push.yml and a newest release +# without its Docker Hub image: with issues disabled (this repository today) and +# with alarm writes failing, nothing is dispatched. Dispatching first and +# recording after let every red watchdog run re-dispatch watchdog.yml, and each +# dispatched run do the same, without end. +redsetup() { + setup "$1" + jq '[.[] | select(.databaseId <= 40686436167)]' "$FIX/build-and-push_schedule_runs.json" > "$C/bp_sched.json" + jq -n --arg t "$(iso '-13 minutes')" '[{databaseId: 4201, conclusion: "failure", status: "completed", + createdAt: $t, url: "https://github.com/88plug/k3d-gpu/actions/runs/4201", headSha: "x", + workflowName: "Watchdog", event: "schedule"}]' > "$C/wd_sched.json" + tags "aur-v0.1.79@-3hours aur-v0.1.80@-1hours" +} +redsetup noissues_red +route '^(issue|label) ' - 1 "the '88plug/k3d-gpu' repository has disabled issues" +watch; rc1=$? +watch; rc2=$? +if [ "$rc1" -eq 1 ] && [ "$rc2" -eq 1 ] && [ "$(ndispatch)" = 0 ] && grep -q '^::error title=ci-alarm red:watchdog.yml::' "$C/out"; then + ok "issues disabled, red scheduled runs and a release gap: ::error, and no re-dispatch it cannot record" +else + rc=$rc2; fail "issues disabled, red: $(why) dispatched=$(tr '\n' ',' < "$C/dispatches.log" 2>/dev/null)" +fi +redsetup alarmwrite_red +route '^(issue (create|edit|comment)|label create) ' - 1 'HTTP 502: Bad Gateway (https://api.github.com/graphql)' +watch; rc1=$? +watch; rc2=$? +if [ "$(ndispatch)" = 0 ] && [ "$rc2" -eq 1 ] && grep -q 'HTTP 502' "$C/err"; then + ok "alarm writes failing: no re-dispatch, the error is reported" +else + rc=$rc2; fail "alarm writes failing: rc1=$rc1 $(why) dispatched=$(tr '\n' ',' < "$C/dispatches.log" 2>/dev/null)" +fi +# Issues on: the same state re-dispatches watchdog.yml, build-and-push.yml and +# the release once each, and the second run adds no dispatch. +redsetup issues_red +watch; rc1=$? +watch; rc2=$? +if [ "$rc1" -eq 0 ] && [ "$rc2" -eq 0 ] && [ "$(ndispatch)" = 3 ] && [ "$(dispatched 'watchdog.yml ref=main')" = 1 ] \ + && [ "$(dispatched 'build-and-push.yml ref=main')" = 1 ] && [ "$(dispatched 'build-and-push.yml ref=main release=0.1.80')" = 1 ] \ + && jq -r .body <<< "$(issue red:watchdog.yml)" | grep -q 'watchdog:dispatched=4201' \ + && jq -r .body <<< "$(issue release:0.1.80)" | grep -q 'watchdog:dispatched=0.1.80'; then + ok "issues on: one re-dispatch each for the red workflows and the release, none on the next run" +else + rc=$rc2; fail "issues on, red: rc1=$rc1 $(why) dispatched=$(tr '\n' ',' < "$C/dispatches.log" 2>/dev/null)" +fi +# A re-dispatch that fails leaves no record: the alarm says so, the next run tries again. +redsetup dispatchfail +route '^workflow run watchdog\.yml ' - 1 'HTTP 422: Workflow does not have workflow_dispatch trigger' +watch; rc1=$? +b1=$(jq -r .body <<< "$(issue red:watchdog.yml)"); e1=$(state watchdog-error) +route '^workflow run watchdog\.yml ' - 0 +watch; rc2=$? +if [ "$rc1" -eq 0 ] && grep -q 'Re-dispatch failed: .*HTTP 422' <<< "$b1" && ! grep -q 'watchdog:dispatched' <<< "$b1" \ + && [ "$e1" = OPEN ] && [ "$(state watchdog-error)" = CLOSED ] \ + && jq -r .body <<< "$(issue red:watchdog.yml)" | grep -q 'watchdog:dispatched=4201'; then + ok "a failed re-dispatch is not recorded: the alarm says so and the next run dispatches" +else + rc=$rc2; fail "dispatch failure: rc1=$rc1 $(why) body1=$(head -c 300 <<< "$b1")" +fi +# A run that cannot list the open alarms (a transient 502 on the watchdog's own +# listing; ci_alarm still works) leaves the dispatching alarms as they are: their +# records survive, so the run after it dispatches nothing new. +redsetup listfail +watch; rc1=$? +route '^issue list .*--json number,title,body$' - 1 'HTTP 502: Bad Gateway (https://api.github.com/graphql)' +watch; rc2=$? +b2=$(jq -r .body <<< "$(issue red:watchdog.yml)") +sed -i '1d' "$C/routes.tsv" +watch; rc3=$? +if [ "$rc1" -eq 0 ] && [ "$rc2" -eq 0 ] && [ "$rc3" -eq 0 ] && [ "$(ndispatch)" = 3 ] \ + && grep -q 'watchdog:dispatched=4201' <<< "$b2" \ + && jq -r .body <<< "$(issue release:0.1.80)" | grep -q 'watchdog:dispatched=0.1.80'; then + ok "a run that cannot list the alarms keeps their dispatch records: no second dispatch" +else + rc=$rc3; fail "listing failure: rc1=$rc1 rc2=$rc2 $(why) dispatched=$(tr '\n' ',' < "$C/dispatches.log" 2>/dev/null)" +fi + +# 13c. An autobump/* branch on a fork is not a bot PR: a fork could otherwise hold +# bot-pr-stuck open for as long as it liked. +setup forkpr +jq -n --arg old "$(iso '-73 hours')" '[ + {number: 244, headRefName: "autobump/convert-pins", headRefOid: "f0f0", createdAt: $old, isCrossRepository: true, + url: "https://github.com/88plug/k3d-gpu/pull/244", title: "fork", author: {login: "someone"}}]' > "$C/prs.json" +route '^run list .*--workflow ci\.yml .*--commit f0f0' - 1 'must not be asked' +watch; rc=$? +if [ "$rc" -eq 0 ] && [ "$(state bot-pr-stuck)" = none ] && ! grep -q 'must not be asked' "$C/err" \ + && grep -q '^pr list .*--json [^ ]*isCrossRepository' "$C/calls.log"; then + ok "a fork's autobump/* PR is not a stuck bot PR" +else + fail "fork PR: $(why) prlist=$(grep '^pr list' "$C/calls.log" | head -1)" +fi + # 14. --list-scheduled: every workflow with on.schedule, with its cron period in hours. setup listsched watch --list-scheduled; rc=$?