ci: alarm on red or missing automation, and harden the release

.github/workflows/aur-deploy.yml

@@ -11,10 +11,14 @@ on:
         required: true
         type: string
 
+permissions:
+  contents: read
+
 jobs:
   deploy:
     name: Publish k3d-gpu to AUR
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -30,12 +34,29 @@ jobs:
             echo "version=$REF" >> "$GITHUB_OUTPUT"
           fi
 
+      # Releases build in parallel (one build-and-push queue per version), so an
+      # older release can reach this deploy after a newer one: AUR never moves back.
+      # When the AUR RPC cannot be read the deploy goes ahead.
+      - name: AUR already newer?
+        id: aur
+        env:
+          VER: ${{ steps.ver.outputs.version }}
+        run: |
+          cur=$(curl -fsSg --retry 3 --max-time 30 'https://aur.archlinux.org/rpc/v5/info?arg[]=k3d-gpu' \
+            | jq -r '.results[0].Version // empty' | cut -d- -f1) || cur=
+          if [ -n "$cur" ] && [ "$cur" != "$VER" ] && [ "$(printf '%s\n%s\n' "$cur" "$VER" | sort -V | tail -1)" = "$cur" ]; then
+            echo "AUR has k3d-gpu $cur, newer than $VER; leaving it"
+            echo "skip=true" >> "$GITHUB_OUTPUT"
+          fi
+
       - name: Update PKGBUILD
+        if: steps.aur.outputs.skip != 'true'
         run: |
           sed -i "s/^pkgver=.*/pkgver=${{ steps.ver.outputs.version }}/" packaging/aur/PKGBUILD
           sed -i "s/^pkgrel=.*/pkgrel=1/" packaging/aur/PKGBUILD
 
       - name: Publish to AUR
+        if: steps.aur.outputs.skip != 'true'
         uses: KSXGitHub/github-actions-deploy-aur@084b0d9b15415bf9cdb65d44dad1efe37a354050 # v4.2.0
         with:
           pkgname: k3d-gpu

.github/workflows/auto-release.yml

@@ -1,22 +1,31 @@
 name: Auto-release AUR
 
-# Auto-tag a release on changes to packaging/, scripts/, share/ or the Dockerfile,
-# and when build-and-push.yml's nightly check bumps upstream pins (dispatch).
+# Auto-tag a release when a commit on main changes a path hack/release-paths lists
+# (what the packages, the formula and the image ship: packaging/, scripts/,
+# share/, the Dockerfile, console/, images.lock and the shipped bench/ and oracle/
+# files), and when build-and-push.yml's nightly check bumps upstream pins
+# (dispatch). The push paths: below repeat hack/release-paths (GitHub reads no
+# file there); tests/ci_wiring_test.sh checks they are equal.
 # Tag format: aur-vX.Y.Z. The launcher at that tag defaults to node image
 # cryptoandcoffee/k3d-gpu:X.Y.Z, so after tagging this dispatches build-and-push.yml
 # with release=X.Y.Z; that run publishes the image and then dispatches aur-deploy.yml.
 # Dispatches bypass the GITHUB_TOKEN downstream-trigger limit.
 #
+# A dispatch's mode: force (the default) always releases; changed releases only
+# when a commit above the last tag changed a shipped path (release_cas.sh runs as
+# for a push), so a merge that a GITHUB_TOKEN push did not announce can still cut
+# its release, and a dispatch with nothing shipped cuts none.
+#
 # hack/release_cas.sh releases from a fresh origin/main and pushes main and the
 # tag in one atomic push (--atomic), retrying when another writer moved main or
 # took the tag. On those lock and non-fast-forward races GitHub rejected both
 # refs together in a branch-ref spike (not yet seen on a live release push); for
 # other rejection classes a release commit can land
 # without its tag, and the next run tags it in place. A push run whose commits
 # an earlier run already released exits with no tag. The
-# concurrency group only queues runs; correctness does not depend on it. Push and
-# dispatch runs queue separately, so a pending forced dispatch is never replaced
-# by a push run that finds nothing to release.
+# concurrency group only queues runs; correctness does not depend on it. Push
+# runs and each dispatch mode queue separately, so a pending forced dispatch is
+# never replaced by a run that finds nothing to release.
 
 on:
   push:
@@ -26,7 +35,28 @@ on:
       - 'scripts/**'
       - 'share/**'
       - 'Dockerfile'
+      - 'console/**'
+      - 'images.lock'
+      - 'bench/vs.py'
+      - 'bench/llm_load.py'
+      - 'bench/energy.sh'
+      - 'bench/agent_trace.py'
+      - 'bench/agent_trace_gen.py'
+      - 'bench/apb/apb.py'
+      - 'bench/apb/capacity.py'
+      - 'bench/competitors.lock'
+      - 'oracle/bundle.py'
+      - 'oracle/project.py'
+      - 'oracle/validate.py'
+      - 'oracle/share.py'
+      - 'oracle/schema/bundle-v1.json'
   workflow_dispatch:
+    inputs:
+      mode:
+        description: 'force: always release; changed: release only when a shipped path changed since the last tag'
+        type: choice
+        options: [force, changed]
+        default: force
 
 permissions:
   contents: write
@@ -36,8 +66,9 @@ jobs:
   release:
     if: "github.actor != '88plug-bot' && !contains(github.event.head_commit.message, '[skip release]')"
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     concurrency:
-      group: auto-release-${{ github.event_name }}
+      group: auto-release-${{ github.event_name }}-${{ inputs.mode || 'push' }}
       cancel-in-progress: false
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -48,7 +79,7 @@ jobs:
       - name: Release (atomic push of main and the tag)
         id: rel
         env:
-          EVENT: ${{ github.event_name }}
+          EVENT: ${{ github.event_name == 'workflow_dispatch' && inputs.mode == 'changed' && 'push' || github.event_name }}
         run: |
           git config user.name  "88plug-bot"
           git config user.email "andrew@88plug.com"

.github/workflows/build-and-push.yml

@@ -5,19 +5,23 @@ name: Auto-update, Build & Release
 #   check   (nightly, or manual without `release`): resolve upstream versions; when
 #           one moved (or `force`), commit the bumped pins to main and dispatch
 #           auto-release.yml, which cuts a launcher release for them.
-#   release (dispatched by auto-release.yml with `release=X.Y.Z`): build the image
-#           from tag aur-vX.Y.Z, publish <repo>:X.Y.Z (the launcher's default
-#           image) next to the floating tags, then dispatch aur-deploy.yml and
-#           packages.yml (deb, rpm, Homebrew) so the launcher reaches every package
-#           channel only once its image exists.
+#   release (dispatched by auto-release.yml with `release=X.Y.Z`, or by
+#           hack/watchdog when a release tag has no image): build the image from
+#           tag aur-vX.Y.Z, publish <repo>:X.Y.Z (the launcher's default image),
+#           move the floating tags when X.Y.Z is the newest release, then dispatch
+#           aur-deploy.yml and packages.yml (deb, rpm, Homebrew) so the launcher
+#           reaches every package channel only once its image exists.
 #
 # Every image is built from a release tag, so every published image has a
 # launcher version. Pushes made with GITHUB_TOKEN start no workflow runs, but
 # workflow_dispatch does, hence the explicit dispatches.
+#
+# Each release version queues on its own (one release never replaces another
+# pending release); the nightly check queues in `nightly`.
 
 on:
   schedule:
-    - cron: '0 6 * * *'
+    - cron: '17 6 * * *'
   workflow_dispatch:
     inputs:
       force:
@@ -30,7 +34,7 @@ on:
         default: ''
 
 concurrency:
-  group: build-and-push
+  group: build-and-push-${{ inputs.release && format('release-{0}', inputs.release) || 'nightly' }}
   cancel-in-progress: false
 
 permissions:
@@ -42,6 +46,7 @@ jobs:
   check:
     if: ${{ !inputs.release }}
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     steps:
       - name: Checkout main
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -292,6 +297,7 @@ jobs:
   release:
     if: ${{ inputs.release }}
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     env:
       RELEASE: ${{ inputs.release }}
     steps:
@@ -347,30 +353,128 @@ jobs:
             exit 1
           fi
 
+      # hack/release-tags comes from main: the release tag checked out above can
+      # predate it (the watchdog re-dispatches an old release whose image is missing).
+      - name: Checkout main's release tooling
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: main
+          path: .ci-main
+          sparse-checkout: hack
+          persist-credentials: false
+
+      # The builds push the versioned tags (<repo>:X.Y.Z, -ubuntu26.04, -ubuntu24.04)
+      # only; the floating tags move after both builds, below.
+      - name: Tags to push
+        id: tags
+        env:
+          K3S: ${{ steps.current.outputs.k3s }}
+          RELEASE_REPOS: cryptoandcoffee/k3d-gpu ghcr.io/${{ github.repository_owner }}/k3d-gpu
+        run: |
+          tags=$(.ci-main/hack/release-tags "$RELEASE" "$K3S")
+          printf '%s\n' "$tags"
+          for v in 26.04 24.04; do
+            echo "v${v/./}<<TAGS_EOF"
+            awk -v v="$v" '$1 == v && $2 == "versioned" { print $3 }' <<< "$tags"
+            echo TAGS_EOF
+          done >> "$GITHUB_OUTPUT"
+
       # Ubuntu 26.04 is the default (:latest and :X.Y.Z); 24.04 is also published.
       # Each carries the baked device plugins, so clusters expose GPUs with no apply.
-      - name: Build & push — Ubuntu 26.04 (default / latest)
+      # Each variant builds in up to 3 attempts with the same inputs (a Docker Hub
+      # 429 stopped release 0.1.76 with its tag cut and no image): attempts 1 and 2
+      # continue on error and are followed by a 2 and a 4 min wait; attempt 3 fails
+      # the job.
+      - name: Build & push — Ubuntu 26.04 (attempt 1)
+        id: b2604_1
+        continue-on-error: true
+        uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
+        with:
+          context: .
+          push: true
+          platforms: linux/amd64
+          labels: |
+            org.opencontainers.image.version=${{ inputs.release }}
+            org.opencontainers.image.revision=aur-v${{ inputs.release }}
+          tags: ${{ steps.tags.outputs.v2604 }}
+
+      - name: Wait 2 min before Ubuntu 26.04 attempt 2
+        if: steps.b2604_1.outcome == 'failure'
+        run: sleep 120
+
+      - name: Build & push — Ubuntu 26.04 (attempt 2)
+        id: b2604_2
+        if: steps.b2604_1.outcome == 'failure'
+        continue-on-error: true
+        uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
+        with:
+          context: .
+          push: true
+          platforms: linux/amd64
+          labels: |
+            org.opencontainers.image.version=${{ inputs.release }}
+            org.opencontainers.image.revision=aur-v${{ inputs.release }}
+          tags: ${{ steps.tags.outputs.v2604 }}
+
+      - name: Wait 4 min before Ubuntu 26.04 attempt 3
+        if: steps.b2604_2.outcome == 'failure'
+        run: sleep 240
+
+      - name: Build & push — Ubuntu 26.04 (attempt 3)
+        id: b2604_3
+        if: steps.b2604_2.outcome == 'failure'
+        uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
+        with:
+          context: .
+          push: true
+          platforms: linux/amd64
+          labels: |
+            org.opencontainers.image.version=${{ inputs.release }}
+            org.opencontainers.image.revision=aur-v${{ inputs.release }}
+          tags: ${{ steps.tags.outputs.v2604 }}
+
+      - name: Build & push — Ubuntu 24.04 (attempt 1)
+        id: b2404_1
+        continue-on-error: true
+        uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
+        with:
+          context: .
+          push: true
+          platforms: linux/amd64
+          build-args: |
+            UBUNTU_IMAGE=${{ steps.current.outputs.ubuntu2404 }}
+          labels: |
+            org.opencontainers.image.version=${{ inputs.release }}
+            org.opencontainers.image.revision=aur-v${{ inputs.release }}
+          tags: ${{ steps.tags.outputs.v2404 }}
+
+      - name: Wait 2 min before Ubuntu 24.04 attempt 2
+        if: steps.b2404_1.outcome == 'failure'
+        run: sleep 120
+
+      - name: Build & push — Ubuntu 24.04 (attempt 2)
+        id: b2404_2
+        if: steps.b2404_1.outcome == 'failure'
+        continue-on-error: true
         uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
         with:
           context: .
           push: true
           platforms: linux/amd64
+          build-args: |
+            UBUNTU_IMAGE=${{ steps.current.outputs.ubuntu2404 }}
           labels: |
             org.opencontainers.image.version=${{ inputs.release }}
             org.opencontainers.image.revision=aur-v${{ inputs.release }}
-          tags: |
-            cryptoandcoffee/k3d-gpu:${{ inputs.release }}
-            cryptoandcoffee/k3d-gpu:${{ inputs.release }}-ubuntu26.04
-            cryptoandcoffee/k3d-gpu:latest
-            cryptoandcoffee/k3d-gpu:ubuntu26.04
-            cryptoandcoffee/k3d-gpu:${{ steps.current.outputs.k3s }}-ubuntu26.04
-            ghcr.io/${{ github.repository_owner }}/k3d-gpu:${{ inputs.release }}
-            ghcr.io/${{ github.repository_owner }}/k3d-gpu:${{ inputs.release }}-ubuntu26.04
-            ghcr.io/${{ github.repository_owner }}/k3d-gpu:latest
-            ghcr.io/${{ github.repository_owner }}/k3d-gpu:ubuntu26.04
-            ghcr.io/${{ github.repository_owner }}/k3d-gpu:${{ steps.current.outputs.k3s }}-ubuntu26.04
-
-      - name: Build & push — Ubuntu 24.04 (variant)
+          tags: ${{ steps.tags.outputs.v2404 }}
+
+      - name: Wait 4 min before Ubuntu 24.04 attempt 3
+        if: steps.b2404_2.outcome == 'failure'
+        run: sleep 240
+
+      - name: Build & push — Ubuntu 24.04 (attempt 3)
+        id: b2404_3
+        if: steps.b2404_2.outcome == 'failure'
         uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
         with:
           context: .
@@ -381,13 +485,26 @@ jobs:
           labels: |
             org.opencontainers.image.version=${{ inputs.release }}
             org.opencontainers.image.revision=aur-v${{ inputs.release }}
-          tags: |
-            cryptoandcoffee/k3d-gpu:${{ inputs.release }}-ubuntu24.04
-            cryptoandcoffee/k3d-gpu:ubuntu24.04
-            cryptoandcoffee/k3d-gpu:${{ steps.current.outputs.k3s }}-ubuntu24.04
-            ghcr.io/${{ github.repository_owner }}/k3d-gpu:${{ inputs.release }}-ubuntu24.04
-            ghcr.io/${{ github.repository_owner }}/k3d-gpu:ubuntu24.04
-            ghcr.io/${{ github.repository_owner }}/k3d-gpu:${{ steps.current.outputs.k3s }}-ubuntu24.04
+          tags: ${{ steps.tags.outputs.v2404 }}
+
+      # The floating tags (latest, ubuntu26.04, ubuntu24.04, <k3s>-ubuntu26.04,
+      # <k3s>-ubuntu24.04) move only when this release is the highest aur-v* tag on
+      # origin now, after both variants pushed: releases build in parallel (one
+      # queue per version), so an older one that finishes late leaves them alone.
+      # Each is a registry-side copy of the versioned tag pushed above.
+      - name: Move the floating tags (newest release only)
+        env:
+          K3S: ${{ steps.current.outputs.k3s }}
+          RELEASE_REPOS: cryptoandcoffee/k3d-gpu ghcr.io/${{ github.repository_owner }}/k3d-gpu
+        run: |
+          tags=$(.ci-main/hack/release-tags "$RELEASE" "$K3S")
+          n=0
+          while read -r _ kind ref src; do
+            [ "$kind" = floating ] || continue
+            docker buildx imagetools create --tag "$ref" "$src"
+            n=$((n + 1))
+          done <<< "$tags"
+          echo "moved $n floating tags"
 
       - name: Create GitHub Release
         uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3
@@ -445,6 +562,7 @@ jobs:
   list-components:
     if: ${{ !inputs.release }}
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     permissions:
       contents: read
     outputs:
@@ -463,6 +581,7 @@ jobs:
     needs: list-components
     if: needs.list-components.outputs.names != '[]'
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     permissions:
       contents: write
       pull-requests: write
@@ -540,6 +659,7 @@ jobs:
   repin-images:
     if: ${{ !inputs.release }}
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     permissions:
       contents: write
       pull-requests: write
@@ -634,6 +754,7 @@ jobs:
   bump-convert-pins:
     if: ${{ !inputs.release }}
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     permissions:
       contents: write
       pull-requests: write