ci: merge auto-eligible bot PRs one at a time, behind a circuit breaker, and batch the release

.github/workflows/bot-sweep.yml

@@ -0,0 +1,79 @@
+name: Bot sweep
+
+# Hourly upkeep of the bot's PRs (branches autobump/*, opened by github-actions[bot]
+# with GITHUB_TOKEN, so their own pull_request runs wait for approval).
+#
+#   ci-fill: hack/bot-ci-dispatch runs ci.yml by workflow_dispatch on each bot PR
+#            branch with no run at its head, and re-runs a first-attempt failure
+#            once, one bot run in flight at a time and none while a human PR's CI
+#            is queued. It fills in what the nightly deferred. Then the same for
+#            canary.yml (engines=auto), on heads whose pins policy maps canary
+#            engines.
+#   merge:   hack/bot-merge merges at most one bot PR whose evidence is green at
+#            its head (policy, CI, canary, current base, circuit breaker, daily
+#            cap), dispatches ci.yml on main after it, and dispatches a batched
+#            auto-release. It runs after ci-fill even when ci-fill had an error:
+#            it reads every piece of evidence itself.
+
+on:
+  schedule:
+    - cron: '13 * * * *'
+  workflow_dispatch:
+
+permissions: {}
+
+concurrency:
+  group: bot-sweep
+  cancel-in-progress: false
+
+jobs:
+  ci-fill:
+    name: CI on bot PRs
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      actions: write
+      contents: read
+      pull-requests: read
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: main
+          fetch-depth: 0 # the canary step's pins policy diffs a PR head against main
+          persist-credentials: false
+
+      - name: Dispatch CI on bot PRs that have none at their head
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: hack/bot-ci-dispatch
+
+      - name: Dispatch the canary on bot PRs whose policy maps canary engines
+        if: ${{ !cancelled() }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: hack/bot-ci-dispatch --workflow canary.yml
+
+  merge:
+    name: Merge one auto-eligible bot PR, batch the release
+    needs: ci-fill
+    if: ${{ !cancelled() }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: write
+      pull-requests: write
+      actions: write
+      issues: write
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: main
+          fetch-depth: 0 # base_ok, today's Bot-merge commits and the aur-v* tags
+          persist-credentials: false
+
+      - name: Merge, then batch the release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: hack/bot-merge

.github/workflows/build-and-push.yml

@@ -440,8 +440,11 @@ jobs:
   # component on branch autobump/<name>. Merging it changes share/** on main,
   # which auto-release.yml turns into a launcher release, and that release
   # dispatches the `release` job above to build the image. A PR opened with
-  # GITHUB_TOKEN starts no CI workflows, so the tests run here before it opens.
-  # A new component is a new hack/components/<name>/ dir; the matrix picks it up.
+  # GITHUB_TOKEN gets pull_request CI runs that wait for a maintainer's approval,
+  # so the tests run here before it opens, and hack/bot-ci-dispatch then runs
+  # ci.yml on the branch by workflow_dispatch (no approval): that run is the PR's
+  # CI evidence. A new component is a new hack/components/<name>/ dir; the matrix
+  # picks it up.
   list-components:
     if: ${{ !inputs.release }}
     runs-on: ubuntu-latest
@@ -466,6 +469,7 @@ jobs:
     permissions:
       contents: write
       pull-requests: write
+      actions: write
     strategy:
       fail-fast: false
       matrix:
@@ -503,6 +507,12 @@ jobs:
       - name: Render tests
         run: tests/render_test.sh
 
+      # create-pull-request force-pushes an existing bump branch, which GitHub
+      # refuses from GITHUB_TOKEN once main's workflow files differ from the
+      # branch's: delete such a branch first (its PR closes; a fresh one opens).
+      - name: Delete a bump branch whose workflows differ from main's
+        run: hack/autobump-branch-reset "autobump/${COMPONENT}"
+
       - name: Open or update the bump PR
         uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
         with:
@@ -522,27 +532,38 @@ jobs:
             - Pins (version, chart sha256, image digests) moved in `hack/components/${{ matrix.component }}/component.conf`.
             - `tests/detect_test.sh` and `tests/render_test.sh` passed in the workflow run
               that opened this PR.
+            - CI: this PR's own pull_request runs wait for approval (a GITHUB_TOKEN PR).
+              The evidence is the ci.yml run dispatched on `autobump/${{ matrix.component }}`
+              at the head SHA: `gh run list -w ci.yml -e workflow_dispatch -b autobump/${{ matrix.component }}`.
             - Merging rebuilds the node image with the new manifest baked in.
 
             Review the manifest diff for new CRDs, RBAC, images and flags before merging.
 
+      - name: Dispatch CI on the bump PR's branch
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: hack/bot-ci-dispatch --branch "autobump/${COMPONENT}"
+
   # ─── Image digests ───────────────────────────────────────────────────────
   # Every image reference is pinned name:tag@sha256 (scripts/images-lock pins).
   # A registry can re-point a tag (a rebuilt base image, a security respin) with
   # no new tag, which the version checks above never see. Daily: re-resolve the
   # tag of every pinned reference, write the registry's digest where it moved,
   # re-pin images.lock, run the image tests, and open or update one PR on branch
-  # autobump/image-digests. A PR opened with GITHUB_TOKEN starts no CI workflows,
-  # so the tests run here before it opens. GITHUB_TOKEN cannot push a change to
-  # .github/workflows/ (no `workflows` permission), so repin leaves those files
-  # alone; a workflow reads its image pins from a file outside them (the
-  # Dockerfile, docs-build/Dockerfile, hack/ci-images.env), which repin moves.
+  # autobump/image-digests. A PR opened with GITHUB_TOKEN gets pull_request CI
+  # runs that wait for a maintainer's approval, so the tests run here before it
+  # opens, and hack/bot-ci-dispatch then runs ci.yml on the branch by
+  # workflow_dispatch: that run is the PR's CI evidence. GITHUB_TOKEN cannot push
+  # a change to .github/workflows/ (no `workflows` permission), so repin leaves
+  # those files alone; a workflow reads its image pins from a file outside them
+  # (the Dockerfile, docs-build/Dockerfile, hack/ci-images.env), which repin moves.
   repin-images:
     if: ${{ !inputs.release }}
     runs-on: ubuntu-latest
     permissions:
       contents: write
       pull-requests: write
+      actions: write
     steps:
       - name: Checkout main
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -594,12 +615,7 @@ jobs:
       # its open PR; the action then cuts a fresh branch off main and opens a new one.
       - name: Delete a digest branch whose workflows differ from main's
         if: steps.repin.outputs.changes != ''
-        run: |
-          if git fetch -q --depth=1 origin autobump/image-digests 2>/dev/null \
-             && ! git diff --quiet FETCH_HEAD HEAD -- .github/workflows/; then
-            echo "autobump/image-digests was cut from a main with other workflow files; deleting it"
-            git push origin --delete autobump/image-digests
-          fi
+        run: hack/autobump-branch-reset autobump/image-digests
 
       - name: Open or update the digest PR
         if: steps.repin.outputs.changes != ''
@@ -622,21 +638,32 @@ jobs:
             - `tests/image_pins_test.sh`, `tests/audit_test.sh`, `tests/detect_test.sh`,
               `tests/render_test.sh` and the render check passed in the workflow run that
               opened this PR.
+            - CI: this PR's own pull_request runs wait for approval (a GITHUB_TOKEN PR).
+              The evidence is the ci.yml run dispatched on `autobump/image-digests` at the
+              head SHA: `gh run list -w ci.yml -e workflow_dispatch -b autobump/image-digests`.
 
             Review what the upstream rebuild changed before merging. A change under
             `share/`, `scripts/` or the Dockerfile cuts a launcher release on merge.
 
+      - name: Dispatch CI on the digest PR's branch
+        if: steps.repin.outputs.changes != ''
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: hack/bot-ci-dispatch --branch autobump/image-digests
+
   # ─── QAT conversion toolchain (console/convert-pins.json) ──────────────────
   # The model optimizer's one-time OpenVINO conversion pins its pip toolchain
   # exactly. Daily: move each pin to the newest release optimum-intel's own
   # requirements allow (hack/bump-convert-pins), run the console tests, and open
-  # or update one PR on branch autobump/convert-pins.
+  # or update one PR on branch autobump/convert-pins, whose CI evidence is the
+  # ci.yml run hack/bot-ci-dispatch runs on the branch.
   bump-convert-pins:
     if: ${{ !inputs.release }}
     runs-on: ubuntu-latest
     permissions:
       contents: write
       pull-requests: write
+      actions: write
     steps:
       - name: Checkout main
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -663,6 +690,12 @@ jobs:
           tests/console_test.sh
           tests/model_opt_test.sh
 
+      # As in repin-images: GitHub refuses create-pull-request's force-push from
+      # GITHUB_TOKEN once main's workflow files differ from the branch's.
+      - name: Delete a pins branch whose workflows differ from main's
+        if: steps.bump.outputs.changes != ''
+        run: hack/autobump-branch-reset autobump/convert-pins
+
       - name: Open or update the pins PR
         if: steps.bump.outputs.changes != ''
         uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
@@ -682,3 +715,13 @@ jobs:
 
             `tests/console_test.sh` and `tests/model_opt_test.sh` passed in the workflow
             run that opened this PR.
+
+            CI: this PR's own pull_request runs wait for approval (a GITHUB_TOKEN PR).
+            The evidence is the ci.yml run dispatched on `autobump/convert-pins` at the
+            head SHA: `gh run list -w ci.yml -e workflow_dispatch -b autobump/convert-pins`.
+
+      - name: Dispatch CI on the pins PR's branch
+        if: steps.bump.outputs.changes != ''
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: hack/bot-ci-dispatch --branch autobump/convert-pins

.github/workflows/ci.yml

@@ -5,11 +5,14 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
+  # hack/bot-ci-dispatch runs CI on a bot PR's branch this way: the PR's own
+  # pull_request runs wait for approval (GITHUB_TOKEN opened it), a dispatch does not.
+  workflow_dispatch:
 
 permissions:
   contents: read
 
-# A new push to a PR supersedes its older run; main runs always finish.
+# A new push to a PR supersedes its older run; main and dispatched runs always finish.
 concurrency:
   group: ci-${{ github.event.pull_request.number || github.sha }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
@@ -55,6 +58,7 @@ jobs:
       - run: tests/prefill_guard_test.sh
       - run: tests/ci_wiring_test.sh
       - run: tests/gate_merge_test.sh
+      - run: tests/bot_ci_dispatch_test.sh
       - run: tests/auto_release_cas_test.sh
 
   quickstart-tests:
@@ -484,3 +488,69 @@ jobs:
       - run: python3 tests/known_issues_test.py
       - run: python3 tests/ledger_freshness_test.py
       - run: hack/known-issues check
+
+  # ─── Bot merge ───────────────────────────────────────────────────────────
+  # hack/bot-merge (bot-sweep.yml, job merge) against a fake gh and fixture repos.
+  bot-merge-tests:
+    name: Bot merge (fake gh, fixture repos)
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - run: tests/bot_merge_test.sh
+
+  # ─── Bot PR CI and alarm surfacing ───────────────────────────────────────
+  # The watchdog files a ci-alarm issue for a red or missing scheduled run or a
+  # broken release (email went unread for 9 nights). Every open alarm, and a
+  # Watchdog workflow that is missing, disabled or without a success in 3 h,
+  # shows as a warning on every CI run. It never fails CI: the script exits 0,
+  # and the job is continue-on-error.
+  alarms:
+    name: Open CI alarms
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    continue-on-error: true
+    permissions:
+      issues: read
+      actions: read
+    env:
+      GH_TOKEN: ${{ github.token }}
+      GH_REPO: ${{ github.repository }}
+    steps:
+      - name: Warn on open ci-alarm issues and a quiet Watchdog
+        run: |
+          set +e
+          err=$(mktemp)
+          # Workflow-command data escapes %, CR and LF.
+          oneline() { tr '\r\n' '  ' < "$err" | sed 's/ *$//; s/%/%25/g'; }
+          if alarms=$(gh issue list --label ci-alarm --state open --limit 100 --json number,title,url,labels 2>"$err"); then
+            jq -r '.[]
+              | "open ci-alarm #\(.number): \(.title) \(.url) (\(if any(.labels[]; .name == "alarm-ack") then "acked"
+                  else "NOT acked; after reading it: gh issue edit \(.number) --add-label alarm-ack" end))"
+              | "::warning::" + (gsub("%"; "%25") | gsub("\r"; "%0D") | gsub("\n"; "%0A"))' <<<"$alarms"
+          else
+            echo "::warning::could not list the open ci-alarm issues: $(oneline)"
+          fi
+          if ! wd=$(gh workflow list --all --json name,path,state \
+                    -q '[.[] | select(.name == "Watchdog")][0] // empty | "\(.state) \(.path)"' 2>"$err"); then
+            echo "::warning::could not list the workflows to check the Watchdog: $(oneline)"
+          elif [ -z "$wd" ]; then
+            echo "::warning::there is no Watchdog workflow, so a red or missing scheduled run raises no ci-alarm"
+          else
+            read -r state path <<<"$wd"
+            if [ "$state" != active ]; then
+              echo "::warning::the Watchdog workflow is $state, so a red or missing scheduled run raises no ci-alarm"
+            else
+              last=$(gh run list -w "${path##*/}" -s success -L 1 --json updatedAt -q '.[0].updatedAt // empty' 2>"$err")
+              ts=$(date -u -d "${last:-@0}" +%s 2>/dev/null) || ts=0
+              if [ $(( $(date -u +%s) - ts )) -gt 10800 ]; then
+                echo "::warning::the Watchdog workflow has no successful run in the last 3 h (last: ${last:-none}); alarms may be missing"
+              fi
+            fi
+          fi
+          rm -f "$err"
+          exit 0

README.md

@@ -1715,7 +1715,13 @@ tests/render_test.sh
 CI fails when a committed manifest differs from its pinned render. The daily
 build-and-push workflow runs `--latest` for every component and opens one PR per
 component (branch `autobump/<name>`); merging it changes `share/`, so auto-release cuts a
-launcher release and the node image is rebuilt for it.
+launcher release and the node image is rebuilt for it. GITHUB_TOKEN opens these PRs, so
+their own pull_request CI runs wait for approval; `hack/bot-ci-dispatch` runs `ci.yml` on
+the branch by workflow_dispatch instead, and that run is the PR's CI. The hourly bot sweep
+(`hack/bot-merge`) merges one such PR at a time when its pins policy says auto (or a
+maintainer approved its head), its CI and canary are green, its base is current and main's
+CI is green; it then dispatches CI on main and a batched release. Any other bot PR stays
+open with one comment that says what blocks it.
 
 ---
 

hack/autobump-branch-reset

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# hack/autobump-branch-reset <branch>: delete origin's autobump/<name> branch when its
+# .github/workflows/ differ from origin/main's. Run from a checkout whose origin
+# push credentials are the workflow's GITHUB_TOKEN, before create-pull-request.
+#
+# create-pull-request resets an existing bump branch (a PR still open, or a merged
+# one whose branch stayed) onto main and force-pushes it. GitHub refuses that push
+# from GITHUB_TOKEN (it has no `workflows` permission) when the branch's workflow
+# files differ from main's, though the bump commit touches none. Deleting the
+# branch closes its open PR; create-pull-request then cuts a fresh branch off main
+# and opens a new PR, whose dispatched CI runs main's current workflows.
+set -euo pipefail
+b=${1:-}
+case $b in
+  autobump/?*) ;;
+  *) echo "usage: hack/autobump-branch-reset autobump/<name> (refusing '${b}')" >&2; exit 2;;
+esac
+git ls-remote --exit-code --heads origin "refs/heads/$b" >/dev/null \
+  || { rc=$?; [ "$rc" -eq 2 ] && { echo "$b: not on origin; nothing to reset"; exit 0; }; exit "$rc"; }
+git fetch -q --depth=1 origin "+refs/heads/main:refs/remotes/origin/main" "+refs/heads/$b:refs/remotes/origin/$b"
+if git diff --quiet "origin/main" "origin/$b" -- .github/workflows; then
+  echo "$b: its workflows match main's; kept"
+else
+  echo "$b: its workflows differ from main's; deleting it (its open PR closes, and a fresh one opens)"
+  git push origin --delete "$b"
+fi