From e44eaa16025da942bd89add22fa5586ba4f08cdd Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 14:25:17 -0700 Subject: [PATCH 1/5] ci: run CI on bot PRs by dispatch; the gate accepts it and surfaces open alarms A PR that GITHUB_TOKEN opens or pushes gets pull_request runs held for approval (#240 sat with CI action_required), while a workflow_dispatch run needs none. ci.yml now takes workflow_dispatch, and hack/bot-ci-dispatch runs it on each same-repo autobump/* branch opened by github-actions[bot] that has no run at its head, re-running a first-attempt failure once. It keeps one bot ci.yml run in flight (BOT_CI_MAX_INFLIGHT) and defers while a human PR's CI is queued, since one run is about 39 jobs against a 20-40 job pool. The three nightly PR jobs call it after create-pull-request, and the new hourly 'Bot sweep' workflow (job ci-fill) fills in what they deferred. hack/autobump-branch-reset generalises the digest job's stale-branch step to all three PR jobs: a bump branch whose workflow files differ from main's is deleted before create-pull-request, whose force-push GitHub would refuse. The merge gate's checks move to hack/gate_lib.sh. ci_status reads ci.yml at a SHA through a pull_request run or a dispatch run on the PR branch; once the dispatched run is green, held entries of 'CI' and 'Auto-merge Dependabot PRs' no longer block. base_ok ignores release-only bot commits on main, so the three release commits per launcher release no longer force a sync. The gate prints every open ci-alarm issue and refuses (exit 4) a PR that changes .github/ or shipped paths while one is not acked; ledger, docs, bench and test PRs are never refused. ci.yml gains a last job, alarms, which warns on every open ci-alarm issue and on a missing, disabled or quiet Watchdog workflow, and never fails. dependabot-auto-merge.yml is deleted: it ran gh pr merge --auto on an unprotected main, merged nothing in 500 runs, and held a run on every bot PR. --- .github/workflows/bot-sweep.yml | 41 ++++ .github/workflows/build-and-push.yml | 71 ++++-- .github/workflows/ci.yml | 58 ++++- .github/workflows/dependabot-auto-merge.yml | 18 -- README.md | 4 +- hack/autobump-branch-reset | 26 +++ hack/bot-ci-dispatch | 110 +++++++++ hack/gate_lib.sh | 186 +++++++++++++++ hack/gate_merge.sh | 127 ++++++----- tests/bot_ci_dispatch_test.sh | 171 ++++++++++++++ tests/ci_wiring_test.sh | 214 ++++++++++++++--- tests/gate_merge_test.sh | 241 +++++++++++++++++++- 12 files changed, 1134 insertions(+), 133 deletions(-) create mode 100644 .github/workflows/bot-sweep.yml delete mode 100644 .github/workflows/dependabot-auto-merge.yml create mode 100755 hack/autobump-branch-reset create mode 100755 hack/bot-ci-dispatch create mode 100644 hack/gate_lib.sh create mode 100755 tests/bot_ci_dispatch_test.sh diff --git a/.github/workflows/bot-sweep.yml b/.github/workflows/bot-sweep.yml new file mode 100644 index 00000000..e19ef05e --- /dev/null +++ b/.github/workflows/bot-sweep.yml @@ -0,0 +1,41 @@ +name: Bot sweep + +# Hourly upkeep of the bot's PRs (branches autobump/*, opened by github-actions[bot] +# with GITHUB_TOKEN, so their own pull_request runs wait for approval). +# +# ci-fill: hack/bot-ci-dispatch runs ci.yml by workflow_dispatch on each bot PR +# branch with no run at its head, and re-runs a first-attempt failure +# once, one bot run in flight at a time and none while a human PR's CI +# is queued. It fills in what the nightly deferred. + +on: + schedule: + - cron: '13 * * * *' + workflow_dispatch: + +permissions: {} + +concurrency: + group: bot-sweep + cancel-in-progress: false + +jobs: + ci-fill: + name: CI on bot PRs + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: write + contents: read + pull-requests: read + steps: + - name: Checkout main + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + persist-credentials: false + + - name: Dispatch CI on bot PRs that have none at their head + env: + GH_TOKEN: ${{ github.token }} + run: hack/bot-ci-dispatch diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 46c75bd8..8a60b807 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -440,8 +440,11 @@ jobs: # component on branch autobump/. Merging it changes share/** on main, # which auto-release.yml turns into a launcher release, and that release # dispatches the `release` job above to build the image. A PR opened with - # GITHUB_TOKEN starts no CI workflows, so the tests run here before it opens. - # A new component is a new hack/components// dir; the matrix picks it up. + # GITHUB_TOKEN gets pull_request CI runs that wait for a maintainer's approval, + # so the tests run here before it opens, and hack/bot-ci-dispatch then runs + # ci.yml on the branch by workflow_dispatch (no approval): that run is the PR's + # CI evidence. A new component is a new hack/components// dir; the matrix + # picks it up. list-components: if: ${{ !inputs.release }} runs-on: ubuntu-latest @@ -466,6 +469,7 @@ jobs: permissions: contents: write pull-requests: write + actions: write strategy: fail-fast: false matrix: @@ -503,6 +507,12 @@ jobs: - name: Render tests run: tests/render_test.sh + # create-pull-request force-pushes an existing bump branch, which GitHub + # refuses from GITHUB_TOKEN once main's workflow files differ from the + # branch's: delete such a branch first (its PR closes; a fresh one opens). + - name: Delete a bump branch whose workflows differ from main's + run: hack/autobump-branch-reset "autobump/${COMPONENT}" + - name: Open or update the bump PR uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 with: @@ -522,27 +532,38 @@ jobs: - Pins (version, chart sha256, image digests) moved in `hack/components/${{ matrix.component }}/component.conf`. - `tests/detect_test.sh` and `tests/render_test.sh` passed in the workflow run that opened this PR. + - CI: this PR's own pull_request runs wait for approval (a GITHUB_TOKEN PR). + The evidence is the ci.yml run dispatched on `autobump/${{ matrix.component }}` + at the head SHA: `gh run list -w ci.yml -e workflow_dispatch -b autobump/${{ matrix.component }}`. - Merging rebuilds the node image with the new manifest baked in. Review the manifest diff for new CRDs, RBAC, images and flags before merging. + - name: Dispatch CI on the bump PR's branch + env: + GH_TOKEN: ${{ github.token }} + run: hack/bot-ci-dispatch --branch "autobump/${COMPONENT}" + # ─── Image digests ─────────────────────────────────────────────────────── # Every image reference is pinned name:tag@sha256 (scripts/images-lock pins). # A registry can re-point a tag (a rebuilt base image, a security respin) with # no new tag, which the version checks above never see. Daily: re-resolve the # tag of every pinned reference, write the registry's digest where it moved, # re-pin images.lock, run the image tests, and open or update one PR on branch - # autobump/image-digests. A PR opened with GITHUB_TOKEN starts no CI workflows, - # so the tests run here before it opens. GITHUB_TOKEN cannot push a change to - # .github/workflows/ (no `workflows` permission), so repin leaves those files - # alone; a workflow reads its image pins from a file outside them (the - # Dockerfile, docs-build/Dockerfile, hack/ci-images.env), which repin moves. + # autobump/image-digests. A PR opened with GITHUB_TOKEN gets pull_request CI + # runs that wait for a maintainer's approval, so the tests run here before it + # opens, and hack/bot-ci-dispatch then runs ci.yml on the branch by + # workflow_dispatch: that run is the PR's CI evidence. GITHUB_TOKEN cannot push + # a change to .github/workflows/ (no `workflows` permission), so repin leaves + # those files alone; a workflow reads its image pins from a file outside them + # (the Dockerfile, docs-build/Dockerfile, hack/ci-images.env), which repin moves. repin-images: if: ${{ !inputs.release }} runs-on: ubuntu-latest permissions: contents: write pull-requests: write + actions: write steps: - name: Checkout main uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -594,12 +615,7 @@ jobs: # its open PR; the action then cuts a fresh branch off main and opens a new one. - name: Delete a digest branch whose workflows differ from main's if: steps.repin.outputs.changes != '' - run: | - if git fetch -q --depth=1 origin autobump/image-digests 2>/dev/null \ - && ! git diff --quiet FETCH_HEAD HEAD -- .github/workflows/; then - echo "autobump/image-digests was cut from a main with other workflow files; deleting it" - git push origin --delete autobump/image-digests - fi + run: hack/autobump-branch-reset autobump/image-digests - name: Open or update the digest PR if: steps.repin.outputs.changes != '' @@ -622,21 +638,32 @@ jobs: - `tests/image_pins_test.sh`, `tests/audit_test.sh`, `tests/detect_test.sh`, `tests/render_test.sh` and the render check passed in the workflow run that opened this PR. + - CI: this PR's own pull_request runs wait for approval (a GITHUB_TOKEN PR). + The evidence is the ci.yml run dispatched on `autobump/image-digests` at the + head SHA: `gh run list -w ci.yml -e workflow_dispatch -b autobump/image-digests`. Review what the upstream rebuild changed before merging. A change under `share/`, `scripts/` or the Dockerfile cuts a launcher release on merge. + - name: Dispatch CI on the digest PR's branch + if: steps.repin.outputs.changes != '' + env: + GH_TOKEN: ${{ github.token }} + run: hack/bot-ci-dispatch --branch autobump/image-digests + # ─── QAT conversion toolchain (console/convert-pins.json) ────────────────── # The model optimizer's one-time OpenVINO conversion pins its pip toolchain # exactly. Daily: move each pin to the newest release optimum-intel's own # requirements allow (hack/bump-convert-pins), run the console tests, and open - # or update one PR on branch autobump/convert-pins. + # or update one PR on branch autobump/convert-pins, whose CI evidence is the + # ci.yml run hack/bot-ci-dispatch runs on the branch. bump-convert-pins: if: ${{ !inputs.release }} runs-on: ubuntu-latest permissions: contents: write pull-requests: write + actions: write steps: - name: Checkout main uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -663,6 +690,12 @@ jobs: tests/console_test.sh tests/model_opt_test.sh + # As in repin-images: GitHub refuses create-pull-request's force-push from + # GITHUB_TOKEN once main's workflow files differ from the branch's. + - name: Delete a pins branch whose workflows differ from main's + if: steps.bump.outputs.changes != '' + run: hack/autobump-branch-reset autobump/convert-pins + - name: Open or update the pins PR if: steps.bump.outputs.changes != '' uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 @@ -682,3 +715,13 @@ jobs: `tests/console_test.sh` and `tests/model_opt_test.sh` passed in the workflow run that opened this PR. + + CI: this PR's own pull_request runs wait for approval (a GITHUB_TOKEN PR). + The evidence is the ci.yml run dispatched on `autobump/convert-pins` at the + head SHA: `gh run list -w ci.yml -e workflow_dispatch -b autobump/convert-pins`. + + - name: Dispatch CI on the pins PR's branch + if: steps.bump.outputs.changes != '' + env: + GH_TOKEN: ${{ github.token }} + run: hack/bot-ci-dispatch --branch autobump/convert-pins diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90f4bde3..3d4ae5bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,11 +5,14 @@ on: branches: [main] pull_request: branches: [main] + # hack/bot-ci-dispatch runs CI on a bot PR's branch this way: the PR's own + # pull_request runs wait for approval (GITHUB_TOKEN opened it), a dispatch does not. + workflow_dispatch: permissions: contents: read -# A new push to a PR supersedes its older run; main runs always finish. +# A new push to a PR supersedes its older run; main and dispatched runs always finish. concurrency: group: ci-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} @@ -54,6 +57,7 @@ jobs: - run: tests/prefill_guard_test.sh - run: tests/ci_wiring_test.sh - run: tests/gate_merge_test.sh + - run: tests/bot_ci_dispatch_test.sh - run: tests/auto_release_cas_test.sh quickstart-tests: @@ -462,3 +466,55 @@ jobs: sudo tar -xzf kubeconform.tar.gz -C /usr/local/bin kubeconform sudo apt-get update && sudo apt-get install -y --no-install-recommends python3-yaml - run: tests/kserve_test.sh + + # ─── Bot PR CI and alarm surfacing ─────────────────────────────────────── + # The watchdog files a ci-alarm issue for a red or missing scheduled run or a + # broken release (email went unread for 9 nights). Every open alarm, and a + # Watchdog workflow that is missing, disabled or without a success in 3 h, + # shows as a warning on every CI run. It never fails CI: the script exits 0, + # and the job is continue-on-error. + alarms: + name: Open CI alarms + runs-on: ubuntu-latest + timeout-minutes: 5 + continue-on-error: true + permissions: + issues: read + actions: read + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + steps: + - name: Warn on open ci-alarm issues and a quiet Watchdog + run: | + set +e + err=$(mktemp) + # Workflow-command data escapes %, CR and LF. + oneline() { tr '\r\n' ' ' < "$err" | sed 's/ *$//; s/%/%25/g'; } + if alarms=$(gh issue list --label ci-alarm --state open --limit 100 --json number,title,url,labels 2>"$err"); then + jq -r '.[] + | "open ci-alarm #\(.number): \(.title) \(.url) (\(if any(.labels[]; .name == "alarm-ack") then "acked" + else "NOT acked; after reading it: gh issue edit \(.number) --add-label alarm-ack" end))" + | "::warning::" + (gsub("%"; "%25") | gsub("\r"; "%0D") | gsub("\n"; "%0A"))' <<<"$alarms" + else + echo "::warning::could not list the open ci-alarm issues: $(oneline)" + fi + if ! wd=$(gh workflow list --all --json name,path,state \ + -q '[.[] | select(.name == "Watchdog")][0] // empty | "\(.state) \(.path)"' 2>"$err"); then + echo "::warning::could not list the workflows to check the Watchdog: $(oneline)" + elif [ -z "$wd" ]; then + echo "::warning::there is no Watchdog workflow, so a red or missing scheduled run raises no ci-alarm" + else + read -r state path <<<"$wd" + if [ "$state" != active ]; then + echo "::warning::the Watchdog workflow is $state, so a red or missing scheduled run raises no ci-alarm" + else + last=$(gh run list -w "${path##*/}" -s success -L 1 --json updatedAt -q '.[0].updatedAt // empty' 2>"$err") + ts=$(date -u -d "${last:-@0}" +%s 2>/dev/null) || ts=0 + if [ $(( $(date -u +%s) - ts )) -gt 10800 ]; then + echo "::warning::the Watchdog workflow has no successful run in the last 3 h (last: ${last:-none}); alarms may be missing" + fi + fi + fi + rm -f "$err" + exit 0 diff --git a/.github/workflows/dependabot-auto-merge.yml b/.github/workflows/dependabot-auto-merge.yml deleted file mode 100644 index cd7a4ecf..00000000 --- a/.github/workflows/dependabot-auto-merge.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Auto-merge Dependabot PRs - -on: pull_request - -permissions: - contents: write - pull-requests: write - -jobs: - automerge: - if: github.actor == 'dependabot[bot]' - runs-on: ubuntu-latest - steps: - - name: Enable auto-merge (squash) - run: gh pr merge --auto --squash "$PR_URL" - env: - PR_URL: ${{ github.event.pull_request.html_url }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index e704433b..c7b924c7 100644 --- a/README.md +++ b/README.md @@ -1704,7 +1704,9 @@ tests/render_test.sh CI fails when a committed manifest differs from its pinned render. The daily build-and-push workflow runs `--latest` for every component and opens one PR per component (branch `autobump/`); merging it changes `share/`, so auto-release cuts a -launcher release and the node image is rebuilt for it. +launcher release and the node image is rebuilt for it. GITHUB_TOKEN opens these PRs, so +their own pull_request CI runs wait for approval; `hack/bot-ci-dispatch` runs `ci.yml` on +the branch by workflow_dispatch instead, and that run is the PR's CI. --- diff --git a/hack/autobump-branch-reset b/hack/autobump-branch-reset new file mode 100755 index 00000000..795943eb --- /dev/null +++ b/hack/autobump-branch-reset @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# hack/autobump-branch-reset : delete origin's autobump/ branch when its +# .github/workflows/ differ from origin/main's. Run from a checkout whose origin +# push credentials are the workflow's GITHUB_TOKEN, before create-pull-request. +# +# create-pull-request resets an existing bump branch (a PR still open, or a merged +# one whose branch stayed) onto main and force-pushes it. GitHub refuses that push +# from GITHUB_TOKEN (it has no `workflows` permission) when the branch's workflow +# files differ from main's, though the bump commit touches none. Deleting the +# branch closes its open PR; create-pull-request then cuts a fresh branch off main +# and opens a new PR, whose dispatched CI runs main's current workflows. +set -euo pipefail +b=${1:-} +case $b in + autobump/?*) ;; + *) echo "usage: hack/autobump-branch-reset autobump/ (refusing '${b}')" >&2; exit 2;; +esac +git ls-remote --exit-code --heads origin "refs/heads/$b" >/dev/null \ + || { rc=$?; [ "$rc" -eq 2 ] && { echo "$b: not on origin; nothing to reset"; exit 0; }; exit "$rc"; } +git fetch -q --depth=1 origin "+refs/heads/main:refs/remotes/origin/main" "+refs/heads/$b:refs/remotes/origin/$b" +if git diff --quiet "origin/main" "origin/$b" -- .github/workflows; then + echo "$b: its workflows match main's; kept" +else + echo "$b: its workflows differ from main's; deleting it (its open PR closes, and a fresh one opens)" + git push origin --delete "$b" +fi diff --git a/hack/bot-ci-dispatch b/hack/bot-ci-dispatch new file mode 100755 index 00000000..96e84560 --- /dev/null +++ b/hack/bot-ci-dispatch @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# hack/bot-ci-dispatch [--branch B] [--workflow ci.yml]: give the bot's PRs real CI. +# +# A PR that GITHUB_TOKEN opened or pushed gets pull_request runs that wait for a +# maintainer's approval (conclusion action_required); a workflow_dispatch run +# needs none. For each open PR into main from a same-repo autobump/* branch opened +# by github-actions[bot] (never a fork, never a human's branch), oldest first: +# - no run of the workflow at the head SHA except held ones: dispatch the +# workflow on the branch (gh workflow run --ref); +# - the latest run at the head failed or timed out on its first attempt, and no +# run there was re-run yet: re-run its failed jobs, once per SHA; +# - otherwise: leave it. +# A run at the head is a pull_request run, or a workflow_dispatch run on the branch. +# +# Runner pool: one ci.yml run is ~39 jobs and a public repo gets 20-40 concurrent +# jobs, so it dispatches or re-runs only while fewer than BOT_CI_MAX_INFLIGHT +# (default 1) runs of the workflow on autobump/* branches are queued or running, +# and not at all while a ci.yml run of a human PR is queued. A deferred PR is +# picked up by the next call (.github/workflows/bot-sweep.yml, hourly). +# +# A branch whose workflow file lacks the workflow_dispatch trigger (cut before +# ci.yml had it) is skipped; the nightly's hack/autobump-branch-reset recreates it. +# Prints one line per decision. Idempotent. Exit 0, or 1 when a gh call failed +# (the rest still ran), 2 on bad usage. +set -uo pipefail + +usage() { echo "usage: hack/bot-ci-dispatch [--branch autobump/] [--workflow ci.yml]" >&2; exit 2; } +wf=ci.yml only= +while [ $# -gt 0 ]; do + case $1 in + --branch) [ $# -ge 2 ] || usage; only=$2; shift 2;; + --workflow) [ $# -ge 2 ] || usage; wf=$2; shift 2;; + -h|--help) sed -n '2,/^set /{/^set /d;s/^# \{0,1\}//;p}' "$0"; exit 0;; + *) usage;; + esac +done +cap=${BOT_CI_MAX_INFLIGHT:-1} +[[ $cap =~ ^[0-9]+$ ]] || { echo "BOT_CI_MAX_INFLIGHT must be a whole number, got '$cap'" >&2; exit 2; } +RUNF=databaseId,event,status,conclusion,headBranch,attempt,createdAt +rc=0 + +prs=$(gh pr list --state open --base main --limit 200 \ + --json number,headRefName,headRefOid,baseRefName,isCrossRepository,author) \ + || { echo "error: cannot list open PRs"; exit 1; } +cands=$(jq -r --arg only "$only" ' + .[] | select(.isCrossRepository == false and .baseRefName == "main" + and (.headRefName | startswith("autobump/")) + and (.author.login | IN("app/github-actions", "github-actions[bot]", "github-actions")) + and (.author.is_bot != false) + and ($only == "" or .headRefName == $only)) + | "\(.number) \(.headRefName) \(.headRefOid)"' <<<"$prs" | sort -n) +[ -n "$cands" ] || { echo "no open github-actions[bot] PR from ${only:-an autobump/ branch}"; exit 0; } + +# The pool: bot runs of the workflow in flight, and queued ci.yml runs of human PRs. +recent=$(gh run list -w "$wf" -L 100 --json status,event,headBranch) || { echo "error: cannot list $wf runs"; exit 1; } +inflight=$(jq '[.[] | select(.status != "completed" and (.headBranch | startswith("autobump/")))] | length' <<<"$recent") +if [ "$wf" != ci.yml ]; then + recent=$(gh run list -w ci.yml -L 100 --json status,event,headBranch) || { echo "error: cannot list ci.yml runs"; exit 1; } +fi +human=$(jq '[.[] | select(.event == "pull_request" and (.status | IN("queued", "requested", "pending", "waiting")) + and (.headBranch | startswith("autobump/") | not))] | length' <<<"$recent") +# busy LINE: print LINE with the reason and succeed when the pool has no room. +busy() { + if [ "$human" -gt 0 ]; then echo "$1: deferred, $human ci.yml run(s) of human PRs queued"; return 0; fi + if [ "$inflight" -ge "$cap" ]; then echo "$1: deferred, $inflight bot $wf run(s) in flight (cap $cap)"; return 0; fi + return 1 +} + +while read -r num br sha; do + tag="#$num $br ${sha:0:7}" + runs=$(gh run list -w "$wf" -c "$sha" -L 100 --json "$RUNF") \ + || { echo "$tag: error: cannot list its $wf runs"; rc=1; continue; } + # " " + read -r count held id st co att reran <<<"$(jq -r --arg b "$br" ' + def word: if . == null or . == "" then "-" else tostring end; + [.[] | select(.event == "pull_request" or (.event == "workflow_dispatch" and .headBranch == $b))] as $at + | ($at | map(select(.conclusion != "action_required"))) as $e + | ($e | max_by(.createdAt // "")) as $l + | "\($e | length) \(($at | length) - ($e | length)) \($l.databaseId | word) \($l.status | word) \($l.conclusion | word) \($l.attempt // 1) \(any($e[]; (.attempt // 1) > 1))"' <<<"$runs")" + if [ "$count" -eq 0 ]; then + why="no $wf run at the head"; [ "$held" -gt 0 ] && why="only held $wf runs at the head" + busy "$tag: $why" && continue + if out=$(gh workflow run "$wf" --ref "$br" 2>&1); then + inflight=$((inflight + 1)) + # Wait until the run shows, so that a call right after this one sees it. + new= + for _ in 1 2 3 4 5 6 7 8 9 10; do + new=$(gh run list -w "$wf" -b "$br" -e workflow_dispatch -c "$sha" -L 1 --json databaseId -q '.[0].databaseId // empty' 2>/dev/null) + [ -n "$new" ] && break; sleep 3 + done + echo "$tag: $why; dispatched $wf${new:+ (run $new)}" + elif grep -qi 'workflow_dispatch' <<<"$out"; then + echo "$tag: skipped, $wf on $br has no workflow_dispatch trigger (the nightly recreates the branch from main)" + else + echo "$tag: error: dispatching $wf failed: $(tr '\n' ' ' <<<"$out")"; rc=1 + fi + elif [ "$st" = completed ] && [[ $co =~ ^(failure|timed_out)$ ]] && [ "$att" = 1 ] && [ "$reran" = false ]; then + busy "$tag: $wf run $id $co on attempt 1" && continue + if out=$(gh run rerun "$id" --failed 2>&1); then + inflight=$((inflight + 1)); echo "$tag: $wf run $id $co on attempt 1; re-ran its failed jobs" + else + echo "$tag: error: re-running run $id failed: $(tr '\n' ' ' <<<"$out")"; rc=1 + fi + elif [ "$st" = completed ] && [ "$co" != success ]; then + echo "$tag: $wf run $id is $co (attempt $att); left for review" + else + echo "$tag: $wf run $id at the head is $st/$co; nothing to do" + fi +done <<<"$cands" +exit "$rc" diff --git a/hack/gate_lib.sh b/hack/gate_lib.sh new file mode 100644 index 00000000..6bdb82e1 --- /dev/null +++ b/hack/gate_lib.sh @@ -0,0 +1,186 @@ +# shellcheck shell=bash +# gate_lib.sh: what the merge gate (hack/gate_merge.sh) reads to decide a PR is +# green and current, for sourcing. Run from inside the repository; gh and git do +# the reading. +# +# READ jq program over `gh pr view --json headRefOid,statusCheckRollup` +# (takes --argjson held_ok BOOL): " +# ", the latest entry per check. +# RUNS jq program over `gh run list -c SHA -e pull_request --json +# workflowName,status,conclusion,createdAt` (takes --argjson +# held_ok BOOL): " ", the latest +# run per workflow. +# ci_status SHA BR green | pending | failed | none: ci.yml at SHA, through a +# pull_request run or a workflow_dispatch run on branch BR. +# ci_eval SHA BR " ", held_ok as below. +# base_ok SHA main is an ancestor of SHA, or main moved only in the ledger or +# by release-only bot commits, or SHA changes only the ledger. +# rules_stale SHA the tree main + SHA would merge to carries a +# console/ledger-rules.json its EXPERIMENTS.md does not generate. +# open_alarms the open ci-alarm issues as a JSON array (gh issue list). +# gated_paths reads paths on stdin, prints those a merge must not ship while +# an alarm is open and not acked (see below). +# +# held_ok: a PR that GITHUB_TOKEN opened or pushed gets pull_request runs held for +# approval (conclusion action_required, no jobs). hack/bot-ci-dispatch runs ci.yml +# on the PR branch by workflow_dispatch instead. Once the latest such dispatched +# run at the SHA succeeded, a held entry of workflow 'CI' or 'Auto-merge Dependabot +# PRs' no longer counts as terminal (READ) or failed (RUNS). Without it, a held +# entry stays terminal, as for a fork PR. + +GATE_LIB_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# The workflows whose held pull_request runs a green dispatched ci.yml run stands in for. +GATE_HELD_WORKFLOWS='["CI", "Auto-merge Dependabot PRs"]' + +# One snapshot: " ", latest run per check. +# An in-progress CheckRun reports conclusion "", a legacy StatusContext only .state. +# READ and RUNS are jq programs ($names are jq's) for the gate that sources this file. +# shellcheck disable=SC2016,SC2034 +READ='def green: IN("SUCCESS", "SKIPPED", "NEUTRAL"); + def terminal: IN("FAILURE", "ERROR", "CANCELLED", "TIMED_OUT", "STARTUP_FAILURE", "ACTION_REQUIRED", "STALE"); + ('"$GATE_HELD_WORKFLOWS"') as $held + | .headRefOid as $sha + | [.statusCheckRollup[] + | {k: [.__typename, (.name // .context), (.workflowName // "")], t: (.startedAt // ""), n: (.name // .context), + w: (.workflowName // ""), + v: ((if (.conclusion // "") != "" then .conclusion else (.state // "PENDING") end) | ascii_upcase)} + | select(($held_ok and .v == "ACTION_REQUIRED" and (.w | IN($held[]))) | not)] + | [group_by(.k)[] | if any(.v | (green or terminal) | not) then .[0] + {v: "PENDING"} else max_by(.t) end] as $c + | ($c | map(select(.v | green)) | length) as $g + | ($c | map(select(.v | terminal))) as $b + | "\($sha) \($g) \($b | length) \($c | length) \($b | map("\(.n)=\(.v)") | join(","))"' +# Every pull_request workflow run for a SHA, latest per workflow: " ", +# state = green | pending | failed. +# shellcheck disable=SC2016,SC2034 +RUNS='('"$GATE_HELD_WORKFLOWS"') as $held + | [.[] | select(($held_ok and .conclusion == "action_required" and (.workflowName | IN($held[]))) | not)] + | [group_by(.workflowName)[] | max_by(.createdAt // "")] as $r + | ($r | map(select(.status != "completed"))) as $p + | ($r | map(select(.status == "completed" and (.conclusion | IN("success", "skipped", "neutral") | not)))) as $f + | if ($f | length) > 0 then "failed \($f | map("\(.workflowName)=\(.conclusion)") | join(","))" + elif ($p | length) > 0 then "pending -" else "green -" end' +# ci.yml runs at a SHA: $p its pull_request runs, $d its workflow_dispatch runs, $b the +# PR branch. Held runs (action_required) are not attempts: the latest completed other +# run decides, a queued or running one means pending, and held runs alone are failed. +# shellcheck disable=SC2016 +CI_EVAL='def ok: .conclusion == "success"; + def word: if . == null or . == "" then "-" else . end; + ($p | map(. + {via: "pull_request"})) + ($d | map(select(.headBranch == $b) | . + {via: "workflow_dispatch"})) as $all + | ($all | map(select(.conclusion != "action_required"))) as $e + | ($e | map(select(.status == "completed")) | max_by(.createdAt // "")) as $last + | ($e | map(select(.via == "workflow_dispatch" and .status == "completed")) | max_by(.createdAt // "")) as $dl + | (if $last != null and ($last | ok) then "green" + elif any($e[]; .status != "completed") then "pending" + elif $last != null or ($all | length) > 0 then "failed" + else "none" end) as $s + | ($s == "green" and $dl != null and ($dl | ok)) as $held_ok + | ($last // ($all | max_by(.createdAt // "")) // {}) as $show + | "\($s) \($held_ok) \($show.via | word):\($show.status | word)/\($show.conclusion | word)"' +ci_eval() { + local p d + p=$(gh run list -w ci.yml -c "$1" -e pull_request -L 100 --json event,headBranch,status,conclusion,createdAt 2>/dev/null) || p= + d=$(gh run list -w ci.yml -c "$1" -e workflow_dispatch -b "$2" -L 100 --json event,headBranch,status,conclusion,createdAt 2>/dev/null) || d= + jq -rn --arg b "$2" --argjson p "${p:-[]}" --argjson d "${d:-[]}" "$CI_EVAL" +} +ci_status() { local s; read -r s _ <<<"$(ci_eval "$1" "$2")"; echo "${s:-none}"; } + +# base_ok SHA: main is an ancestor of SHA, or main moved only in the ledger (or by +# release-only bot commits), or the PR is ledger-only. +code() { git diff --name-only "$1" "$2" | grep -qvxE 'EXPERIMENTS\.md|docs/provenance\.md'; } +# release_only C: C is one of the release commits auto-release.yml and its followers +# push to main: a 'release: ... [skip release]' subject by 88plug-bot or +# github-actions[bot], touching only the release files, and in scripts/k3d-gpu only +# its VERSION="..." line. +release_only() { + local c=$1 f + case $(git log -1 --format=%s "$c") in 'release:'*'[skip release]'*) ;; *) return 1;; esac + case $(git log -1 --format=%an "$c") in 88plug-bot|'github-actions[bot]') ;; *) return 1;; esac + while IFS= read -r f; do + case $f in + packaging/aur/PKGBUILD|packaging/aur/.SRCINFO|Formula/k3d-gpu.rb|packaging/install.sh|console/ledger-rules.json) ;; + scripts/k3d-gpu) + git diff -U0 "$c^" "$c" -- scripts/k3d-gpu \ + | awk '/^@@/ {h = 1; next} h && /^[-+]/ && !/^[-+]VERSION="/ {bad = 1} END {exit bad ? 0 : 1}' && return 1;; + *) return 1;; + esac + done < <(git diff --name-only "$c^" "$c") + return 0 +} +# main_code MB: since MB, main changed a file outside the ledger in a commit that is +# not release-only. When no commit can be listed, that counts as a change. +main_code() { + code "$1" origin/main || return 1 + local c n=0 + while read -r c; do + n=$((n + 1)) + release_only "$c" && continue + code "$c^" "$c" && return 0 + done < <(git rev-list "$1..origin/main") + [ "$n" -eq 0 ] +} +base_ok() { + git fetch -q origin || return 1 + git merge-base --is-ancestor origin/main "$1" && return 0 + local mb; mb=$(git merge-base origin/main "$1") || return 1 + ! { main_code "$mb" && code "$mb" "$1"; } +} + +# rules_stale SHA: the tree main + SHA would merge to carries a console/ledger-rules.json +# that EXPERIMENTS.md there does not generate (read with that tree's own model_opt.py). +# Unknown (a conflict, no such files) is not stale: a conflict syncs anyway. +rules_stale() { + local tree d rc=1 + tree=$(git merge-tree --write-tree origin/main "$1" 2>/dev/null | head -1) + [ -n "$tree" ] || return 1 + d=$(mktemp -d) + if git show "$tree:console/model_opt.py" > "$d/model_opt.py" 2>/dev/null \ + && git show "$tree:console/convert-pins.json" > "$d/convert-pins.json" 2>/dev/null \ + && git show "$tree:console/ledger-rules.json" > "$d/have.json" 2>/dev/null \ + && git show "$tree:EXPERIMENTS.md" > "$d/EXPERIMENTS.md" 2>/dev/null \ + && [ -s "$d/model_opt.py" ] && [ -s "$d/have.json" ] \ + && python3 "$d/model_opt.py" ledger-rules --ledger "$d/EXPERIMENTS.md" --out "$d/want.json" >/dev/null 2>&1 \ + && ! cmp -s "$d/have.json" "$d/want.json"; then + rc=0 + fi + rm -rf "$d" + return "$rc" +} + +# open_alarms: the open ci-alarm issues, [{number, title, url, labels}]; fails when gh does. +open_alarms() { + local a + a=$(gh issue list --label ci-alarm --state open --limit 100 --json number,title,url,labels) || return 1 + jq -c 'if type == "array" then . else error("not an array") end' <<<"${a:-[]}" +} +# gated_paths: of the paths on stdin, print those a merge must not ship while an +# alarm is open and not acked: .github/** and the shipped set. A PR that changes +# only EXPERIMENTS.md, docs/, bench/ or tests/ (the ledger work) prints nothing. +# Shipped set: packaging/, scripts/, share/, Dockerfile, console/, images.lock, plus +# every pattern in hack/release-paths (one per line, # comments, dir/ or dir/** for +# a tree, else a shell glob; an executable file is run for its lines). It is read +# from $GATE_RELEASE_PATHS, else next to this file, else the repository's hack/. +gated_paths() { + local paths p f pats=(packaging/ scripts/ share/ Dockerfile console/ images.lock) rp + paths=$(grep -v '^$') + [ -n "$paths" ] || return 0 + grep -qvE '^(EXPERIMENTS\.md|(docs|bench|tests)/.*)$' <<<"$paths" || return 0 + rp=${GATE_RELEASE_PATHS:-} + [ -n "$rp" ] || for f in "$GATE_LIB_DIR/release-paths" hack/release-paths; do [ -f "$f" ] && { rp=$f; break; }; done + if [ -n "$rp" ] && [ -f "$rp" ]; then + while IFS= read -r p; do + p=${p%%#*}; p=${p#"${p%%[![:space:]]*}"}; p=${p%"${p##*[![:space:]]}"}; p=${p#- }; p=${p//[\"\']/} + [ -n "$p" ] && pats+=("$p") + done < <(if [ -x "$rp" ]; then "$rp"; else cat "$rp"; fi) + fi + while IFS= read -r f; do + case $f in .github/*) echo "$f"; continue;; esac + for p in "${pats[@]}"; do + case $p in */\*\*) p=${p%\*\*};; esac + # shellcheck disable=SC2254 # p is a glob on purpose + case $p in + */) case $f in "$p"*) echo "$f"; break;; esac;; + *) case $f in $p|"$p"/*) echo "$f"; break;; esac;; + esac + done + done <<<"$paths" +} diff --git a/hack/gate_merge.sh b/hack/gate_merge.sh index 94f131bd..6bfcbbaa 100755 --- a/hack/gate_merge.sh +++ b/hack/gate_merge.sh @@ -2,14 +2,28 @@ # gate_merge.sh : keep the PR branch current with main # (ledger/provenance conflicts resolved keep-both; anything else -> stop), # then squash-merge the head SHA whose checks were read. Up to 5 sync rounds. +# The checks it reads live in hack/gate_lib.sh, which it sources from its own +# directory, else from the repository's hack/. # # Green means: every latest rollup entry is SUCCESS, SKIPPED or NEUTRAL (a -# check with any queued or running attempt is pending); the ci.yml -# pull_request run for that exact SHA completed with success; and the latest -# run of every other pull_request workflow for that SHA completed with -# success, skipped or neutral. A terminal non-green entry (FAILURE, ERROR, -# CANCELLED, TIMED_OUT, STARTUP_FAILURE, ACTION_REQUIRED, STALE) or a failed -# workflow run exits 1. +# check with any queued or running attempt is pending); ci.yml for that exact +# SHA succeeded, through a pull_request run or a workflow_dispatch run on the PR +# branch (the latest completed one decides); and the latest run of every other +# pull_request workflow for that SHA completed with success, skipped or neutral. +# A terminal non-green entry (FAILURE, ERROR, CANCELLED, TIMED_OUT, +# STARTUP_FAILURE, ACTION_REQUIRED, STALE), a failed workflow run or a failed +# ci.yml exits 1. +# +# Bot PRs: a PR that GITHUB_TOKEN opened or pushed gets pull_request runs held +# for approval (ACTION_REQUIRED). hack/bot-ci-dispatch runs ci.yml on its branch +# by workflow_dispatch instead; once that run is green, the held entries of 'CI' +# and 'Auto-merge Dependabot PRs' no longer block. Held runs alone still exit 1. +# +# Alarms: the gate first prints every open ci-alarm issue. While any of them +# lacks the label alarm-ack, it refuses (exit 4) a PR that changes .github/ or a +# shipped path (see gated_paths in gate_lib.sh), at the start and again right +# before merging; a PR that changes only EXPERIMENTS.md, docs/, bench/ or tests/ +# is never refused. Read the alarm, then ack it: gh issue edit N --add-label alarm-ack. # # Derived file: console/ledger-rules.json is EXPERIMENTS.md's rule rows as packages # ship them. A ledger-only PR (or base) merges without a CI re-run, so the gate @@ -19,9 +33,10 @@ # Moved base: if main is no longer an ancestor of the head, and both main # (since the merge base) and the PR change files other than EXPERIMENTS.md and # docs/provenance.md, the gate syncs the branch so CI re-runs on the merged -# head. A base that moved only in the ledger, or under a ledger-only PR, is -# merged without a re-run. Another PR can still land between this check and -# the merge; that window stays open. +# head. A base that moved only in the ledger or by release-only bot commits +# ('release: ... [skip release]' on the release files), or under a ledger-only +# PR, is merged without a re-run. Another PR can still land between this check +# and the merge; that window stays open. # # The merge carries --match-head-commit, so GitHub merges only if the head is # still that SHA. On a refusal the gate prints GitHub's reason and re-reads the @@ -29,8 +44,11 @@ # sleeps, and the next round re-checks mergeable (sync on a conflict) and the # base before merging again. # +# Exit: 0 merged, 1 failed or gave up, 2 gh lacks --match-head-commit, 4 refused +# for an open alarm. # Env: GATE_REPO (default: the caller's git toplevel), GATE_WT_ROOT (sync # worktree parent, default $TMPDIR), GATE_POLLS (polls per round, default 60), +# GATE_RELEASE_PATHS (shipped-path patterns, default hack/release-paths), # GATE_SELF_DIR (a directory removed on exit; set by the scratchpad shim). set -u; pr=$1; subj=$2; body=$3 HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) @@ -42,9 +60,38 @@ cleanup() { if [ -n "${GATE_SELF_DIR:-}" ]; then rm -rf "$GATE_SELF_DIR"; fi } trap cleanup EXIT +for lib in "$HERE/gate_lib.sh" hack/gate_lib.sh ""; do [ -f "$lib" ] && break; done +[ -n "$lib" ] || { echo "gate_lib.sh not found next to $0 or in $PWD/hack; copy it beside the gate"; exit 1; } +# shellcheck source=hack/gate_lib.sh +. "$lib" gh pr merge --help 2>/dev/null | grep -q -- --match-head-commit \ || { echo "gh pr merge lacks --match-head-commit; refusing to merge unpinned"; exit 2; } br=$(gh pr view "$pr" --json headRefName -q .headRefName) || exit 1 +# alarm_check [list]: with `list`, print every open ci-alarm issue. Exit 4 when one +# is not acked and the PR changes a gated path (or its paths cannot be read). +alarm_check() { + local a un paths gated e + e=$(mktemp) + if ! a=$(open_alarms 2>"$e"); then + echo "WARNING: could not list the open ci-alarm issues ($(tr '\n' ' ' < "$e" | sed 's/ *$//')); the alarm check was skipped" + rm -f "$e"; return 0 + fi + rm -f "$e" + [ "${1:-}" = list ] && jq -r '.[] | "open ci-alarm #\(.number): \(.title) \(.url)\(if any(.labels[]; .name == "alarm-ack") then " (acked)" else " (NOT acked)" end)"' <<<"$a" + un=$(jq -r '.[] | select(any(.labels[]; .name == "alarm-ack") | not) | " #\(.number) \(.title) \(.url)"' <<<"$a") + [ -n "$un" ] || return 0 + if paths=$(gh api --paginate "repos/{owner}/{repo}/pulls/$pr/files" -q '.[].filename' 2>/dev/null); then + gated=$(gated_paths <<<"$paths") + [ -n "$gated" ] || return 0 + echo "PR$pr REFUSED: it changes $(head -5 <<<"$gated" | paste -sd, -)$([ "$(wc -l <<<"$gated")" -gt 5 ] && echo ', ...') while these ci-alarm issues are open and not acked:" + else + echo "PR$pr REFUSED: its changed files could not be listed, and these ci-alarm issues are open and not acked:" + fi + echo "$un" + jq -r '.[] | select(any(.labels[]; .name == "alarm-ack") | not) | "Read #\(.number), then ack it: gh issue edit \(.number) --add-label alarm-ack"' <<<"$a" + exit 4 +} +alarm_check list sync() { git fetch -q origin git worktree remove --force "$W" 2>/dev/null; git worktree add -q -B "gm/$pr" "$W" "origin/$br" || return 1 @@ -70,69 +117,22 @@ sync() { fi ) || return 1 git -C "$W" push -q origin "HEAD:refs/heads/$br" && echo "synced $br" } -# One snapshot: " ", latest run per check. -# An in-progress CheckRun reports conclusion "", a legacy StatusContext only .state. -READ='def green: IN("SUCCESS", "SKIPPED", "NEUTRAL"); - def terminal: IN("FAILURE", "ERROR", "CANCELLED", "TIMED_OUT", "STARTUP_FAILURE", "ACTION_REQUIRED", "STALE"); - .headRefOid as $sha - | [.statusCheckRollup[] - | {k: [.__typename, (.name // .context), (.workflowName // "")], t: (.startedAt // ""), n: (.name // .context), - v: ((if (.conclusion // "") != "" then .conclusion else (.state // "PENDING") end) | ascii_upcase)}] - | [group_by(.k)[] | if any(.v | (green or terminal) | not) then .[0] + {v: "PENDING"} else max_by(.t) end] as $c - | ($c | map(select(.v | green)) | length) as $g - | ($c | map(select(.v | terminal))) as $b - | "\($sha) \($g) \($b | length) \($c | length) \($b | map("\(.n)=\(.v)") | join(","))"' -# Every pull_request workflow run for a SHA, latest per workflow: " ", -# state = green | pending | failed. -RUNS='[group_by(.workflowName)[] | max_by(.createdAt // "")] as $r - | ($r | map(select(.status != "completed"))) as $p - | ($r | map(select(.status == "completed" and (.conclusion | IN("success", "skipped", "neutral") | not)))) as $f - | if ($f | length) > 0 then "failed \($f | map("\(.workflowName)=\(.conclusion)") | join(","))" - elif ($p | length) > 0 then "pending -" else "green -" end' -# base_ok SHA: main is an ancestor of SHA, or main moved only in the ledger, or the PR is ledger-only. -code() { git diff --name-only "$1" "$2" | grep -qvxE 'EXPERIMENTS\.md|docs/provenance\.md'; } -base_ok() { - git fetch -q origin || return 1 - git merge-base --is-ancestor origin/main "$1" && return 0 - local mb; mb=$(git merge-base origin/main "$1") || return 1 - ! { code "$mb" origin/main && code "$mb" "$1"; } -} -# rules_stale SHA: the tree main + SHA would merge to carries a console/ledger-rules.json -# that EXPERIMENTS.md there does not generate (read with that tree's own model_opt.py). -# Unknown (a conflict, no such files) is not stale: a conflict syncs anyway. -rules_stale() { - local tree d rc=1 - tree=$(git merge-tree --write-tree origin/main "$1" 2>/dev/null | head -1) - [ -n "$tree" ] || return 1 - d=$(mktemp -d) - if git show "$tree:console/model_opt.py" > "$d/model_opt.py" 2>/dev/null \ - && git show "$tree:console/convert-pins.json" > "$d/convert-pins.json" 2>/dev/null \ - && git show "$tree:console/ledger-rules.json" > "$d/have.json" 2>/dev/null \ - && git show "$tree:EXPERIMENTS.md" > "$d/EXPERIMENTS.md" 2>/dev/null \ - && [ -s "$d/model_opt.py" ] && [ -s "$d/have.json" ] \ - && python3 "$d/model_opt.py" ledger-rules --ledger "$d/EXPERIMENTS.md" --out "$d/want.json" >/dev/null 2>&1 \ - && ! cmp -s "$d/have.json" "$d/want.json"; then - rc=0 - fi - rm -rf "$d" - return "$rc" -} [ "$(gh pr view "$pr" --json isDraft -q .isDraft)" = true ] && gh pr ready "$pr" >/dev/null merged() { [ "$(gh pr view "$pr" --json state -q .state)" = MERGED ]; } for round in 1 2 3 4 5; do merged && { echo "PR$pr MERGED"; exit 0; } [ "$(gh pr view "$pr" --json mergeable -q .mergeable)" = CONFLICTING ] && { sync || exit 1; sleep 60; } for _ in $(seq 1 "${GATE_POLLS:-60}"); do - s=$(gh pr view "$pr" --json headRefOid,statusCheckRollup -q "$READ" 2>/dev/null) || { sleep 60; continue; } - read -r sha g b n bad <<<"$s" + pj=$(gh pr view "$pr" --json headRefOid,statusCheckRollup 2>/dev/null) && [ -n "$pj" ] || { sleep 60; continue; } + sha=$(jq -r .headRefOid <<<"$pj") + read -r ci held_ok ci_last <<<"$(ci_eval "$sha" "$br")" + read -r sha g b n bad <<<"$(jq -r --argjson held_ok "${held_ok:-false}" "$READ" <<<"$pj")" [ "$b" -gt 0 ] && { echo "PR$pr FAIL at $sha: $bad"; exit 1; } if [ "$n" -gt 0 ] && [ "$g" -eq "$n" ]; then - ci=$(gh run list -w ci.yml -c "$sha" -e pull_request -L 1 --json status,conclusion \ - -q '.[0] | "\(.status)/\(.conclusion)"' 2>/dev/null) case $ci in - completed/success) + green) read -r rs rbad <<<"$(gh run list -c "$sha" -e pull_request -L 100 \ - --json workflowName,status,conclusion,createdAt -q "$RUNS" 2>/dev/null)" + --json workflowName,status,conclusion,createdAt 2>/dev/null | jq -r --argjson held_ok "$held_ok" "$RUNS" 2>/dev/null)" case $rs in failed) echo "PR$pr FAIL: workflow runs for $sha: $rbad"; exit 1;; green) ;; @@ -146,6 +146,7 @@ for round in 1 2 3 4 5; do echo "PR$pr round $round: console/ledger-rules.json would not match EXPERIMENTS.md after the merge: sync regenerates it" sync || exit 1; sleep 60; break fi + alarm_check why=$(gh pr merge "$pr" --squash --match-head-commit "$sha" --subject "$subj" --body "$body" 2>&1 >/dev/null) merged && { git push -q origin --delete "$br" 2>/dev/null; echo "PR$pr MERGED at $sha"; exit 0; } now=$(gh pr view "$pr" --json headRefOid -q .headRefOid 2>/dev/null) @@ -156,7 +157,7 @@ for round in 1 2 3 4 5; do sleep 60 fi break;; - completed/*) echo "PR$pr FAIL: ci.yml run for $sha is $ci"; exit 1;; + failed) echo "PR$pr FAIL: ci.yml for $sha did not succeed (latest $ci_last)"; exit 1;; esac # no ci.yml run for this SHA yet, or still running: wait fi sleep 60 diff --git a/tests/bot_ci_dispatch_test.sh b/tests/bot_ci_dispatch_test.sh new file mode 100755 index 00000000..9cbcd211 --- /dev/null +++ b/tests/bot_ci_dispatch_test.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# hack/bot-ci-dispatch against a fake gh (no network). +# The fake keeps open PRs in prs.json and workflow runs in runs.json. `gh run +# list` filters runs like the API (-w workflow file, -c head SHA, -b branch, +# -e event, -L count, newest first). `gh workflow run` appends a queued +# workflow_dispatch run at the branch's head, unless the branch is listed in +# no_dispatch (its workflow file lacks the trigger: HTTP 422) or dispatch_fail +# exists (HTTP 500). `gh run rerun ID --failed` queues attempt 2 of that run. +# Every dispatch and rerun is logged to actions. +# +# Usage: tests/bot_ci_dispatch_test.sh + +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +BCD=${BCD:-${ROOT}/hack/bot-ci-dispatch} +T=$(mktemp -d "${TMPDIR:-/tmp}/bot_ci_dispatch.XXXXXX") +trap 'rm -rf "$T"' EXIT +PASS=0 FAILS=0 +ok() { PASS=$((PASS + 1)); echo "ok $1"; } +fail() { FAILS=$((FAILS + 1)); echo "FAIL $1"; } + +mkdir -p "$T/bin" +cat > "$T/bin/gh" <<'STUB' +#!/usr/bin/env bash +F=$FIX +echo "gh $*" >> "$F/calls" +case "$1 $2" in + "pr list") cat "$F/prs.json"; exit 0;; + "run list") + shift 2; wf= c= b= e= q= L=1000 + while [ $# -gt 0 ]; do case $1 in + -w|--workflow) wf=$2; shift 2;; -c|--commit) c=$2; shift 2;; -b|--branch) b=$2; shift 2;; + -e|--event) e=$2; shift 2;; -q|--jq) q=$2; shift 2;; -L|--limit) L=$2; shift 2;; --json) shift 2;; *) shift;; esac; done + out=$(jq --arg w "$wf" --arg c "$c" --arg b "$b" --arg e "$e" --argjson L "$L" \ + '[.[] | select(($w == "" or .workflow == $w) and ($c == "" or .headSha == $c) + and ($b == "" or .headBranch == $b) and ($e == "" or .event == $e))] + | sort_by(.createdAt) | reverse | .[:$L]' "$F/runs.json") + if [ -n "$q" ]; then jq -r "$q" <<<"$out"; else echo "$out"; fi + exit 0;; + "workflow run") + wf=$3; shift 3; ref= + while [ $# -gt 0 ]; do case $1 in -r|--ref) ref=$2; shift 2;; *) shift;; esac; done + if grep -qxF "$ref" "$F/no_dispatch" 2>/dev/null; then + echo "could not create workflow dispatch event: HTTP 422: Workflow does not have 'workflow_dispatch' trigger (https://api.github.com/repos/o/r/actions/workflows/1/dispatches)" >&2 + exit 1 + fi + [ -f "$F/dispatch_fail" ] && { echo "could not create workflow dispatch event: HTTP 500: Server Error" >&2; exit 1; } + sha=$(jq -r --arg b "$ref" '[.[] | select(.headRefName == $b)][0].headRefOid // "unknown"' "$F/prs.json") + n=$(( $(jq length "$F/runs.json") + 900 )) + jq --arg w "$wf" --arg b "$ref" --arg s "$sha" --argjson id "$n" \ + '. + [{databaseId: $id, workflow: $w, event: "workflow_dispatch", headBranch: $b, headSha: $s, + status: "queued", conclusion: "", attempt: 1, createdAt: "2026-10-29T23:00:00Z"}]' \ + "$F/runs.json" > "$F/runs.tmp" && mv "$F/runs.tmp" "$F/runs.json" + echo "dispatch $wf $ref" >> "$F/actions" + exit 0;; + "run rerun") + id=$3; echo "rerun $id ${*:4}" >> "$F/actions" + jq --argjson id "$id" 'map(if .databaseId == $id then .attempt += 1 | .status = "queued" | .conclusion = "" else . end)' \ + "$F/runs.json" > "$F/runs.tmp" && mv "$F/runs.tmp" "$F/runs.json" + exit 0;; +esac +exit 0 +STUB +printf '#!/bin/sh\nexit 0\n' > "$T/bin/sleep" +chmod +x "$T/bin/"* + +BOT='{"login":"app/github-actions","is_bot":true}' +HUMAN='{"login":"88plug","is_bot":false}' +DEPENDABOT='{"login":"app/dependabot","is_bot":true}' +# pr NUMBER BRANCH SHA [AUTHOR] [CROSS]: one open PR into main. +pr() { + printf '{"number":%s,"headRefName":"%s","headRefOid":"%s","baseRefName":"main","isCrossRepository":%s,"author":%s}' \ + "$1" "$2" "$3" "${5:-false}" "${4:-$BOT}" +} +# run ID WORKFLOW EVENT BRANCH SHA STATUS CONCLUSION [ATTEMPT] [CREATED] +run() { + printf '{"databaseId":%s,"workflow":"%s","event":"%s","headBranch":"%s","headSha":"%s","status":"%s","conclusion":"%s","attempt":%s,"createdAt":"%s"}' \ + "$1" "$2" "$3" "$4" "$5" "$6" "$7" "${8:-1}" "${9:-2026-10-29T20:00:00Z}" +} +# fixture NAME PRS RUNS +fixture() { + FIX=$T/$1; export FIX; mkdir -p "$FIX" + echo "$2" > "$FIX/prs.json"; echo "$3" > "$FIX/runs.json"; : > "$FIX/calls"; : > "$FIX/actions" +} +bcd() { OUT=$(PATH="$T/bin:$PATH" timeout 60 bash "$BCD" "$@" 2>&1); RC=$?; } +acts() { grep -c "^$1 " "$FIX/actions"; } +dispatched() { grep -qxF "dispatch ${2:-ci.yml} $1" "$FIX/actions"; } + +# 1. scope: only same-repo autobump/* PRs by github-actions[bot] with no run at the +# head (a held run does not count) get a dispatch; forks, human branches, a +# human's autobump/ branch and dependabot never do. +fixture scope "[$(pr 1 autobump/a a111),$(pr 2 autobump/fork f222 "$BOT" true),$(pr 3 claude/x c333 "$HUMAN"),$(pr 4 autobump/human h444 "$HUMAN"),$(pr 5 dependabot/pip/x d555 "$DEPENDABOT"),$(pr 6 autobump/done e666),$(pr 7 autobump/held g777)]" \ + "[$(run 1 ci.yml pull_request autobump/done e666 completed success),$(run 2 ci.yml pull_request autobump/held g777 completed action_required)]" +BOT_CI_MAX_INFLIGHT=10 bcd +if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 2 ] && dispatched autobump/a && dispatched autobump/held; then + ok "scope: dispatched autobump/a and the held autobump/held only" +else fail "scope: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi +if grep -q '#6 autobump/done' <<<"$OUT" && ! grep -qE '#(2|3|4|5) ' <<<"$OUT"; then + ok "scope: a line for the PR with a run at its head, none for the out-of-scope PRs" +else fail "scope: decision lines: $(tr '\n' ';' <<<"$OUT")"; fi + +# 2. the in-flight cap (default 1): two PRs, nothing running -> one dispatch, one deferred. +fixture cap "[$(pr 1 autobump/a a111),$(pr 2 autobump/b b222)]" '[]' +bcd +if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 1 ] && dispatched autobump/a && grep -q '#2 autobump/b.*deferred' <<<"$OUT"; then + ok "cap: one dispatch, the second deferred" +else fail "cap: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi +# a bot run already in flight on another autobump branch fills the cap. +fixture cap_full "[$(pr 1 autobump/a a111)]" "[$(run 5 ci.yml workflow_dispatch autobump/z z999 in_progress '')]" +bcd +if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 0 ] && grep -q 'deferred' <<<"$OUT"; then ok "cap_full: deferred" +else fail "cap_full: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions")"; fi + +# 3. a queued ci.yml run of a human PR defers every bot dispatch; a running one does not. +fixture human_queued "[$(pr 1 autobump/a a111)]" "[$(run 5 ci.yml pull_request claude/x c333 queued '')]" +BOT_CI_MAX_INFLIGHT=10 bcd +if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 0 ] && grep -q 'human' <<<"$OUT"; then ok "human_queued: deferred" +else fail "human_queued: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi +fixture human_running "[$(pr 1 autobump/a a111)]" "[$(run 5 ci.yml pull_request claude/x c333 in_progress '')]" +bcd +if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 1 ]; then ok "human_running: dispatched" +else fail "human_running: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions")"; fi + +# 4. idempotent: a second call dispatches nothing more. +fixture idempotent "[$(pr 1 autobump/a a111),$(pr 2 autobump/b b222)]" '[]' +BOT_CI_MAX_INFLIGHT=10 bcd; first=$(acts dispatch); BOT_CI_MAX_INFLIGHT=10 bcd +if [ "$first" -eq 2 ] && [ "$(acts dispatch)" -eq 2 ] && [ "$RC" -eq 0 ]; then ok "idempotent: 2 dispatches, none on the second call" +else fail "idempotent: first=$first total=$(acts dispatch) rc=$RC"; fi + +# 5. a run that failed on its first attempt gets its failed jobs re-run once. +fixture rerun "[$(pr 1 autobump/a a111)]" "[$(run 41 ci.yml workflow_dispatch autobump/a a111 completed failure)]" +bcd +if [ "$RC" -eq 0 ] && grep -qxF 'rerun 41 --failed' "$FIX/actions" && [ "$(acts dispatch)" -eq 0 ]; then ok "rerun: failed jobs of run 41 re-run" +else fail "rerun: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi +jq 'map(if .databaseId == 41 then .status = "completed" | .conclusion = "failure" else . end)' "$FIX/runs.json" > "$FIX/r" && mv "$FIX/r" "$FIX/runs.json" +bcd +if [ "$RC" -eq 0 ] && [ "$(acts rerun)" -eq 1 ] && [ "$(acts dispatch)" -eq 0 ]; then ok "rerun: not again after the re-run failed" +else fail "rerun: second call rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi +# a cancelled run is someone's decision: no re-run. +fixture rerun_cancelled "[$(pr 1 autobump/a a111)]" "[$(run 41 ci.yml workflow_dispatch autobump/a a111 completed cancelled)]" +bcd +if [ "$RC" -eq 0 ] && [ "$(acts rerun)" -eq 0 ] && [ "$(acts dispatch)" -eq 0 ]; then ok "rerun_cancelled: left alone" +else fail "rerun_cancelled: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions")"; fi + +# 6. a branch whose ci.yml has no workflow_dispatch: skipped with a reason, exit 0. +fixture no_trigger "[$(pr 1 autobump/old o111),$(pr 2 autobump/b b222)]" '[]'; echo autobump/old > "$FIX/no_dispatch" +BOT_CI_MAX_INFLIGHT=10 bcd +if [ "$RC" -eq 0 ] && grep -q '#1 autobump/old.*workflow_dispatch' <<<"$OUT" && [ "$(acts dispatch)" -eq 1 ] && dispatched autobump/b; then + ok "no_trigger: skipped, the next PR still dispatched" +else fail "no_trigger: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi +# any other dispatch failure is an error: exit 1. +fixture dispatch_error "[$(pr 1 autobump/a a111)]" '[]'; touch "$FIX/dispatch_fail" +bcd +if [ "$RC" -eq 1 ] && grep -q 'HTTP 500' <<<"$OUT"; then ok "dispatch_error: exit 1" +else fail "dispatch_error: rc=$RC out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 7. --branch limits it to one PR; --workflow picks the workflow and its own runs. +fixture one_branch "[$(pr 1 autobump/a a111),$(pr 2 autobump/b b222)]" '[]' +BOT_CI_MAX_INFLIGHT=10 bcd --branch autobump/b +if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 1 ] && dispatched autobump/b; then ok "one_branch: only autobump/b" +else fail "one_branch: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions")"; fi +fixture canary "[$(pr 1 autobump/a a111)]" "[$(run 5 ci.yml workflow_dispatch autobump/a a111 completed success)]" +bcd --workflow canary.yml +if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 1 ] && dispatched autobump/a canary.yml; then ok "workflow: canary.yml dispatched beside a green ci.yml run" +else fail "workflow: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions")"; fi +bcd --bogus +if [ "$RC" -eq 2 ]; then ok "usage: unknown flag exits 2"; else fail "usage: rc=$RC"; fi + +echo "${PASS} passed, ${FAILS} failed" +[ "${FAILS}" -eq 0 ] diff --git a/tests/ci_wiring_test.sh b/tests/ci_wiring_test.sh index 6426f9b7..d021e1e4 100755 --- a/tests/ci_wiring_test.sh +++ b/tests/ci_wiring_test.sh @@ -77,34 +77,72 @@ if grep -qF 'git status --porcelain -- .github/workflows/' "${BP}"; then else fail "repin-images has no 'git status --porcelain -- .github/workflows/' check" fi -# create-pull-request resets an existing digest branch onto main and force-pushes -# it; GitHub refuses that GITHUB_TOKEN push when the branch's workflow files differ -# from main's, though the digest commit touches none. A step before the action, -# under the same if:, deletes such a branch so the action cuts a fresh one off main. -# Its script then runs against a local origin over file:// (no network). +# Every create-pull-request step (a GITHUB_TOKEN PR) sits between two steps under +# its if:. Before it, hack/autobump-branch-reset deletes the branch when its +# workflow files differ from main's: create-pull-request resets the branch onto +# main and force-pushes it, and GitHub refuses that GITHUB_TOKEN push (no +# `workflows` permission) though the bump commit touches no workflow. After it, +# hack/bot-ci-dispatch dispatches ci.yml on the branch: the PR's own pull_request +# runs wait for approval, a workflow_dispatch run does not. The job grants +# actions: write for the dispatch. Each reset step's script then runs against a +# local origin over file:// (no network), for every such branch. T=$(mktemp -d "${TMPDIR:-/tmp}/ci_wiring.XXXXXX") trap 'rm -rf "$T"' EXIT -if python3 - "${BP}" "${T}/drop.sh" <<'PY' -import sys, yaml -steps = yaml.safe_load(open(sys.argv[1]))["jobs"]["repin-images"]["steps"] -cpr = next(i for i, s in enumerate(steps) - if str(s.get("uses", "")).startswith("peter-evans/create-pull-request@")) -branch = steps[cpr]["with"]["branch"] -drop = [s for s in steps[:cpr] if f"git push origin --delete {branch}" in s.get("run", "")] -if len(drop) != 1 or drop[0].get("if") != steps[cpr].get("if"): +if python3 - "${ROOT}" "${T}" > "${T}/cases" 2> "${T}/cpr.err" <<'PY' +import glob, os, sys, yaml +root, out = sys.argv[1], sys.argv[2] +errs, n = [], 0 +for path in sorted(glob.glob(os.path.join(root, ".github/workflows/*.yml"))): + jobs = (yaml.safe_load(open(path)) or {}).get("jobs") or {} + for jid, job in jobs.items(): + steps = job.get("steps") or [] + for i, s in enumerate(steps): + if not str(s.get("uses", "")).startswith("peter-evans/create-pull-request@"): + continue + n += 1 + where = f"{os.path.basename(path)} {jid}" + cond = s.get("if") + reset = [p for p in steps[:i] if "hack/autobump-branch-reset" in p.get("run", "") and p.get("if") == cond] + disp = [p for p in steps[i + 1:] if "hack/bot-ci-dispatch --branch" in p.get("run", "") and p.get("if") == cond] + if len(reset) != 1: + errs.append(f"{where}: no single step before create-pull-request, under its if:, that runs hack/autobump-branch-reset") + if len(disp) != 1 or "GH_TOKEN" not in (disp[0].get("env") or {}): + errs.append(f"{where}: no single step after create-pull-request, under its if:, that runs hack/bot-ci-dispatch --branch with GH_TOKEN") + if (job.get("permissions") or {}).get("actions") != "write": + errs.append(f"{where}: the job lacks permissions actions: write (bot-ci-dispatch dispatches)") + if len(reset) != 1: + continue + # A matrix branch (autobump/${{ matrix.component }}) runs as component "comp". + sub = lambda v: str(v).replace("${{ matrix.component }}", "comp") + branch = sub(s["with"]["branch"]) + env = {k: sub(v) for k, v in {**(job.get("env") or {}), **(reset[0].get("env") or {})}.items()} + script = sub(reset[0]["run"]) + if "${{" in branch + script + "".join(env.values()): + errs.append(f"{where}: the reset step uses an expression this test cannot evaluate") + continue + d = os.path.join(out, f"case{n}") + os.makedirs(d) + open(os.path.join(d, "run.sh"), "w").write(script) + open(os.path.join(d, "env"), "w").write("".join(f"export {k}='{v}'\n" for k, v in env.items())) + print(f"{where}\t{branch}\t{d}") +if errs or n == 0: + sys.stderr.write("\n".join(errs or ["no create-pull-request step found"]) + "\n") sys.exit(1) -open(sys.argv[2], "w").write(drop[0]["run"]) PY then - ok "repin-images deletes a stale digest branch before create-pull-request, under its if:" + ok "every create-pull-request step: autobump-branch-reset before, bot-ci-dispatch after, under its if:, actions: write" else - fail "repin-images has no step before create-pull-request, under its if:, that runs 'git push origin --delete autobump/image-digests'" + fail "create-pull-request wiring: $(tr '\n' ';' < "${T}/cpr.err")" fi -# $1: absent (no digest branch), same (main moved, no workflow change), differ -# (main changed a workflow after the branch was cut). Prints the branch's state -# on origin after the step's script ran in a depth-1 checkout of main. +for b in autobump/image-digests autobump/convert-pins autobump/comp; do + if cut -f2 "${T}/cases" | grep -qxF "$b"; then ok "a reset step covers ${b}"; else fail "no reset step covers ${b}"; fi +done +# $1: absent (no branch), same (main moved, no workflow change), differ (main +# changed a workflow after the branch was cut); $2 the branch; $3 the case dir. +# Prints the branch's state on origin after the step's script ran in a depth-1 +# checkout of main that carries hack/autobump-branch-reset. drop_case() { - local d="${T}/$1" b=autobump/image-digests + local d="$3/$1" b=$2 ( set -e export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 @@ -112,16 +150,17 @@ drop_case() { git init -q --bare -b main "$d/origin.git" git init -q -b main "$d/seed" cd "$d/seed" - mkdir -p .github/workflows + mkdir -p .github/workflows hack echo v1 > .github/workflows/ci.yml echo 1 > images.lock - git add .github/workflows/ci.yml images.lock + cp "${ROOT}/hack/autobump-branch-reset" hack/ + git add .github/workflows/ci.yml images.lock hack/autobump-branch-reset git commit -qm base git push -q "$d/origin.git" main if [ "$1" != absent ]; then git switch -qc "$b" echo 2 > images.lock - git commit -qam digests + git commit -qam bump git push -q "$d/origin.git" "$b" git switch -q main fi @@ -135,17 +174,27 @@ drop_case() { git push -q "$d/origin.git" main git clone -q --depth=1 --branch main "file://$d/origin.git" "$d/work" cd "$d/work" - bash --noprofile --norc -eo pipefail "${T}/drop.sh" >"$d/log" 2>&1 + # shellcheck disable=SC1091 + . "$3/env" + bash --noprofile --norc -eo pipefail "$3/run.sh" >"$d/log" 2>&1 ) || { echo "script-failed"; return; } if [ -n "$(git ls-remote --heads "$d/origin.git" "refs/heads/$b")" ]; then echo present; else echo gone; fi } -for c in absent:gone same:present differ:gone; do - got=$(drop_case "${c%%:*}") - if [ "${got}" = "${c#*:}" ]; then - ok "stale-branch step, ${c%%:*}: digest branch ${got}" - else - fail "stale-branch step, ${c%%:*}: digest branch ${got}, want ${c#*:}" - fi +while IFS=$'\t' read -r where b dir; do + for c in absent:gone same:present differ:gone; do + got=$(drop_case "${c%%:*}" "$b" "$dir") + if [ "${got}" = "${c#*:}" ]; then + ok "${where} reset step, ${c%%:*}: ${b} ${got}" + else + fail "${where} reset step, ${c%%:*}: ${b} ${got}, want ${c#*:}" + fi + done +done < "${T}/cases" +# autobump-branch-reset deletes only autobump/ branches (usage error, exit 2). +for b in main claude/x autobump/ ''; do + (cd "$T" && bash "${ROOT}/hack/autobump-branch-reset" "$b" >/dev/null 2>&1); rc=$? + if [ "$rc" -eq 2 ]; then ok "autobump-branch-reset refuses '${b}'" + else fail "autobump-branch-reset '${b}': exit ${rc}, want 2"; fi done # The release job builds the 24.04 variant from the Dockerfile's pin, which the # nightly moves; a literal pin in the workflow would never move. @@ -156,5 +205,108 @@ else fail "release does not read UBUNTU_2404_IMAGE from the Dockerfile for the 24.04 variant" fi +# ─── CI on bot PRs and alarm surfacing ─────────────────────────────────────── +WF="${ROOT}/.github/workflows" +# ci.yml runs on workflow_dispatch: hack/bot-ci-dispatch runs it on bot PR branches. +# (PyYAML reads the bare key `on` as True.) +if python3 - "${WF}/ci.yml" <<'PY' +import sys, yaml +wf = yaml.safe_load(open(sys.argv[1])) +on = wf.get("on", wf.get(True)) +sys.exit(0 if isinstance(on, dict) and "workflow_dispatch" in on else 1) +PY +then ok "ci.yml has a workflow_dispatch trigger"; else fail "ci.yml has no workflow_dispatch trigger"; fi +# The dependabot auto-merge workflow ran `gh pr merge --auto` on an unprotected main +# (no required checks, so nothing ever merged) and held a run on every bot PR. +if [ ! -e "${WF}/dependabot-auto-merge.yml" ]; then ok "dependabot-auto-merge.yml is gone" +else fail "dependabot-auto-merge.yml still exists"; fi +# bot-sweep.yml: hourly off the hour, never two at once, job ci-fill runs bot-ci-dispatch. +if python3 - "${WF}/bot-sweep.yml" <<'PY' +import sys, yaml +wf = yaml.safe_load(open(sys.argv[1])) +on = wf.get("on", wf.get(True)) or {} +crons = [c["cron"] for c in on.get("schedule") or []] +conc = wf.get("concurrency") or {} +job = (wf.get("jobs") or {}).get("ci-fill") or {} +perms = job.get("permissions") or {} +runs = " ".join(s.get("run", "") for s in job.get("steps") or []) +checks = [ + wf.get("name") == "Bot sweep", + crons == ["13 * * * *"], "workflow_dispatch" in on, + conc.get("group") == "bot-sweep", conc.get("cancel-in-progress") is False, + isinstance(job.get("timeout-minutes"), int), + perms == {"actions": "write", "contents": "read", "pull-requests": "read"}, + "hack/bot-ci-dispatch" in runs, + all("timeout-minutes" in j and "permissions" in j for j in wf["jobs"].values()), +] +sys.exit(0 if all(checks) else 1) +PY +then ok "bot-sweep.yml: 'Bot sweep', cron 13 * * * *, group bot-sweep, job ci-fill runs hack/bot-ci-dispatch" +else fail "bot-sweep.yml is missing or its name, schedule, concurrency or ci-fill job is wrong"; fi + +# ci.yml's last job, alarms: never fails, reads issues and actions only, and warns +# on every open ci-alarm issue and on a missing or stale Watchdog. Its script runs +# against a fake gh. +if python3 - "${WF}/ci.yml" "${T}/alarms.sh" <<'PY' +import sys, yaml +jobs = yaml.safe_load(open(sys.argv[1]))["jobs"] +job = jobs.get("alarms") or {} +ok = (list(jobs)[-1] == "alarms" and job.get("continue-on-error") is True + and isinstance(job.get("timeout-minutes"), int) + and job.get("permissions") == {"issues": "read", "actions": "read"} + and len(job.get("steps") or []) == 1) +if ok: + open(sys.argv[2], "w").write(job["steps"][0]["run"]) +sys.exit(0 if ok else 1) +PY +then ok "ci.yml ends with job alarms: continue-on-error, timeout, issues+actions read" +else fail "ci.yml's last job is not alarms with continue-on-error, timeout-minutes, issues/actions read and one step"; fi +mkdir -p "${T}/abin" +cat > "${T}/abin/gh" <<'STUB' +#!/usr/bin/env bash +A=$AFIX +case "$1 $2" in + "issue list") f=issues;; "workflow list") f=workflows;; "run list") f=runs;; *) exit 0;; +esac +[ -f "$A/$f.fail" ] && { echo "HTTP 502" >&2; exit 1; } +q=; while [ $# -gt 0 ]; do case $1 in -q|--jq) q=$2; shift 2;; *) shift;; esac; done +if [ -n "$q" ]; then jq -r "$q" "$A/$f.json"; else cat "$A/$f.json"; fi +STUB +chmod +x "${T}/abin/gh" +ALARM7='{"number":7,"title":"[ci-alarm] nightly: Auto-update red","url":"https://github.com/o/r/issues/7","labels":[{"name":"ci-alarm"}]}' +ALARM8='{"number":8,"title":"[ci-alarm] release: 100% missing","url":"https://github.com/o/r/issues/8","labels":[{"name":"ci-alarm"},{"name":"alarm-ack"}]}' +WATCHDOG='[{"name":"Watchdog","path":".github/workflows/watchdog.yml","state":"active"}]' +ago() { date -u -d "-$1 hours" +%Y-%m-%dT%H:%M:%SZ; } +# alarm_case NAME ISSUES WORKFLOWS RUNS [fail...]: runs the alarms script; sets AOUT, ARC. +alarm_case() { + AFIX="${T}/alarm-$1"; export AFIX; mkdir -p "$AFIX" + echo "$2" > "$AFIX/issues.json"; echo "$3" > "$AFIX/workflows.json"; echo "$4" > "$AFIX/runs.json" + local f; for f in "${@:5}"; do touch "$AFIX/$f.fail"; done + AOUT=$(PATH="${T}/abin:$PATH" bash --noprofile --norc -eo pipefail "${T}/alarms.sh" 2>&1); ARC=$? +} +alarm_case two "[$ALARM7,$ALARM8]" "$WATCHDOG" "[{\"updatedAt\":\"$(ago 1)\"}]" +if [ "$ARC" -eq 0 ] && [ "$(grep -c '^::warning::' <<<"$AOUT")" -eq 2 ] \ + && grep '^::warning::.*#7' <<<"$AOUT" | grep -F 'https://github.com/o/r/issues/7' | grep -qF 'gh issue edit 7 --add-label alarm-ack' \ + && grep '^::warning::.*#8' <<<"$AOUT" | grep -F 'https://github.com/o/r/issues/8' | grep -qF '100%25 missing' \ + && ! grep -q 'Watchdog' <<<"$AOUT"; then + ok "alarms: one warning per open alarm with its URL and ack state; a fresh Watchdog is quiet" +else fail "alarms two: rc=$ARC out=$(tr '\n' ';' <<<"$AOUT")"; fi +alarm_case none '[]' '[]' '[]' +if [ "$ARC" -eq 0 ] && [ "$(grep -c '^::warning::' <<<"$AOUT")" -eq 1 ] && grep -q '^::warning::.*Watchdog' <<<"$AOUT"; then + ok "alarms: a missing Watchdog workflow warns" +else fail "alarms none: rc=$ARC out=$(tr '\n' ';' <<<"$AOUT")"; fi +alarm_case stale '[]' "$WATCHDOG" "[{\"updatedAt\":\"$(ago 5)\"}]" +if [ "$ARC" -eq 0 ] && grep -q '^::warning::.*Watchdog.*3 h' <<<"$AOUT"; then ok "alarms: a Watchdog success over 3 h old warns" +else fail "alarms stale: rc=$ARC out=$(tr '\n' ';' <<<"$AOUT")"; fi +alarm_case never '[]' "$WATCHDOG" '[]' +if [ "$ARC" -eq 0 ] && grep -q '^::warning::.*Watchdog' <<<"$AOUT"; then ok "alarms: a Watchdog with no success warns" +else fail "alarms never: rc=$ARC out=$(tr '\n' ';' <<<"$AOUT")"; fi +alarm_case disabled '[]' '[{"name":"Watchdog","path":".github/workflows/watchdog.yml","state":"disabled_manually"}]' "[{\"updatedAt\":\"$(ago 1)\"}]" +if [ "$ARC" -eq 0 ] && grep -q '^::warning::.*Watchdog.*disabled_manually' <<<"$AOUT"; then ok "alarms: a disabled Watchdog warns" +else fail "alarms disabled: rc=$ARC out=$(tr '\n' ';' <<<"$AOUT")"; fi +alarm_case gh_down '[]' '[]' '[]' issues workflows runs +if [ "$ARC" -eq 0 ] && [ "$(grep -c '^::warning::' <<<"$AOUT")" -eq 2 ]; then ok "alarms: gh failing warns and still exits 0" +else fail "alarms gh_down: rc=$ARC out=$(tr '\n' ';' <<<"$AOUT")"; fi + echo "${PASS} passed, ${FAILS} failed" [ "${FAILS}" -eq 0 ] diff --git a/tests/gate_merge_test.sh b/tests/gate_merge_test.sh index b43faa27..762e8af3 100755 --- a/tests/gate_merge_test.sh +++ b/tests/gate_merge_test.sh @@ -7,8 +7,13 @@ # files changed on main and on the PR since the merge base). The stub gh answers # both `gh pr view --json statusCheckRollup` and `gh pr checks --json bucket` # from the same rollup (bucket mapping as in cli/cli pkg/cmd/pr/checks/aggregate.go). -# Cases 18-19 run the gate's sync() with real git against a bare origin (gh and +# `gh run list -e/-b` filter runs that carry event/headBranch (bare runs match +# any). Alarms: `gh issue list` answers the next line of alarms_seq (default +# none; issues_off makes it fail), and the PR's changed files are pr_paths +# (pr_paths_fail makes that fail). +# Cases 18-19 and 24-25 run the gate with real git against a bare origin (gh and # sleep still stubbed): the head the stub gh reports is the origin's feat ref. +# The ci_status and rules_stale cases source hack/gate_lib.sh directly. # # Usage: tests/gate_merge_test.sh @@ -65,12 +70,25 @@ case "$1 $2" in if [ -n "$q" ]; then jq -r "$q" <<<"$obj"; else jq "{$fields}" <<<"$obj" 2>/dev/null || echo "$obj"; fi exit 0;; "run list") - shift 2; sha= q= wf= - while [ $# -gt 0 ]; do case $1 in -c|--commit) sha=$2; shift 2;; -w|--workflow) wf=$2; shift 2;; -q) q=$2; shift 2;; *) shift;; esac; done - echo "run-list $sha ${wf:-all}" >> "$F/calls" + shift 2; sha= q= wf= ev= rb= + while [ $# -gt 0 ]; do case $1 in -c|--commit) sha=$2; shift 2;; -w|--workflow) wf=$2; shift 2;; -e|--event) ev=$2; shift 2;; -b|--branch) rb=$2; shift 2;; -q) q=$2; shift 2;; *) shift;; esac; done + echo "run-list $sha ${wf:-all} ${ev:-any}" >> "$F/calls" if [ -n "$wf" ] || [ ! -f "$F/allruns.json" ]; then out=$(cat "$F/runs.json"); else out=$(cat "$F/allruns.json"); fi + # -e and -b filter like the API on runs that carry event / headBranch; a bare run matches any. + out=$(jq --arg e "$ev" --arg b "$rb" \ + 'map(select(($e == "" or (.event // $e) == $e) and ($b == "" or (.headBranch // $b) == $b)))' <<<"$out") if [ -n "$q" ]; then jq -r "$q" <<<"$out"; else echo "$out"; fi exit 0;; + "issue list") + # open ci-alarm issues: the next line of alarms_seq (one JSON array per call), default none + [ -f "$F/issues_off" ] && { echo "the 'o/r' repository has disabled issues" >&2; exit 1; } + if [ -f "$F/alarms_seq" ]; then pop alarms_seq; else echo '[]'; fi + exit 0;; + "api --paginate") + case $* in */pulls/*/files*) + [ -f "$F/pr_paths_fail" ] && { echo "HTTP 502: Bad Gateway" >&2; exit 1; } + cat "$F/pr_paths"; exit 0;; esac + exit 0;; esac exit 0 STUB @@ -130,6 +148,7 @@ fixture() { echo 0 > "$FIX/ancestor"; echo scripts/k3d-gpu > "$FIX/base_files"; echo scripts/k3d-gpu > "$FIX/pr_files" printf '%s\n' ${6:-MERGEABLE} > "$FIX/mergeable"; echo "${7:-$GOOD_HELP}" > "$FIX/merge_help" echo OPEN > "$FIX/state"; : > "$FIX/calls"; : > "$FIX/git"; : > "$FIX/trace" + echo scripts/k3d-gpu > "$FIX/pr_paths" } run_gate() { OUT=$(PATH="$T/bin:$PATH" GATE_REPO="$FIX" GATE_WT_ROOT="$FIX/wt" GATE_POLLS=3 \ @@ -313,8 +332,12 @@ realgit_fixture() { printf '#!/bin/sh\nexit 0\n' > "$R/tests/claims_check.sh" printf '## Hypotheses\n\n| H34 | on the i9-13900T | p | m | SUPPORTED |\n' > "$R/EXPERIMENTS.md" echo base > "$R/docs/provenance.md" + mkdir -p "$R/scripts" "$R/packaging/aur" "$R/Formula" + printf '#!/usr/bin/env bash\nset -euo pipefail\n\nVERSION="0.1.0" # kept in lockstep\nCLUSTER=gpu\n' > "$R/scripts/k3d-gpu" + echo 'pkgver=0.1.0' > "$R/packaging/aur/PKGBUILD"; echo 'pkgver = 0.1.0' > "$R/packaging/aur/.SRCINFO" + echo 'VERSION=0.1.0' > "$R/packaging/install.sh"; echo 'version "0.1.0"' > "$R/Formula/k3d-gpu.rb" "$REALPY" "$R/console/model_opt.py" ledger-rules --ledger "$R/EXPERIMENTS.md" --out "$R/console/ledger-rules.json" - repo_git add console tests docs EXPERIMENTS.md && repo_git commit -q -m base \ + repo_git add console tests docs scripts packaging Formula EXPERIMENTS.md && repo_git commit -q -m base \ && repo_git remote add origin "$FIX/origin.git" && repo_git push -q origin main && repo_git checkout -q -b feat echo "$FIX/origin.git" > "$FIX/origin_git" } @@ -373,5 +396,213 @@ if [ "$RC" -eq 1 ] && ! grep -q 'synced' <<<"$OUT" && [ "$(syncs)" -eq 1 ] && [ ok "sync_merge_commit_fails: exit 1 at once, no push" else fail "sync_merge_commit_fails: rc=$RC syncs=$(syncs) merges=$(merges) origin=$before->${after:-gone} out=$(tr '\n' ';' <<<"$OUT")"; fi +# ─── CI on bot PRs: a ci.yml run dispatched on the PR branch is CI evidence ──── +# A PR that GITHUB_TOKEN opened gets pull_request runs held for approval +# (conclusion action_required); hack/bot-ci-dispatch runs ci.yml on the branch +# by workflow_dispatch instead. The stub PR's branch is feat. +# run_json EVENT BRANCH STATUS CONCLUSION CREATED: one ci.yml run for the head SHA. +run_json() { printf '{"workflowName":"CI","event":"%s","headBranch":"%s","status":"%s","conclusion":"%s","createdAt":"%s"}' "$@"; } +HELD_PR=$(run_json pull_request feat completed action_required 2026-10-20T01:00:00Z) +DISP_OK=$(run_json workflow_dispatch feat completed success 2026-10-20T01:05:00Z) +HELD_CI_CHECK=$(concl ACTION_REQUIRED) +HELD_DEP_CHECK='{"__typename":"CheckRun","name":"automerge","workflowName":"Auto-merge Dependabot PRs","status":"COMPLETED","conclusion":"ACTION_REQUIRED","startedAt":"2026-10-20T01:00:00Z"}' +HELD_DEP_RUN='{"workflowName":"Auto-merge Dependabot PRs","event":"pull_request","headBranch":"feat","status":"completed","conclusion":"action_required","createdAt":"2026-10-20T01:00:00Z"}' + +# 20. no pull_request run at all, a green ci.yml run dispatched on the PR branch: merge. +fixture dispatch_green "[$SUCCESS]" "[$DISP_OK]" abc123; run_gate +if [ "$RC" -eq 0 ] && [ "$(merges)" -eq 1 ] && grep -q -- '--match-head-commit abc123' "$FIX/merges"; then + ok "dispatch_green: a dispatched run on the branch counts as CI" +else fail "dispatch_green: rc=$RC merges=$(merges) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 21. held pull_request runs (CI and the dependabot workflow, in the rollup and the +# run list) plus a green dispatched run: the held entries do not block; merge. +fixture held_plus_dispatch "[$SUCCESS,$HELD_CI_CHECK,$HELD_DEP_CHECK]" "[$HELD_PR,$DISP_OK]" abc123 +echo "[$HELD_PR,$HELD_DEP_RUN,$DISP_OK]" > "$FIX/allruns.json"; run_gate +if [ "$RC" -eq 0 ] && [ "$(merges)" -eq 1 ]; then ok "held_plus_dispatch: held runs ignored once a dispatched run is green" +else fail "held_plus_dispatch: rc=$RC merges=$(merges) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 22. held alone still fails at once, as before: a held CheckRun in the rollup, or +# (no check runs for a held run) only the held ci.yml run. +fixture held_alone_rollup "[$SUCCESS,$HELD_CI_CHECK]" "[$HELD_PR]" abc123; run_gate +if [ "$RC" -eq 1 ] && [ "$(merges)" -eq 0 ] && grep -q FAIL <<<"$OUT"; then ok "held_alone_rollup: exit 1" +else fail "held_alone_rollup: rc=$RC merges=$(merges)"; fi +fixture held_alone_run "[$SUCCESS]" "[$HELD_PR]" abc123; run_gate +if [ "$RC" -eq 1 ] && [ "$(merges)" -eq 0 ] && grep -q FAIL <<<"$OUT"; then ok "held_alone_run: exit 1" +else fail "held_alone_run: rc=$RC merges=$(merges)"; fi + +# 23. a green run dispatched on another branch is not this PR's CI: with nothing +# else no merge (wait), and it does not excuse a held run (exit 1). +fixture dispatch_other_branch "[$SUCCESS]" "[$(run_json workflow_dispatch other completed success 2026-10-20T01:05:00Z)]" abc123; run_gate +if [ "$(merges)" -eq 0 ] && grep -q 'GAVE UP' <<<"$OUT"; then ok "dispatch_other_branch: not accepted, waits" +else fail "dispatch_other_branch: rc=$RC merges=$(merges)"; fi +fixture dispatch_other_branch_held "[$SUCCESS,$HELD_CI_CHECK]" "[$HELD_PR,$(run_json workflow_dispatch other completed success 2026-10-20T01:05:00Z)]" abc123; run_gate +if [ "$RC" -eq 1 ] && [ "$(merges)" -eq 0 ]; then ok "dispatch_other_branch_held: held run still fails" +else fail "dispatch_other_branch_held: rc=$RC merges=$(merges)"; fi + +# 23b. the latest dispatched run failed after an earlier green one: exit 1. +fixture dispatch_latest_failed "[$SUCCESS]" "[$DISP_OK,$(run_json workflow_dispatch feat completed failure 2026-10-20T02:00:00Z)]" abc123; run_gate +if [ "$RC" -eq 1 ] && [ "$(merges)" -eq 0 ]; then ok "dispatch_latest_failed: exit 1" +else fail "dispatch_latest_failed: rc=$RC merges=$(merges)"; fi + +# ─── ci_status and rules_stale, called from hack/gate_lib.sh directly ───────── +lib_ci() { # lib_ci NAME RUNS WANT: ci_status abc123 feat over the ci.yml runs RUNS + fixture "lib_$1" '[]' "$2" abc123 + local got; got=$(PATH="$T/bin:$PATH" bash -c '. "$1"; ci_status abc123 feat' _ "$ROOT/hack/gate_lib.sh" 2>&1) + if [ "$got" = "$3" ]; then ok "ci_status $1: $got"; else fail "ci_status $1: got '$got', want $3"; fi +} +lib_ci none '[]' none +lib_ci pr_green "[$(run_json pull_request feat completed success 2026-10-20T01:00:00Z)]" green +lib_ci dispatch_green "[$DISP_OK]" green +lib_ci held_only "[$HELD_PR]" failed +lib_ci held_plus_dispatch "[$HELD_PR,$DISP_OK]" green +lib_ci held_plus_running "[$HELD_PR,$(run_json workflow_dispatch feat in_progress '' 2026-10-20T01:05:00Z)]" pending +lib_ci dispatch_queued "[$(run_json workflow_dispatch feat queued '' 2026-10-20T01:05:00Z)]" pending +lib_ci other_branch "[$(run_json workflow_dispatch other completed success 2026-10-20T01:05:00Z)]" none +lib_ci latest_failed "[$DISP_OK,$(run_json workflow_dispatch feat completed failure 2026-10-20T02:00:00Z)]" failed +lib_ci failed_then_green "[$(run_json workflow_dispatch feat completed failure 2026-10-20T01:00:00Z),$DISP_OK]" green +lib_ci failed_then_running "[$(run_json workflow_dispatch feat completed failure 2026-10-20T01:00:00Z),$(run_json workflow_dispatch feat in_progress '' 2026-10-20T01:05:00Z)]" pending + +# rules_stale keeps its behaviour: stale tree -> 0, matching tree -> 1, no tree -> 1. +lib_rules() { # lib_rules NAME WANT_RC [current] + fixture "librules_$1" '[]' '[]' abc123 + mkdir -p "$FIX/show/console"; cp "$ROOT/console/model_opt.py" "$ROOT/console/convert-pins.json" "$FIX/show/console/" + printf '## Hypotheses\n\n| H34 | on the i9-13900T | p | m | SUPPORTED |\n' > "$FIX/show/EXPERIMENTS.md" + if [ "${3:-}" = current ]; then + python3 "$ROOT/console/model_opt.py" ledger-rules --ledger "$FIX/show/EXPERIMENTS.md" --out "$FIX/show/console/ledger-rules.json" + else echo '{"schema": "k3d-gpu.ledger-rules/v1", "rows": []}' > "$FIX/show/console/ledger-rules.json"; fi + [ "$1" = notree ] || echo t1 > "$FIX/tree" + PATH="$T/bin:$PATH" bash -c '. "$1"; rules_stale abc123' _ "$ROOT/hack/gate_lib.sh" >/dev/null 2>&1; local rc=$? + if [ "$rc" -eq "$2" ]; then ok "rules_stale $1: rc $rc"; else fail "rules_stale $1: rc $rc, want $2"; fi +} +lib_rules stale 0; lib_rules current 1 current; lib_rules notree 1 + +# ─── base_ok: release-only bot commits on main do not count as a moved base ──── +# release_to_main AUTHOR SUBJECT [code]: one commit on main that moves the version +# in scripts/k3d-gpu's VERSION line and the AUR, install.sh and Formula files; +# with `code` it also changes another line of scripts/k3d-gpu. +release_to_main() { + local R=$FIX/repo + repo_git checkout -q main + sed -i 's/^VERSION="0\.1\.0"/VERSION="0.1.1"/' "$R/scripts/k3d-gpu" + [ "${3:-}" = code ] && sed -i 's/^CLUSTER=gpu$/CLUSTER=cpu/' "$R/scripts/k3d-gpu" + echo 'pkgver=0.1.1' > "$R/packaging/aur/PKGBUILD"; echo 'pkgver = 0.1.1' > "$R/packaging/aur/.SRCINFO" + repo_git add scripts packaging/aur && GIT_AUTHOR_NAME=$1 "$REALGIT" -C "$R" commit -q -m "$2" && repo_git push -q origin main +} +# install_to_main AUTHOR: the install.sh and Homebrew release commits. +install_to_main() { + local R=$FIX/repo + repo_git checkout -q main + echo 'VERSION=0.1.1' > "$R/packaging/install.sh"; repo_git add packaging/install.sh + GIT_AUTHOR_NAME=$1 "$REALGIT" -C "$R" commit -q -m "release: install.sh k3d-gpu 0.1.1 [skip release]" + echo 'version "0.1.1"' > "$R/Formula/k3d-gpu.rb"; repo_git add Formula/k3d-gpu.rb + GIT_AUTHOR_NAME=$1 "$REALGIT" -C "$R" commit -q -m "release: homebrew k3d-gpu 0.1.1 [skip release]" + repo_git push -q origin main +} + +# 24. the PR changes code, and main moved only by release-only bot commits: merge, no sync. +realgit_fixture release_only_base "[$SUCCESS]" "$CI_OK" +commit_to feat scripts.txt $'pr\n' +release_to_main 88plug-bot "release: aur-v0.1.1 [skip release]"; install_to_main 'github-actions[bot]' +run_gate_real +if [ "$RC" -eq 0 ] && [ "$(merges)" -eq 1 ] && [ "$(syncs)" -eq 0 ] && ! grep -q 'main moved under code' <<<"$OUT"; then + ok "release_only_base: merged, no sync" +else fail "release_only_base: rc=$RC merges=$(merges) syncs=$(syncs) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 25. a release commit that also changes another launcher line is a code move: sync first. +realgit_fixture release_with_code "[$SUCCESS]" "$CI_OK" +commit_to feat scripts.txt $'pr\n' +release_to_main 88plug-bot "release: aur-v0.1.1 [skip release]" code +run_gate_real +if [ "$RC" -eq 0 ] && [ "$(merges)" -eq 1 ] && [ "$(syncs)" -eq 1 ] && grep -q 'main moved under code' <<<"$OUT"; then + ok "release_with_code: synced before the merge" +else fail "release_with_code: rc=$RC merges=$(merges) syncs=$(syncs) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 25b. the same release files under a release subject, but by a human: a code move. +realgit_fixture release_by_human "[$SUCCESS]" "$CI_OK" +commit_to feat scripts.txt $'pr\n' +release_to_main gate-test "release: aur-v0.1.1 [skip release]" +run_gate_real +if [ "$RC" -eq 0 ] && [ "$(syncs)" -eq 1 ] && grep -q 'main moved under code' <<<"$OUT"; then + ok "release_by_human: synced before the merge" +else fail "release_by_human: rc=$RC merges=$(merges) syncs=$(syncs) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 25c. a bot commit on release files without [skip release] in its subject: a code move. +realgit_fixture release_no_skip "[$SUCCESS]" "$CI_OK" +commit_to feat scripts.txt $'pr\n' +release_to_main 88plug-bot "release: aur-v0.1.1" +run_gate_real +if [ "$RC" -eq 0 ] && [ "$(syncs)" -eq 1 ]; then ok "release_no_skip: synced before the merge" +else fail "release_no_skip: rc=$RC merges=$(merges) syncs=$(syncs) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# ─── alarms: open ci-alarm issues are printed, and refuse shipped changes ────── +UNACKED='{"number":7,"title":"[ci-alarm] nightly: Auto-update red","url":"https://github.com/o/r/issues/7","labels":[{"name":"ci-alarm"}]}' +ACKED='{"number":8,"title":"[ci-alarm] release: 0.1.79 missing on GHCR","url":"https://github.com/o/r/issues/8","labels":[{"name":"ci-alarm"},{"name":"alarm-ack"}]}' + +# 26. an un-acked alarm and a PR that changes the launcher: exit 4, URL and ack command shown. +fixture alarm_unacked "[$SUCCESS]" "$CI_OK" abc123; echo "[$UNACKED]" > "$FIX/alarms_seq"; run_gate +if [ "$RC" -eq 4 ] && [ "$(merges)" -eq 0 ] && grep -qF 'https://github.com/o/r/issues/7' <<<"$OUT" \ + && grep -qF 'gh issue edit 7 --add-label alarm-ack' <<<"$OUT"; then + ok "alarm_unacked: refused with exit 4, URL and ack command printed" +else fail "alarm_unacked: rc=$RC merges=$(merges) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 26b. a workflow-only PR is refused too; so is a PR whose files cannot be listed. +fixture alarm_workflow "[$SUCCESS]" "$CI_OK" abc123; echo "[$UNACKED]" > "$FIX/alarms_seq" +echo .github/workflows/ci.yml > "$FIX/pr_paths"; run_gate +if [ "$RC" -eq 4 ] && [ "$(merges)" -eq 0 ]; then ok "alarm_workflow: refused" +else fail "alarm_workflow: rc=$RC merges=$(merges)"; fi +fixture alarm_files_unknown "[$SUCCESS]" "$CI_OK" abc123; echo "[$UNACKED]" > "$FIX/alarms_seq"; touch "$FIX/pr_paths_fail"; run_gate +if [ "$RC" -eq 4 ] && [ "$(merges)" -eq 0 ]; then ok "alarm_files_unknown: refused" +else fail "alarm_files_unknown: rc=$RC merges=$(merges)"; fi + +# 26c. GATE_RELEASE_PATHS adds shipped paths (hack/release-paths format: one +# pattern per line, # comments, dir/** for a tree). +fixture alarm_release_paths "[$SUCCESS]" "$CI_OK" abc123; echo "[$UNACKED]" > "$FIX/alarms_seq" +printf '# shipped by nfpm\noracle/**\n' > "$FIX/release-paths"; echo oracle/bundle.py > "$FIX/pr_paths" +OUT=$(PATH="$T/bin:$PATH" GATE_REPO="$FIX" GATE_WT_ROOT="$FIX/wt" GATE_POLLS=3 GATE_RELEASE_PATHS="$FIX/release-paths" \ + timeout 60 bash "$GATE" 1 subj body 2>&1); RC=$? +if [ "$RC" -eq 4 ] && [ "$(merges)" -eq 0 ]; then ok "alarm_release_paths: a listed path is refused" +else fail "alarm_release_paths: rc=$RC merges=$(merges) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 26d. no alarm at the start, an un-acked one by the merge: refused before merging. +fixture alarm_opens_later "[$SUCCESS]" "$CI_OK" abc123; printf '[]\n[%s]\n' "$UNACKED" > "$FIX/alarms_seq"; run_gate +if [ "$RC" -eq 4 ] && [ "$(merges)" -eq 0 ]; then ok "alarm_opens_later: re-checked and refused before the merge" +else fail "alarm_opens_later: rc=$RC merges=$(merges) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 27. every open alarm acked: listed at the start, and the PR merges. +fixture alarm_acked "[$SUCCESS]" "$CI_OK" abc123; echo "[$ACKED]" > "$FIX/alarms_seq"; run_gate +if [ "$RC" -eq 0 ] && [ "$(merges)" -eq 1 ] && grep -F '#8' <<<"$OUT" | grep -qF 'https://github.com/o/r/issues/8'; then + ok "alarm_acked: listed, merged" +else fail "alarm_acked: rc=$RC merges=$(merges) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 28. ledger, docs, bench and test changes are never refused; nor is a path outside +# the shipped set. +fixture alarm_ledger_only "[$SUCCESS]" "$CI_OK" abc123; echo "[$UNACKED,$ACKED]" > "$FIX/alarms_seq" +printf 'EXPERIMENTS.md\ndocs/provenance.md\nbench/h54/run.sh\ntests/bench_test.sh\n' > "$FIX/pr_paths"; run_gate +if [ "$RC" -eq 0 ] && [ "$(merges)" -eq 1 ] && grep -qF 'issues/7' <<<"$OUT"; then ok "alarm_ledger_only: listed, merged" +else fail "alarm_ledger_only: rc=$RC merges=$(merges) out=$(tr '\n' ';' <<<"$OUT")"; fi +fixture alarm_unshipped "[$SUCCESS]" "$CI_OK" abc123; echo "[$UNACKED]" > "$FIX/alarms_seq" +echo README.md > "$FIX/pr_paths"; run_gate +if [ "$RC" -eq 0 ] && [ "$(merges)" -eq 1 ]; then ok "alarm_unshipped: merged" +else fail "alarm_unshipped: rc=$RC merges=$(merges)"; fi + +# 30. the gate copied alone (a scratchpad shim) finds gate_lib.sh in the repository's +# hack/; with neither copy it stops before merging. +fixture solo_gate "[$SUCCESS]" "$CI_OK" abc123; mkdir -p "$FIX/solo" "$FIX/hack" +cp "$GATE" "$FIX/solo/gate_merge.sh"; cp "$ROOT/hack/gate_lib.sh" "$FIX/hack/" +OUT=$(PATH="$T/bin:$PATH" GATE_REPO="$FIX" GATE_WT_ROOT="$FIX/wt" GATE_POLLS=3 timeout 60 bash "$FIX/solo/gate_merge.sh" 1 subj body 2>&1); RC=$? +if [ "$RC" -eq 0 ] && [ "$(merges)" -eq 1 ]; then ok "solo_gate: gate_lib.sh read from the repository's hack/" +else fail "solo_gate: rc=$RC merges=$(merges) out=$(tr '\n' ';' <<<"$OUT")"; fi +rm "$FIX/hack/gate_lib.sh"; echo OPEN > "$FIX/state"; rm -f "$FIX/merges" +OUT=$(PATH="$T/bin:$PATH" GATE_REPO="$FIX" GATE_WT_ROOT="$FIX/wt" GATE_POLLS=3 timeout 60 bash "$FIX/solo/gate_merge.sh" 1 subj body 2>&1); RC=$? +if [ "$RC" -eq 1 ] && [ "$(merges)" -eq 0 ] && grep -q 'gate_lib.sh not found' <<<"$OUT"; then ok "solo_gate: no gate_lib.sh, exit 1" +else fail "solo_gate missing lib: rc=$RC merges=$(merges) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 29. issues cannot be listed (the repository has them disabled): the gate says so, +# with gh's reason, and goes on. +fixture alarm_issues_off "[$SUCCESS]" "$CI_OK" abc123; touch "$FIX/issues_off"; run_gate +if [ "$RC" -eq 0 ] && [ "$(merges)" -eq 1 ] && grep -q 'WARNING: could not list the open ci-alarm issues.*disabled issues' <<<"$OUT"; then + ok "alarm_issues_off: warned with the reason, merged" +else fail "alarm_issues_off: rc=$RC merges=$(merges) out=$(tr '\n' ';' <<<"$OUT")"; fi + echo "${PASS} passed, ${FAILS} failed" [ "${FAILS}" -eq 0 ] From c2ffee04ece061bf41ce205cb4a4a8cba3cd6e06 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 14:28:32 -0700 Subject: [PATCH 2/5] bot-ci-dispatch: a run without a head branch counts as a human run; stop on an unreadable run pool --- hack/bot-ci-dispatch | 5 +++-- tests/bot_ci_dispatch_test.sh | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/hack/bot-ci-dispatch b/hack/bot-ci-dispatch index 96e84560..f756a589 100755 --- a/hack/bot-ci-dispatch +++ b/hack/bot-ci-dispatch @@ -53,12 +53,13 @@ cands=$(jq -r --arg only "$only" ' # The pool: bot runs of the workflow in flight, and queued ci.yml runs of human PRs. recent=$(gh run list -w "$wf" -L 100 --json status,event,headBranch) || { echo "error: cannot list $wf runs"; exit 1; } -inflight=$(jq '[.[] | select(.status != "completed" and (.headBranch | startswith("autobump/")))] | length' <<<"$recent") +inflight=$(jq '[.[] | select(.status != "completed" and ((.headBranch // "") | startswith("autobump/")))] | length' <<<"$recent") if [ "$wf" != ci.yml ]; then recent=$(gh run list -w ci.yml -L 100 --json status,event,headBranch) || { echo "error: cannot list ci.yml runs"; exit 1; } fi human=$(jq '[.[] | select(.event == "pull_request" and (.status | IN("queued", "requested", "pending", "waiting")) - and (.headBranch | startswith("autobump/") | not))] | length' <<<"$recent") + and ((.headBranch // "") | startswith("autobump/") | not))] | length' <<<"$recent") +[[ $inflight =~ ^[0-9]+$ && $human =~ ^[0-9]+$ ]] || { echo "error: cannot read the run pool from gh run list"; exit 1; } # busy LINE: print LINE with the reason and succeed when the pool has no room. busy() { if [ "$human" -gt 0 ]; then echo "$1: deferred, $human ci.yml run(s) of human PRs queued"; return 0; fi diff --git a/tests/bot_ci_dispatch_test.sh b/tests/bot_ci_dispatch_test.sh index 9cbcd211..0c70bb74 100755 --- a/tests/bot_ci_dispatch_test.sh +++ b/tests/bot_ci_dispatch_test.sh @@ -117,6 +117,13 @@ fixture human_queued "[$(pr 1 autobump/a a111)]" "[$(run 5 ci.yml pull_request c BOT_CI_MAX_INFLIGHT=10 bcd if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 0 ] && grep -q 'human' <<<"$OUT"; then ok "human_queued: deferred" else fail "human_queued: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi +# a queued run whose head branch is gone (headBranch null) counts as a human run. +fixture human_null_branch "[$(pr 1 autobump/a a111)]" \ + "[{\"databaseId\":5,\"workflow\":\"ci.yml\",\"event\":\"pull_request\",\"headBranch\":null,\"headSha\":\"c333\",\"status\":\"queued\",\"conclusion\":\"\",\"attempt\":1,\"createdAt\":\"2026-10-29T20:00:00Z\"}]" +BOT_CI_MAX_INFLIGHT=10 bcd +if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 0 ] && grep -q 'deferred, 1 ci.yml run(s) of human PRs queued' <<<"$OUT"; then + ok "human_null_branch: deferred" +else fail "human_null_branch: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi fixture human_running "[$(pr 1 autobump/a a111)]" "[$(run 5 ci.yml pull_request claude/x c333 in_progress '')]" bcd if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 1 ]; then ok "human_running: dispatched" From af9bbbbdcf0bdb62f561439c7db8928397719df5 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 14:43:29 -0700 Subject: [PATCH 3/5] ci: merge auto-eligible bot PRs one at a time, behind a circuit breaker, and batch the release hack/bot-merge runs as job merge of the hourly bot sweep. Each pass merges at most one open autobump/* PR that github-actions[bot] opened from this repository (never autobump/amend-*, never a ledger-amendment, Dependabot or fork PR, never a PR that touches .github/). A PR merges only when, at its head SHA: hack/pins policy origin/main SHA says auto, or a maintainer (a User with OWNER, MEMBER or COLLABORATOR association) approved that SHA and no maintainer's latest review requests changes; ci_status is green; every canary engine the policy lists passed hard and exact in the newest canary.yml run's canary-result artifact for that SHA (exact false labels canary-drift and needs an approval; isa-skip, too-big and fail never merge); base_ok holds and ledger-rules.json would stay current, else freshness.yml is dispatched to regenerate the branch (three times per group per UTC day at most, never while a freshness run for it is in flight); main's newest completed ci.yml run is green and no ci-main, ci-missing, release: or red:canary alarm is open; and fewer than MAX_BOT_MERGES_PER_DAY (6) bot merges landed today, unless the policy marks the PR urgent. Urgent PRs go first, then the oldest. The merge is a squash pinned with --match-head-commit, and its body ends with a "Bot-merge: pr= group= urgent=" line that later passes count. A GITHUB_TOKEN merge starts no push workflow, so the pass dispatches ci.yml on main after it. Every candidate keeps one comment that states what blocks it, or the evidence it merged on, edited in place. A refused merge prints GitHub's reason and ends the pass. Release batching follows the merge step: auto-release.yml is dispatched with mode=changed when main holds Bot-merge commits no aur-v* tag contains, main's HEAD ci.yml run succeeded, no release workflow is queued or running, no auto-release run succeeded since the newest of those merges, and an urgent fix is among them, or no other bot PR is queued with an allowed verdict and green CI, or the newest tag is 6 h old. hack/bot-ci-dispatch --workflow canary.yml now dispatches with engines=auto and only for heads whose policy lists canary engines; ci-fill runs it after the ci.yml step. The merge job checks out main only, with no persisted credentials, and runs after ci-fill even when ci-fill had an error, since it reads every piece of evidence itself. tests/bot_merge_test.sh runs the script against a fake GitHub (tests/bot_merge_fake_gh.py) and real fixture repositories, one case per rule and per release-batching condition (54 cases); 20 mutants of the script each fail at least one case. ci_wiring_test.sh checks that the merge job needs ci-fill, checks out main only and runs hack/bot-merge, and that the script never checks out a ref. --- .github/workflows/bot-sweep.yml | 40 ++- .github/workflows/ci.yml | 14 + README.md | 6 +- hack/bot-ci-dispatch | 34 ++- hack/bot-merge | 366 ++++++++++++++++++++++++++ tests/bot_ci_dispatch_test.sh | 61 ++++- tests/bot_merge_fake_gh.py | 342 ++++++++++++++++++++++++ tests/bot_merge_test.sh | 446 ++++++++++++++++++++++++++++++++ tests/ci_wiring_test.sh | 35 +++ 9 files changed, 1332 insertions(+), 12 deletions(-) create mode 100755 hack/bot-merge create mode 100755 tests/bot_merge_fake_gh.py create mode 100755 tests/bot_merge_test.sh diff --git a/.github/workflows/bot-sweep.yml b/.github/workflows/bot-sweep.yml index e19ef05e..487015a4 100644 --- a/.github/workflows/bot-sweep.yml +++ b/.github/workflows/bot-sweep.yml @@ -6,7 +6,14 @@ name: Bot sweep # ci-fill: hack/bot-ci-dispatch runs ci.yml by workflow_dispatch on each bot PR # branch with no run at its head, and re-runs a first-attempt failure # once, one bot run in flight at a time and none while a human PR's CI -# is queued. It fills in what the nightly deferred. +# is queued. It fills in what the nightly deferred. Then the same for +# canary.yml (engines=auto), on heads whose pins policy maps canary +# engines. +# merge: hack/bot-merge merges at most one bot PR whose evidence is green at +# its head (policy, CI, canary, current base, circuit breaker, daily +# cap), dispatches ci.yml on main after it, and dispatches a batched +# auto-release. It runs after ci-fill even when ci-fill had an error: +# it reads every piece of evidence itself. on: schedule: @@ -33,9 +40,40 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: main + fetch-depth: 0 # the canary step's pins policy diffs a PR head against main persist-credentials: false - name: Dispatch CI on bot PRs that have none at their head env: GH_TOKEN: ${{ github.token }} run: hack/bot-ci-dispatch + + - name: Dispatch the canary on bot PRs whose policy maps canary engines + if: ${{ !cancelled() }} + env: + GH_TOKEN: ${{ github.token }} + run: hack/bot-ci-dispatch --workflow canary.yml + + merge: + name: Merge one auto-eligible bot PR, batch the release + needs: ci-fill + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: write + pull-requests: write + actions: write + issues: write + steps: + - name: Checkout main + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + fetch-depth: 0 # base_ok, today's Bot-merge commits and the aur-v* tags + persist-credentials: false + + - name: Merge, then batch the release + env: + GH_TOKEN: ${{ github.token }} + run: hack/bot-merge diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d4ae5bc..8c7fd99d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -467,6 +467,20 @@ jobs: sudo apt-get update && sudo apt-get install -y --no-install-recommends python3-yaml - run: tests/kserve_test.sh + # ─── Bot merge ─────────────────────────────────────────────────────────── + # hack/bot-merge (bot-sweep.yml, job merge) against a fake gh and fixture repos. + bot-merge-tests: + name: Bot merge (fake gh, fixture repos) + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: tests/bot_merge_test.sh + # ─── Bot PR CI and alarm surfacing ─────────────────────────────────────── # The watchdog files a ci-alarm issue for a red or missing scheduled run or a # broken release (email went unread for 9 nights). Every open alarm, and a diff --git a/README.md b/README.md index c7b924c7..72d4f592 100644 --- a/README.md +++ b/README.md @@ -1706,7 +1706,11 @@ build-and-push workflow runs `--latest` for every component and opens one PR per component (branch `autobump/`); merging it changes `share/`, so auto-release cuts a launcher release and the node image is rebuilt for it. GITHUB_TOKEN opens these PRs, so their own pull_request CI runs wait for approval; `hack/bot-ci-dispatch` runs `ci.yml` on -the branch by workflow_dispatch instead, and that run is the PR's CI. +the branch by workflow_dispatch instead, and that run is the PR's CI. The hourly bot sweep +(`hack/bot-merge`) merges one such PR at a time when its pins policy says auto (or a +maintainer approved its head), its CI and canary are green, its base is current and main's +CI is green; it then dispatches CI on main and a batched release. Any other bot PR stays +open with one comment that says what blocks it. --- diff --git a/hack/bot-ci-dispatch b/hack/bot-ci-dispatch index f756a589..44c2a597 100755 --- a/hack/bot-ci-dispatch +++ b/hack/bot-ci-dispatch @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# hack/bot-ci-dispatch [--branch B] [--workflow ci.yml]: give the bot's PRs real CI. +# hack/bot-ci-dispatch [--branch B] [--workflow ci.yml|canary.yml]: give the bot's PRs real CI. # # A PR that GITHUB_TOKEN opened or pushed gets pull_request runs that wait for a # maintainer's approval (conclusion action_required); a workflow_dispatch run @@ -18,13 +18,19 @@ # and not at all while a ci.yml run of a human PR is queued. A deferred PR is # picked up by the next call (.github/workflows/bot-sweep.yml, hourly). # +# --workflow canary.yml dispatches with -f engines=auto, and only for a head whose +# `hack/pins policy origin/main SHA` maps canary engines (a non-empty +# canary list); the head is fetched from origin for that, never checked out. A +# head that moved on origin since the PR list was read waits for the next call; a +# policy that fails is an error for that PR. +# # A branch whose workflow file lacks the workflow_dispatch trigger (cut before # ci.yml had it) is skipped; the nightly's hack/autobump-branch-reset recreates it. # Prints one line per decision. Idempotent. Exit 0, or 1 when a gh call failed # (the rest still ran), 2 on bad usage. set -uo pipefail -usage() { echo "usage: hack/bot-ci-dispatch [--branch autobump/] [--workflow ci.yml]" >&2; exit 2; } +usage() { echo "usage: hack/bot-ci-dispatch [--branch autobump/] [--workflow ci.yml|canary.yml]" >&2; exit 2; } wf=ci.yml only= while [ $# -gt 0 ]; do case $1 in @@ -38,6 +44,19 @@ cap=${BOT_CI_MAX_INFLIGHT:-1} [[ $cap =~ ^[0-9]+$ ]] || { echo "BOT_CI_MAX_INFLIGHT must be a whole number, got '$cap'" >&2; exit 2; } RUNF=databaseId,event,status,conclusion,headBranch,attempt,createdAt rc=0 +inputs=() +if [ "$wf" = canary.yml ]; then + inputs=(-f engines=auto) + cd "$(git -C "$(dirname "${BASH_SOURCE[0]}")" rev-parse --show-toplevel)" || exit 1 + err=$(mktemp); trap 'rm -f "$err"' EXIT +fi +# canary_engines BRANCH SHA: the policy's canary list for SHA (a JSON array), or +# "moved" when origin's BRANCH is no longer SHA; fails when the policy does. +canary_engines() { + git fetch -q origin "+refs/heads/$1:refs/remotes/origin/$1" || return 1 + [ "$(git rev-parse -q --verify "refs/remotes/origin/$1^{commit}")" = "$2" ] || { echo moved; return 0; } + hack/pins policy origin/main "$2" | jq -ce '.canary | arrays' +} prs=$(gh pr list --state open --base main --limit 200 \ --json number,headRefName,headRefOid,baseRefName,isCrossRepository,author) \ @@ -80,8 +99,17 @@ while read -r num br sha; do | "\($e | length) \(($at | length) - ($e | length)) \($l.databaseId | word) \($l.status | word) \($l.conclusion | word) \($l.attempt // 1) \(any($e[]; (.attempt // 1) > 1))"' <<<"$runs")" if [ "$count" -eq 0 ]; then why="no $wf run at the head"; [ "$held" -gt 0 ] && why="only held $wf runs at the head" + if [ "$wf" = canary.yml ]; then + if ! eng=$(canary_engines "$br" "$sha" 2>"$err"); then + echo "$tag: error: hack/pins policy failed: $(tr '\n' ' ' < "$err")"; rc=1; continue + fi + case $eng in + moved) echo "$tag: skipped, origin's $br moved since the PR list was read"; continue;; + '[]') echo "$tag: the policy maps no canary engine; no canary"; continue;; + esac + fi busy "$tag: $why" && continue - if out=$(gh workflow run "$wf" --ref "$br" 2>&1); then + if out=$(gh workflow run "$wf" --ref "$br" "${inputs[@]}" 2>&1); then inflight=$((inflight + 1)) # Wait until the run shows, so that a call right after this one sees it. new= diff --git a/hack/bot-merge b/hack/bot-merge new file mode 100755 index 00000000..da2859ba --- /dev/null +++ b/hack/bot-merge @@ -0,0 +1,366 @@ +#!/usr/bin/env bash +# hack/bot-merge: one pass of the bot-PR merge (bot-sweep.yml, job merge). It merges +# at most one PR, then decides whether to dispatch a batched release. +# +# Candidates are open PRs into main from this repository's own autobump/* branches +# (never autobump/amend-*), opened by github-actions[bot], labelled autobump and not +# ledger-amendment. Dependabot PRs, ledger amendments and forks are left to humans. +# A candidate that changes anything under .github/ is never merged. +# +# A candidate merges only when all of these hold at its head SHA: +# a. `hack/pins policy origin/main SHA` says auto, or a human approved SHA: a +# review APPROVED with commit_id SHA by a User whose author_association is +# OWNER, MEMBER or COLLABORATOR. No such human's latest review may request +# changes. Labels never change the verdict. +# b. ci_status SHA BRANCH is green (ci.yml carries pin-guard and the pins checks). +# c. When the policy names canary engines (its canary list): the newest +# canary.yml run for SHA succeeded, and its canary-result artifact, made for +# SHA, shows verdict pass, hard and exact for every one of them. A hard pass +# with exact false is a review condition: the PR is labelled canary-drift and +# merges on a human approval at SHA. Any other verdict (fail, isa-skip, +# too-big, unparseable) never merges. +# d. base_ok SHA and not rules_stale SHA. Otherwise the pass dispatches +# freshness.yml -f group= to regenerate the branch from main: at most 3 +# per group per UTC day, counted from today's dispatched freshness runs whose +# title names the group (a title naming no group counts for every group), and +# never while such a freshness run is queued or running. +# e. Circuit breaker: the newest completed ci.yml run on main (push or dispatch) +# succeeded, and no open ci-alarm issue's key starts with ci-main, ci-missing, +# release: or red:canary. +# f. Fewer than MAX_BOT_MERGES_PER_DAY (default 6) bot merges on main today (UTC). +# A PR the policy marks urgent is exempt. +# Urgent PRs merge first, then the oldest. The merge is a squash pinned with +# --match-head-commit, and its body ends "Bot-merge: pr=N group=G urgent=B": later +# passes count today's merges and find unreleased ones by that line. A GITHUB_TOKEN +# merge starts no push workflow, so the pass then dispatches ci.yml on main. A +# refused merge prints GitHub's reason and ends the pass. Every candidate keeps one +# comment, starting , edited in place: why it is not merged, or +# the evidence it merged on. +# +# Release batching, after the merge step: dispatch auto-release.yml -f mode=changed +# when main has Bot-merge commits that no aur-v* tag contains, the newest ci.yml run +# for main's HEAD (push or dispatch) succeeded, no release workflow run is queued or +# running, no auto-release run has succeeded since the newest of those merges, and +# one of: one of them is urgent; no other open candidate has an allowed verdict and +# green CI; the newest aur-v* tag is 6 h old or more. +# +# Reads only main's checkout: PR heads are fetched as objects, never checked out. +# rules_stale runs the merged tree's console/model_opt.py, without the token. +# +# Env: GITHUB_REPOSITORY (owner/repo), GH_TOKEN, MAX_BOT_MERGES_PER_DAY, +# BOT_MERGE_NOW (epoch seconds, for tests), GITHUB_SERVER_URL and GITHUB_RUN_ID +# (the sweep run named in comments). +# Exit: 0 pass done; 1 a read or dispatch failed; 2 gh cannot pin a merge. +set -uo pipefail +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +cd "$(git -C "$HERE" rev-parse --show-toplevel)" || exit 1 +# shellcheck source=hack/gate_lib.sh +. "$HERE/gate_lib.sh" + +R=${GITHUB_REPOSITORY:?set GITHUB_REPOSITORY to owner/repo} +NOW=${BOT_MERGE_NOW:-$(date -u +%s)} +DAY=$(date -u -d "@$NOW" +%Y-%m-%d) +DAY0=$(date -u -d "$DAY" +%s) +MAX=${MAX_BOT_MERGES_PER_DAY:-6} +REGEN_CAP=3 +BOT='github-actions[bot]' +MARK='' +SWEEP=${GITHUB_SERVER_URL:-https://github.com}/$R/actions/runs/${GITHUB_RUN_ID:-0} +W=$(mktemp -d); trap 'rm -rf "$W"' EXIT +RC=0 + +gh pr merge --help 2>/dev/null | grep -q -- --match-head-commit \ + || { echo "gh pr merge lacks --match-head-commit; refusing to merge unpinned"; exit 2; } + +# --- reads shared by every candidate ------------------------------------------ +gh api --paginate "repos/$R/pulls?state=open&base=main&per_page=100" --jq '.[]' > "$W/pulls" \ + || { echo "cannot list open pull requests"; exit 1; } +jq -s -c --arg bot "$BOT" ' + .[] | select((.head.repo // {}).full_name == .base.repo.full_name + and (.head.ref | startswith("autobump/")) and (.head.ref | startswith("autobump/amend-") | not) + and .user.login == $bot + and any(.labels[]; .name == "autobump") and (any(.labels[]; .name == "ledger-amendment") | not)) + | {n: .number, br: .head.ref, sha: .head.sha, at: .created_at, title, draft, + labels: [.labels[].name]}' "$W/pulls" > "$W/cands" || { echo "cannot read the pull request list"; exit 1; } +git fetch -q --force origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/aur-v*:refs/tags/aur-v*' \ + || { echo "git fetch origin failed"; exit 1; } + +# breaker: why merging stops for every PR (empty when closed). +breaker() { + local ci alarms + ci=$(gh run list -w ci.yml -b main -s completed -L 20 --json event,conclusion,url,createdAt \ + | jq -r '[.[] | select(.event == "push" or .event == "workflow_dispatch")] | max_by(.createdAt) // empty + | "\(.conclusion) \(.url)"') || { echo "cannot read ci.yml runs on main"; return; } + case $ci in + "") echo "no completed ci.yml run on main"; return;; + success\ *) ;; + *) echo "the newest completed ci.yml run on main is ${ci%% *}: ${ci#* }"; return;; + esac + alarms=$(open_alarms) || { echo "cannot read ci-alarm issues"; return; } + jq -r '[.[] | (.title | capture("^\\[ci-alarm\\] (?.+?): ") | .k) as $k + | select($k | test("^(ci-main|ci-missing|release:|red:canary)"; "i")) | "\($k) (#\(.number))"] + | if length > 0 then "open alarm " + join(", ") else empty end' <<<"$alarms" +} +BREAKER=$(breaker) +[ -n "$BREAKER" ] && echo "circuit breaker open: $BREAKER" +TODAY=$(git log origin/main --format=%ct --grep='^Bot-merge: ' | awk -v d="$DAY0" '$1 >= d' | wc -l) +echo "bot merges today ($DAY UTC): $TODAY of $MAX" + +# freshness_runs GROUP JQ_FILTER [gh run list flags...]: how many freshness.yml runs, +# among those the flags select, name GROUP (or no group) in their title and pass JQ_FILTER. +freshness_runs() { + local g=$1 f=$2; shift 2 + gh run list -w freshness.yml "$@" --json displayTitle,status \ + | jq --arg g "group=$g" "[.[] | select(([(.displayTitle // \"\") | scan(\"group=[^ ,;)]+\")] as \$t + | (\$t | length) == 0 or any(\$t[]; . == \$g))) | select($f)] | length" +} +# regen GROUP WHY: dispatch freshness.yml for GROUP within the cap; prints what +# happened, returns 1 when a read or the dispatch failed. +regen() { + local g=$1 busy used + busy=$(freshness_runs "$g" '.status != "completed"' -L 30) || { echo "cannot read freshness runs"; return 1; } + if [ "$busy" -gt 0 ]; then echo "a freshness run that covers $g is queued or running"; return 0; fi + used=$(freshness_runs "$g" true -e workflow_dispatch --created ">=${DAY}T00:00:00Z" -L 200) \ + || { echo "cannot read freshness runs"; return 1; } + if [ "$used" -ge "$REGEN_CAP" ]; then + echo "regeneration cap reached ($used of $REGEN_CAP for $g today); the scheduled freshness run regenerates it" + elif gh workflow run freshness.yml --ref main -f group="$g" -f reason="bot-merge:$2" >/dev/null; then + echo "regeneration dispatched ($((used + 1)) of $REGEN_CAP for $g today)" + else echo "the freshness.yml dispatch failed"; return 1; fi +} +# canary SHA ENGINES_JSON DIR: one line per finding: "ok E", "drift E", "block WHY", +# then "url URL" when a run exists. +canary() { + local runs id st concl url + runs=$(gh run list -w canary.yml -c "$1" -L 20 --json databaseId,status,conclusion,createdAt,url) \ + || { echo "block cannot read canary.yml runs"; return; } + read -r id st concl url < <(jq -r 'max_by(.createdAt) // empty + | "\(.databaseId) \(.status) \(if (.conclusion // "") == "" then "-" else .conclusion end) \(.url)"' <<<"$runs") + [ -n "${id:-}" ] || { echo "block canary has not run at this head"; return; } + echo "url $url" + [ "$st" = completed ] || { echo "block canary is pending ($url)"; return; } + if gh run download "$id" -n canary-result -D "$3/artifact" >/dev/null 2>&1 && [ -f "$3/artifact/canary-result.json" ]; then + jq -r --arg sha "$1" --argjson want "$2" ' + if .sha != $sha then "block canary result is for \(.sha), not this head" + else $want[] as $e | (.engines[$e] // null) as $r + | if $r == null then "block canary result lacks engine \($e)" + elif $r.verdict == "pass" and $r.hard == true and $r.exact == true then "ok \($e)" + elif $r.verdict == "pass" and $r.hard == true then "drift \($e)" + else "block canary \($e): \($r.verdict)" end + end' "$3/artifact/canary-result.json" 2>/dev/null || echo "block canary-result.json is unreadable ($url)" + else + echo "block canary run has no canary-result artifact ($url)" + fi + [ "$concl" = success ] || echo "block canary run concluded $concl ($url)" +} +# ci_url SHA BRANCH: the newest successful ci.yml run that ci_status counts. +ci_url() { + gh run list -w ci.yml -c "$1" -L 100 --json event,conclusion,headBranch,createdAt,url \ + | jq -r --arg b "$2" '[.[] | select(.conclusion == "success") + | select(.event == "pull_request" or (.event == "workflow_dispatch" and .headBranch == $b))] + | max_by(.createdAt) // empty | .url' +} +# comment N FILE: the PR's one bot-merge comment becomes FILE's text. +comment() { + local body ex + body="$MARK"$'\n'"$(cat "$2")" + ex=$(gh api --paginate "repos/$R/issues/$1/comments?per_page=100" --jq '.[]' \ + | jq -s -c --arg bot "$BOT" --arg m "$MARK" \ + '[.[] | select(.user.login == $bot and (.body | startswith($m)))] | first // empty') \ + || { echo "PR#$1: cannot read comments"; RC=1; return; } + if [ -z "$ex" ]; then + gh api -X POST "repos/$R/issues/$1/comments" -f body="$body" >/dev/null || { echo "PR#$1: comment failed"; RC=1; } + elif [ "$(jq -r .body <<<"$ex")" != "$body" ]; then + gh api -X PATCH "repos/$R/issues/comments/$(jq -r .id <<<"$ex")" -f body="$body" >/dev/null \ + || { echo "PR#$1: comment edit failed"; RC=1; } + fi +} + +# --- assess every candidate --------------------------------------------------- +: > "$W/eligible"; : > "$W/queue" +while read -r c <&3; do + n=$(jq -r .n <<<"$c"); br=$(jq -r .br <<<"$c"); sha=$(jq -r .sha <<<"$c") + D=$W/$n; mkdir -p "$D"; : > "$D/why" + why() { echo "- $*" >> "$D/why"; } + gh api --paginate "repos/$R/pulls/$n/files?per_page=100" --jq '.[] | .filename, (.previous_filename // empty)' > "$D/files" \ + || { why "cannot read the changed files"; RC=1; continue; } + if grep -q '^\.github/' "$D/files"; then + why "changes files under .github/ ($(grep '^\.github/' "$D/files" | head -3 | paste -sd, -)): a bot never merges workflow changes; a human merges this" + continue + fi + [ "$(jq -r .draft <<<"$c")" = true ] && why "draft" + if [ "$(git rev-parse -q --verify "refs/remotes/origin/$br^{commit}")" != "$sha" ]; then + why "the branch no longer points at $sha; the next pass reads the new head"; continue + fi + if ! "$HERE/pins" policy origin/main "$sha" > "$D/policy" 2> "$D/policy.err"; then + why "hack/pins policy failed: $(tail -1 "$D/policy.err")"; continue + fi + if ! jq -e '.verdict == "auto" or .verdict == "review"' "$D/policy" >/dev/null 2>&1; then + why "hack/pins policy gave no auto or review verdict"; continue + fi + if ! jq -e '.canary | type == "array"' "$D/policy" >/dev/null; then + why "hack/pins policy output has no canary list, so the canary evidence is unknown"; continue + fi + verdict=$(jq -r .verdict "$D/policy"); group=$(jq -r '.group // "unknown"' "$D/policy") + upd=$(jq -r '.update_type // "unknown"' "$D/policy"); urgent=$(jq -r '.urgent == true' "$D/policy") + engines=$(jq -c .canary "$D/policy") + review=() + [ "$verdict" = review ] && review+=("policy: $(jq -r '.reasons // [] | join("; ") | if . == "" then "review" else . end' "$D/policy")") + + # human reviews: each human's latest deciding review + gh api --paginate "repos/$R/pulls/$n/reviews?per_page=100" --jq '.[]' > "$D/reviews" \ + || { why "cannot read reviews"; RC=1; continue; } + jq -s '[.[] | select(.user.type == "User" and (.author_association | IN("OWNER", "MEMBER", "COLLABORATOR")) + and (.state | IN("APPROVED", "CHANGES_REQUESTED", "DISMISSED")))] + | group_by(.user.login) | map(max_by(.submitted_at))' "$D/reviews" > "$D/latest" + approver=$(jq -r --arg sha "$sha" '[.[] | select(.state == "APPROVED" and .commit_id == $sha) | .user.login] | first // empty' "$D/latest") + changes=$(jq -r '[.[] | select(.state == "CHANGES_REQUESTED") | "@" + .user.login] | join(", ")' "$D/latest") + [ -n "$changes" ] && why "changes requested by $changes" + + # canary + curl_= + if [ "$engines" != "[]" ]; then + canary "$sha" "$engines" "$D" > "$D/canary" + curl_=$(sed -n 's/^url //p' "$D/canary") + while read -r w; do why "${w#block }"; done < <(grep '^block ' "$D/canary") + drift=$(sed -n 's/^drift //p' "$D/canary" | paste -sd, -) + if [ -n "$drift" ]; then + review+=("canary-drift: $drift passed the hard gate but its output differs from the golden") + if ! jq -e '.labels | index("canary-drift")' <<<"$c" >/dev/null; then + gh api -X POST "repos/$R/issues/$n/labels" -f 'labels[]=canary-drift' >/dev/null || { echo "PR#$n: label failed"; RC=1; } + fi + fi + fi + + # CI + ci=$(ci_status "$sha" "$br") + case $ci in + green) ;; + pending) why "CI is pending at this head";; + failed) why "CI failed at this head";; + *) why "no ci.yml run at this head yet (bot-ci-dispatch starts one)";; + esac + + allowed=y + if [ "${#review[@]}" -gt 0 ] && [ -z "$approver" ]; then + allowed=n + why "needs review: $(printf '%s; ' "${review[@]}" | sed 's/; $//'); an approval of $sha by a maintainer merges it" + fi + [ -n "$changes" ] && allowed=n + [ "$allowed" = y ] && [ "$ci" = green ] && echo "$n" >> "$W/queue" + + [ -n "$BREAKER" ] && why "circuit breaker: $BREAKER" + if [ "$allowed" = y ] && [ -z "$BREAKER" ]; then + stale= + if ! base_ok "$sha"; then stale="main moved under code since $sha was tested" + elif ( unset GH_TOKEN GITHUB_TOKEN; rules_stale "$sha" ); then + stale="console/ledger-rules.json would not match EXPERIMENTS.md after the merge" + fi + if [ -n "$stale" ]; then + msg=$(regen "$group" "$([ "${stale#main}" != "$stale" ] && echo base-moved || echo rules-stale)") || RC=1 + why "$stale; $msg" + fi + fi + if [ "$urgent" != true ] && [ "$TODAY" -ge "$MAX" ]; then + why "daily cap reached ($TODAY of $MAX bot merges today, UTC); it merges tomorrow" + fi + + { + echo "verdict=$verdict"; echo "group=$group"; echo "upd=$upd"; echo "urgent=$urgent" + echo "approver=$approver"; echo "canary=$curl_"; echo "engines=$engines" + } > "$D/facts" + if [ ! -s "$D/why" ]; then + printf '%s %s %s\n' "$([ "$urgent" = true ] && echo 0 || echo 1)" "$(jq -r .at <<<"$c")" "$n" >> "$W/eligible" + fi +done 3< "$W/cands" + +# --- merge one ------------------------------------------------------------------ +MERGED=; REFUSED= +pick=$(sort -k1,1n -k2,2 "$W/eligible" | head -1 | cut -d' ' -f3) +if [ -n "$pick" ]; then + c=$(jq -c --argjson n "$pick" 'select(.n == $n)' "$W/cands"); D=$W/$pick + sha=$(jq -r .sha <<<"$c"); br=$(jq -r .br <<<"$c"); title=$(jq -r .title <<<"$c") + get() { sed -n "s/^$1=//p" "$D/facts"; } + group=$(get group); upd=$(get upd); urgent=$(get urgent); approver=$(get approver); curl_=$(get canary) + if [[ $title =~ ^[a-z]+(\([^\)]*\))?!?:\ ]]; then subj="$title (#$pick)"; else subj="chore($group): $title (#$pick)"; fi + cu=$(ci_url "$sha" "$br") + policy_line="policy $(get verdict) ($upd update of $group$([ "$urgent" = true ] && echo ', urgent'))" + [ -n "$approver" ] && policy_line+=", approved at ${sha:0:7} by @$approver" + body="Merged by the bot sweep: $policy_line; CI ${cu:-green}$([ -n "$curl_" ] && echo "; canary $curl_")." + body+=$'\n\n'"Bot-merge: pr=$pick group=$group urgent=$urgent" + why=$(gh pr merge "$pick" --squash --match-head-commit "$sha" --subject "$subj" --body "$body" 2>&1 >/dev/null); mrc=$? + if [ "$mrc" -eq 0 ] || [ "$(gh pr view "$pick" --json state -q .state 2>/dev/null)" = MERGED ]; then + MERGED=$pick + echo "PR#$pick MERGED at $sha" + grep -vxF "$pick" "$W/queue" > "$W/queue.left"; mv "$W/queue.left" "$W/queue" + gh workflow run ci.yml --ref main >/dev/null || { echo "ci.yml dispatch on main failed"; RC=1; } + { + echo "**Merged by the bot sweep** at \`$sha\`." + echo + echo "- Policy: $policy_line" + echo "- CI: ${cu:-green}" + if [ -n "$curl_" ]; then echo "- Canary: $curl_ ($(jq -r '. | join(", ")' <<<"$(get engines)"): pass, hard, exact)" + else echo "- Canary: no engine mapped for this group"; fi + echo "- Base: current with main (base_ok, ledger-rules.json current)" + echo "- Sweep run: $SWEEP" + } > "$D/body" + else + REFUSED=$pick + echo "PR#$pick merge at $sha refused: $(tr '\n' ' ' <<<"${why:-no reason given}")" + echo "- GitHub refused the merge at \`$sha\`: $(tr '\n' ' ' <<<"${why:-no reason given}")" > "$D/why" + fi +fi + +# --- one comment per candidate ------------------------------------------------ +while read -r c <&3; do + n=$(jq -r .n <<<"$c"); D=$W/$n + if [ "$n" != "$MERGED" ]; then + if [ ! -s "$D/why" ]; then + echo "- eligible; waiting for the next pass (one merge per pass; #${MERGED:-$REFUSED} went first)" > "$D/why" + fi + { echo "**Not merged by the bot sweep** at \`$(jq -r .sha <<<"$c")\`:"; echo; cat "$D/why"; } > "$D/body" + sed 's/^- /PR#'"$n"' blocked: /' "$D/why" + fi + comment "$n" "$D/body" +done 3< "$W/cands" +[ -n "$REFUSED" ] && exit "$RC" + +# --- release batching ----------------------------------------------------------- +release() { + git fetch -q --force origin '+refs/heads/main:refs/remotes/origin/main' '+refs/tags/aur-v*:refs/tags/aur-v*' \ + || { echo "release: git fetch failed"; RC=1; return; } + local head tags unrel newest ci wf busy ok_at last last_ct why + head=$(git rev-parse origin/main) + mapfile -t tags < <(git tag -l 'aur-v*') + unrel=$(git log origin/main --format=%ct --grep='^Bot-merge: ' --not "${tags[@]}") + [ -n "$unrel" ] || { echo "release: no bot merge above the last aur-v* tag"; return; } + newest=$(sort -n <<<"$unrel" | tail -1) + ci=$(gh run list -w ci.yml -c "$head" -L 20 --json event,status,conclusion,createdAt \ + | jq -r '[.[] | select(.event != "pull_request")] | max_by(.createdAt) // empty | "\(.status)/\(.conclusion)"') \ + || { echo "release: cannot read ci.yml runs"; RC=1; return; } + [ "$ci" = completed/success ] || { echo "release: wait: ci.yml for main ${head:0:7} is ${ci:-not run yet}"; return; } + for wf in auto-release.yml build-and-push.yml aur-deploy.yml packages.yml; do + busy=$(gh run list -w "$wf" -L 20 --json status | jq '[.[] | select(.status != "completed")] | length') \ + || { echo "release: cannot read $wf runs"; RC=1; return; } + [ "$busy" -eq 0 ] || { echo "release: wait: $wf has a run queued or in progress"; return; } + done + ok_at=$(gh run list -w auto-release.yml -s success -L 1 --json createdAt | jq -r '.[0].createdAt // empty') \ + || { echo "release: cannot read auto-release runs"; RC=1; return; } + if [ -n "$ok_at" ] && [ "$(date -u -d "$ok_at" +%s)" -ge "$newest" ]; then + echo "release: auto-release succeeded at $ok_at, after the newest bot merge"; return + fi + last=$(git tag -l 'aur-v*' --sort=-v:refname | head -1) + last_ct=$(git log -1 --format=%ct "$last" 2>/dev/null || echo 0) + if [ -n "$(git log origin/main --format=%H --grep='^Bot-merge: .* urgent=true' --not "${tags[@]}")" ]; then + why="an urgent fix merged since ${last:-the first release}" + elif [ ! -s "$W/queue" ]; then why="no other bot PR is queued" + elif [ $((NOW - last_ct)) -ge 21600 ]; then why="$last is $(( (NOW - last_ct) / 3600 )) h old" + else + echo "release: wait: $(wc -l < "$W/queue") bot PR(s) still queued and $last is under 6 h old"; return + fi + if gh workflow run auto-release.yml --ref main -f mode=changed >/dev/null; then + echo "release: dispatched auto-release.yml mode=changed ($why)" + else echo "release: the auto-release.yml dispatch failed"; RC=1; fi +} +release +exit "$RC" diff --git a/tests/bot_ci_dispatch_test.sh b/tests/bot_ci_dispatch_test.sh index 0c70bb74..9a0f134b 100755 --- a/tests/bot_ci_dispatch_test.sh +++ b/tests/bot_ci_dispatch_test.sh @@ -39,8 +39,9 @@ case "$1 $2" in if [ -n "$q" ]; then jq -r "$q" <<<"$out"; else echo "$out"; fi exit 0;; "workflow run") - wf=$3; shift 3; ref= - while [ $# -gt 0 ]; do case $1 in -r|--ref) ref=$2; shift 2;; *) shift;; esac; done + wf=$3; shift 3; ref= fields= + while [ $# -gt 0 ]; do case $1 in -r|--ref) ref=$2; shift 2;; -f|-F|--raw-field|--field) fields="$fields $2"; shift 2;; *) shift;; esac; done + echo "$wf $ref$fields" >> "$F/fields" if grep -qxF "$ref" "$F/no_dispatch" 2>/dev/null; then echo "could not create workflow dispatch event: HTTP 422: Workflow does not have 'workflow_dispatch' trigger (https://api.github.com/repos/o/r/actions/workflows/1/dispatches)" >&2 exit 1 @@ -162,15 +163,61 @@ bcd if [ "$RC" -eq 1 ] && grep -q 'HTTP 500' <<<"$OUT"; then ok "dispatch_error: exit 1" else fail "dispatch_error: rc=$RC out=$(tr '\n' ';' <<<"$OUT")"; fi -# 7. --branch limits it to one PR; --workflow picks the workflow and its own runs. +# 7. --branch limits it to one PR. fixture one_branch "[$(pr 1 autobump/a a111),$(pr 2 autobump/b b222)]" '[]' BOT_CI_MAX_INFLIGHT=10 bcd --branch autobump/b if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 1 ] && dispatched autobump/b; then ok "one_branch: only autobump/b" else fail "one_branch: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions")"; fi -fixture canary "[$(pr 1 autobump/a a111)]" "[$(run 5 ci.yml workflow_dispatch autobump/a a111 completed success)]" -bcd --workflow canary.yml -if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 1 ] && dispatched autobump/a canary.yml; then ok "workflow: canary.yml dispatched beside a green ci.yml run" -else fail "workflow: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions")"; fi + +# 8. --workflow canary.yml: a head gets canary.yml, with engines=auto, only when +# `hack/pins policy origin/main SHA` maps canary engines for it, read on the head +# fetched from origin. canary_repo NAME: a bare origin whose branches autobump/a, +# autobump/b and autobump/c each carry one commit off main, and a clone on main +# whose hack/ holds bot-ci-dispatch and a fixture pins that prints +# policy/.json (no file: it fails) and a note on stderr (which must not +# read as the policy). Sets SHA_A, SHA_B, SHA_C. +canary_repo() { + fixture "$1" '[]' '[]'; CR=$FIX/repo + ( set -e + export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 GIT_AUTHOR_NAME=t GIT_AUTHOR_EMAIL=t@t \ + GIT_COMMITTER_NAME=t GIT_COMMITTER_EMAIL=t@t + git init -q --bare -b main "$FIX/origin.git"; git init -q -b main "$CR"; cd "$CR" + echo base > f; git add f; git commit -qm base; git remote add origin "$FIX/origin.git"; git push -q origin main + for b in a b c; do git switch -qc "autobump/$b" main; echo "$b" > f; git commit -qam "$b"; git push -q origin "autobump/$b"; done + git switch -q main; git fetch -q origin + mkdir -p hack policy; cp "$BCD" hack/bot-ci-dispatch + printf '%s\n' '#!/usr/bin/env bash' \ + '[ "$1" = policy ] && git rev-parse -q --verify "$2^{commit}" >/dev/null && git cat-file -e "$3^{commit}" || exit 1' \ + 'echo "pins: a note on stderr" >&2' \ + 'cat "$(dirname "$0")/../policy/$3.json"' > hack/pins + chmod +x hack/pins hack/bot-ci-dispatch ) + SHA_A=$(git -C "$CR" rev-parse origin/autobump/a); SHA_B=$(git -C "$CR" rev-parse origin/autobump/b) + SHA_C=$(git -C "$CR" rev-parse origin/autobump/c) + echo '{"verdict":"auto","canary":["vllm-cpu"]}' > "$CR/policy/$SHA_A.json" + echo '{"verdict":"auto","canary":[]}' > "$CR/policy/$SHA_B.json" +} +bcd_repo() { OUT=$(cd "$CR" && PATH="$T/bin:$PATH" timeout 60 bash hack/bot-ci-dispatch "$@" 2>&1); RC=$?; } +canary_repo canary_mapped +echo "[$(pr 1 autobump/a "$SHA_A"),$(pr 2 autobump/b "$SHA_B"),$(pr 3 autobump/c "$SHA_C")]" > "$FIX/prs.json" +echo "[$(run 5 ci.yml workflow_dispatch autobump/a "$SHA_A" completed success)]" > "$FIX/runs.json" +BOT_CI_MAX_INFLIGHT=10 bcd_repo --workflow canary.yml +if [ "$RC" -eq 1 ] && [ "$(acts dispatch)" -eq 1 ] && dispatched autobump/a canary.yml \ + && grep -qxF 'canary.yml autobump/a engines=auto' "$FIX/fields" \ + && grep -q '#2 autobump/b.*no canary engine' <<<"$OUT" && grep -q '#3 autobump/c.*policy' <<<"$OUT"; then + ok "canary: dispatched with engines=auto where the policy maps engines; none mapped: skipped; policy failed: exit 1" +else fail "canary: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") fields=$(tr '\n' ';' < "$FIX/fields" 2>/dev/null) out=$(tr '\n' ';' <<<"$OUT")"; fi +# a head that moved on origin since the PR list was read: no canary for the stale SHA. +canary_repo canary_moved; echo "[$(pr 1 autobump/a "$SHA_B")]" > "$FIX/prs.json" +cp "$CR/policy/$SHA_A.json" "$CR/policy/$SHA_B.json"; bcd_repo --workflow canary.yml +if [ "$(acts dispatch)" -eq 0 ] && grep -q '#1 autobump/a.*moved' <<<"$OUT"; then ok "canary_moved: no dispatch" +else fail "canary_moved: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi +# ci.yml never consults the policy, and its dispatch carries no inputs. +canary_repo ci_no_policy; rm "$CR/hack/pins" +echo "[$(pr 1 autobump/a "$SHA_A"),$(pr 2 autobump/b "$SHA_B")]" > "$FIX/prs.json" +BOT_CI_MAX_INFLIGHT=10 bcd_repo +if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 2 ] && grep -qxF 'ci.yml autobump/a' "$FIX/fields"; then ok "ci_no_policy: ci.yml dispatched for both, no inputs" +else fail "ci_no_policy: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi + bcd --bogus if [ "$RC" -eq 2 ]; then ok "usage: unknown flag exits 2"; else fail "usage: rc=$RC"; fi diff --git a/tests/bot_merge_fake_gh.py b/tests/bot_merge_fake_gh.py new file mode 100755 index 00000000..07675298 --- /dev/null +++ b/tests/bot_merge_fake_gh.py @@ -0,0 +1,342 @@ +#!/usr/bin/env python3 +"""A stand-in for `gh` that answers from fixture files (tests/bot_merge_test.sh). + +State lives under $FIX/gh: + pulls.json REST pull objects (number, title, user, head, base, labels, + created_at, draft, state, merged) + files/.json REST pull files ([{filename}]) + reviews/.json REST reviews + comments/.json REST issue comments (POST and PATCH edit this file) + runs.json `gh run list --json` rows, plus "workflow" (the file name) + artifacts/// files `gh run download -n ` extracts + issues.json [{number, title, state, labels: [{name}]}] + merge_rc exit code sequence for `gh pr merge` (last one repeats) +Calls are logged to $FIX/gh/calls; dispatches to $FIX/gh/dispatches, merges to +$FIX/gh/merges, label additions to $FIX/gh/labels. + +A successful merge marks the pull merged and writes the squash commit onto the +bare origin's main ($FIX/origin.git): main's tree merged with the head's, authored +by github-actions[bot], committed at $BOT_MERGE_NOW. +""" +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +FIX = Path(os.environ["FIX"]) +G = FIX / "gh" + + +def load(p, default): + try: + return json.loads(p.read_text()) + except FileNotFoundError: + return default + + +def save(p, obj): + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(json.dumps(obj, indent=1)) + + +def log(name, line): + with open(G / name, "a") as f: + f.write(line + "\n") + + +def pop(name, default): + p = G / name + if not p.exists(): + return default + seq = p.read_text().split() + i_path = G / (name + ".i") + i = int(i_path.read_text()) if i_path.exists() else 0 + i_path.write_text(str(i + 1)) + return seq[min(i, len(seq) - 1)] + + +def emit(obj, jq_expr): + """Print obj as gh does: --jq output through jq (strings raw), else JSON.""" + text = json.dumps(obj) + if jq_expr is None: + print(text) + return 0 + r = subprocess.run(["jq", "-c", "-r", jq_expr], input=text, text=True, capture_output=True) + sys.stdout.write(r.stdout) + sys.stderr.write(r.stderr) + return r.returncode + + +def flags(argv, with_value): + """Split argv into positionals and a {flag: [values]} map.""" + pos, opts = [], {} + i = 0 + while i < len(argv): + a = argv[i] + if a in with_value: + opts.setdefault(a, []).append(argv[i + 1]) + i += 2 + elif a.startswith("-"): + opts.setdefault(a, []).append(True) + i += 1 + else: + pos.append(a) + i += 1 + return pos, opts + + +def one(opts, *names): + for n in names: + if n in opts: + return opts[n][-1] + return None + + +def pull(n): + for p in load(G / "pulls.json", []): + if p["number"] == n: + return p + return None + + +def set_pull(n, **kv): + pulls = load(G / "pulls.json", []) + for p in pulls: + if p["number"] == n: + p.update(kv) + save(G / "pulls.json", pulls) + + +# ---- gh api --------------------------------------------------------------- +def api(argv): + pos, opts = flags(argv, {"-X", "--method", "-f", "--raw-field", "-F", "--field", + "-q", "--jq", "-H", "--header", "--input"}) + endpoint = pos[0].lstrip("/") + method = (one(opts, "-X", "--method") or "GET").upper() + fields = {} + for kv in opts.get("-f", []) + opts.get("--raw-field", []): # raw: taken as given + k, v = kv.split("=", 1) + fields.setdefault(k, []).append(v) + for kv in opts.get("-F", []) + opts.get("--field", []): # typed: @path reads a file + k, v = kv.split("=", 1) + if v.startswith("@"): + v = Path(v[1:]).read_text() if v != "@-" else sys.stdin.read() + fields.setdefault(k, []).append(v) + if "-f" in opts or "-F" in opts or "--field" in opts or "--raw-field" in opts: + if method == "GET" and "-X" not in opts and "--method" not in opts: + method = "POST" + jq_expr = one(opts, "-q", "--jq") + path, _, query = endpoint.partition("?") + parts = path.split("/") + # repos///... + rest = parts[3:] + if rest[:1] == ["pulls"] and len(rest) == 1: + q = dict(kv.split("=", 1) for kv in query.split("&") if kv) + pulls = [p for p in load(G / "pulls.json", []) + if q.get("state", "open") in ("all", p.get("state", "open")) + and q.get("base", p["base"]["ref"]) == p["base"]["ref"]] + return emit(pulls, jq_expr) + if rest[:1] == ["pulls"] and len(rest) == 2: + p = pull(int(rest[1])) + if p is None: + print("gh: Not Found (HTTP 404)", file=sys.stderr) + return 1 + return emit(p, jq_expr) + if rest[:1] == ["pulls"] and rest[2] == "files": + return emit(load(G / "files" / f"{rest[1]}.json", []), jq_expr) + if rest[:1] == ["pulls"] and rest[2] == "reviews": + return emit(load(G / "reviews" / f"{rest[1]}.json", []), jq_expr) + if rest[:1] == ["issues"] and len(rest) == 3 and rest[2] == "comments": + cpath = G / "comments" / f"{rest[1]}.json" + comments = load(cpath, []) + if method == "POST": + ids = [c["id"] for f in (G / "comments").glob("*.json") for c in load(f, [])] + c = {"id": max(ids, default=1000) + 1, "user": {"login": "github-actions[bot]", "type": "Bot"}, + "body": fields["body"][-1]} + comments.append(c) + save(cpath, comments) + log("calls", f"comment-post {rest[1]}") + return emit(c, jq_expr) + return emit(comments, jq_expr) + if rest[:2] == ["issues", "comments"] and method == "PATCH": + cid = int(rest[2]) + for f in (G / "comments").glob("*.json"): + comments = load(f, []) + for c in comments: + if c["id"] == cid: + c["body"] = fields["body"][-1] + save(f, comments) + log("calls", f"comment-patch {f.stem} {cid}") + return emit(c, jq_expr) + print("gh: Not Found (HTTP 404)", file=sys.stderr) + return 1 + if rest[:1] == ["issues"] and len(rest) == 3 and rest[2] == "labels" and method == "POST": + n = int(rest[1]) + names = fields.get("labels[]", []) + p = pull(n) + have = [lab["name"] for lab in p["labels"]] + set_pull(n, labels=p["labels"] + [{"name": x} for x in names if x not in have]) + for x in names: + log("labels", f"{n} {x}") + return emit([{"name": x} for x in have + names], jq_expr) + print(f"bot_merge_fake_gh: unhandled api {method} {endpoint}", file=sys.stderr) + return 99 + + +# ---- gh run --------------------------------------------------------------- +def wf_match(run, wf): + return wf in (run.get("workflow"), run.get("workflowName")) + + +def run_list(argv): + pos, opts = flags(argv, {"-w", "--workflow", "-c", "--commit", "-b", "--branch", "-e", "--event", + "-s", "--status", "-u", "--user", "--created", "-L", "--limit", + "--json", "-q", "--jq"}) + runs = load(G / "runs.json", []) + wf = one(opts, "-w", "--workflow") + sha = one(opts, "-c", "--commit") + br = one(opts, "-b", "--branch") + ev = one(opts, "-e", "--event") + st = one(opts, "-s", "--status") + created = one(opts, "--created") + log("calls", "run-list " + " ".join(f"{k}={v}" for k, v in + (("w", wf), ("c", sha), ("b", br), ("e", ev), ("s", st), ("created", created)) if v)) + out = [] + for r in runs: + if wf and not wf_match(r, wf): + continue + if sha and r["headSha"] != sha: + continue + if br and r["headBranch"] != br: + continue + if ev and r["event"] != ev: + continue + if st and st not in (r["status"], r.get("conclusion")): + continue + if created: + assert created.startswith(">="), created + if r["createdAt"] < created[2:]: + continue + out.append(r) + out.sort(key=lambda r: r["createdAt"], reverse=True) + out = out[: int(one(opts, "-L", "--limit") or 20)] + fields = (one(opts, "--json") or "").split(",") + out = [{k: r.get(k) for k in fields} for r in out] + return emit(out, one(opts, "-q", "--jq")) + + +def run_download(argv): + pos, opts = flags(argv, {"-n", "--name", "-D", "--dir", "-p", "--pattern"}) + rid = pos[0] + names = opts.get("-n", []) + opts.get("--name", []) + dest = Path(one(opts, "-D", "--dir") or ".") + log("calls", f"run-download {rid} {' '.join(names)}") + src = G / "artifacts" / rid + found = [n for n in names if (src / n).is_dir()] + if not found: + print("no valid artifacts found to download", file=sys.stderr) + return 1 + dest.mkdir(parents=True, exist_ok=True) + for f in (src / found[0]).iterdir(): + shutil.copy(f, dest / f.name) + return 0 + + +# ---- gh pr ---------------------------------------------------------------- +HELP = " --match-head-commit SHA Commit SHA that the pull request head must match to allow merge\n" + + +def git_origin(*args, env=None): + return subprocess.run(["git", f"--git-dir={FIX / 'origin.git'}", *args], text=True, + capture_output=True, check=True, env=env).stdout.strip() + + +def pr_merge(argv): + if "--help" in argv: + sys.stdout.write((G / "merge_help").read_text() if (G / "merge_help").exists() else HELP) + return 0 + pos, opts = flags(argv, {"--match-head-commit", "-t", "--subject", "-b", "--body", "-F", "--body-file"}) + n = int(pos[0]) + sha = one(opts, "--match-head-commit") + subj = one(opts, "-t", "--subject") + body = one(opts, "-b", "--body") + log("merges", " ".join(argv).replace("\n", "\\n")) + p = pull(n) + rc = int(pop("merge_rc", "0")) + if rc != 0: + print("GraphQL: Base branch was modified. Review and try the merge again. (mergePullRequest)", file=sys.stderr) + return rc + if sha != p["head"]["sha"]: + print("GraphQL: Head branch was modified. Review and try the merge again. (mergePullRequest)", file=sys.stderr) + return 1 + tree = git_origin("merge-tree", "--write-tree", "main", sha).splitlines()[0] + when = os.environ["BOT_MERGE_NOW"] + " +0000" + env = dict(os.environ, GIT_AUTHOR_NAME="github-actions[bot]", + GIT_AUTHOR_EMAIL="41898282+github-actions[bot]@users.noreply.github.com", + GIT_COMMITTER_NAME="GitHub", GIT_COMMITTER_EMAIL="noreply@github.com", + GIT_AUTHOR_DATE=when, GIT_COMMITTER_DATE=when) + commit = git_origin("commit-tree", tree, "-p", "main", "-m", subj, "-m", body, env=env) + git_origin("update-ref", "refs/heads/main", commit) + set_pull(n, state="closed", merged=True) + return 0 + + +def pr_view(argv): + pos, opts = flags(argv, {"--json", "-q", "--jq"}) + p = pull(int(pos[0])) + obj = {"state": "MERGED" if p.get("merged") else p.get("state", "open").upper(), + "headRefOid": p["head"]["sha"], "headRefName": p["head"]["ref"]} + return emit(obj, one(opts, "-q", "--jq")) + + +# ---- gh issue / workflow / repo ------------------------------------------- +def issue_list(argv): + pos, opts = flags(argv, {"--label", "-l", "--state", "-s", "--json", "-L", "--limit", "-q", "--jq"}) + label = one(opts, "--label", "-l") + state = one(opts, "--state", "-s") or "open" + issues = [i for i in load(G / "issues.json", []) + if (state == "all" or i.get("state", "open") == state) + and (label is None or any(lab["name"] == label for lab in i.get("labels", [])))] + fields = (one(opts, "--json") or "number,title").split(",") + return emit([{k: i.get(k) for k in fields} for i in issues], one(opts, "-q", "--jq")) + + +def workflow_run(argv): + pos, opts = flags(argv, {"-f", "--raw-field", "-F", "--field", "-r", "--ref"}) + kv = opts.get("-f", []) + opts.get("--raw-field", []) + opts.get("-F", []) + opts.get("--field", []) + log("dispatches", " ".join([pos[0], "ref=" + str(one(opts, "-r", "--ref"))] + kv)) + rc = int(pop("dispatch_rc", "0")) + if rc == 0: + print(f"https://github.com/o/r/actions/runs/9{len((G / 'dispatches').read_text().splitlines())}") + return rc + + +def main(argv): + G.mkdir(parents=True, exist_ok=True) + log("trace", ("gh " + " ".join(argv)).replace("\n", "\\n")) + if "-R" in argv or "--repo" in argv: + print("bot_merge_fake_gh: -R is not expected", file=sys.stderr) + return 98 + head = argv[:2] + if argv[:1] == ["api"]: + return api(argv[1:]) + if head == ["run", "list"]: + return run_list(argv[2:]) + if head == ["run", "download"]: + return run_download(argv[2:]) + if head == ["pr", "merge"]: + return pr_merge(argv[2:]) + if head == ["pr", "view"]: + return pr_view(argv[2:]) + if head == ["issue", "list"]: + return issue_list(argv[2:]) + if head == ["workflow", "run"]: + return workflow_run(argv[2:]) + print(f"bot_merge_fake_gh: unhandled {' '.join(argv)}", file=sys.stderr) + return 99 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/bot_merge_test.sh b/tests/bot_merge_test.sh new file mode 100755 index 00000000..61f44a18 --- /dev/null +++ b/tests/bot_merge_test.sh @@ -0,0 +1,446 @@ +#!/usr/bin/env bash +# hack/bot-merge against a fake gh (tests/bot_merge_fake_gh.py) and real git. +# Each case builds a fixture: a bare origin whose main holds the ledger files +# rules_stale reads (console/model_opt.py, EXPERIMENTS.md, the ledger-rules.json +# it generates), scripts/k3d-gpu and tag aur-v0.1.79; a clone ($FIX/repo) on main +# whose hack/ holds bot-merge, gate_lib.sh and a fixture `pins` that prints +# $FIX/policy/.json; and the fake GitHub's state under $FIX/gh (pulls, +# files, reviews, comments, workflow runs, canary artifacts, alarm issues). A +# successful fake merge writes the squash commit onto origin's main. +# +# Usage: tests/bot_merge_test.sh + +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +BOT_MERGE=${BOT_MERGE:-${ROOT}/hack/bot-merge} +GATE_LIB=${GATE_LIB:-${ROOT}/hack/gate_lib.sh} +T=$(mktemp -d "${TMPDIR:-/tmp}/bot_merge_test.XXXXXX") +trap '[ -n "${KEEP:-}" ] || rm -rf "$T"' EXIT +PASS=0 FAILS=0 +ok() { PASS=$((PASS + 1)); echo "ok $1"; } +fail() { FAILS=$((FAILS + 1)); echo "FAIL $1"; } + +export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 \ + GIT_AUTHOR_NAME=t GIT_AUTHOR_EMAIL=t@t GIT_COMMITTER_NAME=t GIT_COMMITTER_EMAIL=t@t +mkdir -p "$T/bin" +cp "$ROOT/tests/bot_merge_fake_gh.py" "$T/bin/gh"; chmod +x "$T/bin/gh" + +NOW=$(date -u -d 2026-10-29T14:13:00Z +%s) # the pass runs at 14:13 UTC +iso() { date -u -d "@$1" +%Y-%m-%dT%H:%M:%SZ; } +ago() { iso $((NOW - $1 * 60)); } # ago MINUTES: an ISO time before the pass + +# --- fixture builders ------------------------------------------------------ +G() { echo "$FIX/gh/$1"; } +jqi() { local f; f=$(G "$1"); shift; jq "$@" "$f" > "$f.tmp" && mv "$f.tmp" "$f"; } +rgit() { git -C "$FIX/repo" "$@"; } +# commit_at MINUTES_AGO MSG [AUTHOR]: commit the clone's staged changes at that time. +commit_at() { + local when; when="$(( NOW - $1 * 60 )) +0000" + GIT_AUTHOR_DATE=$when GIT_COMMITTER_DATE=$when GIT_AUTHOR_NAME=${3:-t} \ + git -C "$FIX/repo" commit -q -m "$2" +} + +# fx NAME [TAG_MINUTES_AGO]: fresh fixture; main's base commit (tagged aur-v0.1.79) +# was made TAG_MINUTES_AGO (default 60) and its ci.yml push run succeeded. +fx() { + FIX=$T/$1; export FIX + mkdir -p "$FIX/gh/files" "$FIX/gh/reviews" "$FIX/gh/comments" "$FIX/policy" + echo '[]' > "$(G pulls.json)"; echo '[]' > "$(G runs.json)"; echo '[]' > "$(G issues.json)" + : > "$(G dispatches)"; : > "$(G merges)"; : > "$(G labels)"; : > "$(G calls)" + RUNID=1000 + git init -q --bare -b main "$FIX/origin.git"; git init -q -b main "$FIX/repo" + local R=$FIX/repo + mkdir -p "$R/console" "$R/scripts" "$R/docs" "$R/hack" + cp "$ROOT/console/model_opt.py" "$ROOT/console/convert-pins.json" "$R/console/" + printf '## Hypotheses\n\n| H34 | on the i9-13900T | p | m | SUPPORTED |\n' > "$R/EXPERIMENTS.md" + python3 "$R/console/model_opt.py" ledger-rules --ledger "$R/EXPERIMENTS.md" --out "$R/console/ledger-rules.json" + printf '#!/bin/sh\nVERSION="0.1.79"\n' > "$R/scripts/k3d-gpu" + echo base > "$R/docs/provenance.md"; echo 'k3s v1.35.0' > "$R/pins.txt" + rgit add console scripts docs EXPERIMENTS.md pins.txt + commit_at "${2:-60}" base + rgit tag aur-v0.1.79 + rgit remote add origin "$FIX/origin.git"; rgit push -q origin main aur-v0.1.79; rgit fetch -q origin + # the code under test, plus the fixture policy (untracked in the clone) + cp "$BOT_MERGE" "$R/hack/bot-merge"; cp "$GATE_LIB" "$R/hack/gate_lib.sh" + cat > "$R/hack/pins" <<'PINS' +#!/usr/bin/env bash +# fixture: `pins policy BASE HEAD` prints $FIX/policy/HEAD.json once both refs resolve +[ "$1" = policy ] || exit 2 +echo "pins $*" >> "$FIX/gh/trace" +git rev-parse -q --verify "$2^{commit}" >/dev/null || { echo "pins: no base $2" >&2; exit 1; } +git cat-file -e "$3^{commit}" 2>/dev/null || { echo "pins: head $3 not fetched" >&2; exit 1; } +cat "$FIX/policy/$3.json" +PINS + chmod +x "$R/hack/pins" "$R/hack/bot-merge" + run ci.yml push "$(rgit rev-parse HEAD)" main completed success "$(ago "${2:-60}")" +} + +wfname() { case $1 in ci.yml) echo CI;; canary.yml) echo Canary;; freshness.yml) echo Freshness;; + auto-release.yml) echo 'Auto-release AUR';; build-and-push.yml) echo 'Auto-update, Build & Release';; *) echo "$1";; esac; } +# run WORKFLOW EVENT SHA BRANCH STATUS CONCLUSION [CREATED] [TITLE]: add a workflow run; sets LASTRUN. +run() { + RUNID=$((RUNID + 1)); LASTRUN=$RUNID + jqi runs.json --argjson id "$RUNID" --arg wf "$1" --arg name "$(wfname "$1")" --arg ev "$2" --arg sha "$3" \ + --arg br "$4" --arg st "$5" --arg c "$6" --arg at "${7:-$(ago 30)}" --arg t "${8:-$(wfname "$1")}" \ + '. + [{databaseId: $id, workflow: $wf, workflowName: $name, event: $ev, headSha: $sha, headBranch: $br, + status: $st, conclusion: $c, createdAt: $at, displayTitle: $t, attempt: 1, + url: ("https://github.com/o/r/actions/runs/\($id)")}]' +} +# mkpr N [FILE] [CREATED_MINUTES_AGO]: bot PR #N on autobump/gN, one commit changing FILE +# (default pins.txt) off origin/main; policy auto with no canary engines, a green +# ci.yml dispatch run on the branch. Sets SHA. +mkpr() { + local n=$1 f=${2:-pins.txt} br=autobump/g$1 + rgit fetch -q origin; rgit checkout -q -B "$br" origin/main + mkdir -p "$(dirname "$FIX/repo/$f")"; echo "bump $n" >> "$FIX/repo/$f"; rgit add "$f" + commit_at "${3:-50}" "chore(g$n): bump"; rgit push -q -f origin "$br"; SHA=$(rgit rev-parse HEAD) + rgit checkout -q main + jqi pulls.json --argjson n "$n" --arg br "$br" --arg sha "$SHA" --arg at "$(ago "${3:-50}")" \ + '. + [{number: $n, title: "chore(g\($n)): bump g\($n) to its newest patch", state: "open", draft: false, + user: {login: "github-actions[bot]", type: "Bot"}, created_at: $at, + head: {ref: $br, sha: $sha, repo: {full_name: "o/r"}}, base: {ref: "main", repo: {full_name: "o/r"}}, + labels: [{name: "autobump"}]}]' + jq -n --arg f "$f" '[{filename: $f}]' > "$FIX/gh/files/$n.json" + jq -n --arg g "g$n" '{verdict: "auto", reasons: [], group: $g, update_type: "patch", urgent: false, canary: []}' \ + > "$FIX/policy/$SHA.json" + run ci.yml workflow_dispatch "$SHA" "$br" completed success "$(ago 20)" "CI" +} +no_ci() { jqi runs.json --arg s "$1" 'map(select(.workflow != "ci.yml" or .headSha != $s))'; } # drop SHA's ci.yml runs +pr_set() { jqi pulls.json --argjson n "$1" "map(if .number == \$n then $2 else . end)"; } +pol() { local f=$FIX/policy/$1.json; jq "$2" "$f" > "$f.tmp" && mv "$f.tmp" "$f"; } +# review N LOGIN TYPE ASSOCIATION STATE COMMIT [SUBMITTED_MINUTES_AGO] +review() { + local f=$FIX/gh/reviews/$1.json; [ -f "$f" ] || echo '[]' > "$f" + jq --arg u "$2" --arg t "$3" --arg a "$4" --arg s "$5" --arg c "$6" --arg at "$(ago "${7:-10}")" \ + '. + [{user: {login: $u, type: $t}, author_association: $a, state: $s, commit_id: $c, submitted_at: $at}]' \ + "$f" > "$f.tmp" && mv "$f.tmp" "$f" +} +# canary SHA BRANCH ENGINES_JSON [STATUS] [CONCLUSION]: a canary.yml run for SHA and its artifact. +canary() { + run canary.yml workflow_dispatch "$1" "$2" "${4:-completed}" "${5:-success}" "$(ago 15)" Canary + mkdir -p "$FIX/gh/artifacts/$LASTRUN/canary-result" + jq -n --arg sha "$1" --arg ref "$2" --argjson e "$3" '{sha: $sha, ref: $ref, engines: $e}' \ + > "$FIX/gh/artifacts/$LASTRUN/canary-result/canary-result.json" +} +alarm() { jqi issues.json --arg k "$1" '. + [{number: (length + 500), title: "[ci-alarm] \($k): see the run", state: "open", labels: [{name: "ci-alarm"}]}]'; } +# botcommit MINUTES_AGO URGENT [PR]: a bot-merge squash commit on origin's main, with +# a green ci.yml run for it. +botcommit() { + rgit fetch -q origin; rgit checkout -q main; rgit reset -q --hard origin/main + echo "merged $1" >> "$FIX/repo/pins.txt"; rgit add pins.txt + local when; when="$(( NOW - $1 * 60 )) +0000" + GIT_AUTHOR_DATE=$when GIT_COMMITTER_DATE=$when GIT_AUTHOR_NAME='github-actions[bot]' \ + git -C "$FIX/repo" commit -q -m "chore(g${3:-9}): bump (#${3:-9})" \ + -m "Bump of g${3:-9}." -m "Bot-merge: pr=${3:-9} group=g${3:-9} urgent=$2" + rgit push -q origin main + run ci.yml workflow_dispatch "$(rgit rev-parse HEAD)" main completed success "$(ago $(( $1 > 2 ? $1 - 2 : 0 )))" +} +# main_commit MINUTES_AGO FILE CONTENT [AUTHOR] [MSG]: a commit on origin's main by someone else. +main_commit() { + rgit fetch -q origin; rgit checkout -q main; rgit reset -q --hard origin/main + printf '%s' "$3" > "$FIX/repo/$2"; rgit add "$2"; commit_at "$1" "${5:-edit $2}" "${4:-t}"; rgit push -q origin main +} + +# run_bm: one pass. CAP=N run_bm sets MAX_BOT_MERGES_PER_DAY (unset: the script's default). +run_bm() { + OUT=$(cd "$FIX/repo" && PATH="$T/bin:$PATH" GITHUB_REPOSITORY=o/r BOT_MERGE_NOW=$NOW \ + GITHUB_SERVER_URL=https://github.com GITHUB_RUN_ID=777 env ${CAP:+MAX_BOT_MERGES_PER_DAY=$CAP} \ + timeout 120 bash hack/bot-merge 2>&1); RC=$? +} +merges() { wc -l < "$(G merges)"; } +merged() { grep -cE "^$1 " "$(G merges)"; } +pinned() { grep -E "^$1 " "$(G merges)" | grep -oE -- '--match-head-commit [0-9a-f]+' | cut -d' ' -f2; } +disp() { grep -cE "$1" "$(G dispatches)"; } +comment() { jq -r '.[].body' "$FIX/gh/comments/$1.json" 2>/dev/null; } +ncomments() { jq length "$FIX/gh/comments/$1.json" 2>/dev/null || echo 0; } +show() { echo "rc=$RC merges=$(merges) dispatches=[$(tr '\n' ';' < "$(G dispatches)")] out=$(tr '\n' ';' <<<"$OUT")"; } + +# 1. auto verdict, all green: one merge pinned to the head, then ci.yml dispatched on main; +# the squash subject is conventional and the body carries the Bot-merge line. +fx auto_green; mkpr 1; run_bm +main_head=$(git --git-dir="$FIX/origin.git" log -1 --format=%B main) +if [ "$RC" -eq 0 ] && [ "$(merges)" -eq 1 ] && [ "$(pinned 1)" = "$SHA" ] && grep -q -- '--squash' "$(G merges)" \ + && [ "$(disp '^ci.yml ref=main$')" -eq 1 ] && head -1 <<<"$main_head" | grep -qxE 'chore\(g1\): bump g1 to its newest patch \(#1\)' \ + && grep -qx 'Bot-merge: pr=1 group=g1 urgent=false' <<<"$main_head" \ + && comment 1 | grep -q 'Merged' && comment 1 | grep -qF 'actions/runs/'; then + ok "auto_green: merged at the head, ci.yml dispatched on main, evidence comment" +else fail "auto_green: $(show) msg=$(tr '\n' '|' <<<"$main_head")"; fi + +# 2. review verdict, no approval: no merge, a blocking comment that names the reason. +fx review_no_approval; mkpr 1; pol "$SHA" '.verdict = "review" | .reasons = ["minor update of a static-only engine"]'; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'needs review' && comment 1 | grep -q 'static-only engine' \ + && [ "$(disp ci.yml)" -eq 0 ]; then ok "review_no_approval: blocked with a comment" +else fail "review_no_approval: $(show) comment=$(comment 1)"; fi + +# 3. review verdict, approved only by a bot account: no merge. +fx review_bot_approval; mkpr 1; pol "$SHA" '.verdict = "review"' +review 1 'github-actions[bot]' Bot NONE APPROVED "$SHA"; review 1 'some-app[bot]' Bot MEMBER APPROVED "$SHA"; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'needs review'; then ok "review_bot_approval: a bot approval never counts" +else fail "review_bot_approval: $(show)"; fi + +# 4. review verdict with an OWNER approval at the head: merges. +fx review_owner_approval; mkpr 1; pol "$SHA" '.verdict = "review"'; review 1 88plug User OWNER APPROVED "$SHA"; run_bm +if [ "$(merged 1)" -eq 1 ] && [ "$(pinned 1)" = "$SHA" ] && comment 1 | grep -q '@88plug'; then + ok "review_owner_approval: merged on the human approval at the head" +else fail "review_owner_approval: $(show)"; fi + +# 4b. an approval from a user without write access (CONTRIBUTOR) does not count. +fx review_contributor; mkpr 1; pol "$SHA" '.verdict = "review"'; review 1 someone User CONTRIBUTOR APPROVED "$SHA"; run_bm +if [ "$(merges)" -eq 0 ]; then ok "review_contributor: no merge"; else fail "review_contributor: $(show)"; fi + +# 5. an approval at an older SHA does not count for the new head. +fx approval_old_sha; mkpr 1; old=$SHA; mkpr 1 pins.txt 45; pol "$SHA" '.verdict = "review"' +review 1 88plug User OWNER APPROVED "$old"; jqi pulls.json 'map(select(.head.sha != "'"$old"'"))'; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'needs review'; then ok "approval_old_sha: no merge" +else fail "approval_old_sha: $(show)"; fi + +# 5b. a human's later changes-requested review blocks even an auto verdict, and such a PR +# is not regenerated when main moves. +fx changes_requested; mkpr 1; review 1 88plug User OWNER APPROVED "$SHA" 30; review 1 88plug User OWNER CHANGES_REQUESTED "$SHA" 5 +main_commit 10 scripts/other.sh 'echo moved'; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'changes requested' && [ "$(disp freshness.yml)" -eq 0 ]; then ok "changes_requested: blocks, no regeneration" +else fail "changes_requested: $(show) comment=$(comment 1)"; fi + +# 6. a held (action_required) pull_request run newer than a green dispatch run: merges. +fx held_plus_dispatch; mkpr 1; run ci.yml pull_request "$SHA" autobump/g1 completed action_required "$(ago 5)"; run_bm +if [ "$(merged 1)" -eq 1 ]; then ok "held_plus_dispatch: the dispatch run counts, the held run is ignored" +else fail "held_plus_dispatch: $(show)"; fi + +# 7. CI still running at the head: skip. +fx ci_pending; mkpr 1; no_ci "$SHA"; run ci.yml workflow_dispatch "$SHA" autobump/g1 in_progress '' "$(ago 2)"; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'CI is pending'; then ok "ci_pending: skipped" +else fail "ci_pending: $(show) comment=$(comment 1)"; fi + +# 7b. CI failed at the head: skip. +fx ci_failed; mkpr 1; run ci.yml workflow_dispatch "$SHA" autobump/g1 completed failure "$(ago 2)"; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'CI failed'; then ok "ci_failed: skipped" +else fail "ci_failed: $(show)"; fi + +# 8. canary engines mapped but no canary run, or one still running: skip. +fx canary_missing; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]'; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -qi 'canary has not run'; then ok "canary_missing: skipped" +else fail "canary_missing: $(show) comment=$(comment 1)"; fi +fx canary_pending; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]'; canary "$SHA" autobump/g1 '{}' in_progress ''; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -qi 'canary is pending'; then ok "canary_pending: skipped" +else fail "canary_pending: $(show) comment=$(comment 1)"; fi + +# 8b. canary green for every mapped engine: merges, and the comment links the canary run. +fx canary_pass; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu","node"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"kind":"runtime","verdict":"pass","hard":true,"exact":true},"node":{"kind":"node","verdict":"pass","hard":true,"exact":true}}' +crun=$LASTRUN; run_bm +if [ "$(merged 1)" -eq 1 ] && comment 1 | grep -qF "actions/runs/$crun"; then ok "canary_pass: merged" +else fail "canary_pass: $(show)"; fi + +# 8c. a canary result for another SHA, or one lacking a mapped engine, is not green. +fx canary_wrong_sha; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":true}}' +jq '.sha = "0000000"' "$FIX/gh/artifacts/$LASTRUN/canary-result/canary-result.json" > "$T/x" && mv "$T/x" "$FIX/gh/artifacts/$LASTRUN/canary-result/canary-result.json"; run_bm +if [ "$(merges)" -eq 0 ]; then ok "canary_wrong_sha: no merge"; else fail "canary_wrong_sha: $(show)"; fi +fx canary_lacks_engine; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu","ovms-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":true}}'; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'ovms-cpu'; then ok "canary_lacks_engine: no merge"; else fail "canary_lacks_engine: $(show)"; fi + +# 9. canary hard pass with exact false: label canary-drift, no merge (review)... +fx canary_drift; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":false,"detail":"golden differs"}}'; run_bm +if [ "$(merges)" -eq 0 ] && grep -qx '1 canary-drift' "$(G labels)" && comment 1 | grep -q 'canary-drift'; then + ok "canary_drift: labelled canary-drift, no merge" +else fail "canary_drift: $(show) labels=$(cat "$(G labels)")"; fi +# ...and with an OWNER approval at the head it merges (drift is a review condition). +fx canary_drift_approved; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":false}}' +review 1 88plug User OWNER APPROVED "$SHA"; run_bm +if [ "$(merged 1)" -eq 1 ]; then ok "canary_drift_approved: merged on the human approval" +else fail "canary_drift_approved: $(show)"; fi + +# 10. canary isa-skip (and too-big) is never green, even with an approval. +for v in isa-skip too-big; do + fx "canary_$v"; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' + canary "$SHA" autobump/g1 "{\"vllm-cpu\":{\"verdict\":\"$v\",\"hard\":false,\"exact\":false}}" + review 1 88plug User OWNER APPROVED "$SHA"; run_bm + if [ "$(merges)" -eq 0 ] && comment 1 | grep -q -- "$v" && ! grep -q canary-drift "$(G labels)"; then ok "canary_$v: no merge" + else fail "canary_$v: $(show)"; fi +done + +# 11. main moved under code since the head was tested: regeneration dispatched (freshness.yml +# group=g1); a second PR's group counts separately... +fx base_moved_code; mkpr 1; main_commit 10 scripts/other.sh 'echo moved'; run_bm +if [ "$(merges)" -eq 0 ] && [ "$(disp '^freshness.yml ref=main group=g1 reason=')" -eq 1 ] && comment 1 | grep -q 'main moved'; then + ok "base_moved_code: regeneration dispatched" +else fail "base_moved_code: $(show)"; fi +# ...two of today's dispatched runs name g1 and one names g10: a third g1 regeneration goes out... +fx regen_under_cap; mkpr 1; main_commit 10 scripts/other.sh 'echo moved' +run freshness.yml workflow_dispatch "$(rgit rev-parse origin/main)" main completed success "$(ago 120)" "Freshness group=g1 reason=x" +run freshness.yml workflow_dispatch "$(rgit rev-parse origin/main)" main completed success "$(ago 100)" "Freshness group=g1 reason=x" +run freshness.yml workflow_dispatch "$(rgit rev-parse origin/main)" main completed success "$(ago 90)" "Freshness group=g10 reason=x" +run freshness.yml workflow_dispatch "$(rgit rev-parse origin/main)" main completed success "$(iso $((NOW - 15 * 3600)))" "Freshness group=g1 reason=x" +run_bm +if [ "$(disp '^freshness.yml ref=main group=g1 ')" -eq 1 ]; then ok "regen_under_cap: 2 of 3 used today, dispatched" +else fail "regen_under_cap: $(show)"; fi +# ...with three today the cap holds. +fx regen_cap; mkpr 1; main_commit 10 scripts/other.sh 'echo moved' +for m in 120 100 80; do run freshness.yml workflow_dispatch "$(rgit rev-parse origin/main)" main completed success "$(ago $m)" "Freshness group=g1 reason=x"; done +run_bm +if [ "$(disp freshness.yml)" -eq 0 ] && [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'regeneration cap'; then ok "regen_cap: 3 regenerations today, no fourth" +else fail "regen_cap: $(show) comment=$(comment 1)"; fi +# 11b. a freshness run still in flight: wait for it, dispatch nothing. +fx regen_in_flight; mkpr 1; main_commit 10 scripts/other.sh 'echo moved' +run freshness.yml schedule "$(rgit rev-parse origin/main)" main in_progress '' "$(ago 5)" Freshness; run_bm +if [ "$(disp freshness.yml)" -eq 0 ] && [ "$(merges)" -eq 0 ]; then ok "regen_in_flight: no second dispatch" +else fail "regen_in_flight: $(show)"; fi + +# 12. main moved only by release commits (and the ledger): merges without regeneration. +fx base_moved_release; mkpr 1 +main_commit 20 scripts/k3d-gpu $'#!/bin/sh\nVERSION="0.1.80"\n' 88plug-bot 'release: aur-v0.1.80 [skip release]' +main_commit 15 docs/provenance.md $'ledger\n' +run ci.yml push "$(rgit rev-parse origin/main)" main completed success "$(ago 12)"; run_bm +if [ "$(merged 1)" -eq 1 ] && [ "$(disp freshness.yml)" -eq 0 ]; then ok "base_moved_release: merged, no regeneration" +else fail "base_moved_release: $(show)"; fi + +# 13. main's ledger moved (a new SUPPORTED row) without ledger-rules.json: the merged tree's +# rules are stale, so regenerate. +fx rules_stale; mkpr 1 +main_commit 10 EXPERIMENTS.md $'## Hypotheses\n\n| H34 | on the i9-13900T | p | m | SUPPORTED |\n| H34b | on the i5-12500T | q | n | SUPPORTED |\n' +run ci.yml push "$(rgit rev-parse origin/main)" main completed success "$(ago 8)"; run_bm +if [ "$(merges)" -eq 0 ] && [ "$(disp '^freshness.yml ref=main group=g1 ')" -eq 1 ] && comment 1 | grep -q 'ledger-rules.json'; then + ok "rules_stale: regeneration dispatched" +else fail "rules_stale: $(show) comment=$(comment 1)"; fi + +# 14. the latest completed ci.yml run on main failed: nothing merges, nothing regenerates. +fx main_ci_red; mkpr 1; mkpr 2; main_commit 10 scripts/other.sh 'echo moved' +run ci.yml push "$(rgit rev-parse origin/main)" main completed failure "$(ago 8)"; run_bm +if [ "$(merges)" -eq 0 ] && [ "$(disp freshness.yml)" -eq 0 ] && comment 1 | grep -q 'circuit breaker'; then ok "main_ci_red: nothing merges" +else fail "main_ci_red: $(show)"; fi +# a later green pull_request run whose head branch is named main (a fork's) does not close it. +fx main_ci_red_fork_main; mkpr 1 +run ci.yml push "$(rgit rev-parse origin/main)" main completed failure "$(ago 8)" +run ci.yml pull_request deadbeef main completed success "$(ago 4)"; run_bm +if [ "$(merges)" -eq 0 ]; then ok "main_ci_red_fork_main: still blocked"; else fail "main_ci_red_fork_main: $(show)"; fi + +# 15. an open ci-main alarm (and every other breaker key): nothing merges. +for k in ci-main ci-missing:freshness.yml release:aur-v0.1.79 red:canary; do + fx "alarm_${k//[:.]/_}"; mkpr 1; alarm "$k"; run_bm + if [ "$(merges)" -eq 0 ] && comment 1 | grep -qF "$k"; then ok "alarm $k: nothing merges" + else fail "alarm $k: $(show)"; fi +done + +# 16. an alarm whose key is not a breaker key: merges. +fx alarm_other; mkpr 1; alarm 'blind-spot:k3s'; run_bm +if [ "$(merged 1)" -eq 1 ]; then ok "alarm_other: merged"; else fail "alarm_other: $(show)"; fi + +# 17. daily cap reached: the non-urgent PR waits, the urgent one merges. Yesterday's merge +# does not count. +fx daily_cap; botcommit $((15 * 60)) false 7; botcommit 300 false 8; botcommit 200 false 9 +mkpr 1 pins.txt 40; mkpr 2 other.txt 30; pol "$SHA" '.urgent = true' +CAP=2 run_bm +if [ "$(merged 1)" -eq 0 ] && [ "$(merged 2)" -eq 1 ] && comment 1 | grep -q 'daily cap'; then + ok "daily_cap: non-urgent skipped, urgent merged" +else fail "daily_cap: $(show) comment=$(comment 1)"; fi + +# 18. a PR touching .github/: never merged, and its comment says why. +fx touches_github; mkpr 1 .github/workflows/ci.yml; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -qF '.github/'; then ok "touches_github: never" +else fail "touches_github: $(show)"; fi +# a file renamed out of .github/ counts too. +fx renames_github; mkpr 1 ci-moved.yml +echo '[{"filename": "ci-moved.yml", "previous_filename": ".github/workflows/ci.yml", "status": "renamed"}]' > "$FIX/gh/files/1.json"; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -qF '.github/'; then ok "renames_github: never" +else fail "renames_github: $(show)"; fi + +# 19. ledger amendments, dependabot and fork PRs are never bot-merged (and get no comment). +fx not_candidates; mkpr 1; mkpr 2; mkpr 3; mkpr 4 +pr_set 1 '.head.ref = "autobump/amend-H54q" | .labels += [{name: "ledger-amendment"}]' +pr_set 2 '.labels += [{name: "ledger-amendment"}]' +pr_set 3 '.user = {login: "dependabot[bot]", type: "Bot"}' +pr_set 4 '.head.repo.full_name = "someone/r"' +mkpr 5; pr_set 5 '.labels = []' +mkpr 6; pr_set 6 '.head.repo = null' +run_bm +if [ "$(merges)" -eq 0 ] && [ "$(ncomments 1)$(ncomments 2)$(ncomments 3)$(ncomments 4)$(ncomments 5)$(ncomments 6)" = 000000 ]; then + ok "not_candidates: amendment, dependabot, fork, unlabelled: untouched" +else fail "not_candidates: $(show)"; fi + +# 20. two eligible PRs: one merge per pass, the urgent one first though it is newer... +fx two_urgent_first; mkpr 1 pins.txt 40; mkpr 2 other.txt 30; pol "$SHA" '.urgent = true'; run_bm +if [ "$(merges)" -eq 1 ] && [ "$(merged 2)" -eq 1 ] && comment 1 | grep -q 'one merge per pass'; then + ok "two_urgent_first: only the urgent PR merged" +else fail "two_urgent_first: $(show)"; fi +# ...else the oldest. +fx two_oldest_first; mkpr 2 other.txt 30; mkpr 1 pins.txt 40; run_bm +if [ "$(merges)" -eq 1 ] && [ "$(merged 1)" -eq 1 ]; then ok "two_oldest_first: the older PR merged" +else fail "two_oldest_first: $(show)"; fi + +# 21. a refused merge: GitHub's reason printed, the pass stops (no ci.yml dispatch, no +# release), the comment says so. +fx merge_refused; mkpr 1; echo 1 > "$(G merge_rc)"; run_bm +if [ "$(merges)" -eq 1 ] && [ "$(disp ci.yml)" -eq 0 ] && grep -q 'Base branch was modified' <<<"$OUT" \ + && comment 1 | grep -q 'refused'; then ok "merge_refused: reason shown, pass stopped" +else fail "merge_refused: $(show)"; fi + +# 22. a gh without --match-head-commit never merges. +fx old_gh; mkpr 1; echo ' --squash Squash the commits' > "$(G merge_help)"; run_bm +if [ "$RC" -eq 2 ] && [ "$(merges)" -eq 0 ]; then ok "old_gh: exit 2"; else fail "old_gh: $(show)"; fi + +# 23. a policy without a canary list fails closed; a failing policy blocks. +fx policy_no_engines; mkpr 1; pol "$SHA" 'del(.canary)'; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'no canary list'; then ok "policy_no_engines: blocked" +else fail "policy_no_engines: $(show)"; fi +fx policy_fails; mkpr 1; rm "$FIX/policy/$SHA.json"; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'policy'; then ok "policy_fails: blocked" +else fail "policy_fails: $(show)"; fi + +# 24. one comment per PR, edited in place when the reason changes, untouched when not. +fx comment_in_place; mkpr 1; no_ci "$SHA"; run ci.yml workflow_dispatch "$SHA" autobump/g1 in_progress '' "$(ago 2)"; run_bm; run_bm +patches1=$(grep -c comment-patch "$(G calls)") +jqi runs.json --argjson id "$LASTRUN" 'map(if .databaseId == $id then .status = "completed" | .conclusion = "failure" else . end)'; run_bm +if [ "$(ncomments 1)" -eq 1 ] && [ "$patches1" -eq 0 ] && [ "$(grep -c comment-patch "$(G calls)")" -eq 1 ] \ + && comment 1 | grep -q 'CI failed'; then ok "comment_in_place: one comment, edited only on change" +else fail "comment_in_place: comments=$(ncomments 1) patches1=$patches1 patches=$(grep -c comment-patch "$(G calls)")"; fi + +# 25. a draft PR is not merged. +fx draft; mkpr 1; pr_set 1 '.draft = true'; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q draft; then ok "draft: no merge"; else fail "draft: $(show)"; fi + +# --- release batching --------------------------------------------------------- +REL='^auto-release.yml ref=main mode=changed$' +# R1. an urgent bot merge since the last tag: release at once, though the queue is not empty +# (PR 1 is auto-eligible with green CI, held by the daily cap) and the tag is 1 h old. +fx rel_urgent; botcommit 20 true; mkpr 1; CAP=1 run_bm +if [ "$(merges)" -eq 0 ] && [ "$(disp "$REL")" -eq 1 ]; then ok "rel_urgent: dispatched" +else fail "rel_urgent: $(show)"; fi +# R2. queue empty: dispatch. +fx rel_queue_empty; botcommit 20 false; run_bm +if [ "$(disp "$REL")" -eq 1 ]; then ok "rel_queue_empty: dispatched"; else fail "rel_queue_empty: $(show)"; fi +# R3. queue non-empty, last tag under 6 h old: wait. +fx rel_queue_wait; botcommit 20 false; mkpr 1; CAP=1 run_bm +if [ "$(disp auto-release)" -eq 0 ] && grep -q 'release: wait' <<<"$OUT"; then ok "rel_queue_wait: waits" +else fail "rel_queue_wait: $(show)"; fi +# R3b. queue non-empty but the last tag is 7 h old: dispatch. +fx rel_six_hours 420; botcommit 20 false; mkpr 1; CAP=1 run_bm +if [ "$(merges)" -eq 0 ] && [ "$(disp "$REL")" -eq 1 ]; then ok "rel_six_hours: dispatched"; else fail "rel_six_hours: $(show)"; fi +# R4. main HEAD's ci.yml run has not finished green: wait. +fx rel_main_ci; botcommit 20 false +run ci.yml workflow_dispatch "$(rgit rev-parse origin/main)" main in_progress '' "$(ago 1)"; run_bm +if [ "$(disp auto-release)" -eq 0 ]; then ok "rel_main_ci: waits"; else fail "rel_main_ci: $(show)"; fi +# R5. a release run in progress: wait. +for wf in auto-release.yml build-and-push.yml; do + fx "rel_in_progress_${wf%%.*}"; botcommit 20 false + run "$wf" workflow_dispatch "$(rgit rev-parse origin/main)" main in_progress '' "$(ago 1)"; run_bm + if [ "$(disp auto-release)" -eq 0 ]; then ok "rel_in_progress ($wf): waits"; else fail "rel_in_progress ($wf): $(show)"; fi +done +# R6. no bot merge above the last tag: nothing to release. +fx rel_nothing; run_bm +if [ "$(disp auto-release)" -eq 0 ]; then ok "rel_nothing: no dispatch"; else fail "rel_nothing: $(show)"; fi +# R7. a release run already succeeded after the newest bot merge (it found nothing to ship): +# do not dispatch again every pass. +fx rel_already; botcommit 20 false +run auto-release.yml workflow_dispatch "$(rgit rev-parse origin/main)" main completed success "$(ago 10)"; run_bm +if [ "$(disp auto-release)" -eq 0 ]; then ok "rel_already: no repeat dispatch"; else fail "rel_already: $(show)"; fi +# R8. a merge in this pass leaves main's new head without CI: the release waits for it. +fx rel_after_merge; mkpr 1; run_bm +if [ "$(merged 1)" -eq 1 ] && [ "$(disp auto-release)" -eq 0 ]; then ok "rel_after_merge: waits for main CI" +else fail "rel_after_merge: $(show)"; fi + +echo "${PASS} passed, ${FAILS} failed" +[ "${FAILS}" -eq 0 ] diff --git a/tests/ci_wiring_test.sh b/tests/ci_wiring_test.sh index d021e1e4..87b4b2b8 100755 --- a/tests/ci_wiring_test.sh +++ b/tests/ci_wiring_test.sh @@ -308,5 +308,40 @@ alarm_case gh_down '[]' '[]' '[]' issues workflows runs if [ "$ARC" -eq 0 ] && [ "$(grep -c '^::warning::' <<<"$AOUT")" -eq 2 ]; then ok "alarms: gh failing warns and still exits 0" else fail "alarms gh_down: rc=$ARC out=$(tr '\n' ';' <<<"$AOUT")"; fi +# bot-sweep.yml job merge runs hack/bot-merge after ci-fill. Its token can merge, +# so it checks out main only (never a PR's code), and the script never checks out, +# switches to or resets onto another ref. +BS="${ROOT}/.github/workflows/bot-sweep.yml" +if python3 - "${BS}" 2>/dev/null <<'PY' +import sys, yaml +jobs = yaml.safe_load(open(sys.argv[1]))["jobs"] +needs = jobs["merge"].get("needs", []) +sys.exit(0 if "ci-fill" in jobs and "ci-fill" in ([needs] if isinstance(needs, str) else needs) else 1) +PY +then + ok "bot-sweep.yml job merge needs ci-fill" +else + fail "bot-sweep.yml has no job merge that needs ci-fill" +fi +if python3 - "${BS}" 2>/dev/null <<'PY' +import sys, yaml +steps = yaml.safe_load(open(sys.argv[1]))["jobs"]["merge"]["steps"] +co = [s for s in steps if str(s.get("uses", "")).startswith("actions/checkout@")] +text = yaml.safe_dump(steps) +sys.exit(0 if co and all((s.get("with") or {}).get("ref") == "main" for s in co) + and not any(k in text for k in ("github.head_ref", "github.event", "refs/pull/", "pull_request")) + and any("hack/bot-merge" in s.get("run", "") for s in steps) else 1) +PY +then + ok "bot-sweep.yml job merge checks out main only and runs hack/bot-merge" +else + fail "bot-sweep.yml job merge checks out a ref other than main, reads a PR ref, or does not run hack/bot-merge" +fi +if [ -f "${ROOT}/hack/bot-merge" ] && ! grep -qE 'git (checkout|switch|worktree|reset)|gh pr checkout' "${ROOT}/hack/bot-merge"; then + ok "hack/bot-merge never checks out a ref" +else + fail "hack/bot-merge is missing or runs git checkout/switch/worktree/reset or gh pr checkout" +fi + echo "${PASS} passed, ${FAILS} failed" [ "${FAILS}" -eq 0 ] From f49ad673b32511685edbaf3aaaf1451edec4cc96 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 17:40:39 -0700 Subject: [PATCH 4/5] bot-merge: read the canary per engine across the head's runs, never a held or foreign run The canary evidence came from the newest canary.yml run at the head SHA, whatever it was. Two runs the canary workflow makes broke that. A pull_request run of a bot PR is held for approval (action_required, no artifact); when one is newer than the dispatched run, the PR stayed blocked. And an isa-skip re-dispatch reruns only the skipped engines, so its artifact lacks the engines the first run passed, and a multi-engine group could never merge. A dispatched run on another branch at the same SHA also counted as this PR's evidence. The runs at the head are now its pull_request runs and its workflow_dispatch runs on the PR branch, held ones aside, the same set ci_status and bot-ci-dispatch read. All must be complete and the newest must have succeeded. Each mapped engine is decided by the newest canary-result artifact for the SHA that holds it, so the first run's pass and the re-dispatch's pass combine. A newer run with no readable artifact for the SHA blocks, since what it found is unknown, and an engine entry that is not a result object is not a verdict. The evidence comment links every run whose result counted. Eight new cases cover the held run, the re-dispatch (passing and failing), a newer failure superseding a pass, a newest run red on an unmapped engine, a newest run without an artifact, another branch's run and a null entry (62 cases in all). Nine mutants of the canary reader each fail at least one case. --- hack/bot-merge | 77 +++++++++++++++++++++++++---------------- tests/bot_merge_test.sh | 56 ++++++++++++++++++++++++++++-- 2 files changed, 101 insertions(+), 32 deletions(-) diff --git a/hack/bot-merge b/hack/bot-merge index da2859ba..7418ccb8 100755 --- a/hack/bot-merge +++ b/hack/bot-merge @@ -13,12 +13,14 @@ # OWNER, MEMBER or COLLABORATOR. No such human's latest review may request # changes. Labels never change the verdict. # b. ci_status SHA BRANCH is green (ci.yml carries pin-guard and the pins checks). -# c. When the policy names canary engines (its canary list): the newest -# canary.yml run for SHA succeeded, and its canary-result artifact, made for -# SHA, shows verdict pass, hard and exact for every one of them. A hard pass -# with exact false is a review condition: the PR is labelled canary-drift and -# merges on a human approval at SHA. Any other verdict (fail, isa-skip, -# too-big, unparseable) never merges. +# c. When the policy names canary engines (its canary list): every canary.yml run +# for SHA (its pull_request runs and its dispatches on the PR branch, held ones +# aside) completed, the newest succeeded, and for every one of those engines the +# newest canary-result artifact, made for SHA, that holds it shows verdict +# pass, hard and exact (an isa-skip re-dispatch reruns only the skipped +# engines). A hard pass with exact false is a review condition: the PR is +# labelled canary-drift and merges on a human approval at SHA. Any other +# verdict (fail, isa-skip, too-big, unparseable) never merges. # d. base_ok SHA and not rules_stale SHA. Otherwise the pass dispatches # freshness.yml -f group= to regenerate the branch from main: at most 3 # per group per UTC day, counted from today's dispatched freshness runs whose @@ -128,30 +130,45 @@ regen() { echo "regeneration dispatched ($((used + 1)) of $REGEN_CAP for $g today)" else echo "the freshness.yml dispatch failed"; return 1; fi } -# canary SHA ENGINES_JSON DIR: one line per finding: "ok E", "drift E", "block WHY", -# then "url URL" when a run exists. +# canary SHA BRANCH ENGINES_JSON DIR: one line per finding: "ok E", "drift E", +# "block WHY", and "url URL" for each run whose result counted. The canary runs at SHA +# are its pull_request runs and its workflow_dispatch runs on BRANCH; a held +# (action_required) run is not an attempt. Every one must be complete and the newest +# must have succeeded. Each engine's newest result decides, because an isa-skip +# re-dispatch reruns only the skipped engines; a run, newer than the one deciding an +# engine, without a canary-result for SHA blocks, since what it found is unknown. canary() { - local runs id st concl url - runs=$(gh run list -w canary.yml -c "$1" -L 20 --json databaseId,status,conclusion,createdAt,url) \ - || { echo "block cannot read canary.yml runs"; return; } - read -r id st concl url < <(jq -r 'max_by(.createdAt) // empty - | "\(.databaseId) \(.status) \(if (.conclusion // "") == "" then "-" else .conclusion end) \(.url)"' <<<"$runs") - [ -n "${id:-}" ] || { echo "block canary has not run at this head"; return; } - echo "url $url" - [ "$st" = completed ] || { echo "block canary is pending ($url)"; return; } - if gh run download "$id" -n canary-result -D "$3/artifact" >/dev/null 2>&1 && [ -f "$3/artifact/canary-result.json" ]; then - jq -r --arg sha "$1" --argjson want "$2" ' - if .sha != $sha then "block canary result is for \(.sha), not this head" - else $want[] as $e | (.engines[$e] // null) as $r - | if $r == null then "block canary result lacks engine \($e)" - elif $r.verdict == "pass" and $r.hard == true and $r.exact == true then "ok \($e)" - elif $r.verdict == "pass" and $r.hard == true then "drift \($e)" - else "block canary \($e): \($r.verdict)" end - end' "$3/artifact/canary-result.json" 2>/dev/null || echo "block canary-result.json is unreadable ($url)" - else - echo "block canary run has no canary-result artifact ($url)" - fi + local runs concl url id f got left found + runs=$(gh run list -w canary.yml -c "$1" -L 50 --json databaseId,event,headBranch,status,conclusion,createdAt,url \ + | jq -c --arg b "$2" '[.[] | select(.conclusion != "action_required") + | select(.event == "pull_request" or (.event == "workflow_dispatch" and .headBranch == $b))] + | sort_by(.createdAt) | reverse') || { echo "block cannot read canary.yml runs"; return; } + [ "$(jq length <<<"$runs")" -gt 0 ] || { echo "block canary has not run at this head"; return; } + url=$(jq -r '[.[] | select(.status != "completed") | .url] | first // empty' <<<"$runs") + [ -z "$url" ] || { echo "block canary is pending ($url)"; return; } + read -r concl url < <(jq -r '.[0] | "\(if (.conclusion // "") == "" then "-" else .conclusion end) \(.url)"' <<<"$runs") [ "$concl" = success ] || echo "block canary run concluded $concl ($url)" + left=$3 + while read -r id url; do + [ "$(jq length <<<"$left")" -gt 0 ] || break + f=$4/canary-$id/canary-result.json + if ! gh run download "$id" -n canary-result -D "$4/canary-$id" >/dev/null 2>&1 || [ ! -f "$f" ]; then + echo "block canary run has no canary-result artifact ($url)"; return + fi + got=$(jq -r 'if (.engines | type) == "object" then .sha else error("no engines") end' "$f" 2>/dev/null) \ + || { echo "block canary-result.json is unreadable ($url)"; return; } + [ "$got" = "$1" ] || { echo "block canary result is for $got, not this head ($url)"; return; } + if ! found=$(jq -r --argjson want "$left" '.engines as $m | $want[] as $e | select($m | has($e)) | $m[$e] + | if type != "object" then "block canary \($e): no result object" + elif .verdict == "pass" and .hard == true and .exact == true then "ok \($e)" + elif .verdict == "pass" and .hard == true then "drift \($e)" + else "block canary \($e): \(.verdict)" end' "$f" 2>/dev/null) \ + || ! left=$(jq -c --argjson want "$left" '.engines as $m | [$want[] | select(. as $e | $m | has($e) | not)]' "$f"); then + echo "block canary-result.json is unreadable ($url)"; return + fi + [ -z "$found" ] || { echo "$found"; echo "url $url"; } + done < <(jq -r '.[] | "\(.databaseId) \(.url)"' <<<"$runs") + jq -r '.[] | "block canary result lacks engine \(.)"' <<<"$left" } # ci_url SHA BRANCH: the newest successful ci.yml run that ci_status counts. ci_url() { @@ -220,8 +237,8 @@ while read -r c <&3; do # canary curl_= if [ "$engines" != "[]" ]; then - canary "$sha" "$engines" "$D" > "$D/canary" - curl_=$(sed -n 's/^url //p' "$D/canary") + canary "$sha" "$br" "$engines" "$D" > "$D/canary" + curl_=$(sed -n 's/^url //p' "$D/canary" | paste -sd' ' -) while read -r w; do why "${w#block }"; done < <(grep '^block ' "$D/canary") drift=$(sed -n 's/^drift //p' "$D/canary" | paste -sd, -) if [ -n "$drift" ]; then diff --git a/tests/bot_merge_test.sh b/tests/bot_merge_test.sh index 61f44a18..1dffc1bd 100755 --- a/tests/bot_merge_test.sh +++ b/tests/bot_merge_test.sh @@ -116,9 +116,10 @@ review() { '. + [{user: {login: $u, type: $t}, author_association: $a, state: $s, commit_id: $c, submitted_at: $at}]' \ "$f" > "$f.tmp" && mv "$f.tmp" "$f" } -# canary SHA BRANCH ENGINES_JSON [STATUS] [CONCLUSION]: a canary.yml run for SHA and its artifact. +# canary SHA BRANCH ENGINES_JSON [STATUS] [CONCLUSION] [MINUTES_AGO]: a canary.yml run for SHA +# (dispatched on BRANCH) and its artifact. canary() { - run canary.yml workflow_dispatch "$1" "$2" "${4:-completed}" "${5:-success}" "$(ago 15)" Canary + run canary.yml workflow_dispatch "$1" "$2" "${4:-completed}" "${5:-success}" "$(ago "${6:-15}")" Canary mkdir -p "$FIX/gh/artifacts/$LASTRUN/canary-result" jq -n --arg sha "$1" --arg ref "$2" --argjson e "$3" '{sha: $sha, ref: $ref, engines: $e}' \ > "$FIX/gh/artifacts/$LASTRUN/canary-result/canary-result.json" @@ -240,6 +241,57 @@ if [ "$(merges)" -eq 0 ]; then ok "canary_wrong_sha: no merge"; else fail "canar fx canary_lacks_engine; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu","ovms-cpu"]' canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":true}}'; run_bm if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'ovms-cpu'; then ok "canary_lacks_engine: no merge"; else fail "canary_lacks_engine: $(show)"; fi +# an engine entry that is not a result object (null) is not a verdict. +fx canary_null_entry; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu","ovms-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":true},"ovms-cpu":null}'; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'ovms-cpu'; then ok "canary_null_entry: no merge"; else fail "canary_null_entry: $(show)"; fi + +# 8d. a held (action_required) pull_request canary run newer than the green dispatched one +# is not an attempt: merges. +fx canary_held_newer; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":true}}' completed success 20 +run canary.yml pull_request "$SHA" autobump/g1 completed action_required "$(ago 5)" Canary; run_bm +if [ "$(merged 1)" -eq 1 ]; then ok "canary_held_newer: the held run is ignored" +else fail "canary_held_newer: $(show) comment=$(comment 1)"; fi +# 8e. an isa-skip re-dispatch reruns only the skipped engine: each engine's newest result at +# the head decides, so the first run's pass for vllm-cpu and the re-dispatch's pass for +# ovms-cpu merge, and the comment links both runs. +fx canary_redispatch; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu","ovms-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":true},"ovms-cpu":{"verdict":"isa-skip","hard":false,"exact":false}}' completed failure 40 +c1=$LASTRUN +canary "$SHA" autobump/g1 '{"ovms-cpu":{"verdict":"pass","hard":true,"exact":true}}' completed success 15 +c2=$LASTRUN; run_bm +if [ "$(merged 1)" -eq 1 ] && comment 1 | grep -qF "actions/runs/$c1" && comment 1 | grep -qF "actions/runs/$c2"; then + ok "canary_redispatch: per-engine newest result, merged" +else fail "canary_redispatch: $(show) comment=$(comment 1)"; fi +# ...but a re-dispatch that fails blocks... +fx canary_redispatch_fail; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu","ovms-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":true},"ovms-cpu":{"verdict":"isa-skip","hard":false,"exact":false}}' completed failure 40 +canary "$SHA" autobump/g1 '{"ovms-cpu":{"verdict":"fail","hard":false,"exact":false}}' completed failure 15; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'ovms-cpu: fail'; then ok "canary_redispatch_fail: no merge" +else fail "canary_redispatch_fail: $(show) comment=$(comment 1)"; fi +# ...a newer result for the same engine supersedes an older pass... +fx canary_newer_fail; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":true}}' completed success 40 +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"fail","hard":false,"exact":false}}' completed failure 15; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'vllm-cpu: fail'; then ok "canary_newer_fail: no merge" +else fail "canary_newer_fail: $(show) comment=$(comment 1)"; fi +# ...a newer completed run without a canary-result artifact blocks: what it found is unknown... +fx canary_newest_no_artifact; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":true}}' completed success 40 +run canary.yml workflow_dispatch "$SHA" autobump/g1 completed cancelled "$(ago 15)" Canary; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'no canary-result artifact'; then ok "canary_newest_no_artifact: no merge" +else fail "canary_newest_no_artifact: $(show) comment=$(comment 1)"; fi +# ...a newest run that is red on an engine the policy does not map still blocks... +fx canary_run_red; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":true},"node":{"verdict":"fail","hard":false,"exact":false}}' completed failure; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'canary run concluded failure'; then ok "canary_run_red: no merge" +else fail "canary_run_red: $(show) comment=$(comment 1)"; fi +# ...and a canary dispatched on another branch at the same SHA is not this PR's run. +fx canary_other_branch; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' +canary "$SHA" autobump/other '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":true}}'; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -qi 'canary has not run'; then ok "canary_other_branch: no merge" +else fail "canary_other_branch: $(show) comment=$(comment 1)"; fi # 9. canary hard pass with exact false: label canary-drift, no merge (review)... fx canary_drift; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' From aa71b7ea8fd71475c50afbdeea809d940b685863 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 22:29:11 -0700 Subject: [PATCH 5/5] bot-merge: judge the PR's own diff, regenerate a head that conflicts, read only this PR's canary runs The pins policy now reads merge-base(main, head)..head, in bot-merge and in the canary dispatch. Against origin/main it read main's later commits backwards: one ledger commit turned an auto patch into changes-defaults (checked with the freshness branch's hack/pins), and a stale head's group became "mixed", so it was neither merged nor regenerated. base_ok passes over release commits, but those can touch the head's lines: each release rewrites K3D_GPU_VERSION in packaging/install.sh, on the line above the k3d pin. GitHub refused such a merge on every pass, and since the refused PR was the oldest, nothing else merged and no release was batched. A head that git merge-tree cannot merge onto main is now regenerated like a stale one. The regeneration dispatch sends group only: freshness.yml takes reason as known-issue: and fails any other value. The canary reader lists runs through the REST API and keeps only those whose head is this repository's PR branch. canary.yml runs on pull requests into any base, so a fork could open one from the bot's commit and its result counted as this PR's. A breaker that cannot read main's CI runs or the alarm issues still blocks every merge, and now also fails the pass, so the sweep goes red instead of staying green while merging nothing. --- hack/bot-ci-dispatch | 12 +++-- hack/bot-merge | 74 ++++++++++++++++++----------- tests/bot_ci_dispatch_test.sh | 22 +++++++-- tests/bot_merge_fake_gh.py | 30 +++++++++++- tests/bot_merge_test.sh | 88 ++++++++++++++++++++++++++++++----- 5 files changed, 176 insertions(+), 50 deletions(-) diff --git a/hack/bot-ci-dispatch b/hack/bot-ci-dispatch index 44c2a597..5fcb78e6 100755 --- a/hack/bot-ci-dispatch +++ b/hack/bot-ci-dispatch @@ -19,10 +19,11 @@ # picked up by the next call (.github/workflows/bot-sweep.yml, hourly). # # --workflow canary.yml dispatches with -f engines=auto, and only for a head whose -# `hack/pins policy origin/main SHA` maps canary engines (a non-empty -# canary list); the head is fetched from origin for that, never checked out. A -# head that moved on origin since the PR list was read waits for the next call; a -# policy that fails is an error for that PR. +# `hack/pins policy MB SHA` (MB: the merge-base of origin/main and SHA, so the PR's +# own diff) maps canary engines (a non-empty canary list); the head is fetched +# from origin for that, never checked out. A head that moved on origin since the +# PR list was read waits for the next call; a policy that fails is an error for +# that PR. # # A branch whose workflow file lacks the workflow_dispatch trigger (cut before # ci.yml had it) is skipped; the nightly's hack/autobump-branch-reset recreates it. @@ -55,7 +56,8 @@ fi canary_engines() { git fetch -q origin "+refs/heads/$1:refs/remotes/origin/$1" || return 1 [ "$(git rev-parse -q --verify "refs/remotes/origin/$1^{commit}")" = "$2" ] || { echo moved; return 0; } - hack/pins policy origin/main "$2" | jq -ce '.canary | arrays' + local mb; mb=$(git merge-base origin/main "$2") || return 1 + hack/pins policy "$mb" "$2" | jq -ce '.canary | arrays' } prs=$(gh pr list --state open --base main --limit 200 \ diff --git a/hack/bot-merge b/hack/bot-merge index 7418ccb8..9817b067 100755 --- a/hack/bot-merge +++ b/hack/bot-merge @@ -8,21 +8,25 @@ # A candidate that changes anything under .github/ is never merged. # # A candidate merges only when all of these hold at its head SHA: -# a. `hack/pins policy origin/main SHA` says auto, or a human approved SHA: a -# review APPROVED with commit_id SHA by a User whose author_association is -# OWNER, MEMBER or COLLABORATOR. No such human's latest review may request -# changes. Labels never change the verdict. +# a. `hack/pins policy MB SHA` says auto, MB being the merge-base of origin/main +# and SHA (the PR's own diff: main's later ledger or release lines are not the +# PR's changes), or a human approved SHA: a review APPROVED with commit_id SHA +# by a User whose author_association is OWNER, MEMBER or COLLABORATOR. No such +# human's latest review may request changes. Labels never change the verdict. # b. ci_status SHA BRANCH is green (ci.yml carries pin-guard and the pins checks). # c. When the policy names canary engines (its canary list): every canary.yml run -# for SHA (its pull_request runs and its dispatches on the PR branch, held ones -# aside) completed, the newest succeeded, and for every one of those engines the -# newest canary-result artifact, made for SHA, that holds it shows verdict -# pass, hard and exact (an isa-skip re-dispatch reruns only the skipped -# engines). A hard pass with exact false is a review condition: the PR is -# labelled canary-drift and merges on a human approval at SHA. Any other -# verdict (fail, isa-skip, too-big, unparseable) never merges. -# d. base_ok SHA and not rules_stale SHA. Otherwise the pass dispatches -# freshness.yml -f group= to regenerate the branch from main: at most 3 +# for SHA (its pull_request and workflow_dispatch runs whose head is this +# repository's PR branch, never a fork's; held ones aside) completed, the +# newest succeeded, and for every one of those engines the newest +# canary-result artifact, made for SHA, that holds it shows verdict pass, hard +# and exact (an isa-skip re-dispatch reruns only the skipped engines). A hard +# pass with exact false is a review condition: the PR is labelled canary-drift +# and merges on a human approval at SHA. Any other verdict (fail, isa-skip, +# too-big, unparseable) never merges. +# d. base_ok SHA, SHA merges onto main without a conflict (base_ok passes over +# release commits, which can touch the head's lines), and not rules_stale SHA. +# Otherwise the pass dispatches freshness.yml -f group= (no reason: that +# input is known-issue: only) to regenerate the branch from main: at most 3 # per group per UTC day, counted from today's dispatched freshness runs whose # title names the group (a title naming no group counts for every group), and # never while such a freshness run is queued or running. @@ -105,6 +109,8 @@ breaker() { } BREAKER=$(breaker) [ -n "$BREAKER" ] && echo "circuit breaker open: $BREAKER" +# A breaker it cannot read stops every merge: fail the pass so the sweep shows red. +case $BREAKER in "cannot read "*) RC=1;; esac TODAY=$(git log origin/main --format=%ct --grep='^Bot-merge: ' | awk -v d="$DAY0" '$1 >= d' | wc -l) echo "bot merges today ($DAY UTC): $TODAY of $MAX" @@ -116,7 +122,7 @@ freshness_runs() { | jq --arg g "group=$g" "[.[] | select(([(.displayTitle // \"\") | scan(\"group=[^ ,;)]+\")] as \$t | (\$t | length) == 0 or any(\$t[]; . == \$g))) | select($f)] | length" } -# regen GROUP WHY: dispatch freshness.yml for GROUP within the cap; prints what +# regen GROUP: dispatch freshness.yml -f group=GROUP within the cap; prints what # happened, returns 1 when a read or the dispatch failed. regen() { local g=$1 busy used @@ -126,22 +132,28 @@ regen() { || { echo "cannot read freshness runs"; return 1; } if [ "$used" -ge "$REGEN_CAP" ]; then echo "regeneration cap reached ($used of $REGEN_CAP for $g today); the scheduled freshness run regenerates it" - elif gh workflow run freshness.yml --ref main -f group="$g" -f reason="bot-merge:$2" >/dev/null; then + elif gh workflow run freshness.yml --ref main -f group="$g" >/dev/null; then echo "regeneration dispatched ($((used + 1)) of $REGEN_CAP for $g today)" else echo "the freshness.yml dispatch failed"; return 1; fi } # canary SHA BRANCH ENGINES_JSON DIR: one line per finding: "ok E", "drift E", # "block WHY", and "url URL" for each run whose result counted. The canary runs at SHA -# are its pull_request runs and its workflow_dispatch runs on BRANCH; a held -# (action_required) run is not an attempt. Every one must be complete and the newest -# must have succeeded. Each engine's newest result decides, because an isa-skip -# re-dispatch reruns only the skipped engines; a run, newer than the one deciding an -# engine, without a canary-result for SHA blocks, since what it found is unknown. +# are its pull_request and workflow_dispatch runs whose head is this repository's +# BRANCH (canary.yml runs on a pull request into any base, and a fork can open one +# from SHA); a held (action_required) run is not an attempt. Every one must be +# complete and the newest must have succeeded. Each engine's newest result decides, +# because an isa-skip re-dispatch reruns only the skipped engines; a run, newer than +# the one deciding an engine, without a canary-result for SHA blocks, since what it +# found is unknown. canary() { local runs concl url id f got left found - runs=$(gh run list -w canary.yml -c "$1" -L 50 --json databaseId,event,headBranch,status,conclusion,createdAt,url \ - | jq -c --arg b "$2" '[.[] | select(.conclusion != "action_required") - | select(.event == "pull_request" or (.event == "workflow_dispatch" and .headBranch == $b))] + # REST: gh run list does not say which repository a run's head came from. + runs=$(gh api "repos/$R/actions/workflows/canary.yml/runs?head_sha=$1&per_page=100" \ + --jq '[.workflow_runs[] | {databaseId: .id, event, headBranch: .head_branch, status, conclusion, + repo: (.head_repository.full_name // ""), createdAt: .created_at, url: .html_url}]' \ + | jq -c --arg b "$2" --arg r "$R" '[.[] | select(.conclusion != "action_required") + | select(.repo == $r and .headBranch == $b) + | select(.event == "pull_request" or .event == "workflow_dispatch")] | sort_by(.createdAt) | reverse') || { echo "block cannot read canary.yml runs"; return; } [ "$(jq length <<<"$runs")" -gt 0 ] || { echo "block canary has not run at this head"; return; } url=$(jq -r '[.[] | select(.status != "completed") | .url] | first // empty' <<<"$runs") @@ -209,7 +221,10 @@ while read -r c <&3; do if [ "$(git rev-parse -q --verify "refs/remotes/origin/$br^{commit}")" != "$sha" ]; then why "the branch no longer points at $sha; the next pass reads the new head"; continue fi - if ! "$HERE/pins" policy origin/main "$sha" > "$D/policy" 2> "$D/policy.err"; then + if ! mb=$(git merge-base origin/main "$sha"); then + why "no merge-base of main and $sha"; RC=1; continue + fi + if ! "$HERE/pins" policy "$mb" "$sha" > "$D/policy" 2> "$D/policy.err"; then why "hack/pins policy failed: $(tail -1 "$D/policy.err")"; continue fi if ! jq -e '.verdict == "auto" or .verdict == "review"' "$D/policy" >/dev/null 2>&1; then @@ -270,11 +285,16 @@ while read -r c <&3; do if [ "$allowed" = y ] && [ -z "$BREAKER" ]; then stale= if ! base_ok "$sha"; then stale="main moved under code since $sha was tested" - elif ( unset GH_TOKEN GITHUB_TOKEN; rules_stale "$sha" ); then - stale="console/ledger-rules.json would not match EXPERIMENTS.md after the merge" + else + git merge-tree --write-tree origin/main "$sha" >/dev/null 2>&1; mt=$? + if [ "$mt" -eq 1 ]; then stale="$sha conflicts with main" + elif [ "$mt" -ne 0 ]; then why "git merge-tree cannot merge $sha onto main (exit $mt)"; RC=1 + elif ( unset GH_TOKEN GITHUB_TOKEN; rules_stale "$sha" ); then + stale="console/ledger-rules.json would not match EXPERIMENTS.md after the merge" + fi fi if [ -n "$stale" ]; then - msg=$(regen "$group" "$([ "${stale#main}" != "$stale" ] && echo base-moved || echo rules-stale)") || RC=1 + msg=$(regen "$group") || RC=1 why "$stale; $msg" fi fi diff --git a/tests/bot_ci_dispatch_test.sh b/tests/bot_ci_dispatch_test.sh index 9a0f134b..682a4d86 100755 --- a/tests/bot_ci_dispatch_test.sh +++ b/tests/bot_ci_dispatch_test.sh @@ -170,11 +170,12 @@ if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 1 ] && dispatched autobump/b; the else fail "one_branch: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions")"; fi # 8. --workflow canary.yml: a head gets canary.yml, with engines=auto, only when -# `hack/pins policy origin/main SHA` maps canary engines for it, read on the head -# fetched from origin. canary_repo NAME: a bare origin whose branches autobump/a, -# autobump/b and autobump/c each carry one commit off main, and a clone on main -# whose hack/ holds bot-ci-dispatch and a fixture pins that prints -# policy/.json (no file: it fails) and a note on stderr (which must not +# `hack/pins policy MERGE_BASE SHA` (MERGE_BASE: of origin/main and SHA) maps canary +# engines for it, read on the head fetched from origin. canary_repo NAME: a bare +# origin whose branches autobump/a, autobump/b and autobump/c each carry one commit +# off main, and a clone on main whose hack/ holds bot-ci-dispatch and a fixture +# pins that prints policy/.json (no file, or a base that is not an ancestor of +# the head, so not the PR's own diff: it fails) and a note on stderr (which must not # read as the policy). Sets SHA_A, SHA_B, SHA_C. canary_repo() { fixture "$1" '[]' '[]'; CR=$FIX/repo @@ -188,6 +189,7 @@ canary_repo() { mkdir -p hack policy; cp "$BCD" hack/bot-ci-dispatch printf '%s\n' '#!/usr/bin/env bash' \ '[ "$1" = policy ] && git rev-parse -q --verify "$2^{commit}" >/dev/null && git cat-file -e "$3^{commit}" || exit 1' \ + 'git merge-base --is-ancestor "$2" "$3" || { echo "pins: $2 is not an ancestor of $3" >&2; exit 1; }' \ 'echo "pins: a note on stderr" >&2' \ 'cat "$(dirname "$0")/../policy/$3.json"' > hack/pins chmod +x hack/pins hack/bot-ci-dispatch ) @@ -211,6 +213,16 @@ canary_repo canary_moved; echo "[$(pr 1 autobump/a "$SHA_B")]" > "$FIX/prs.json" cp "$CR/policy/$SHA_A.json" "$CR/policy/$SHA_B.json"; bcd_repo --workflow canary.yml if [ "$(acts dispatch)" -eq 0 ] && grep -q '#1 autobump/a.*moved' <<<"$OUT"; then ok "canary_moved: no dispatch" else fail "canary_moved: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi +# main moved (a ledger commit) since the heads were cut: the policy still reads each +# head's own diff, from its merge-base with main. +canary_repo canary_main_moved +( cd "$CR" && export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 GIT_AUTHOR_NAME=t GIT_AUTHOR_EMAIL=t@t \ + GIT_COMMITTER_NAME=t GIT_COMMITTER_EMAIL=t@t && echo ledger > g && git add g && git commit -qm ledger \ + && git push -q origin main && git fetch -q origin ) +echo "[$(pr 1 autobump/a "$SHA_A")]" > "$FIX/prs.json"; bcd_repo --workflow canary.yml +if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 1 ] && dispatched autobump/a canary.yml; then + ok "canary_main_moved: policy from the merge-base, dispatched" +else fail "canary_main_moved: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi # ci.yml never consults the policy, and its dispatch carries no inputs. canary_repo ci_no_policy; rm "$CR/hack/pins" echo "[$(pr 1 autobump/a "$SHA_A"),$(pr 2 autobump/b "$SHA_B")]" > "$FIX/prs.json" diff --git a/tests/bot_merge_fake_gh.py b/tests/bot_merge_fake_gh.py index 07675298..e54382b3 100755 --- a/tests/bot_merge_fake_gh.py +++ b/tests/bot_merge_fake_gh.py @@ -7,10 +7,13 @@ files/.json REST pull files ([{filename}]) reviews/.json REST reviews comments/.json REST issue comments (POST and PATCH edit this file) - runs.json `gh run list --json` rows, plus "workflow" (the file name) + runs.json `gh run list --json` rows, plus "workflow" (the file name) and + "headRepository" (owner/repo the head came from); the REST + workflow runs list is served from them too artifacts/// files `gh run download -n ` extracts issues.json [{number, title, state, labels: [{name}]}] merge_rc exit code sequence for `gh pr merge` (last one repeats) + issue_list_rc exit code sequence for `gh issue list` (last one repeats) Calls are logged to $FIX/gh/calls; dispatches to $FIX/gh/dispatches, merges to $FIX/gh/merges, label additions to $FIX/gh/labels. @@ -172,6 +175,21 @@ def api(argv): return emit(c, jq_expr) print("gh: Not Found (HTTP 404)", file=sys.stderr) return 1 + if rest[:2] == ["actions", "workflows"] and len(rest) == 4 and rest[3] == "runs" and method == "GET": + q = dict(kv.split("=", 1) for kv in query.split("&") if kv) + log("calls", f"api-runs {rest[2]} " + " ".join(f"{k}={v}" for k, v in sorted(q.items()))) + out = [r for r in load(G / "runs.json", []) if wf_match(r, rest[2]) + and q.get("head_sha", r["headSha"]) == r["headSha"] + and q.get("branch", r["headBranch"]) == r["headBranch"] + and q.get("event", r["event"]) == r["event"]] + out.sort(key=lambda r: r["createdAt"], reverse=True) + out = out[: int(q.get("per_page", 30))] + return emit({"total_count": len(out), "workflow_runs": [ + {"id": r["databaseId"], "name": r.get("workflowName"), "event": r["event"], + "head_sha": r["headSha"], "head_branch": r["headBranch"], + "head_repository": {"full_name": r.get("headRepository", "o/r")}, + "status": r["status"], "conclusion": r.get("conclusion") or None, + "created_at": r["createdAt"], "html_url": r["url"]} for r in out]}, jq_expr) if rest[:1] == ["issues"] and len(rest) == 3 and rest[2] == "labels" and method == "POST": n = int(rest[1]) names = fields.get("labels[]", []) @@ -271,7 +289,11 @@ def pr_merge(argv): if sha != p["head"]["sha"]: print("GraphQL: Head branch was modified. Review and try the merge again. (mergePullRequest)", file=sys.stderr) return 1 - tree = git_origin("merge-tree", "--write-tree", "main", sha).splitlines()[0] + try: + tree = git_origin("merge-tree", "--write-tree", "main", sha).splitlines()[0] + except subprocess.CalledProcessError: + print(f"Pull request #{n} is not mergeable: the merge commit cannot be cleanly created.", file=sys.stderr) + return 1 when = os.environ["BOT_MERGE_NOW"] + " +0000" env = dict(os.environ, GIT_AUTHOR_NAME="github-actions[bot]", GIT_AUTHOR_EMAIL="41898282+github-actions[bot]@users.noreply.github.com", @@ -294,6 +316,10 @@ def pr_view(argv): # ---- gh issue / workflow / repo ------------------------------------------- def issue_list(argv): pos, opts = flags(argv, {"--label", "-l", "--state", "-s", "--json", "-L", "--limit", "-q", "--jq"}) + rc = int(pop("issue_list_rc", "0")) + if rc != 0: + print("HTTP 502: Server Error (https://api.github.com/graphql)", file=sys.stderr) + return rc label = one(opts, "--label", "-l") state = one(opts, "--state", "-s") or "open" issues = [i for i in load(G / "issues.json", []) diff --git a/tests/bot_merge_test.sh b/tests/bot_merge_test.sh index 1dffc1bd..036ddb24 100755 --- a/tests/bot_merge_test.sh +++ b/tests/bot_merge_test.sh @@ -65,12 +65,20 @@ fx() { cp "$BOT_MERGE" "$R/hack/bot-merge"; cp "$GATE_LIB" "$R/hack/gate_lib.sh" cat > "$R/hack/pins" <<'PINS' #!/usr/bin/env bash -# fixture: `pins policy BASE HEAD` prints $FIX/policy/HEAD.json once both refs resolve +# fixture: `pins policy BASE HEAD` prints $FIX/policy/HEAD.json once both refs resolve. +# Like hack/pins it judges the diff BASE..HEAD: a file that diff changes and HEAD's own +# diff (from its merge-base with BASE) does not is a line that is no pin move, so the +# verdict is review (changes-defaults) and the group mixed. [ "$1" = policy ] || exit 2 echo "pins $*" >> "$FIX/gh/trace" git rev-parse -q --verify "$2^{commit}" >/dev/null || { echo "pins: no base $2" >&2; exit 1; } git cat-file -e "$3^{commit}" 2>/dev/null || { echo "pins: head $3 not fetched" >&2; exit 1; } -cat "$FIX/policy/$3.json" +mb=$(git merge-base "$2" "$3") || exit 1 +extra=$(git diff --name-only "$2" "$3" | grep -vxF -f <(git diff --name-only "$mb" "$3") | paste -sd' ' -) +if [ -n "$extra" ]; then + jq --arg x "$extra" '.verdict = "review" | .group = "mixed" + | .reasons += ["changes-defaults: changed lines that are no pin move in " + $x]' "$FIX/policy/$3.json" +else cat "$FIX/policy/$3.json"; fi PINS chmod +x "$R/hack/pins" "$R/hack/bot-merge" run ci.yml push "$(rgit rev-parse HEAD)" main completed success "$(ago "${2:-60}")" @@ -78,22 +86,27 @@ PINS wfname() { case $1 in ci.yml) echo CI;; canary.yml) echo Canary;; freshness.yml) echo Freshness;; auto-release.yml) echo 'Auto-release AUR';; build-and-push.yml) echo 'Auto-update, Build & Release';; *) echo "$1";; esac; } -# run WORKFLOW EVENT SHA BRANCH STATUS CONCLUSION [CREATED] [TITLE]: add a workflow run; sets LASTRUN. +# run WORKFLOW EVENT SHA BRANCH STATUS CONCLUSION [CREATED] [TITLE] [HEAD_REPO]: add a workflow +# run whose head came from HEAD_REPO (default o/r, this repository); sets LASTRUN. run() { RUNID=$((RUNID + 1)); LASTRUN=$RUNID jqi runs.json --argjson id "$RUNID" --arg wf "$1" --arg name "$(wfname "$1")" --arg ev "$2" --arg sha "$3" \ --arg br "$4" --arg st "$5" --arg c "$6" --arg at "${7:-$(ago 30)}" --arg t "${8:-$(wfname "$1")}" \ + --arg hr "${9:-o/r}" \ '. + [{databaseId: $id, workflow: $wf, workflowName: $name, event: $ev, headSha: $sha, headBranch: $br, - status: $st, conclusion: $c, createdAt: $at, displayTitle: $t, attempt: 1, + headRepository: $hr, status: $st, conclusion: $c, createdAt: $at, displayTitle: $t, attempt: 1, url: ("https://github.com/o/r/actions/runs/\($id)")}]' } -# mkpr N [FILE] [CREATED_MINUTES_AGO]: bot PR #N on autobump/gN, one commit changing FILE -# (default pins.txt) off origin/main; policy auto with no canary engines, a green -# ci.yml dispatch run on the branch. Sets SHA. +# mkpr N [FILE] [CREATED_MINUTES_AGO] [CONTENT]: bot PR #N on autobump/gN, one commit off +# origin/main changing FILE (default pins.txt: a line appended, or the file set to +# CONTENT); policy auto with no canary engines, a green ci.yml dispatch run on the +# branch. Sets SHA. mkpr() { local n=$1 f=${2:-pins.txt} br=autobump/g$1 rgit fetch -q origin; rgit checkout -q -B "$br" origin/main - mkdir -p "$(dirname "$FIX/repo/$f")"; echo "bump $n" >> "$FIX/repo/$f"; rgit add "$f" + mkdir -p "$(dirname "$FIX/repo/$f")" + if [ -n "${4:-}" ]; then printf '%s' "$4" > "$FIX/repo/$f"; else echo "bump $n" >> "$FIX/repo/$f"; fi + rgit add "$f" commit_at "${3:-50}" "chore(g$n): bump"; rgit push -q -f origin "$br"; SHA=$(rgit rev-parse HEAD) rgit checkout -q main jqi pulls.json --argjson n "$n" --arg br "$br" --arg sha "$SHA" --arg at "$(ago "${3:-50}")" \ @@ -140,6 +153,7 @@ botcommit() { # main_commit MINUTES_AGO FILE CONTENT [AUTHOR] [MSG]: a commit on origin's main by someone else. main_commit() { rgit fetch -q origin; rgit checkout -q main; rgit reset -q --hard origin/main + mkdir -p "$(dirname "$FIX/repo/$2")" printf '%s' "$3" > "$FIX/repo/$2"; rgit add "$2"; commit_at "$1" "${5:-edit $2}" "${4:-t}"; rgit push -q origin main } @@ -293,6 +307,26 @@ canary "$SHA" autobump/other '{"vllm-cpu":{"verdict":"pass","hard":true,"exact": if [ "$(merges)" -eq 0 ] && comment 1 | grep -qi 'canary has not run'; then ok "canary_other_branch: no merge" else fail "canary_other_branch: $(show) comment=$(comment 1)"; fi +# ...nor is a pull_request run at the same SHA whose head is a fork's (a fork can push the +# bot's commit to a branch it names autobump/g1 and open a PR into any base): newer and +# exact, it does not override this PR's own drift, so the PR still needs a review... +fx canary_fork_run; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":false}}' completed success 30 +run canary.yml pull_request "$SHA" autobump/g1 completed success "$(ago 10)" Canary someone/r +mkdir -p "$FIX/gh/artifacts/$LASTRUN/canary-result" +jq -n --arg sha "$SHA" '{sha: $sha, ref: "autobump/g1", engines: {"vllm-cpu": {verdict: "pass", hard: true, exact: true}}}' \ + > "$FIX/gh/artifacts/$LASTRUN/canary-result/canary-result.json"; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'canary-drift'; then ok "canary_fork_run: a fork's run is not this PR's evidence" +else fail "canary_fork_run: $(show) comment=$(comment 1)"; fi +# ...while this PR's own pull_request run (approved, so not held) is. +fx canary_own_pr_run; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' +run canary.yml pull_request "$SHA" autobump/g1 completed success "$(ago 10)" Canary +mkdir -p "$FIX/gh/artifacts/$LASTRUN/canary-result" +jq -n --arg sha "$SHA" '{sha: $sha, ref: "autobump/g1", engines: {"vllm-cpu": {verdict: "pass", hard: true, exact: true}}}' \ + > "$FIX/gh/artifacts/$LASTRUN/canary-result/canary-result.json"; run_bm +if [ "$(merged 1)" -eq 1 ]; then ok "canary_own_pr_run: this PR's own pull_request run counts" +else fail "canary_own_pr_run: $(show) comment=$(comment 1)"; fi + # 9. canary hard pass with exact false: label canary-drift, no merge (review)... fx canary_drift; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":false,"detail":"golden differs"}}'; run_bm @@ -318,7 +352,7 @@ done # 11. main moved under code since the head was tested: regeneration dispatched (freshness.yml # group=g1); a second PR's group counts separately... fx base_moved_code; mkpr 1; main_commit 10 scripts/other.sh 'echo moved'; run_bm -if [ "$(merges)" -eq 0 ] && [ "$(disp '^freshness.yml ref=main group=g1 reason=')" -eq 1 ] && comment 1 | grep -q 'main moved'; then +if [ "$(merges)" -eq 0 ] && [ "$(disp '^freshness.yml ref=main group=g1$')" -eq 1 ] && comment 1 | grep -q 'main moved'; then ok "base_moved_code: regeneration dispatched" else fail "base_moved_code: $(show)"; fi # ...two of today's dispatched runs name g1 and one names g10: a third g1 regeneration goes out... @@ -328,7 +362,7 @@ run freshness.yml workflow_dispatch "$(rgit rev-parse origin/main)" main complet run freshness.yml workflow_dispatch "$(rgit rev-parse origin/main)" main completed success "$(ago 90)" "Freshness group=g10 reason=x" run freshness.yml workflow_dispatch "$(rgit rev-parse origin/main)" main completed success "$(iso $((NOW - 15 * 3600)))" "Freshness group=g1 reason=x" run_bm -if [ "$(disp '^freshness.yml ref=main group=g1 ')" -eq 1 ]; then ok "regen_under_cap: 2 of 3 used today, dispatched" +if [ "$(disp '^freshness.yml ref=main group=g1$')" -eq 1 ]; then ok "regen_under_cap: 2 of 3 used today, dispatched" else fail "regen_under_cap: $(show)"; fi # ...with three today the cap holds. fx regen_cap; mkpr 1; main_commit 10 scripts/other.sh 'echo moved' @@ -350,12 +384,37 @@ run ci.yml push "$(rgit rev-parse origin/main)" main completed success "$(ago 12 if [ "$(merged 1)" -eq 1 ] && [ "$(disp freshness.yml)" -eq 0 ]; then ok "base_moved_release: merged, no regeneration" else fail "base_moved_release: $(show)"; fi +# 12b. main moved only in the ledger since the head was cut: the policy judges the PR's own +# diff (from its merge-base with main), not main's ledger lines read backwards, so an +# auto bump still merges. +fx ledger_moved_auto; mkpr 1; mb=$(rgit rev-parse origin/main) +main_commit 15 docs/provenance.md $'ledger\n' +run ci.yml push "$(rgit rev-parse origin/main)" main completed success "$(ago 12)"; run_bm +if [ "$(merged 1)" -eq 1 ] && [ "$(disp freshness.yml)" -eq 0 ] && grep -qxF "pins policy $mb $SHA" "$(G trace)"; then + ok "ledger_moved_auto: policy from the merge-base, merged" +else fail "ledger_moved_auto: $(show) trace=$(grep '^pins' "$(G trace)" | tr '\n' ';') comment=$(comment 1)"; fi +# 12c. a release commit that conflicts with the head (each release rewrites K3D_GPU_VERSION in +# packaging/install.sh, on the line above the k3d pin) passes base_ok, but GitHub would +# refuse that merge on every pass: regenerate instead, and the next eligible PR merges +# in the same pass. +fx base_conflict_release +main_commit 55 packaging/install.sh $'K3D_GPU_VERSION=0.1.79\nK3D_VERSION=v5.9.0\n' +mkpr 1 packaging/install.sh 50 $'K3D_GPU_VERSION=0.1.79\nK3D_VERSION=v5.9.1\n' +main_commit 20 packaging/install.sh $'K3D_GPU_VERSION=0.1.80\nK3D_VERSION=v5.9.0\n' 88plug-bot \ + 'release: install.sh k3d-gpu 0.1.80 [skip release]' +run ci.yml push "$(rgit rev-parse origin/main)" main completed success "$(ago 12)" +mkpr 2 other.txt 10; run_bm +if [ "$(merged 1)" -eq 0 ] && [ "$(merged 2)" -eq 1 ] && [ "$(disp '^freshness.yml ref=main group=g1$')" -eq 1 ] \ + && comment 1 | grep -q 'conflicts with main' && ! grep -q refused <<<"$OUT"; then + ok "base_conflict_release: no merge attempt, regenerated; the next PR merged" +else fail "base_conflict_release: $(show) comment=$(comment 1)"; fi + # 13. main's ledger moved (a new SUPPORTED row) without ledger-rules.json: the merged tree's # rules are stale, so regenerate. fx rules_stale; mkpr 1 main_commit 10 EXPERIMENTS.md $'## Hypotheses\n\n| H34 | on the i9-13900T | p | m | SUPPORTED |\n| H34b | on the i5-12500T | q | n | SUPPORTED |\n' run ci.yml push "$(rgit rev-parse origin/main)" main completed success "$(ago 8)"; run_bm -if [ "$(merges)" -eq 0 ] && [ "$(disp '^freshness.yml ref=main group=g1 ')" -eq 1 ] && comment 1 | grep -q 'ledger-rules.json'; then +if [ "$(merges)" -eq 0 ] && [ "$(disp '^freshness.yml ref=main group=g1$')" -eq 1 ] && comment 1 | grep -q 'ledger-rules.json'; then ok "rules_stale: regeneration dispatched" else fail "rules_stale: $(show) comment=$(comment 1)"; fi @@ -377,6 +436,13 @@ for k in ci-main ci-missing:freshness.yml release:aur-v0.1.79 red:canary; do else fail "alarm $k: $(show)"; fi done +# 15b. the alarm issues cannot be read: nothing merges, and the pass fails, so the sweep +# goes red instead of sitting green while it merges nothing. +fx breaker_unreadable; mkpr 1; echo 1 > "$(G issue_list_rc)"; run_bm +if [ "$(merges)" -eq 0 ] && [ "$RC" -eq 1 ] && comment 1 | grep -q 'cannot read ci-alarm issues'; then + ok "breaker_unreadable: no merge, exit 1" +else fail "breaker_unreadable: $(show) comment=$(comment 1)"; fi + # 16. an alarm whose key is not a breaker key: merges. fx alarm_other; mkpr 1; alarm 'blind-spot:k3s'; run_bm if [ "$(merged 1)" -eq 1 ]; then ok "alarm_other: merged"; else fail "alarm_other: $(show)"; fi