diff --git a/.github/workflows/bot-sweep.yml b/.github/workflows/bot-sweep.yml new file mode 100644 index 00000000..487015a4 --- /dev/null +++ b/.github/workflows/bot-sweep.yml @@ -0,0 +1,79 @@ +name: Bot sweep + +# Hourly upkeep of the bot's PRs (branches autobump/*, opened by github-actions[bot] +# with GITHUB_TOKEN, so their own pull_request runs wait for approval). +# +# ci-fill: hack/bot-ci-dispatch runs ci.yml by workflow_dispatch on each bot PR +# branch with no run at its head, and re-runs a first-attempt failure +# once, one bot run in flight at a time and none while a human PR's CI +# is queued. It fills in what the nightly deferred. Then the same for +# canary.yml (engines=auto), on heads whose pins policy maps canary +# engines. +# merge: hack/bot-merge merges at most one bot PR whose evidence is green at +# its head (policy, CI, canary, current base, circuit breaker, daily +# cap), dispatches ci.yml on main after it, and dispatches a batched +# auto-release. It runs after ci-fill even when ci-fill had an error: +# it reads every piece of evidence itself. + +on: + schedule: + - cron: '13 * * * *' + workflow_dispatch: + +permissions: {} + +concurrency: + group: bot-sweep + cancel-in-progress: false + +jobs: + ci-fill: + name: CI on bot PRs + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: write + contents: read + pull-requests: read + steps: + - name: Checkout main + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + fetch-depth: 0 # the canary step's pins policy diffs a PR head against main + persist-credentials: false + + - name: Dispatch CI on bot PRs that have none at their head + env: + GH_TOKEN: ${{ github.token }} + run: hack/bot-ci-dispatch + + - name: Dispatch the canary on bot PRs whose policy maps canary engines + if: ${{ !cancelled() }} + env: + GH_TOKEN: ${{ github.token }} + run: hack/bot-ci-dispatch --workflow canary.yml + + merge: + name: Merge one auto-eligible bot PR, batch the release + needs: ci-fill + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: write + pull-requests: write + actions: write + issues: write + steps: + - name: Checkout main + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + fetch-depth: 0 # base_ok, today's Bot-merge commits and the aur-v* tags + persist-credentials: false + + - name: Merge, then batch the release + env: + GH_TOKEN: ${{ github.token }} + run: hack/bot-merge diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 46c75bd8..8a60b807 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -440,8 +440,11 @@ jobs: # component on branch autobump/. Merging it changes share/** on main, # which auto-release.yml turns into a launcher release, and that release # dispatches the `release` job above to build the image. A PR opened with - # GITHUB_TOKEN starts no CI workflows, so the tests run here before it opens. - # A new component is a new hack/components// dir; the matrix picks it up. + # GITHUB_TOKEN gets pull_request CI runs that wait for a maintainer's approval, + # so the tests run here before it opens, and hack/bot-ci-dispatch then runs + # ci.yml on the branch by workflow_dispatch (no approval): that run is the PR's + # CI evidence. A new component is a new hack/components// dir; the matrix + # picks it up. list-components: if: ${{ !inputs.release }} runs-on: ubuntu-latest @@ -466,6 +469,7 @@ jobs: permissions: contents: write pull-requests: write + actions: write strategy: fail-fast: false matrix: @@ -503,6 +507,12 @@ jobs: - name: Render tests run: tests/render_test.sh + # create-pull-request force-pushes an existing bump branch, which GitHub + # refuses from GITHUB_TOKEN once main's workflow files differ from the + # branch's: delete such a branch first (its PR closes; a fresh one opens). + - name: Delete a bump branch whose workflows differ from main's + run: hack/autobump-branch-reset "autobump/${COMPONENT}" + - name: Open or update the bump PR uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 with: @@ -522,27 +532,38 @@ jobs: - Pins (version, chart sha256, image digests) moved in `hack/components/${{ matrix.component }}/component.conf`. - `tests/detect_test.sh` and `tests/render_test.sh` passed in the workflow run that opened this PR. + - CI: this PR's own pull_request runs wait for approval (a GITHUB_TOKEN PR). + The evidence is the ci.yml run dispatched on `autobump/${{ matrix.component }}` + at the head SHA: `gh run list -w ci.yml -e workflow_dispatch -b autobump/${{ matrix.component }}`. - Merging rebuilds the node image with the new manifest baked in. Review the manifest diff for new CRDs, RBAC, images and flags before merging. + - name: Dispatch CI on the bump PR's branch + env: + GH_TOKEN: ${{ github.token }} + run: hack/bot-ci-dispatch --branch "autobump/${COMPONENT}" + # ─── Image digests ─────────────────────────────────────────────────────── # Every image reference is pinned name:tag@sha256 (scripts/images-lock pins). # A registry can re-point a tag (a rebuilt base image, a security respin) with # no new tag, which the version checks above never see. Daily: re-resolve the # tag of every pinned reference, write the registry's digest where it moved, # re-pin images.lock, run the image tests, and open or update one PR on branch - # autobump/image-digests. A PR opened with GITHUB_TOKEN starts no CI workflows, - # so the tests run here before it opens. GITHUB_TOKEN cannot push a change to - # .github/workflows/ (no `workflows` permission), so repin leaves those files - # alone; a workflow reads its image pins from a file outside them (the - # Dockerfile, docs-build/Dockerfile, hack/ci-images.env), which repin moves. + # autobump/image-digests. A PR opened with GITHUB_TOKEN gets pull_request CI + # runs that wait for a maintainer's approval, so the tests run here before it + # opens, and hack/bot-ci-dispatch then runs ci.yml on the branch by + # workflow_dispatch: that run is the PR's CI evidence. GITHUB_TOKEN cannot push + # a change to .github/workflows/ (no `workflows` permission), so repin leaves + # those files alone; a workflow reads its image pins from a file outside them + # (the Dockerfile, docs-build/Dockerfile, hack/ci-images.env), which repin moves. repin-images: if: ${{ !inputs.release }} runs-on: ubuntu-latest permissions: contents: write pull-requests: write + actions: write steps: - name: Checkout main uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -594,12 +615,7 @@ jobs: # its open PR; the action then cuts a fresh branch off main and opens a new one. - name: Delete a digest branch whose workflows differ from main's if: steps.repin.outputs.changes != '' - run: | - if git fetch -q --depth=1 origin autobump/image-digests 2>/dev/null \ - && ! git diff --quiet FETCH_HEAD HEAD -- .github/workflows/; then - echo "autobump/image-digests was cut from a main with other workflow files; deleting it" - git push origin --delete autobump/image-digests - fi + run: hack/autobump-branch-reset autobump/image-digests - name: Open or update the digest PR if: steps.repin.outputs.changes != '' @@ -622,21 +638,32 @@ jobs: - `tests/image_pins_test.sh`, `tests/audit_test.sh`, `tests/detect_test.sh`, `tests/render_test.sh` and the render check passed in the workflow run that opened this PR. + - CI: this PR's own pull_request runs wait for approval (a GITHUB_TOKEN PR). + The evidence is the ci.yml run dispatched on `autobump/image-digests` at the + head SHA: `gh run list -w ci.yml -e workflow_dispatch -b autobump/image-digests`. Review what the upstream rebuild changed before merging. A change under `share/`, `scripts/` or the Dockerfile cuts a launcher release on merge. + - name: Dispatch CI on the digest PR's branch + if: steps.repin.outputs.changes != '' + env: + GH_TOKEN: ${{ github.token }} + run: hack/bot-ci-dispatch --branch autobump/image-digests + # ─── QAT conversion toolchain (console/convert-pins.json) ────────────────── # The model optimizer's one-time OpenVINO conversion pins its pip toolchain # exactly. Daily: move each pin to the newest release optimum-intel's own # requirements allow (hack/bump-convert-pins), run the console tests, and open - # or update one PR on branch autobump/convert-pins. + # or update one PR on branch autobump/convert-pins, whose CI evidence is the + # ci.yml run hack/bot-ci-dispatch runs on the branch. bump-convert-pins: if: ${{ !inputs.release }} runs-on: ubuntu-latest permissions: contents: write pull-requests: write + actions: write steps: - name: Checkout main uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -663,6 +690,12 @@ jobs: tests/console_test.sh tests/model_opt_test.sh + # As in repin-images: GitHub refuses create-pull-request's force-push from + # GITHUB_TOKEN once main's workflow files differ from the branch's. + - name: Delete a pins branch whose workflows differ from main's + if: steps.bump.outputs.changes != '' + run: hack/autobump-branch-reset autobump/convert-pins + - name: Open or update the pins PR if: steps.bump.outputs.changes != '' uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 @@ -682,3 +715,13 @@ jobs: `tests/console_test.sh` and `tests/model_opt_test.sh` passed in the workflow run that opened this PR. + + CI: this PR's own pull_request runs wait for approval (a GITHUB_TOKEN PR). + The evidence is the ci.yml run dispatched on `autobump/convert-pins` at the + head SHA: `gh run list -w ci.yml -e workflow_dispatch -b autobump/convert-pins`. + + - name: Dispatch CI on the pins PR's branch + if: steps.bump.outputs.changes != '' + env: + GH_TOKEN: ${{ github.token }} + run: hack/bot-ci-dispatch --branch autobump/convert-pins diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81cd1259..afbf390e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,11 +5,14 @@ on: branches: [main] pull_request: branches: [main] + # hack/bot-ci-dispatch runs CI on a bot PR's branch this way: the PR's own + # pull_request runs wait for approval (GITHUB_TOKEN opened it), a dispatch does not. + workflow_dispatch: permissions: contents: read -# A new push to a PR supersedes its older run; main runs always finish. +# A new push to a PR supersedes its older run; main and dispatched runs always finish. concurrency: group: ci-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} @@ -55,6 +58,7 @@ jobs: - run: tests/prefill_guard_test.sh - run: tests/ci_wiring_test.sh - run: tests/gate_merge_test.sh + - run: tests/bot_ci_dispatch_test.sh - run: tests/auto_release_cas_test.sh quickstart-tests: @@ -484,3 +488,69 @@ jobs: - run: python3 tests/known_issues_test.py - run: python3 tests/ledger_freshness_test.py - run: hack/known-issues check + + # ─── Bot merge ─────────────────────────────────────────────────────────── + # hack/bot-merge (bot-sweep.yml, job merge) against a fake gh and fixture repos. + bot-merge-tests: + name: Bot merge (fake gh, fixture repos) + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: tests/bot_merge_test.sh + + # ─── Bot PR CI and alarm surfacing ─────────────────────────────────────── + # The watchdog files a ci-alarm issue for a red or missing scheduled run or a + # broken release (email went unread for 9 nights). Every open alarm, and a + # Watchdog workflow that is missing, disabled or without a success in 3 h, + # shows as a warning on every CI run. It never fails CI: the script exits 0, + # and the job is continue-on-error. + alarms: + name: Open CI alarms + runs-on: ubuntu-latest + timeout-minutes: 5 + continue-on-error: true + permissions: + issues: read + actions: read + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + steps: + - name: Warn on open ci-alarm issues and a quiet Watchdog + run: | + set +e + err=$(mktemp) + # Workflow-command data escapes %, CR and LF. + oneline() { tr '\r\n' ' ' < "$err" | sed 's/ *$//; s/%/%25/g'; } + if alarms=$(gh issue list --label ci-alarm --state open --limit 100 --json number,title,url,labels 2>"$err"); then + jq -r '.[] + | "open ci-alarm #\(.number): \(.title) \(.url) (\(if any(.labels[]; .name == "alarm-ack") then "acked" + else "NOT acked; after reading it: gh issue edit \(.number) --add-label alarm-ack" end))" + | "::warning::" + (gsub("%"; "%25") | gsub("\r"; "%0D") | gsub("\n"; "%0A"))' <<<"$alarms" + else + echo "::warning::could not list the open ci-alarm issues: $(oneline)" + fi + if ! wd=$(gh workflow list --all --json name,path,state \ + -q '[.[] | select(.name == "Watchdog")][0] // empty | "\(.state) \(.path)"' 2>"$err"); then + echo "::warning::could not list the workflows to check the Watchdog: $(oneline)" + elif [ -z "$wd" ]; then + echo "::warning::there is no Watchdog workflow, so a red or missing scheduled run raises no ci-alarm" + else + read -r state path <<<"$wd" + if [ "$state" != active ]; then + echo "::warning::the Watchdog workflow is $state, so a red or missing scheduled run raises no ci-alarm" + else + last=$(gh run list -w "${path##*/}" -s success -L 1 --json updatedAt -q '.[0].updatedAt // empty' 2>"$err") + ts=$(date -u -d "${last:-@0}" +%s 2>/dev/null) || ts=0 + if [ $(( $(date -u +%s) - ts )) -gt 10800 ]; then + echo "::warning::the Watchdog workflow has no successful run in the last 3 h (last: ${last:-none}); alarms may be missing" + fi + fi + fi + rm -f "$err" + exit 0 diff --git a/.github/workflows/dependabot-auto-merge.yml b/.github/workflows/dependabot-auto-merge.yml deleted file mode 100644 index cd7a4ecf..00000000 --- a/.github/workflows/dependabot-auto-merge.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Auto-merge Dependabot PRs - -on: pull_request - -permissions: - contents: write - pull-requests: write - -jobs: - automerge: - if: github.actor == 'dependabot[bot]' - runs-on: ubuntu-latest - steps: - - name: Enable auto-merge (squash) - run: gh pr merge --auto --squash "$PR_URL" - env: - PR_URL: ${{ github.event.pull_request.html_url }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index 8e7aad3f..bac0074d 100644 --- a/README.md +++ b/README.md @@ -1715,7 +1715,13 @@ tests/render_test.sh CI fails when a committed manifest differs from its pinned render. The daily build-and-push workflow runs `--latest` for every component and opens one PR per component (branch `autobump/`); merging it changes `share/`, so auto-release cuts a -launcher release and the node image is rebuilt for it. +launcher release and the node image is rebuilt for it. GITHUB_TOKEN opens these PRs, so +their own pull_request CI runs wait for approval; `hack/bot-ci-dispatch` runs `ci.yml` on +the branch by workflow_dispatch instead, and that run is the PR's CI. The hourly bot sweep +(`hack/bot-merge`) merges one such PR at a time when its pins policy says auto (or a +maintainer approved its head), its CI and canary are green, its base is current and main's +CI is green; it then dispatches CI on main and a batched release. Any other bot PR stays +open with one comment that says what blocks it. --- diff --git a/hack/autobump-branch-reset b/hack/autobump-branch-reset new file mode 100755 index 00000000..795943eb --- /dev/null +++ b/hack/autobump-branch-reset @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# hack/autobump-branch-reset : delete origin's autobump/ branch when its +# .github/workflows/ differ from origin/main's. Run from a checkout whose origin +# push credentials are the workflow's GITHUB_TOKEN, before create-pull-request. +# +# create-pull-request resets an existing bump branch (a PR still open, or a merged +# one whose branch stayed) onto main and force-pushes it. GitHub refuses that push +# from GITHUB_TOKEN (it has no `workflows` permission) when the branch's workflow +# files differ from main's, though the bump commit touches none. Deleting the +# branch closes its open PR; create-pull-request then cuts a fresh branch off main +# and opens a new PR, whose dispatched CI runs main's current workflows. +set -euo pipefail +b=${1:-} +case $b in + autobump/?*) ;; + *) echo "usage: hack/autobump-branch-reset autobump/ (refusing '${b}')" >&2; exit 2;; +esac +git ls-remote --exit-code --heads origin "refs/heads/$b" >/dev/null \ + || { rc=$?; [ "$rc" -eq 2 ] && { echo "$b: not on origin; nothing to reset"; exit 0; }; exit "$rc"; } +git fetch -q --depth=1 origin "+refs/heads/main:refs/remotes/origin/main" "+refs/heads/$b:refs/remotes/origin/$b" +if git diff --quiet "origin/main" "origin/$b" -- .github/workflows; then + echo "$b: its workflows match main's; kept" +else + echo "$b: its workflows differ from main's; deleting it (its open PR closes, and a fresh one opens)" + git push origin --delete "$b" +fi diff --git a/hack/bot-ci-dispatch b/hack/bot-ci-dispatch new file mode 100755 index 00000000..5fcb78e6 --- /dev/null +++ b/hack/bot-ci-dispatch @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# hack/bot-ci-dispatch [--branch B] [--workflow ci.yml|canary.yml]: give the bot's PRs real CI. +# +# A PR that GITHUB_TOKEN opened or pushed gets pull_request runs that wait for a +# maintainer's approval (conclusion action_required); a workflow_dispatch run +# needs none. For each open PR into main from a same-repo autobump/* branch opened +# by github-actions[bot] (never a fork, never a human's branch), oldest first: +# - no run of the workflow at the head SHA except held ones: dispatch the +# workflow on the branch (gh workflow run --ref); +# - the latest run at the head failed or timed out on its first attempt, and no +# run there was re-run yet: re-run its failed jobs, once per SHA; +# - otherwise: leave it. +# A run at the head is a pull_request run, or a workflow_dispatch run on the branch. +# +# Runner pool: one ci.yml run is ~39 jobs and a public repo gets 20-40 concurrent +# jobs, so it dispatches or re-runs only while fewer than BOT_CI_MAX_INFLIGHT +# (default 1) runs of the workflow on autobump/* branches are queued or running, +# and not at all while a ci.yml run of a human PR is queued. A deferred PR is +# picked up by the next call (.github/workflows/bot-sweep.yml, hourly). +# +# --workflow canary.yml dispatches with -f engines=auto, and only for a head whose +# `hack/pins policy MB SHA` (MB: the merge-base of origin/main and SHA, so the PR's +# own diff) maps canary engines (a non-empty canary list); the head is fetched +# from origin for that, never checked out. A head that moved on origin since the +# PR list was read waits for the next call; a policy that fails is an error for +# that PR. +# +# A branch whose workflow file lacks the workflow_dispatch trigger (cut before +# ci.yml had it) is skipped; the nightly's hack/autobump-branch-reset recreates it. +# Prints one line per decision. Idempotent. Exit 0, or 1 when a gh call failed +# (the rest still ran), 2 on bad usage. +set -uo pipefail + +usage() { echo "usage: hack/bot-ci-dispatch [--branch autobump/] [--workflow ci.yml|canary.yml]" >&2; exit 2; } +wf=ci.yml only= +while [ $# -gt 0 ]; do + case $1 in + --branch) [ $# -ge 2 ] || usage; only=$2; shift 2;; + --workflow) [ $# -ge 2 ] || usage; wf=$2; shift 2;; + -h|--help) sed -n '2,/^set /{/^set /d;s/^# \{0,1\}//;p}' "$0"; exit 0;; + *) usage;; + esac +done +cap=${BOT_CI_MAX_INFLIGHT:-1} +[[ $cap =~ ^[0-9]+$ ]] || { echo "BOT_CI_MAX_INFLIGHT must be a whole number, got '$cap'" >&2; exit 2; } +RUNF=databaseId,event,status,conclusion,headBranch,attempt,createdAt +rc=0 +inputs=() +if [ "$wf" = canary.yml ]; then + inputs=(-f engines=auto) + cd "$(git -C "$(dirname "${BASH_SOURCE[0]}")" rev-parse --show-toplevel)" || exit 1 + err=$(mktemp); trap 'rm -f "$err"' EXIT +fi +# canary_engines BRANCH SHA: the policy's canary list for SHA (a JSON array), or +# "moved" when origin's BRANCH is no longer SHA; fails when the policy does. +canary_engines() { + git fetch -q origin "+refs/heads/$1:refs/remotes/origin/$1" || return 1 + [ "$(git rev-parse -q --verify "refs/remotes/origin/$1^{commit}")" = "$2" ] || { echo moved; return 0; } + local mb; mb=$(git merge-base origin/main "$2") || return 1 + hack/pins policy "$mb" "$2" | jq -ce '.canary | arrays' +} + +prs=$(gh pr list --state open --base main --limit 200 \ + --json number,headRefName,headRefOid,baseRefName,isCrossRepository,author) \ + || { echo "error: cannot list open PRs"; exit 1; } +cands=$(jq -r --arg only "$only" ' + .[] | select(.isCrossRepository == false and .baseRefName == "main" + and (.headRefName | startswith("autobump/")) + and (.author.login | IN("app/github-actions", "github-actions[bot]", "github-actions")) + and (.author.is_bot != false) + and ($only == "" or .headRefName == $only)) + | "\(.number) \(.headRefName) \(.headRefOid)"' <<<"$prs" | sort -n) +[ -n "$cands" ] || { echo "no open github-actions[bot] PR from ${only:-an autobump/ branch}"; exit 0; } + +# The pool: bot runs of the workflow in flight, and queued ci.yml runs of human PRs. +recent=$(gh run list -w "$wf" -L 100 --json status,event,headBranch) || { echo "error: cannot list $wf runs"; exit 1; } +inflight=$(jq '[.[] | select(.status != "completed" and ((.headBranch // "") | startswith("autobump/")))] | length' <<<"$recent") +if [ "$wf" != ci.yml ]; then + recent=$(gh run list -w ci.yml -L 100 --json status,event,headBranch) || { echo "error: cannot list ci.yml runs"; exit 1; } +fi +human=$(jq '[.[] | select(.event == "pull_request" and (.status | IN("queued", "requested", "pending", "waiting")) + and ((.headBranch // "") | startswith("autobump/") | not))] | length' <<<"$recent") +[[ $inflight =~ ^[0-9]+$ && $human =~ ^[0-9]+$ ]] || { echo "error: cannot read the run pool from gh run list"; exit 1; } +# busy LINE: print LINE with the reason and succeed when the pool has no room. +busy() { + if [ "$human" -gt 0 ]; then echo "$1: deferred, $human ci.yml run(s) of human PRs queued"; return 0; fi + if [ "$inflight" -ge "$cap" ]; then echo "$1: deferred, $inflight bot $wf run(s) in flight (cap $cap)"; return 0; fi + return 1 +} + +while read -r num br sha; do + tag="#$num $br ${sha:0:7}" + runs=$(gh run list -w "$wf" -c "$sha" -L 100 --json "$RUNF") \ + || { echo "$tag: error: cannot list its $wf runs"; rc=1; continue; } + # " " + read -r count held id st co att reran <<<"$(jq -r --arg b "$br" ' + def word: if . == null or . == "" then "-" else tostring end; + [.[] | select(.event == "pull_request" or (.event == "workflow_dispatch" and .headBranch == $b))] as $at + | ($at | map(select(.conclusion != "action_required"))) as $e + | ($e | max_by(.createdAt // "")) as $l + | "\($e | length) \(($at | length) - ($e | length)) \($l.databaseId | word) \($l.status | word) \($l.conclusion | word) \($l.attempt // 1) \(any($e[]; (.attempt // 1) > 1))"' <<<"$runs")" + if [ "$count" -eq 0 ]; then + why="no $wf run at the head"; [ "$held" -gt 0 ] && why="only held $wf runs at the head" + if [ "$wf" = canary.yml ]; then + if ! eng=$(canary_engines "$br" "$sha" 2>"$err"); then + echo "$tag: error: hack/pins policy failed: $(tr '\n' ' ' < "$err")"; rc=1; continue + fi + case $eng in + moved) echo "$tag: skipped, origin's $br moved since the PR list was read"; continue;; + '[]') echo "$tag: the policy maps no canary engine; no canary"; continue;; + esac + fi + busy "$tag: $why" && continue + if out=$(gh workflow run "$wf" --ref "$br" "${inputs[@]}" 2>&1); then + inflight=$((inflight + 1)) + # Wait until the run shows, so that a call right after this one sees it. + new= + for _ in 1 2 3 4 5 6 7 8 9 10; do + new=$(gh run list -w "$wf" -b "$br" -e workflow_dispatch -c "$sha" -L 1 --json databaseId -q '.[0].databaseId // empty' 2>/dev/null) + [ -n "$new" ] && break; sleep 3 + done + echo "$tag: $why; dispatched $wf${new:+ (run $new)}" + elif grep -qi 'workflow_dispatch' <<<"$out"; then + echo "$tag: skipped, $wf on $br has no workflow_dispatch trigger (the nightly recreates the branch from main)" + else + echo "$tag: error: dispatching $wf failed: $(tr '\n' ' ' <<<"$out")"; rc=1 + fi + elif [ "$st" = completed ] && [[ $co =~ ^(failure|timed_out)$ ]] && [ "$att" = 1 ] && [ "$reran" = false ]; then + busy "$tag: $wf run $id $co on attempt 1" && continue + if out=$(gh run rerun "$id" --failed 2>&1); then + inflight=$((inflight + 1)); echo "$tag: $wf run $id $co on attempt 1; re-ran its failed jobs" + else + echo "$tag: error: re-running run $id failed: $(tr '\n' ' ' <<<"$out")"; rc=1 + fi + elif [ "$st" = completed ] && [ "$co" != success ]; then + echo "$tag: $wf run $id is $co (attempt $att); left for review" + else + echo "$tag: $wf run $id at the head is $st/$co; nothing to do" + fi +done <<<"$cands" +exit "$rc" diff --git a/hack/bot-merge b/hack/bot-merge new file mode 100755 index 00000000..9817b067 --- /dev/null +++ b/hack/bot-merge @@ -0,0 +1,403 @@ +#!/usr/bin/env bash +# hack/bot-merge: one pass of the bot-PR merge (bot-sweep.yml, job merge). It merges +# at most one PR, then decides whether to dispatch a batched release. +# +# Candidates are open PRs into main from this repository's own autobump/* branches +# (never autobump/amend-*), opened by github-actions[bot], labelled autobump and not +# ledger-amendment. Dependabot PRs, ledger amendments and forks are left to humans. +# A candidate that changes anything under .github/ is never merged. +# +# A candidate merges only when all of these hold at its head SHA: +# a. `hack/pins policy MB SHA` says auto, MB being the merge-base of origin/main +# and SHA (the PR's own diff: main's later ledger or release lines are not the +# PR's changes), or a human approved SHA: a review APPROVED with commit_id SHA +# by a User whose author_association is OWNER, MEMBER or COLLABORATOR. No such +# human's latest review may request changes. Labels never change the verdict. +# b. ci_status SHA BRANCH is green (ci.yml carries pin-guard and the pins checks). +# c. When the policy names canary engines (its canary list): every canary.yml run +# for SHA (its pull_request and workflow_dispatch runs whose head is this +# repository's PR branch, never a fork's; held ones aside) completed, the +# newest succeeded, and for every one of those engines the newest +# canary-result artifact, made for SHA, that holds it shows verdict pass, hard +# and exact (an isa-skip re-dispatch reruns only the skipped engines). A hard +# pass with exact false is a review condition: the PR is labelled canary-drift +# and merges on a human approval at SHA. Any other verdict (fail, isa-skip, +# too-big, unparseable) never merges. +# d. base_ok SHA, SHA merges onto main without a conflict (base_ok passes over +# release commits, which can touch the head's lines), and not rules_stale SHA. +# Otherwise the pass dispatches freshness.yml -f group= (no reason: that +# input is known-issue: only) to regenerate the branch from main: at most 3 +# per group per UTC day, counted from today's dispatched freshness runs whose +# title names the group (a title naming no group counts for every group), and +# never while such a freshness run is queued or running. +# e. Circuit breaker: the newest completed ci.yml run on main (push or dispatch) +# succeeded, and no open ci-alarm issue's key starts with ci-main, ci-missing, +# release: or red:canary. +# f. Fewer than MAX_BOT_MERGES_PER_DAY (default 6) bot merges on main today (UTC). +# A PR the policy marks urgent is exempt. +# Urgent PRs merge first, then the oldest. The merge is a squash pinned with +# --match-head-commit, and its body ends "Bot-merge: pr=N group=G urgent=B": later +# passes count today's merges and find unreleased ones by that line. A GITHUB_TOKEN +# merge starts no push workflow, so the pass then dispatches ci.yml on main. A +# refused merge prints GitHub's reason and ends the pass. Every candidate keeps one +# comment, starting , edited in place: why it is not merged, or +# the evidence it merged on. +# +# Release batching, after the merge step: dispatch auto-release.yml -f mode=changed +# when main has Bot-merge commits that no aur-v* tag contains, the newest ci.yml run +# for main's HEAD (push or dispatch) succeeded, no release workflow run is queued or +# running, no auto-release run has succeeded since the newest of those merges, and +# one of: one of them is urgent; no other open candidate has an allowed verdict and +# green CI; the newest aur-v* tag is 6 h old or more. +# +# Reads only main's checkout: PR heads are fetched as objects, never checked out. +# rules_stale runs the merged tree's console/model_opt.py, without the token. +# +# Env: GITHUB_REPOSITORY (owner/repo), GH_TOKEN, MAX_BOT_MERGES_PER_DAY, +# BOT_MERGE_NOW (epoch seconds, for tests), GITHUB_SERVER_URL and GITHUB_RUN_ID +# (the sweep run named in comments). +# Exit: 0 pass done; 1 a read or dispatch failed; 2 gh cannot pin a merge. +set -uo pipefail +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +cd "$(git -C "$HERE" rev-parse --show-toplevel)" || exit 1 +# shellcheck source=hack/gate_lib.sh +. "$HERE/gate_lib.sh" + +R=${GITHUB_REPOSITORY:?set GITHUB_REPOSITORY to owner/repo} +NOW=${BOT_MERGE_NOW:-$(date -u +%s)} +DAY=$(date -u -d "@$NOW" +%Y-%m-%d) +DAY0=$(date -u -d "$DAY" +%s) +MAX=${MAX_BOT_MERGES_PER_DAY:-6} +REGEN_CAP=3 +BOT='github-actions[bot]' +MARK='' +SWEEP=${GITHUB_SERVER_URL:-https://github.com}/$R/actions/runs/${GITHUB_RUN_ID:-0} +W=$(mktemp -d); trap 'rm -rf "$W"' EXIT +RC=0 + +gh pr merge --help 2>/dev/null | grep -q -- --match-head-commit \ + || { echo "gh pr merge lacks --match-head-commit; refusing to merge unpinned"; exit 2; } + +# --- reads shared by every candidate ------------------------------------------ +gh api --paginate "repos/$R/pulls?state=open&base=main&per_page=100" --jq '.[]' > "$W/pulls" \ + || { echo "cannot list open pull requests"; exit 1; } +jq -s -c --arg bot "$BOT" ' + .[] | select((.head.repo // {}).full_name == .base.repo.full_name + and (.head.ref | startswith("autobump/")) and (.head.ref | startswith("autobump/amend-") | not) + and .user.login == $bot + and any(.labels[]; .name == "autobump") and (any(.labels[]; .name == "ledger-amendment") | not)) + | {n: .number, br: .head.ref, sha: .head.sha, at: .created_at, title, draft, + labels: [.labels[].name]}' "$W/pulls" > "$W/cands" || { echo "cannot read the pull request list"; exit 1; } +git fetch -q --force origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/aur-v*:refs/tags/aur-v*' \ + || { echo "git fetch origin failed"; exit 1; } + +# breaker: why merging stops for every PR (empty when closed). +breaker() { + local ci alarms + ci=$(gh run list -w ci.yml -b main -s completed -L 20 --json event,conclusion,url,createdAt \ + | jq -r '[.[] | select(.event == "push" or .event == "workflow_dispatch")] | max_by(.createdAt) // empty + | "\(.conclusion) \(.url)"') || { echo "cannot read ci.yml runs on main"; return; } + case $ci in + "") echo "no completed ci.yml run on main"; return;; + success\ *) ;; + *) echo "the newest completed ci.yml run on main is ${ci%% *}: ${ci#* }"; return;; + esac + alarms=$(open_alarms) || { echo "cannot read ci-alarm issues"; return; } + jq -r '[.[] | (.title | capture("^\\[ci-alarm\\] (?.+?): ") | .k) as $k + | select($k | test("^(ci-main|ci-missing|release:|red:canary)"; "i")) | "\($k) (#\(.number))"] + | if length > 0 then "open alarm " + join(", ") else empty end' <<<"$alarms" +} +BREAKER=$(breaker) +[ -n "$BREAKER" ] && echo "circuit breaker open: $BREAKER" +# A breaker it cannot read stops every merge: fail the pass so the sweep shows red. +case $BREAKER in "cannot read "*) RC=1;; esac +TODAY=$(git log origin/main --format=%ct --grep='^Bot-merge: ' | awk -v d="$DAY0" '$1 >= d' | wc -l) +echo "bot merges today ($DAY UTC): $TODAY of $MAX" + +# freshness_runs GROUP JQ_FILTER [gh run list flags...]: how many freshness.yml runs, +# among those the flags select, name GROUP (or no group) in their title and pass JQ_FILTER. +freshness_runs() { + local g=$1 f=$2; shift 2 + gh run list -w freshness.yml "$@" --json displayTitle,status \ + | jq --arg g "group=$g" "[.[] | select(([(.displayTitle // \"\") | scan(\"group=[^ ,;)]+\")] as \$t + | (\$t | length) == 0 or any(\$t[]; . == \$g))) | select($f)] | length" +} +# regen GROUP: dispatch freshness.yml -f group=GROUP within the cap; prints what +# happened, returns 1 when a read or the dispatch failed. +regen() { + local g=$1 busy used + busy=$(freshness_runs "$g" '.status != "completed"' -L 30) || { echo "cannot read freshness runs"; return 1; } + if [ "$busy" -gt 0 ]; then echo "a freshness run that covers $g is queued or running"; return 0; fi + used=$(freshness_runs "$g" true -e workflow_dispatch --created ">=${DAY}T00:00:00Z" -L 200) \ + || { echo "cannot read freshness runs"; return 1; } + if [ "$used" -ge "$REGEN_CAP" ]; then + echo "regeneration cap reached ($used of $REGEN_CAP for $g today); the scheduled freshness run regenerates it" + elif gh workflow run freshness.yml --ref main -f group="$g" >/dev/null; then + echo "regeneration dispatched ($((used + 1)) of $REGEN_CAP for $g today)" + else echo "the freshness.yml dispatch failed"; return 1; fi +} +# canary SHA BRANCH ENGINES_JSON DIR: one line per finding: "ok E", "drift E", +# "block WHY", and "url URL" for each run whose result counted. The canary runs at SHA +# are its pull_request and workflow_dispatch runs whose head is this repository's +# BRANCH (canary.yml runs on a pull request into any base, and a fork can open one +# from SHA); a held (action_required) run is not an attempt. Every one must be +# complete and the newest must have succeeded. Each engine's newest result decides, +# because an isa-skip re-dispatch reruns only the skipped engines; a run, newer than +# the one deciding an engine, without a canary-result for SHA blocks, since what it +# found is unknown. +canary() { + local runs concl url id f got left found + # REST: gh run list does not say which repository a run's head came from. + runs=$(gh api "repos/$R/actions/workflows/canary.yml/runs?head_sha=$1&per_page=100" \ + --jq '[.workflow_runs[] | {databaseId: .id, event, headBranch: .head_branch, status, conclusion, + repo: (.head_repository.full_name // ""), createdAt: .created_at, url: .html_url}]' \ + | jq -c --arg b "$2" --arg r "$R" '[.[] | select(.conclusion != "action_required") + | select(.repo == $r and .headBranch == $b) + | select(.event == "pull_request" or .event == "workflow_dispatch")] + | sort_by(.createdAt) | reverse') || { echo "block cannot read canary.yml runs"; return; } + [ "$(jq length <<<"$runs")" -gt 0 ] || { echo "block canary has not run at this head"; return; } + url=$(jq -r '[.[] | select(.status != "completed") | .url] | first // empty' <<<"$runs") + [ -z "$url" ] || { echo "block canary is pending ($url)"; return; } + read -r concl url < <(jq -r '.[0] | "\(if (.conclusion // "") == "" then "-" else .conclusion end) \(.url)"' <<<"$runs") + [ "$concl" = success ] || echo "block canary run concluded $concl ($url)" + left=$3 + while read -r id url; do + [ "$(jq length <<<"$left")" -gt 0 ] || break + f=$4/canary-$id/canary-result.json + if ! gh run download "$id" -n canary-result -D "$4/canary-$id" >/dev/null 2>&1 || [ ! -f "$f" ]; then + echo "block canary run has no canary-result artifact ($url)"; return + fi + got=$(jq -r 'if (.engines | type) == "object" then .sha else error("no engines") end' "$f" 2>/dev/null) \ + || { echo "block canary-result.json is unreadable ($url)"; return; } + [ "$got" = "$1" ] || { echo "block canary result is for $got, not this head ($url)"; return; } + if ! found=$(jq -r --argjson want "$left" '.engines as $m | $want[] as $e | select($m | has($e)) | $m[$e] + | if type != "object" then "block canary \($e): no result object" + elif .verdict == "pass" and .hard == true and .exact == true then "ok \($e)" + elif .verdict == "pass" and .hard == true then "drift \($e)" + else "block canary \($e): \(.verdict)" end' "$f" 2>/dev/null) \ + || ! left=$(jq -c --argjson want "$left" '.engines as $m | [$want[] | select(. as $e | $m | has($e) | not)]' "$f"); then + echo "block canary-result.json is unreadable ($url)"; return + fi + [ -z "$found" ] || { echo "$found"; echo "url $url"; } + done < <(jq -r '.[] | "\(.databaseId) \(.url)"' <<<"$runs") + jq -r '.[] | "block canary result lacks engine \(.)"' <<<"$left" +} +# ci_url SHA BRANCH: the newest successful ci.yml run that ci_status counts. +ci_url() { + gh run list -w ci.yml -c "$1" -L 100 --json event,conclusion,headBranch,createdAt,url \ + | jq -r --arg b "$2" '[.[] | select(.conclusion == "success") + | select(.event == "pull_request" or (.event == "workflow_dispatch" and .headBranch == $b))] + | max_by(.createdAt) // empty | .url' +} +# comment N FILE: the PR's one bot-merge comment becomes FILE's text. +comment() { + local body ex + body="$MARK"$'\n'"$(cat "$2")" + ex=$(gh api --paginate "repos/$R/issues/$1/comments?per_page=100" --jq '.[]' \ + | jq -s -c --arg bot "$BOT" --arg m "$MARK" \ + '[.[] | select(.user.login == $bot and (.body | startswith($m)))] | first // empty') \ + || { echo "PR#$1: cannot read comments"; RC=1; return; } + if [ -z "$ex" ]; then + gh api -X POST "repos/$R/issues/$1/comments" -f body="$body" >/dev/null || { echo "PR#$1: comment failed"; RC=1; } + elif [ "$(jq -r .body <<<"$ex")" != "$body" ]; then + gh api -X PATCH "repos/$R/issues/comments/$(jq -r .id <<<"$ex")" -f body="$body" >/dev/null \ + || { echo "PR#$1: comment edit failed"; RC=1; } + fi +} + +# --- assess every candidate --------------------------------------------------- +: > "$W/eligible"; : > "$W/queue" +while read -r c <&3; do + n=$(jq -r .n <<<"$c"); br=$(jq -r .br <<<"$c"); sha=$(jq -r .sha <<<"$c") + D=$W/$n; mkdir -p "$D"; : > "$D/why" + why() { echo "- $*" >> "$D/why"; } + gh api --paginate "repos/$R/pulls/$n/files?per_page=100" --jq '.[] | .filename, (.previous_filename // empty)' > "$D/files" \ + || { why "cannot read the changed files"; RC=1; continue; } + if grep -q '^\.github/' "$D/files"; then + why "changes files under .github/ ($(grep '^\.github/' "$D/files" | head -3 | paste -sd, -)): a bot never merges workflow changes; a human merges this" + continue + fi + [ "$(jq -r .draft <<<"$c")" = true ] && why "draft" + if [ "$(git rev-parse -q --verify "refs/remotes/origin/$br^{commit}")" != "$sha" ]; then + why "the branch no longer points at $sha; the next pass reads the new head"; continue + fi + if ! mb=$(git merge-base origin/main "$sha"); then + why "no merge-base of main and $sha"; RC=1; continue + fi + if ! "$HERE/pins" policy "$mb" "$sha" > "$D/policy" 2> "$D/policy.err"; then + why "hack/pins policy failed: $(tail -1 "$D/policy.err")"; continue + fi + if ! jq -e '.verdict == "auto" or .verdict == "review"' "$D/policy" >/dev/null 2>&1; then + why "hack/pins policy gave no auto or review verdict"; continue + fi + if ! jq -e '.canary | type == "array"' "$D/policy" >/dev/null; then + why "hack/pins policy output has no canary list, so the canary evidence is unknown"; continue + fi + verdict=$(jq -r .verdict "$D/policy"); group=$(jq -r '.group // "unknown"' "$D/policy") + upd=$(jq -r '.update_type // "unknown"' "$D/policy"); urgent=$(jq -r '.urgent == true' "$D/policy") + engines=$(jq -c .canary "$D/policy") + review=() + [ "$verdict" = review ] && review+=("policy: $(jq -r '.reasons // [] | join("; ") | if . == "" then "review" else . end' "$D/policy")") + + # human reviews: each human's latest deciding review + gh api --paginate "repos/$R/pulls/$n/reviews?per_page=100" --jq '.[]' > "$D/reviews" \ + || { why "cannot read reviews"; RC=1; continue; } + jq -s '[.[] | select(.user.type == "User" and (.author_association | IN("OWNER", "MEMBER", "COLLABORATOR")) + and (.state | IN("APPROVED", "CHANGES_REQUESTED", "DISMISSED")))] + | group_by(.user.login) | map(max_by(.submitted_at))' "$D/reviews" > "$D/latest" + approver=$(jq -r --arg sha "$sha" '[.[] | select(.state == "APPROVED" and .commit_id == $sha) | .user.login] | first // empty' "$D/latest") + changes=$(jq -r '[.[] | select(.state == "CHANGES_REQUESTED") | "@" + .user.login] | join(", ")' "$D/latest") + [ -n "$changes" ] && why "changes requested by $changes" + + # canary + curl_= + if [ "$engines" != "[]" ]; then + canary "$sha" "$br" "$engines" "$D" > "$D/canary" + curl_=$(sed -n 's/^url //p' "$D/canary" | paste -sd' ' -) + while read -r w; do why "${w#block }"; done < <(grep '^block ' "$D/canary") + drift=$(sed -n 's/^drift //p' "$D/canary" | paste -sd, -) + if [ -n "$drift" ]; then + review+=("canary-drift: $drift passed the hard gate but its output differs from the golden") + if ! jq -e '.labels | index("canary-drift")' <<<"$c" >/dev/null; then + gh api -X POST "repos/$R/issues/$n/labels" -f 'labels[]=canary-drift' >/dev/null || { echo "PR#$n: label failed"; RC=1; } + fi + fi + fi + + # CI + ci=$(ci_status "$sha" "$br") + case $ci in + green) ;; + pending) why "CI is pending at this head";; + failed) why "CI failed at this head";; + *) why "no ci.yml run at this head yet (bot-ci-dispatch starts one)";; + esac + + allowed=y + if [ "${#review[@]}" -gt 0 ] && [ -z "$approver" ]; then + allowed=n + why "needs review: $(printf '%s; ' "${review[@]}" | sed 's/; $//'); an approval of $sha by a maintainer merges it" + fi + [ -n "$changes" ] && allowed=n + [ "$allowed" = y ] && [ "$ci" = green ] && echo "$n" >> "$W/queue" + + [ -n "$BREAKER" ] && why "circuit breaker: $BREAKER" + if [ "$allowed" = y ] && [ -z "$BREAKER" ]; then + stale= + if ! base_ok "$sha"; then stale="main moved under code since $sha was tested" + else + git merge-tree --write-tree origin/main "$sha" >/dev/null 2>&1; mt=$? + if [ "$mt" -eq 1 ]; then stale="$sha conflicts with main" + elif [ "$mt" -ne 0 ]; then why "git merge-tree cannot merge $sha onto main (exit $mt)"; RC=1 + elif ( unset GH_TOKEN GITHUB_TOKEN; rules_stale "$sha" ); then + stale="console/ledger-rules.json would not match EXPERIMENTS.md after the merge" + fi + fi + if [ -n "$stale" ]; then + msg=$(regen "$group") || RC=1 + why "$stale; $msg" + fi + fi + if [ "$urgent" != true ] && [ "$TODAY" -ge "$MAX" ]; then + why "daily cap reached ($TODAY of $MAX bot merges today, UTC); it merges tomorrow" + fi + + { + echo "verdict=$verdict"; echo "group=$group"; echo "upd=$upd"; echo "urgent=$urgent" + echo "approver=$approver"; echo "canary=$curl_"; echo "engines=$engines" + } > "$D/facts" + if [ ! -s "$D/why" ]; then + printf '%s %s %s\n' "$([ "$urgent" = true ] && echo 0 || echo 1)" "$(jq -r .at <<<"$c")" "$n" >> "$W/eligible" + fi +done 3< "$W/cands" + +# --- merge one ------------------------------------------------------------------ +MERGED=; REFUSED= +pick=$(sort -k1,1n -k2,2 "$W/eligible" | head -1 | cut -d' ' -f3) +if [ -n "$pick" ]; then + c=$(jq -c --argjson n "$pick" 'select(.n == $n)' "$W/cands"); D=$W/$pick + sha=$(jq -r .sha <<<"$c"); br=$(jq -r .br <<<"$c"); title=$(jq -r .title <<<"$c") + get() { sed -n "s/^$1=//p" "$D/facts"; } + group=$(get group); upd=$(get upd); urgent=$(get urgent); approver=$(get approver); curl_=$(get canary) + if [[ $title =~ ^[a-z]+(\([^\)]*\))?!?:\ ]]; then subj="$title (#$pick)"; else subj="chore($group): $title (#$pick)"; fi + cu=$(ci_url "$sha" "$br") + policy_line="policy $(get verdict) ($upd update of $group$([ "$urgent" = true ] && echo ', urgent'))" + [ -n "$approver" ] && policy_line+=", approved at ${sha:0:7} by @$approver" + body="Merged by the bot sweep: $policy_line; CI ${cu:-green}$([ -n "$curl_" ] && echo "; canary $curl_")." + body+=$'\n\n'"Bot-merge: pr=$pick group=$group urgent=$urgent" + why=$(gh pr merge "$pick" --squash --match-head-commit "$sha" --subject "$subj" --body "$body" 2>&1 >/dev/null); mrc=$? + if [ "$mrc" -eq 0 ] || [ "$(gh pr view "$pick" --json state -q .state 2>/dev/null)" = MERGED ]; then + MERGED=$pick + echo "PR#$pick MERGED at $sha" + grep -vxF "$pick" "$W/queue" > "$W/queue.left"; mv "$W/queue.left" "$W/queue" + gh workflow run ci.yml --ref main >/dev/null || { echo "ci.yml dispatch on main failed"; RC=1; } + { + echo "**Merged by the bot sweep** at \`$sha\`." + echo + echo "- Policy: $policy_line" + echo "- CI: ${cu:-green}" + if [ -n "$curl_" ]; then echo "- Canary: $curl_ ($(jq -r '. | join(", ")' <<<"$(get engines)"): pass, hard, exact)" + else echo "- Canary: no engine mapped for this group"; fi + echo "- Base: current with main (base_ok, ledger-rules.json current)" + echo "- Sweep run: $SWEEP" + } > "$D/body" + else + REFUSED=$pick + echo "PR#$pick merge at $sha refused: $(tr '\n' ' ' <<<"${why:-no reason given}")" + echo "- GitHub refused the merge at \`$sha\`: $(tr '\n' ' ' <<<"${why:-no reason given}")" > "$D/why" + fi +fi + +# --- one comment per candidate ------------------------------------------------ +while read -r c <&3; do + n=$(jq -r .n <<<"$c"); D=$W/$n + if [ "$n" != "$MERGED" ]; then + if [ ! -s "$D/why" ]; then + echo "- eligible; waiting for the next pass (one merge per pass; #${MERGED:-$REFUSED} went first)" > "$D/why" + fi + { echo "**Not merged by the bot sweep** at \`$(jq -r .sha <<<"$c")\`:"; echo; cat "$D/why"; } > "$D/body" + sed 's/^- /PR#'"$n"' blocked: /' "$D/why" + fi + comment "$n" "$D/body" +done 3< "$W/cands" +[ -n "$REFUSED" ] && exit "$RC" + +# --- release batching ----------------------------------------------------------- +release() { + git fetch -q --force origin '+refs/heads/main:refs/remotes/origin/main' '+refs/tags/aur-v*:refs/tags/aur-v*' \ + || { echo "release: git fetch failed"; RC=1; return; } + local head tags unrel newest ci wf busy ok_at last last_ct why + head=$(git rev-parse origin/main) + mapfile -t tags < <(git tag -l 'aur-v*') + unrel=$(git log origin/main --format=%ct --grep='^Bot-merge: ' --not "${tags[@]}") + [ -n "$unrel" ] || { echo "release: no bot merge above the last aur-v* tag"; return; } + newest=$(sort -n <<<"$unrel" | tail -1) + ci=$(gh run list -w ci.yml -c "$head" -L 20 --json event,status,conclusion,createdAt \ + | jq -r '[.[] | select(.event != "pull_request")] | max_by(.createdAt) // empty | "\(.status)/\(.conclusion)"') \ + || { echo "release: cannot read ci.yml runs"; RC=1; return; } + [ "$ci" = completed/success ] || { echo "release: wait: ci.yml for main ${head:0:7} is ${ci:-not run yet}"; return; } + for wf in auto-release.yml build-and-push.yml aur-deploy.yml packages.yml; do + busy=$(gh run list -w "$wf" -L 20 --json status | jq '[.[] | select(.status != "completed")] | length') \ + || { echo "release: cannot read $wf runs"; RC=1; return; } + [ "$busy" -eq 0 ] || { echo "release: wait: $wf has a run queued or in progress"; return; } + done + ok_at=$(gh run list -w auto-release.yml -s success -L 1 --json createdAt | jq -r '.[0].createdAt // empty') \ + || { echo "release: cannot read auto-release runs"; RC=1; return; } + if [ -n "$ok_at" ] && [ "$(date -u -d "$ok_at" +%s)" -ge "$newest" ]; then + echo "release: auto-release succeeded at $ok_at, after the newest bot merge"; return + fi + last=$(git tag -l 'aur-v*' --sort=-v:refname | head -1) + last_ct=$(git log -1 --format=%ct "$last" 2>/dev/null || echo 0) + if [ -n "$(git log origin/main --format=%H --grep='^Bot-merge: .* urgent=true' --not "${tags[@]}")" ]; then + why="an urgent fix merged since ${last:-the first release}" + elif [ ! -s "$W/queue" ]; then why="no other bot PR is queued" + elif [ $((NOW - last_ct)) -ge 21600 ]; then why="$last is $(( (NOW - last_ct) / 3600 )) h old" + else + echo "release: wait: $(wc -l < "$W/queue") bot PR(s) still queued and $last is under 6 h old"; return + fi + if gh workflow run auto-release.yml --ref main -f mode=changed >/dev/null; then + echo "release: dispatched auto-release.yml mode=changed ($why)" + else echo "release: the auto-release.yml dispatch failed"; RC=1; fi +} +release +exit "$RC" diff --git a/hack/gate_lib.sh b/hack/gate_lib.sh new file mode 100644 index 00000000..6bdb82e1 --- /dev/null +++ b/hack/gate_lib.sh @@ -0,0 +1,186 @@ +# shellcheck shell=bash +# gate_lib.sh: what the merge gate (hack/gate_merge.sh) reads to decide a PR is +# green and current, for sourcing. Run from inside the repository; gh and git do +# the reading. +# +# READ jq program over `gh pr view --json headRefOid,statusCheckRollup` +# (takes --argjson held_ok BOOL): " +# ", the latest entry per check. +# RUNS jq program over `gh run list -c SHA -e pull_request --json +# workflowName,status,conclusion,createdAt` (takes --argjson +# held_ok BOOL): " ", the latest +# run per workflow. +# ci_status SHA BR green | pending | failed | none: ci.yml at SHA, through a +# pull_request run or a workflow_dispatch run on branch BR. +# ci_eval SHA BR " ", held_ok as below. +# base_ok SHA main is an ancestor of SHA, or main moved only in the ledger or +# by release-only bot commits, or SHA changes only the ledger. +# rules_stale SHA the tree main + SHA would merge to carries a +# console/ledger-rules.json its EXPERIMENTS.md does not generate. +# open_alarms the open ci-alarm issues as a JSON array (gh issue list). +# gated_paths reads paths on stdin, prints those a merge must not ship while +# an alarm is open and not acked (see below). +# +# held_ok: a PR that GITHUB_TOKEN opened or pushed gets pull_request runs held for +# approval (conclusion action_required, no jobs). hack/bot-ci-dispatch runs ci.yml +# on the PR branch by workflow_dispatch instead. Once the latest such dispatched +# run at the SHA succeeded, a held entry of workflow 'CI' or 'Auto-merge Dependabot +# PRs' no longer counts as terminal (READ) or failed (RUNS). Without it, a held +# entry stays terminal, as for a fork PR. + +GATE_LIB_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# The workflows whose held pull_request runs a green dispatched ci.yml run stands in for. +GATE_HELD_WORKFLOWS='["CI", "Auto-merge Dependabot PRs"]' + +# One snapshot: " ", latest run per check. +# An in-progress CheckRun reports conclusion "", a legacy StatusContext only .state. +# READ and RUNS are jq programs ($names are jq's) for the gate that sources this file. +# shellcheck disable=SC2016,SC2034 +READ='def green: IN("SUCCESS", "SKIPPED", "NEUTRAL"); + def terminal: IN("FAILURE", "ERROR", "CANCELLED", "TIMED_OUT", "STARTUP_FAILURE", "ACTION_REQUIRED", "STALE"); + ('"$GATE_HELD_WORKFLOWS"') as $held + | .headRefOid as $sha + | [.statusCheckRollup[] + | {k: [.__typename, (.name // .context), (.workflowName // "")], t: (.startedAt // ""), n: (.name // .context), + w: (.workflowName // ""), + v: ((if (.conclusion // "") != "" then .conclusion else (.state // "PENDING") end) | ascii_upcase)} + | select(($held_ok and .v == "ACTION_REQUIRED" and (.w | IN($held[]))) | not)] + | [group_by(.k)[] | if any(.v | (green or terminal) | not) then .[0] + {v: "PENDING"} else max_by(.t) end] as $c + | ($c | map(select(.v | green)) | length) as $g + | ($c | map(select(.v | terminal))) as $b + | "\($sha) \($g) \($b | length) \($c | length) \($b | map("\(.n)=\(.v)") | join(","))"' +# Every pull_request workflow run for a SHA, latest per workflow: " ", +# state = green | pending | failed. +# shellcheck disable=SC2016,SC2034 +RUNS='('"$GATE_HELD_WORKFLOWS"') as $held + | [.[] | select(($held_ok and .conclusion == "action_required" and (.workflowName | IN($held[]))) | not)] + | [group_by(.workflowName)[] | max_by(.createdAt // "")] as $r + | ($r | map(select(.status != "completed"))) as $p + | ($r | map(select(.status == "completed" and (.conclusion | IN("success", "skipped", "neutral") | not)))) as $f + | if ($f | length) > 0 then "failed \($f | map("\(.workflowName)=\(.conclusion)") | join(","))" + elif ($p | length) > 0 then "pending -" else "green -" end' +# ci.yml runs at a SHA: $p its pull_request runs, $d its workflow_dispatch runs, $b the +# PR branch. Held runs (action_required) are not attempts: the latest completed other +# run decides, a queued or running one means pending, and held runs alone are failed. +# shellcheck disable=SC2016 +CI_EVAL='def ok: .conclusion == "success"; + def word: if . == null or . == "" then "-" else . end; + ($p | map(. + {via: "pull_request"})) + ($d | map(select(.headBranch == $b) | . + {via: "workflow_dispatch"})) as $all + | ($all | map(select(.conclusion != "action_required"))) as $e + | ($e | map(select(.status == "completed")) | max_by(.createdAt // "")) as $last + | ($e | map(select(.via == "workflow_dispatch" and .status == "completed")) | max_by(.createdAt // "")) as $dl + | (if $last != null and ($last | ok) then "green" + elif any($e[]; .status != "completed") then "pending" + elif $last != null or ($all | length) > 0 then "failed" + else "none" end) as $s + | ($s == "green" and $dl != null and ($dl | ok)) as $held_ok + | ($last // ($all | max_by(.createdAt // "")) // {}) as $show + | "\($s) \($held_ok) \($show.via | word):\($show.status | word)/\($show.conclusion | word)"' +ci_eval() { + local p d + p=$(gh run list -w ci.yml -c "$1" -e pull_request -L 100 --json event,headBranch,status,conclusion,createdAt 2>/dev/null) || p= + d=$(gh run list -w ci.yml -c "$1" -e workflow_dispatch -b "$2" -L 100 --json event,headBranch,status,conclusion,createdAt 2>/dev/null) || d= + jq -rn --arg b "$2" --argjson p "${p:-[]}" --argjson d "${d:-[]}" "$CI_EVAL" +} +ci_status() { local s; read -r s _ <<<"$(ci_eval "$1" "$2")"; echo "${s:-none}"; } + +# base_ok SHA: main is an ancestor of SHA, or main moved only in the ledger (or by +# release-only bot commits), or the PR is ledger-only. +code() { git diff --name-only "$1" "$2" | grep -qvxE 'EXPERIMENTS\.md|docs/provenance\.md'; } +# release_only C: C is one of the release commits auto-release.yml and its followers +# push to main: a 'release: ... [skip release]' subject by 88plug-bot or +# github-actions[bot], touching only the release files, and in scripts/k3d-gpu only +# its VERSION="..." line. +release_only() { + local c=$1 f + case $(git log -1 --format=%s "$c") in 'release:'*'[skip release]'*) ;; *) return 1;; esac + case $(git log -1 --format=%an "$c") in 88plug-bot|'github-actions[bot]') ;; *) return 1;; esac + while IFS= read -r f; do + case $f in + packaging/aur/PKGBUILD|packaging/aur/.SRCINFO|Formula/k3d-gpu.rb|packaging/install.sh|console/ledger-rules.json) ;; + scripts/k3d-gpu) + git diff -U0 "$c^" "$c" -- scripts/k3d-gpu \ + | awk '/^@@/ {h = 1; next} h && /^[-+]/ && !/^[-+]VERSION="/ {bad = 1} END {exit bad ? 0 : 1}' && return 1;; + *) return 1;; + esac + done < <(git diff --name-only "$c^" "$c") + return 0 +} +# main_code MB: since MB, main changed a file outside the ledger in a commit that is +# not release-only. When no commit can be listed, that counts as a change. +main_code() { + code "$1" origin/main || return 1 + local c n=0 + while read -r c; do + n=$((n + 1)) + release_only "$c" && continue + code "$c^" "$c" && return 0 + done < <(git rev-list "$1..origin/main") + [ "$n" -eq 0 ] +} +base_ok() { + git fetch -q origin || return 1 + git merge-base --is-ancestor origin/main "$1" && return 0 + local mb; mb=$(git merge-base origin/main "$1") || return 1 + ! { main_code "$mb" && code "$mb" "$1"; } +} + +# rules_stale SHA: the tree main + SHA would merge to carries a console/ledger-rules.json +# that EXPERIMENTS.md there does not generate (read with that tree's own model_opt.py). +# Unknown (a conflict, no such files) is not stale: a conflict syncs anyway. +rules_stale() { + local tree d rc=1 + tree=$(git merge-tree --write-tree origin/main "$1" 2>/dev/null | head -1) + [ -n "$tree" ] || return 1 + d=$(mktemp -d) + if git show "$tree:console/model_opt.py" > "$d/model_opt.py" 2>/dev/null \ + && git show "$tree:console/convert-pins.json" > "$d/convert-pins.json" 2>/dev/null \ + && git show "$tree:console/ledger-rules.json" > "$d/have.json" 2>/dev/null \ + && git show "$tree:EXPERIMENTS.md" > "$d/EXPERIMENTS.md" 2>/dev/null \ + && [ -s "$d/model_opt.py" ] && [ -s "$d/have.json" ] \ + && python3 "$d/model_opt.py" ledger-rules --ledger "$d/EXPERIMENTS.md" --out "$d/want.json" >/dev/null 2>&1 \ + && ! cmp -s "$d/have.json" "$d/want.json"; then + rc=0 + fi + rm -rf "$d" + return "$rc" +} + +# open_alarms: the open ci-alarm issues, [{number, title, url, labels}]; fails when gh does. +open_alarms() { + local a + a=$(gh issue list --label ci-alarm --state open --limit 100 --json number,title,url,labels) || return 1 + jq -c 'if type == "array" then . else error("not an array") end' <<<"${a:-[]}" +} +# gated_paths: of the paths on stdin, print those a merge must not ship while an +# alarm is open and not acked: .github/** and the shipped set. A PR that changes +# only EXPERIMENTS.md, docs/, bench/ or tests/ (the ledger work) prints nothing. +# Shipped set: packaging/, scripts/, share/, Dockerfile, console/, images.lock, plus +# every pattern in hack/release-paths (one per line, # comments, dir/ or dir/** for +# a tree, else a shell glob; an executable file is run for its lines). It is read +# from $GATE_RELEASE_PATHS, else next to this file, else the repository's hack/. +gated_paths() { + local paths p f pats=(packaging/ scripts/ share/ Dockerfile console/ images.lock) rp + paths=$(grep -v '^$') + [ -n "$paths" ] || return 0 + grep -qvE '^(EXPERIMENTS\.md|(docs|bench|tests)/.*)$' <<<"$paths" || return 0 + rp=${GATE_RELEASE_PATHS:-} + [ -n "$rp" ] || for f in "$GATE_LIB_DIR/release-paths" hack/release-paths; do [ -f "$f" ] && { rp=$f; break; }; done + if [ -n "$rp" ] && [ -f "$rp" ]; then + while IFS= read -r p; do + p=${p%%#*}; p=${p#"${p%%[![:space:]]*}"}; p=${p%"${p##*[![:space:]]}"}; p=${p#- }; p=${p//[\"\']/} + [ -n "$p" ] && pats+=("$p") + done < <(if [ -x "$rp" ]; then "$rp"; else cat "$rp"; fi) + fi + while IFS= read -r f; do + case $f in .github/*) echo "$f"; continue;; esac + for p in "${pats[@]}"; do + case $p in */\*\*) p=${p%\*\*};; esac + # shellcheck disable=SC2254 # p is a glob on purpose + case $p in + */) case $f in "$p"*) echo "$f"; break;; esac;; + *) case $f in $p|"$p"/*) echo "$f"; break;; esac;; + esac + done + done <<<"$paths" +} diff --git a/hack/gate_merge.sh b/hack/gate_merge.sh index 94f131bd..6bfcbbaa 100755 --- a/hack/gate_merge.sh +++ b/hack/gate_merge.sh @@ -2,14 +2,28 @@ # gate_merge.sh : keep the PR branch current with main # (ledger/provenance conflicts resolved keep-both; anything else -> stop), # then squash-merge the head SHA whose checks were read. Up to 5 sync rounds. +# The checks it reads live in hack/gate_lib.sh, which it sources from its own +# directory, else from the repository's hack/. # # Green means: every latest rollup entry is SUCCESS, SKIPPED or NEUTRAL (a -# check with any queued or running attempt is pending); the ci.yml -# pull_request run for that exact SHA completed with success; and the latest -# run of every other pull_request workflow for that SHA completed with -# success, skipped or neutral. A terminal non-green entry (FAILURE, ERROR, -# CANCELLED, TIMED_OUT, STARTUP_FAILURE, ACTION_REQUIRED, STALE) or a failed -# workflow run exits 1. +# check with any queued or running attempt is pending); ci.yml for that exact +# SHA succeeded, through a pull_request run or a workflow_dispatch run on the PR +# branch (the latest completed one decides); and the latest run of every other +# pull_request workflow for that SHA completed with success, skipped or neutral. +# A terminal non-green entry (FAILURE, ERROR, CANCELLED, TIMED_OUT, +# STARTUP_FAILURE, ACTION_REQUIRED, STALE), a failed workflow run or a failed +# ci.yml exits 1. +# +# Bot PRs: a PR that GITHUB_TOKEN opened or pushed gets pull_request runs held +# for approval (ACTION_REQUIRED). hack/bot-ci-dispatch runs ci.yml on its branch +# by workflow_dispatch instead; once that run is green, the held entries of 'CI' +# and 'Auto-merge Dependabot PRs' no longer block. Held runs alone still exit 1. +# +# Alarms: the gate first prints every open ci-alarm issue. While any of them +# lacks the label alarm-ack, it refuses (exit 4) a PR that changes .github/ or a +# shipped path (see gated_paths in gate_lib.sh), at the start and again right +# before merging; a PR that changes only EXPERIMENTS.md, docs/, bench/ or tests/ +# is never refused. Read the alarm, then ack it: gh issue edit N --add-label alarm-ack. # # Derived file: console/ledger-rules.json is EXPERIMENTS.md's rule rows as packages # ship them. A ledger-only PR (or base) merges without a CI re-run, so the gate @@ -19,9 +33,10 @@ # Moved base: if main is no longer an ancestor of the head, and both main # (since the merge base) and the PR change files other than EXPERIMENTS.md and # docs/provenance.md, the gate syncs the branch so CI re-runs on the merged -# head. A base that moved only in the ledger, or under a ledger-only PR, is -# merged without a re-run. Another PR can still land between this check and -# the merge; that window stays open. +# head. A base that moved only in the ledger or by release-only bot commits +# ('release: ... [skip release]' on the release files), or under a ledger-only +# PR, is merged without a re-run. Another PR can still land between this check +# and the merge; that window stays open. # # The merge carries --match-head-commit, so GitHub merges only if the head is # still that SHA. On a refusal the gate prints GitHub's reason and re-reads the @@ -29,8 +44,11 @@ # sleeps, and the next round re-checks mergeable (sync on a conflict) and the # base before merging again. # +# Exit: 0 merged, 1 failed or gave up, 2 gh lacks --match-head-commit, 4 refused +# for an open alarm. # Env: GATE_REPO (default: the caller's git toplevel), GATE_WT_ROOT (sync # worktree parent, default $TMPDIR), GATE_POLLS (polls per round, default 60), +# GATE_RELEASE_PATHS (shipped-path patterns, default hack/release-paths), # GATE_SELF_DIR (a directory removed on exit; set by the scratchpad shim). set -u; pr=$1; subj=$2; body=$3 HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) @@ -42,9 +60,38 @@ cleanup() { if [ -n "${GATE_SELF_DIR:-}" ]; then rm -rf "$GATE_SELF_DIR"; fi } trap cleanup EXIT +for lib in "$HERE/gate_lib.sh" hack/gate_lib.sh ""; do [ -f "$lib" ] && break; done +[ -n "$lib" ] || { echo "gate_lib.sh not found next to $0 or in $PWD/hack; copy it beside the gate"; exit 1; } +# shellcheck source=hack/gate_lib.sh +. "$lib" gh pr merge --help 2>/dev/null | grep -q -- --match-head-commit \ || { echo "gh pr merge lacks --match-head-commit; refusing to merge unpinned"; exit 2; } br=$(gh pr view "$pr" --json headRefName -q .headRefName) || exit 1 +# alarm_check [list]: with `list`, print every open ci-alarm issue. Exit 4 when one +# is not acked and the PR changes a gated path (or its paths cannot be read). +alarm_check() { + local a un paths gated e + e=$(mktemp) + if ! a=$(open_alarms 2>"$e"); then + echo "WARNING: could not list the open ci-alarm issues ($(tr '\n' ' ' < "$e" | sed 's/ *$//')); the alarm check was skipped" + rm -f "$e"; return 0 + fi + rm -f "$e" + [ "${1:-}" = list ] && jq -r '.[] | "open ci-alarm #\(.number): \(.title) \(.url)\(if any(.labels[]; .name == "alarm-ack") then " (acked)" else " (NOT acked)" end)"' <<<"$a" + un=$(jq -r '.[] | select(any(.labels[]; .name == "alarm-ack") | not) | " #\(.number) \(.title) \(.url)"' <<<"$a") + [ -n "$un" ] || return 0 + if paths=$(gh api --paginate "repos/{owner}/{repo}/pulls/$pr/files" -q '.[].filename' 2>/dev/null); then + gated=$(gated_paths <<<"$paths") + [ -n "$gated" ] || return 0 + echo "PR$pr REFUSED: it changes $(head -5 <<<"$gated" | paste -sd, -)$([ "$(wc -l <<<"$gated")" -gt 5 ] && echo ', ...') while these ci-alarm issues are open and not acked:" + else + echo "PR$pr REFUSED: its changed files could not be listed, and these ci-alarm issues are open and not acked:" + fi + echo "$un" + jq -r '.[] | select(any(.labels[]; .name == "alarm-ack") | not) | "Read #\(.number), then ack it: gh issue edit \(.number) --add-label alarm-ack"' <<<"$a" + exit 4 +} +alarm_check list sync() { git fetch -q origin git worktree remove --force "$W" 2>/dev/null; git worktree add -q -B "gm/$pr" "$W" "origin/$br" || return 1 @@ -70,69 +117,22 @@ sync() { fi ) || return 1 git -C "$W" push -q origin "HEAD:refs/heads/$br" && echo "synced $br" } -# One snapshot: " ", latest run per check. -# An in-progress CheckRun reports conclusion "", a legacy StatusContext only .state. -READ='def green: IN("SUCCESS", "SKIPPED", "NEUTRAL"); - def terminal: IN("FAILURE", "ERROR", "CANCELLED", "TIMED_OUT", "STARTUP_FAILURE", "ACTION_REQUIRED", "STALE"); - .headRefOid as $sha - | [.statusCheckRollup[] - | {k: [.__typename, (.name // .context), (.workflowName // "")], t: (.startedAt // ""), n: (.name // .context), - v: ((if (.conclusion // "") != "" then .conclusion else (.state // "PENDING") end) | ascii_upcase)}] - | [group_by(.k)[] | if any(.v | (green or terminal) | not) then .[0] + {v: "PENDING"} else max_by(.t) end] as $c - | ($c | map(select(.v | green)) | length) as $g - | ($c | map(select(.v | terminal))) as $b - | "\($sha) \($g) \($b | length) \($c | length) \($b | map("\(.n)=\(.v)") | join(","))"' -# Every pull_request workflow run for a SHA, latest per workflow: " ", -# state = green | pending | failed. -RUNS='[group_by(.workflowName)[] | max_by(.createdAt // "")] as $r - | ($r | map(select(.status != "completed"))) as $p - | ($r | map(select(.status == "completed" and (.conclusion | IN("success", "skipped", "neutral") | not)))) as $f - | if ($f | length) > 0 then "failed \($f | map("\(.workflowName)=\(.conclusion)") | join(","))" - elif ($p | length) > 0 then "pending -" else "green -" end' -# base_ok SHA: main is an ancestor of SHA, or main moved only in the ledger, or the PR is ledger-only. -code() { git diff --name-only "$1" "$2" | grep -qvxE 'EXPERIMENTS\.md|docs/provenance\.md'; } -base_ok() { - git fetch -q origin || return 1 - git merge-base --is-ancestor origin/main "$1" && return 0 - local mb; mb=$(git merge-base origin/main "$1") || return 1 - ! { code "$mb" origin/main && code "$mb" "$1"; } -} -# rules_stale SHA: the tree main + SHA would merge to carries a console/ledger-rules.json -# that EXPERIMENTS.md there does not generate (read with that tree's own model_opt.py). -# Unknown (a conflict, no such files) is not stale: a conflict syncs anyway. -rules_stale() { - local tree d rc=1 - tree=$(git merge-tree --write-tree origin/main "$1" 2>/dev/null | head -1) - [ -n "$tree" ] || return 1 - d=$(mktemp -d) - if git show "$tree:console/model_opt.py" > "$d/model_opt.py" 2>/dev/null \ - && git show "$tree:console/convert-pins.json" > "$d/convert-pins.json" 2>/dev/null \ - && git show "$tree:console/ledger-rules.json" > "$d/have.json" 2>/dev/null \ - && git show "$tree:EXPERIMENTS.md" > "$d/EXPERIMENTS.md" 2>/dev/null \ - && [ -s "$d/model_opt.py" ] && [ -s "$d/have.json" ] \ - && python3 "$d/model_opt.py" ledger-rules --ledger "$d/EXPERIMENTS.md" --out "$d/want.json" >/dev/null 2>&1 \ - && ! cmp -s "$d/have.json" "$d/want.json"; then - rc=0 - fi - rm -rf "$d" - return "$rc" -} [ "$(gh pr view "$pr" --json isDraft -q .isDraft)" = true ] && gh pr ready "$pr" >/dev/null merged() { [ "$(gh pr view "$pr" --json state -q .state)" = MERGED ]; } for round in 1 2 3 4 5; do merged && { echo "PR$pr MERGED"; exit 0; } [ "$(gh pr view "$pr" --json mergeable -q .mergeable)" = CONFLICTING ] && { sync || exit 1; sleep 60; } for _ in $(seq 1 "${GATE_POLLS:-60}"); do - s=$(gh pr view "$pr" --json headRefOid,statusCheckRollup -q "$READ" 2>/dev/null) || { sleep 60; continue; } - read -r sha g b n bad <<<"$s" + pj=$(gh pr view "$pr" --json headRefOid,statusCheckRollup 2>/dev/null) && [ -n "$pj" ] || { sleep 60; continue; } + sha=$(jq -r .headRefOid <<<"$pj") + read -r ci held_ok ci_last <<<"$(ci_eval "$sha" "$br")" + read -r sha g b n bad <<<"$(jq -r --argjson held_ok "${held_ok:-false}" "$READ" <<<"$pj")" [ "$b" -gt 0 ] && { echo "PR$pr FAIL at $sha: $bad"; exit 1; } if [ "$n" -gt 0 ] && [ "$g" -eq "$n" ]; then - ci=$(gh run list -w ci.yml -c "$sha" -e pull_request -L 1 --json status,conclusion \ - -q '.[0] | "\(.status)/\(.conclusion)"' 2>/dev/null) case $ci in - completed/success) + green) read -r rs rbad <<<"$(gh run list -c "$sha" -e pull_request -L 100 \ - --json workflowName,status,conclusion,createdAt -q "$RUNS" 2>/dev/null)" + --json workflowName,status,conclusion,createdAt 2>/dev/null | jq -r --argjson held_ok "$held_ok" "$RUNS" 2>/dev/null)" case $rs in failed) echo "PR$pr FAIL: workflow runs for $sha: $rbad"; exit 1;; green) ;; @@ -146,6 +146,7 @@ for round in 1 2 3 4 5; do echo "PR$pr round $round: console/ledger-rules.json would not match EXPERIMENTS.md after the merge: sync regenerates it" sync || exit 1; sleep 60; break fi + alarm_check why=$(gh pr merge "$pr" --squash --match-head-commit "$sha" --subject "$subj" --body "$body" 2>&1 >/dev/null) merged && { git push -q origin --delete "$br" 2>/dev/null; echo "PR$pr MERGED at $sha"; exit 0; } now=$(gh pr view "$pr" --json headRefOid -q .headRefOid 2>/dev/null) @@ -156,7 +157,7 @@ for round in 1 2 3 4 5; do sleep 60 fi break;; - completed/*) echo "PR$pr FAIL: ci.yml run for $sha is $ci"; exit 1;; + failed) echo "PR$pr FAIL: ci.yml for $sha did not succeed (latest $ci_last)"; exit 1;; esac # no ci.yml run for this SHA yet, or still running: wait fi sleep 60 diff --git a/tests/bot_ci_dispatch_test.sh b/tests/bot_ci_dispatch_test.sh new file mode 100755 index 00000000..682a4d86 --- /dev/null +++ b/tests/bot_ci_dispatch_test.sh @@ -0,0 +1,237 @@ +#!/usr/bin/env bash +# hack/bot-ci-dispatch against a fake gh (no network). +# The fake keeps open PRs in prs.json and workflow runs in runs.json. `gh run +# list` filters runs like the API (-w workflow file, -c head SHA, -b branch, +# -e event, -L count, newest first). `gh workflow run` appends a queued +# workflow_dispatch run at the branch's head, unless the branch is listed in +# no_dispatch (its workflow file lacks the trigger: HTTP 422) or dispatch_fail +# exists (HTTP 500). `gh run rerun ID --failed` queues attempt 2 of that run. +# Every dispatch and rerun is logged to actions. +# +# Usage: tests/bot_ci_dispatch_test.sh + +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +BCD=${BCD:-${ROOT}/hack/bot-ci-dispatch} +T=$(mktemp -d "${TMPDIR:-/tmp}/bot_ci_dispatch.XXXXXX") +trap 'rm -rf "$T"' EXIT +PASS=0 FAILS=0 +ok() { PASS=$((PASS + 1)); echo "ok $1"; } +fail() { FAILS=$((FAILS + 1)); echo "FAIL $1"; } + +mkdir -p "$T/bin" +cat > "$T/bin/gh" <<'STUB' +#!/usr/bin/env bash +F=$FIX +echo "gh $*" >> "$F/calls" +case "$1 $2" in + "pr list") cat "$F/prs.json"; exit 0;; + "run list") + shift 2; wf= c= b= e= q= L=1000 + while [ $# -gt 0 ]; do case $1 in + -w|--workflow) wf=$2; shift 2;; -c|--commit) c=$2; shift 2;; -b|--branch) b=$2; shift 2;; + -e|--event) e=$2; shift 2;; -q|--jq) q=$2; shift 2;; -L|--limit) L=$2; shift 2;; --json) shift 2;; *) shift;; esac; done + out=$(jq --arg w "$wf" --arg c "$c" --arg b "$b" --arg e "$e" --argjson L "$L" \ + '[.[] | select(($w == "" or .workflow == $w) and ($c == "" or .headSha == $c) + and ($b == "" or .headBranch == $b) and ($e == "" or .event == $e))] + | sort_by(.createdAt) | reverse | .[:$L]' "$F/runs.json") + if [ -n "$q" ]; then jq -r "$q" <<<"$out"; else echo "$out"; fi + exit 0;; + "workflow run") + wf=$3; shift 3; ref= fields= + while [ $# -gt 0 ]; do case $1 in -r|--ref) ref=$2; shift 2;; -f|-F|--raw-field|--field) fields="$fields $2"; shift 2;; *) shift;; esac; done + echo "$wf $ref$fields" >> "$F/fields" + if grep -qxF "$ref" "$F/no_dispatch" 2>/dev/null; then + echo "could not create workflow dispatch event: HTTP 422: Workflow does not have 'workflow_dispatch' trigger (https://api.github.com/repos/o/r/actions/workflows/1/dispatches)" >&2 + exit 1 + fi + [ -f "$F/dispatch_fail" ] && { echo "could not create workflow dispatch event: HTTP 500: Server Error" >&2; exit 1; } + sha=$(jq -r --arg b "$ref" '[.[] | select(.headRefName == $b)][0].headRefOid // "unknown"' "$F/prs.json") + n=$(( $(jq length "$F/runs.json") + 900 )) + jq --arg w "$wf" --arg b "$ref" --arg s "$sha" --argjson id "$n" \ + '. + [{databaseId: $id, workflow: $w, event: "workflow_dispatch", headBranch: $b, headSha: $s, + status: "queued", conclusion: "", attempt: 1, createdAt: "2026-10-29T23:00:00Z"}]' \ + "$F/runs.json" > "$F/runs.tmp" && mv "$F/runs.tmp" "$F/runs.json" + echo "dispatch $wf $ref" >> "$F/actions" + exit 0;; + "run rerun") + id=$3; echo "rerun $id ${*:4}" >> "$F/actions" + jq --argjson id "$id" 'map(if .databaseId == $id then .attempt += 1 | .status = "queued" | .conclusion = "" else . end)' \ + "$F/runs.json" > "$F/runs.tmp" && mv "$F/runs.tmp" "$F/runs.json" + exit 0;; +esac +exit 0 +STUB +printf '#!/bin/sh\nexit 0\n' > "$T/bin/sleep" +chmod +x "$T/bin/"* + +BOT='{"login":"app/github-actions","is_bot":true}' +HUMAN='{"login":"88plug","is_bot":false}' +DEPENDABOT='{"login":"app/dependabot","is_bot":true}' +# pr NUMBER BRANCH SHA [AUTHOR] [CROSS]: one open PR into main. +pr() { + printf '{"number":%s,"headRefName":"%s","headRefOid":"%s","baseRefName":"main","isCrossRepository":%s,"author":%s}' \ + "$1" "$2" "$3" "${5:-false}" "${4:-$BOT}" +} +# run ID WORKFLOW EVENT BRANCH SHA STATUS CONCLUSION [ATTEMPT] [CREATED] +run() { + printf '{"databaseId":%s,"workflow":"%s","event":"%s","headBranch":"%s","headSha":"%s","status":"%s","conclusion":"%s","attempt":%s,"createdAt":"%s"}' \ + "$1" "$2" "$3" "$4" "$5" "$6" "$7" "${8:-1}" "${9:-2026-10-29T20:00:00Z}" +} +# fixture NAME PRS RUNS +fixture() { + FIX=$T/$1; export FIX; mkdir -p "$FIX" + echo "$2" > "$FIX/prs.json"; echo "$3" > "$FIX/runs.json"; : > "$FIX/calls"; : > "$FIX/actions" +} +bcd() { OUT=$(PATH="$T/bin:$PATH" timeout 60 bash "$BCD" "$@" 2>&1); RC=$?; } +acts() { grep -c "^$1 " "$FIX/actions"; } +dispatched() { grep -qxF "dispatch ${2:-ci.yml} $1" "$FIX/actions"; } + +# 1. scope: only same-repo autobump/* PRs by github-actions[bot] with no run at the +# head (a held run does not count) get a dispatch; forks, human branches, a +# human's autobump/ branch and dependabot never do. +fixture scope "[$(pr 1 autobump/a a111),$(pr 2 autobump/fork f222 "$BOT" true),$(pr 3 claude/x c333 "$HUMAN"),$(pr 4 autobump/human h444 "$HUMAN"),$(pr 5 dependabot/pip/x d555 "$DEPENDABOT"),$(pr 6 autobump/done e666),$(pr 7 autobump/held g777)]" \ + "[$(run 1 ci.yml pull_request autobump/done e666 completed success),$(run 2 ci.yml pull_request autobump/held g777 completed action_required)]" +BOT_CI_MAX_INFLIGHT=10 bcd +if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 2 ] && dispatched autobump/a && dispatched autobump/held; then + ok "scope: dispatched autobump/a and the held autobump/held only" +else fail "scope: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi +if grep -q '#6 autobump/done' <<<"$OUT" && ! grep -qE '#(2|3|4|5) ' <<<"$OUT"; then + ok "scope: a line for the PR with a run at its head, none for the out-of-scope PRs" +else fail "scope: decision lines: $(tr '\n' ';' <<<"$OUT")"; fi + +# 2. the in-flight cap (default 1): two PRs, nothing running -> one dispatch, one deferred. +fixture cap "[$(pr 1 autobump/a a111),$(pr 2 autobump/b b222)]" '[]' +bcd +if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 1 ] && dispatched autobump/a && grep -q '#2 autobump/b.*deferred' <<<"$OUT"; then + ok "cap: one dispatch, the second deferred" +else fail "cap: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi +# a bot run already in flight on another autobump branch fills the cap. +fixture cap_full "[$(pr 1 autobump/a a111)]" "[$(run 5 ci.yml workflow_dispatch autobump/z z999 in_progress '')]" +bcd +if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 0 ] && grep -q 'deferred' <<<"$OUT"; then ok "cap_full: deferred" +else fail "cap_full: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions")"; fi + +# 3. a queued ci.yml run of a human PR defers every bot dispatch; a running one does not. +fixture human_queued "[$(pr 1 autobump/a a111)]" "[$(run 5 ci.yml pull_request claude/x c333 queued '')]" +BOT_CI_MAX_INFLIGHT=10 bcd +if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 0 ] && grep -q 'human' <<<"$OUT"; then ok "human_queued: deferred" +else fail "human_queued: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi +# a queued run whose head branch is gone (headBranch null) counts as a human run. +fixture human_null_branch "[$(pr 1 autobump/a a111)]" \ + "[{\"databaseId\":5,\"workflow\":\"ci.yml\",\"event\":\"pull_request\",\"headBranch\":null,\"headSha\":\"c333\",\"status\":\"queued\",\"conclusion\":\"\",\"attempt\":1,\"createdAt\":\"2026-10-29T20:00:00Z\"}]" +BOT_CI_MAX_INFLIGHT=10 bcd +if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 0 ] && grep -q 'deferred, 1 ci.yml run(s) of human PRs queued' <<<"$OUT"; then + ok "human_null_branch: deferred" +else fail "human_null_branch: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi +fixture human_running "[$(pr 1 autobump/a a111)]" "[$(run 5 ci.yml pull_request claude/x c333 in_progress '')]" +bcd +if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 1 ]; then ok "human_running: dispatched" +else fail "human_running: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions")"; fi + +# 4. idempotent: a second call dispatches nothing more. +fixture idempotent "[$(pr 1 autobump/a a111),$(pr 2 autobump/b b222)]" '[]' +BOT_CI_MAX_INFLIGHT=10 bcd; first=$(acts dispatch); BOT_CI_MAX_INFLIGHT=10 bcd +if [ "$first" -eq 2 ] && [ "$(acts dispatch)" -eq 2 ] && [ "$RC" -eq 0 ]; then ok "idempotent: 2 dispatches, none on the second call" +else fail "idempotent: first=$first total=$(acts dispatch) rc=$RC"; fi + +# 5. a run that failed on its first attempt gets its failed jobs re-run once. +fixture rerun "[$(pr 1 autobump/a a111)]" "[$(run 41 ci.yml workflow_dispatch autobump/a a111 completed failure)]" +bcd +if [ "$RC" -eq 0 ] && grep -qxF 'rerun 41 --failed' "$FIX/actions" && [ "$(acts dispatch)" -eq 0 ]; then ok "rerun: failed jobs of run 41 re-run" +else fail "rerun: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi +jq 'map(if .databaseId == 41 then .status = "completed" | .conclusion = "failure" else . end)' "$FIX/runs.json" > "$FIX/r" && mv "$FIX/r" "$FIX/runs.json" +bcd +if [ "$RC" -eq 0 ] && [ "$(acts rerun)" -eq 1 ] && [ "$(acts dispatch)" -eq 0 ]; then ok "rerun: not again after the re-run failed" +else fail "rerun: second call rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi +# a cancelled run is someone's decision: no re-run. +fixture rerun_cancelled "[$(pr 1 autobump/a a111)]" "[$(run 41 ci.yml workflow_dispatch autobump/a a111 completed cancelled)]" +bcd +if [ "$RC" -eq 0 ] && [ "$(acts rerun)" -eq 0 ] && [ "$(acts dispatch)" -eq 0 ]; then ok "rerun_cancelled: left alone" +else fail "rerun_cancelled: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions")"; fi + +# 6. a branch whose ci.yml has no workflow_dispatch: skipped with a reason, exit 0. +fixture no_trigger "[$(pr 1 autobump/old o111),$(pr 2 autobump/b b222)]" '[]'; echo autobump/old > "$FIX/no_dispatch" +BOT_CI_MAX_INFLIGHT=10 bcd +if [ "$RC" -eq 0 ] && grep -q '#1 autobump/old.*workflow_dispatch' <<<"$OUT" && [ "$(acts dispatch)" -eq 1 ] && dispatched autobump/b; then + ok "no_trigger: skipped, the next PR still dispatched" +else fail "no_trigger: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi +# any other dispatch failure is an error: exit 1. +fixture dispatch_error "[$(pr 1 autobump/a a111)]" '[]'; touch "$FIX/dispatch_fail" +bcd +if [ "$RC" -eq 1 ] && grep -q 'HTTP 500' <<<"$OUT"; then ok "dispatch_error: exit 1" +else fail "dispatch_error: rc=$RC out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 7. --branch limits it to one PR. +fixture one_branch "[$(pr 1 autobump/a a111),$(pr 2 autobump/b b222)]" '[]' +BOT_CI_MAX_INFLIGHT=10 bcd --branch autobump/b +if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 1 ] && dispatched autobump/b; then ok "one_branch: only autobump/b" +else fail "one_branch: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions")"; fi + +# 8. --workflow canary.yml: a head gets canary.yml, with engines=auto, only when +# `hack/pins policy MERGE_BASE SHA` (MERGE_BASE: of origin/main and SHA) maps canary +# engines for it, read on the head fetched from origin. canary_repo NAME: a bare +# origin whose branches autobump/a, autobump/b and autobump/c each carry one commit +# off main, and a clone on main whose hack/ holds bot-ci-dispatch and a fixture +# pins that prints policy/.json (no file, or a base that is not an ancestor of +# the head, so not the PR's own diff: it fails) and a note on stderr (which must not +# read as the policy). Sets SHA_A, SHA_B, SHA_C. +canary_repo() { + fixture "$1" '[]' '[]'; CR=$FIX/repo + ( set -e + export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 GIT_AUTHOR_NAME=t GIT_AUTHOR_EMAIL=t@t \ + GIT_COMMITTER_NAME=t GIT_COMMITTER_EMAIL=t@t + git init -q --bare -b main "$FIX/origin.git"; git init -q -b main "$CR"; cd "$CR" + echo base > f; git add f; git commit -qm base; git remote add origin "$FIX/origin.git"; git push -q origin main + for b in a b c; do git switch -qc "autobump/$b" main; echo "$b" > f; git commit -qam "$b"; git push -q origin "autobump/$b"; done + git switch -q main; git fetch -q origin + mkdir -p hack policy; cp "$BCD" hack/bot-ci-dispatch + printf '%s\n' '#!/usr/bin/env bash' \ + '[ "$1" = policy ] && git rev-parse -q --verify "$2^{commit}" >/dev/null && git cat-file -e "$3^{commit}" || exit 1' \ + 'git merge-base --is-ancestor "$2" "$3" || { echo "pins: $2 is not an ancestor of $3" >&2; exit 1; }' \ + 'echo "pins: a note on stderr" >&2' \ + 'cat "$(dirname "$0")/../policy/$3.json"' > hack/pins + chmod +x hack/pins hack/bot-ci-dispatch ) + SHA_A=$(git -C "$CR" rev-parse origin/autobump/a); SHA_B=$(git -C "$CR" rev-parse origin/autobump/b) + SHA_C=$(git -C "$CR" rev-parse origin/autobump/c) + echo '{"verdict":"auto","canary":["vllm-cpu"]}' > "$CR/policy/$SHA_A.json" + echo '{"verdict":"auto","canary":[]}' > "$CR/policy/$SHA_B.json" +} +bcd_repo() { OUT=$(cd "$CR" && PATH="$T/bin:$PATH" timeout 60 bash hack/bot-ci-dispatch "$@" 2>&1); RC=$?; } +canary_repo canary_mapped +echo "[$(pr 1 autobump/a "$SHA_A"),$(pr 2 autobump/b "$SHA_B"),$(pr 3 autobump/c "$SHA_C")]" > "$FIX/prs.json" +echo "[$(run 5 ci.yml workflow_dispatch autobump/a "$SHA_A" completed success)]" > "$FIX/runs.json" +BOT_CI_MAX_INFLIGHT=10 bcd_repo --workflow canary.yml +if [ "$RC" -eq 1 ] && [ "$(acts dispatch)" -eq 1 ] && dispatched autobump/a canary.yml \ + && grep -qxF 'canary.yml autobump/a engines=auto' "$FIX/fields" \ + && grep -q '#2 autobump/b.*no canary engine' <<<"$OUT" && grep -q '#3 autobump/c.*policy' <<<"$OUT"; then + ok "canary: dispatched with engines=auto where the policy maps engines; none mapped: skipped; policy failed: exit 1" +else fail "canary: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") fields=$(tr '\n' ';' < "$FIX/fields" 2>/dev/null) out=$(tr '\n' ';' <<<"$OUT")"; fi +# a head that moved on origin since the PR list was read: no canary for the stale SHA. +canary_repo canary_moved; echo "[$(pr 1 autobump/a "$SHA_B")]" > "$FIX/prs.json" +cp "$CR/policy/$SHA_A.json" "$CR/policy/$SHA_B.json"; bcd_repo --workflow canary.yml +if [ "$(acts dispatch)" -eq 0 ] && grep -q '#1 autobump/a.*moved' <<<"$OUT"; then ok "canary_moved: no dispatch" +else fail "canary_moved: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi +# main moved (a ledger commit) since the heads were cut: the policy still reads each +# head's own diff, from its merge-base with main. +canary_repo canary_main_moved +( cd "$CR" && export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 GIT_AUTHOR_NAME=t GIT_AUTHOR_EMAIL=t@t \ + GIT_COMMITTER_NAME=t GIT_COMMITTER_EMAIL=t@t && echo ledger > g && git add g && git commit -qm ledger \ + && git push -q origin main && git fetch -q origin ) +echo "[$(pr 1 autobump/a "$SHA_A")]" > "$FIX/prs.json"; bcd_repo --workflow canary.yml +if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 1 ] && dispatched autobump/a canary.yml; then + ok "canary_main_moved: policy from the merge-base, dispatched" +else fail "canary_main_moved: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi +# ci.yml never consults the policy, and its dispatch carries no inputs. +canary_repo ci_no_policy; rm "$CR/hack/pins" +echo "[$(pr 1 autobump/a "$SHA_A"),$(pr 2 autobump/b "$SHA_B")]" > "$FIX/prs.json" +BOT_CI_MAX_INFLIGHT=10 bcd_repo +if [ "$RC" -eq 0 ] && [ "$(acts dispatch)" -eq 2 ] && grep -qxF 'ci.yml autobump/a' "$FIX/fields"; then ok "ci_no_policy: ci.yml dispatched for both, no inputs" +else fail "ci_no_policy: rc=$RC actions=$(tr '\n' ';' < "$FIX/actions") out=$(tr '\n' ';' <<<"$OUT")"; fi + +bcd --bogus +if [ "$RC" -eq 2 ]; then ok "usage: unknown flag exits 2"; else fail "usage: rc=$RC"; fi + +echo "${PASS} passed, ${FAILS} failed" +[ "${FAILS}" -eq 0 ] diff --git a/tests/bot_merge_fake_gh.py b/tests/bot_merge_fake_gh.py new file mode 100755 index 00000000..e54382b3 --- /dev/null +++ b/tests/bot_merge_fake_gh.py @@ -0,0 +1,368 @@ +#!/usr/bin/env python3 +"""A stand-in for `gh` that answers from fixture files (tests/bot_merge_test.sh). + +State lives under $FIX/gh: + pulls.json REST pull objects (number, title, user, head, base, labels, + created_at, draft, state, merged) + files/.json REST pull files ([{filename}]) + reviews/.json REST reviews + comments/.json REST issue comments (POST and PATCH edit this file) + runs.json `gh run list --json` rows, plus "workflow" (the file name) and + "headRepository" (owner/repo the head came from); the REST + workflow runs list is served from them too + artifacts/// files `gh run download -n ` extracts + issues.json [{number, title, state, labels: [{name}]}] + merge_rc exit code sequence for `gh pr merge` (last one repeats) + issue_list_rc exit code sequence for `gh issue list` (last one repeats) +Calls are logged to $FIX/gh/calls; dispatches to $FIX/gh/dispatches, merges to +$FIX/gh/merges, label additions to $FIX/gh/labels. + +A successful merge marks the pull merged and writes the squash commit onto the +bare origin's main ($FIX/origin.git): main's tree merged with the head's, authored +by github-actions[bot], committed at $BOT_MERGE_NOW. +""" +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +FIX = Path(os.environ["FIX"]) +G = FIX / "gh" + + +def load(p, default): + try: + return json.loads(p.read_text()) + except FileNotFoundError: + return default + + +def save(p, obj): + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(json.dumps(obj, indent=1)) + + +def log(name, line): + with open(G / name, "a") as f: + f.write(line + "\n") + + +def pop(name, default): + p = G / name + if not p.exists(): + return default + seq = p.read_text().split() + i_path = G / (name + ".i") + i = int(i_path.read_text()) if i_path.exists() else 0 + i_path.write_text(str(i + 1)) + return seq[min(i, len(seq) - 1)] + + +def emit(obj, jq_expr): + """Print obj as gh does: --jq output through jq (strings raw), else JSON.""" + text = json.dumps(obj) + if jq_expr is None: + print(text) + return 0 + r = subprocess.run(["jq", "-c", "-r", jq_expr], input=text, text=True, capture_output=True) + sys.stdout.write(r.stdout) + sys.stderr.write(r.stderr) + return r.returncode + + +def flags(argv, with_value): + """Split argv into positionals and a {flag: [values]} map.""" + pos, opts = [], {} + i = 0 + while i < len(argv): + a = argv[i] + if a in with_value: + opts.setdefault(a, []).append(argv[i + 1]) + i += 2 + elif a.startswith("-"): + opts.setdefault(a, []).append(True) + i += 1 + else: + pos.append(a) + i += 1 + return pos, opts + + +def one(opts, *names): + for n in names: + if n in opts: + return opts[n][-1] + return None + + +def pull(n): + for p in load(G / "pulls.json", []): + if p["number"] == n: + return p + return None + + +def set_pull(n, **kv): + pulls = load(G / "pulls.json", []) + for p in pulls: + if p["number"] == n: + p.update(kv) + save(G / "pulls.json", pulls) + + +# ---- gh api --------------------------------------------------------------- +def api(argv): + pos, opts = flags(argv, {"-X", "--method", "-f", "--raw-field", "-F", "--field", + "-q", "--jq", "-H", "--header", "--input"}) + endpoint = pos[0].lstrip("/") + method = (one(opts, "-X", "--method") or "GET").upper() + fields = {} + for kv in opts.get("-f", []) + opts.get("--raw-field", []): # raw: taken as given + k, v = kv.split("=", 1) + fields.setdefault(k, []).append(v) + for kv in opts.get("-F", []) + opts.get("--field", []): # typed: @path reads a file + k, v = kv.split("=", 1) + if v.startswith("@"): + v = Path(v[1:]).read_text() if v != "@-" else sys.stdin.read() + fields.setdefault(k, []).append(v) + if "-f" in opts or "-F" in opts or "--field" in opts or "--raw-field" in opts: + if method == "GET" and "-X" not in opts and "--method" not in opts: + method = "POST" + jq_expr = one(opts, "-q", "--jq") + path, _, query = endpoint.partition("?") + parts = path.split("/") + # repos///... + rest = parts[3:] + if rest[:1] == ["pulls"] and len(rest) == 1: + q = dict(kv.split("=", 1) for kv in query.split("&") if kv) + pulls = [p for p in load(G / "pulls.json", []) + if q.get("state", "open") in ("all", p.get("state", "open")) + and q.get("base", p["base"]["ref"]) == p["base"]["ref"]] + return emit(pulls, jq_expr) + if rest[:1] == ["pulls"] and len(rest) == 2: + p = pull(int(rest[1])) + if p is None: + print("gh: Not Found (HTTP 404)", file=sys.stderr) + return 1 + return emit(p, jq_expr) + if rest[:1] == ["pulls"] and rest[2] == "files": + return emit(load(G / "files" / f"{rest[1]}.json", []), jq_expr) + if rest[:1] == ["pulls"] and rest[2] == "reviews": + return emit(load(G / "reviews" / f"{rest[1]}.json", []), jq_expr) + if rest[:1] == ["issues"] and len(rest) == 3 and rest[2] == "comments": + cpath = G / "comments" / f"{rest[1]}.json" + comments = load(cpath, []) + if method == "POST": + ids = [c["id"] for f in (G / "comments").glob("*.json") for c in load(f, [])] + c = {"id": max(ids, default=1000) + 1, "user": {"login": "github-actions[bot]", "type": "Bot"}, + "body": fields["body"][-1]} + comments.append(c) + save(cpath, comments) + log("calls", f"comment-post {rest[1]}") + return emit(c, jq_expr) + return emit(comments, jq_expr) + if rest[:2] == ["issues", "comments"] and method == "PATCH": + cid = int(rest[2]) + for f in (G / "comments").glob("*.json"): + comments = load(f, []) + for c in comments: + if c["id"] == cid: + c["body"] = fields["body"][-1] + save(f, comments) + log("calls", f"comment-patch {f.stem} {cid}") + return emit(c, jq_expr) + print("gh: Not Found (HTTP 404)", file=sys.stderr) + return 1 + if rest[:2] == ["actions", "workflows"] and len(rest) == 4 and rest[3] == "runs" and method == "GET": + q = dict(kv.split("=", 1) for kv in query.split("&") if kv) + log("calls", f"api-runs {rest[2]} " + " ".join(f"{k}={v}" for k, v in sorted(q.items()))) + out = [r for r in load(G / "runs.json", []) if wf_match(r, rest[2]) + and q.get("head_sha", r["headSha"]) == r["headSha"] + and q.get("branch", r["headBranch"]) == r["headBranch"] + and q.get("event", r["event"]) == r["event"]] + out.sort(key=lambda r: r["createdAt"], reverse=True) + out = out[: int(q.get("per_page", 30))] + return emit({"total_count": len(out), "workflow_runs": [ + {"id": r["databaseId"], "name": r.get("workflowName"), "event": r["event"], + "head_sha": r["headSha"], "head_branch": r["headBranch"], + "head_repository": {"full_name": r.get("headRepository", "o/r")}, + "status": r["status"], "conclusion": r.get("conclusion") or None, + "created_at": r["createdAt"], "html_url": r["url"]} for r in out]}, jq_expr) + if rest[:1] == ["issues"] and len(rest) == 3 and rest[2] == "labels" and method == "POST": + n = int(rest[1]) + names = fields.get("labels[]", []) + p = pull(n) + have = [lab["name"] for lab in p["labels"]] + set_pull(n, labels=p["labels"] + [{"name": x} for x in names if x not in have]) + for x in names: + log("labels", f"{n} {x}") + return emit([{"name": x} for x in have + names], jq_expr) + print(f"bot_merge_fake_gh: unhandled api {method} {endpoint}", file=sys.stderr) + return 99 + + +# ---- gh run --------------------------------------------------------------- +def wf_match(run, wf): + return wf in (run.get("workflow"), run.get("workflowName")) + + +def run_list(argv): + pos, opts = flags(argv, {"-w", "--workflow", "-c", "--commit", "-b", "--branch", "-e", "--event", + "-s", "--status", "-u", "--user", "--created", "-L", "--limit", + "--json", "-q", "--jq"}) + runs = load(G / "runs.json", []) + wf = one(opts, "-w", "--workflow") + sha = one(opts, "-c", "--commit") + br = one(opts, "-b", "--branch") + ev = one(opts, "-e", "--event") + st = one(opts, "-s", "--status") + created = one(opts, "--created") + log("calls", "run-list " + " ".join(f"{k}={v}" for k, v in + (("w", wf), ("c", sha), ("b", br), ("e", ev), ("s", st), ("created", created)) if v)) + out = [] + for r in runs: + if wf and not wf_match(r, wf): + continue + if sha and r["headSha"] != sha: + continue + if br and r["headBranch"] != br: + continue + if ev and r["event"] != ev: + continue + if st and st not in (r["status"], r.get("conclusion")): + continue + if created: + assert created.startswith(">="), created + if r["createdAt"] < created[2:]: + continue + out.append(r) + out.sort(key=lambda r: r["createdAt"], reverse=True) + out = out[: int(one(opts, "-L", "--limit") or 20)] + fields = (one(opts, "--json") or "").split(",") + out = [{k: r.get(k) for k in fields} for r in out] + return emit(out, one(opts, "-q", "--jq")) + + +def run_download(argv): + pos, opts = flags(argv, {"-n", "--name", "-D", "--dir", "-p", "--pattern"}) + rid = pos[0] + names = opts.get("-n", []) + opts.get("--name", []) + dest = Path(one(opts, "-D", "--dir") or ".") + log("calls", f"run-download {rid} {' '.join(names)}") + src = G / "artifacts" / rid + found = [n for n in names if (src / n).is_dir()] + if not found: + print("no valid artifacts found to download", file=sys.stderr) + return 1 + dest.mkdir(parents=True, exist_ok=True) + for f in (src / found[0]).iterdir(): + shutil.copy(f, dest / f.name) + return 0 + + +# ---- gh pr ---------------------------------------------------------------- +HELP = " --match-head-commit SHA Commit SHA that the pull request head must match to allow merge\n" + + +def git_origin(*args, env=None): + return subprocess.run(["git", f"--git-dir={FIX / 'origin.git'}", *args], text=True, + capture_output=True, check=True, env=env).stdout.strip() + + +def pr_merge(argv): + if "--help" in argv: + sys.stdout.write((G / "merge_help").read_text() if (G / "merge_help").exists() else HELP) + return 0 + pos, opts = flags(argv, {"--match-head-commit", "-t", "--subject", "-b", "--body", "-F", "--body-file"}) + n = int(pos[0]) + sha = one(opts, "--match-head-commit") + subj = one(opts, "-t", "--subject") + body = one(opts, "-b", "--body") + log("merges", " ".join(argv).replace("\n", "\\n")) + p = pull(n) + rc = int(pop("merge_rc", "0")) + if rc != 0: + print("GraphQL: Base branch was modified. Review and try the merge again. (mergePullRequest)", file=sys.stderr) + return rc + if sha != p["head"]["sha"]: + print("GraphQL: Head branch was modified. Review and try the merge again. (mergePullRequest)", file=sys.stderr) + return 1 + try: + tree = git_origin("merge-tree", "--write-tree", "main", sha).splitlines()[0] + except subprocess.CalledProcessError: + print(f"Pull request #{n} is not mergeable: the merge commit cannot be cleanly created.", file=sys.stderr) + return 1 + when = os.environ["BOT_MERGE_NOW"] + " +0000" + env = dict(os.environ, GIT_AUTHOR_NAME="github-actions[bot]", + GIT_AUTHOR_EMAIL="41898282+github-actions[bot]@users.noreply.github.com", + GIT_COMMITTER_NAME="GitHub", GIT_COMMITTER_EMAIL="noreply@github.com", + GIT_AUTHOR_DATE=when, GIT_COMMITTER_DATE=when) + commit = git_origin("commit-tree", tree, "-p", "main", "-m", subj, "-m", body, env=env) + git_origin("update-ref", "refs/heads/main", commit) + set_pull(n, state="closed", merged=True) + return 0 + + +def pr_view(argv): + pos, opts = flags(argv, {"--json", "-q", "--jq"}) + p = pull(int(pos[0])) + obj = {"state": "MERGED" if p.get("merged") else p.get("state", "open").upper(), + "headRefOid": p["head"]["sha"], "headRefName": p["head"]["ref"]} + return emit(obj, one(opts, "-q", "--jq")) + + +# ---- gh issue / workflow / repo ------------------------------------------- +def issue_list(argv): + pos, opts = flags(argv, {"--label", "-l", "--state", "-s", "--json", "-L", "--limit", "-q", "--jq"}) + rc = int(pop("issue_list_rc", "0")) + if rc != 0: + print("HTTP 502: Server Error (https://api.github.com/graphql)", file=sys.stderr) + return rc + label = one(opts, "--label", "-l") + state = one(opts, "--state", "-s") or "open" + issues = [i for i in load(G / "issues.json", []) + if (state == "all" or i.get("state", "open") == state) + and (label is None or any(lab["name"] == label for lab in i.get("labels", [])))] + fields = (one(opts, "--json") or "number,title").split(",") + return emit([{k: i.get(k) for k in fields} for i in issues], one(opts, "-q", "--jq")) + + +def workflow_run(argv): + pos, opts = flags(argv, {"-f", "--raw-field", "-F", "--field", "-r", "--ref"}) + kv = opts.get("-f", []) + opts.get("--raw-field", []) + opts.get("-F", []) + opts.get("--field", []) + log("dispatches", " ".join([pos[0], "ref=" + str(one(opts, "-r", "--ref"))] + kv)) + rc = int(pop("dispatch_rc", "0")) + if rc == 0: + print(f"https://github.com/o/r/actions/runs/9{len((G / 'dispatches').read_text().splitlines())}") + return rc + + +def main(argv): + G.mkdir(parents=True, exist_ok=True) + log("trace", ("gh " + " ".join(argv)).replace("\n", "\\n")) + if "-R" in argv or "--repo" in argv: + print("bot_merge_fake_gh: -R is not expected", file=sys.stderr) + return 98 + head = argv[:2] + if argv[:1] == ["api"]: + return api(argv[1:]) + if head == ["run", "list"]: + return run_list(argv[2:]) + if head == ["run", "download"]: + return run_download(argv[2:]) + if head == ["pr", "merge"]: + return pr_merge(argv[2:]) + if head == ["pr", "view"]: + return pr_view(argv[2:]) + if head == ["issue", "list"]: + return issue_list(argv[2:]) + if head == ["workflow", "run"]: + return workflow_run(argv[2:]) + print(f"bot_merge_fake_gh: unhandled {' '.join(argv)}", file=sys.stderr) + return 99 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/bot_merge_test.sh b/tests/bot_merge_test.sh new file mode 100755 index 00000000..036ddb24 --- /dev/null +++ b/tests/bot_merge_test.sh @@ -0,0 +1,564 @@ +#!/usr/bin/env bash +# hack/bot-merge against a fake gh (tests/bot_merge_fake_gh.py) and real git. +# Each case builds a fixture: a bare origin whose main holds the ledger files +# rules_stale reads (console/model_opt.py, EXPERIMENTS.md, the ledger-rules.json +# it generates), scripts/k3d-gpu and tag aur-v0.1.79; a clone ($FIX/repo) on main +# whose hack/ holds bot-merge, gate_lib.sh and a fixture `pins` that prints +# $FIX/policy/.json; and the fake GitHub's state under $FIX/gh (pulls, +# files, reviews, comments, workflow runs, canary artifacts, alarm issues). A +# successful fake merge writes the squash commit onto origin's main. +# +# Usage: tests/bot_merge_test.sh + +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +BOT_MERGE=${BOT_MERGE:-${ROOT}/hack/bot-merge} +GATE_LIB=${GATE_LIB:-${ROOT}/hack/gate_lib.sh} +T=$(mktemp -d "${TMPDIR:-/tmp}/bot_merge_test.XXXXXX") +trap '[ -n "${KEEP:-}" ] || rm -rf "$T"' EXIT +PASS=0 FAILS=0 +ok() { PASS=$((PASS + 1)); echo "ok $1"; } +fail() { FAILS=$((FAILS + 1)); echo "FAIL $1"; } + +export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 \ + GIT_AUTHOR_NAME=t GIT_AUTHOR_EMAIL=t@t GIT_COMMITTER_NAME=t GIT_COMMITTER_EMAIL=t@t +mkdir -p "$T/bin" +cp "$ROOT/tests/bot_merge_fake_gh.py" "$T/bin/gh"; chmod +x "$T/bin/gh" + +NOW=$(date -u -d 2026-10-29T14:13:00Z +%s) # the pass runs at 14:13 UTC +iso() { date -u -d "@$1" +%Y-%m-%dT%H:%M:%SZ; } +ago() { iso $((NOW - $1 * 60)); } # ago MINUTES: an ISO time before the pass + +# --- fixture builders ------------------------------------------------------ +G() { echo "$FIX/gh/$1"; } +jqi() { local f; f=$(G "$1"); shift; jq "$@" "$f" > "$f.tmp" && mv "$f.tmp" "$f"; } +rgit() { git -C "$FIX/repo" "$@"; } +# commit_at MINUTES_AGO MSG [AUTHOR]: commit the clone's staged changes at that time. +commit_at() { + local when; when="$(( NOW - $1 * 60 )) +0000" + GIT_AUTHOR_DATE=$when GIT_COMMITTER_DATE=$when GIT_AUTHOR_NAME=${3:-t} \ + git -C "$FIX/repo" commit -q -m "$2" +} + +# fx NAME [TAG_MINUTES_AGO]: fresh fixture; main's base commit (tagged aur-v0.1.79) +# was made TAG_MINUTES_AGO (default 60) and its ci.yml push run succeeded. +fx() { + FIX=$T/$1; export FIX + mkdir -p "$FIX/gh/files" "$FIX/gh/reviews" "$FIX/gh/comments" "$FIX/policy" + echo '[]' > "$(G pulls.json)"; echo '[]' > "$(G runs.json)"; echo '[]' > "$(G issues.json)" + : > "$(G dispatches)"; : > "$(G merges)"; : > "$(G labels)"; : > "$(G calls)" + RUNID=1000 + git init -q --bare -b main "$FIX/origin.git"; git init -q -b main "$FIX/repo" + local R=$FIX/repo + mkdir -p "$R/console" "$R/scripts" "$R/docs" "$R/hack" + cp "$ROOT/console/model_opt.py" "$ROOT/console/convert-pins.json" "$R/console/" + printf '## Hypotheses\n\n| H34 | on the i9-13900T | p | m | SUPPORTED |\n' > "$R/EXPERIMENTS.md" + python3 "$R/console/model_opt.py" ledger-rules --ledger "$R/EXPERIMENTS.md" --out "$R/console/ledger-rules.json" + printf '#!/bin/sh\nVERSION="0.1.79"\n' > "$R/scripts/k3d-gpu" + echo base > "$R/docs/provenance.md"; echo 'k3s v1.35.0' > "$R/pins.txt" + rgit add console scripts docs EXPERIMENTS.md pins.txt + commit_at "${2:-60}" base + rgit tag aur-v0.1.79 + rgit remote add origin "$FIX/origin.git"; rgit push -q origin main aur-v0.1.79; rgit fetch -q origin + # the code under test, plus the fixture policy (untracked in the clone) + cp "$BOT_MERGE" "$R/hack/bot-merge"; cp "$GATE_LIB" "$R/hack/gate_lib.sh" + cat > "$R/hack/pins" <<'PINS' +#!/usr/bin/env bash +# fixture: `pins policy BASE HEAD` prints $FIX/policy/HEAD.json once both refs resolve. +# Like hack/pins it judges the diff BASE..HEAD: a file that diff changes and HEAD's own +# diff (from its merge-base with BASE) does not is a line that is no pin move, so the +# verdict is review (changes-defaults) and the group mixed. +[ "$1" = policy ] || exit 2 +echo "pins $*" >> "$FIX/gh/trace" +git rev-parse -q --verify "$2^{commit}" >/dev/null || { echo "pins: no base $2" >&2; exit 1; } +git cat-file -e "$3^{commit}" 2>/dev/null || { echo "pins: head $3 not fetched" >&2; exit 1; } +mb=$(git merge-base "$2" "$3") || exit 1 +extra=$(git diff --name-only "$2" "$3" | grep -vxF -f <(git diff --name-only "$mb" "$3") | paste -sd' ' -) +if [ -n "$extra" ]; then + jq --arg x "$extra" '.verdict = "review" | .group = "mixed" + | .reasons += ["changes-defaults: changed lines that are no pin move in " + $x]' "$FIX/policy/$3.json" +else cat "$FIX/policy/$3.json"; fi +PINS + chmod +x "$R/hack/pins" "$R/hack/bot-merge" + run ci.yml push "$(rgit rev-parse HEAD)" main completed success "$(ago "${2:-60}")" +} + +wfname() { case $1 in ci.yml) echo CI;; canary.yml) echo Canary;; freshness.yml) echo Freshness;; + auto-release.yml) echo 'Auto-release AUR';; build-and-push.yml) echo 'Auto-update, Build & Release';; *) echo "$1";; esac; } +# run WORKFLOW EVENT SHA BRANCH STATUS CONCLUSION [CREATED] [TITLE] [HEAD_REPO]: add a workflow +# run whose head came from HEAD_REPO (default o/r, this repository); sets LASTRUN. +run() { + RUNID=$((RUNID + 1)); LASTRUN=$RUNID + jqi runs.json --argjson id "$RUNID" --arg wf "$1" --arg name "$(wfname "$1")" --arg ev "$2" --arg sha "$3" \ + --arg br "$4" --arg st "$5" --arg c "$6" --arg at "${7:-$(ago 30)}" --arg t "${8:-$(wfname "$1")}" \ + --arg hr "${9:-o/r}" \ + '. + [{databaseId: $id, workflow: $wf, workflowName: $name, event: $ev, headSha: $sha, headBranch: $br, + headRepository: $hr, status: $st, conclusion: $c, createdAt: $at, displayTitle: $t, attempt: 1, + url: ("https://github.com/o/r/actions/runs/\($id)")}]' +} +# mkpr N [FILE] [CREATED_MINUTES_AGO] [CONTENT]: bot PR #N on autobump/gN, one commit off +# origin/main changing FILE (default pins.txt: a line appended, or the file set to +# CONTENT); policy auto with no canary engines, a green ci.yml dispatch run on the +# branch. Sets SHA. +mkpr() { + local n=$1 f=${2:-pins.txt} br=autobump/g$1 + rgit fetch -q origin; rgit checkout -q -B "$br" origin/main + mkdir -p "$(dirname "$FIX/repo/$f")" + if [ -n "${4:-}" ]; then printf '%s' "$4" > "$FIX/repo/$f"; else echo "bump $n" >> "$FIX/repo/$f"; fi + rgit add "$f" + commit_at "${3:-50}" "chore(g$n): bump"; rgit push -q -f origin "$br"; SHA=$(rgit rev-parse HEAD) + rgit checkout -q main + jqi pulls.json --argjson n "$n" --arg br "$br" --arg sha "$SHA" --arg at "$(ago "${3:-50}")" \ + '. + [{number: $n, title: "chore(g\($n)): bump g\($n) to its newest patch", state: "open", draft: false, + user: {login: "github-actions[bot]", type: "Bot"}, created_at: $at, + head: {ref: $br, sha: $sha, repo: {full_name: "o/r"}}, base: {ref: "main", repo: {full_name: "o/r"}}, + labels: [{name: "autobump"}]}]' + jq -n --arg f "$f" '[{filename: $f}]' > "$FIX/gh/files/$n.json" + jq -n --arg g "g$n" '{verdict: "auto", reasons: [], group: $g, update_type: "patch", urgent: false, canary: []}' \ + > "$FIX/policy/$SHA.json" + run ci.yml workflow_dispatch "$SHA" "$br" completed success "$(ago 20)" "CI" +} +no_ci() { jqi runs.json --arg s "$1" 'map(select(.workflow != "ci.yml" or .headSha != $s))'; } # drop SHA's ci.yml runs +pr_set() { jqi pulls.json --argjson n "$1" "map(if .number == \$n then $2 else . end)"; } +pol() { local f=$FIX/policy/$1.json; jq "$2" "$f" > "$f.tmp" && mv "$f.tmp" "$f"; } +# review N LOGIN TYPE ASSOCIATION STATE COMMIT [SUBMITTED_MINUTES_AGO] +review() { + local f=$FIX/gh/reviews/$1.json; [ -f "$f" ] || echo '[]' > "$f" + jq --arg u "$2" --arg t "$3" --arg a "$4" --arg s "$5" --arg c "$6" --arg at "$(ago "${7:-10}")" \ + '. + [{user: {login: $u, type: $t}, author_association: $a, state: $s, commit_id: $c, submitted_at: $at}]' \ + "$f" > "$f.tmp" && mv "$f.tmp" "$f" +} +# canary SHA BRANCH ENGINES_JSON [STATUS] [CONCLUSION] [MINUTES_AGO]: a canary.yml run for SHA +# (dispatched on BRANCH) and its artifact. +canary() { + run canary.yml workflow_dispatch "$1" "$2" "${4:-completed}" "${5:-success}" "$(ago "${6:-15}")" Canary + mkdir -p "$FIX/gh/artifacts/$LASTRUN/canary-result" + jq -n --arg sha "$1" --arg ref "$2" --argjson e "$3" '{sha: $sha, ref: $ref, engines: $e}' \ + > "$FIX/gh/artifacts/$LASTRUN/canary-result/canary-result.json" +} +alarm() { jqi issues.json --arg k "$1" '. + [{number: (length + 500), title: "[ci-alarm] \($k): see the run", state: "open", labels: [{name: "ci-alarm"}]}]'; } +# botcommit MINUTES_AGO URGENT [PR]: a bot-merge squash commit on origin's main, with +# a green ci.yml run for it. +botcommit() { + rgit fetch -q origin; rgit checkout -q main; rgit reset -q --hard origin/main + echo "merged $1" >> "$FIX/repo/pins.txt"; rgit add pins.txt + local when; when="$(( NOW - $1 * 60 )) +0000" + GIT_AUTHOR_DATE=$when GIT_COMMITTER_DATE=$when GIT_AUTHOR_NAME='github-actions[bot]' \ + git -C "$FIX/repo" commit -q -m "chore(g${3:-9}): bump (#${3:-9})" \ + -m "Bump of g${3:-9}." -m "Bot-merge: pr=${3:-9} group=g${3:-9} urgent=$2" + rgit push -q origin main + run ci.yml workflow_dispatch "$(rgit rev-parse HEAD)" main completed success "$(ago $(( $1 > 2 ? $1 - 2 : 0 )))" +} +# main_commit MINUTES_AGO FILE CONTENT [AUTHOR] [MSG]: a commit on origin's main by someone else. +main_commit() { + rgit fetch -q origin; rgit checkout -q main; rgit reset -q --hard origin/main + mkdir -p "$(dirname "$FIX/repo/$2")" + printf '%s' "$3" > "$FIX/repo/$2"; rgit add "$2"; commit_at "$1" "${5:-edit $2}" "${4:-t}"; rgit push -q origin main +} + +# run_bm: one pass. CAP=N run_bm sets MAX_BOT_MERGES_PER_DAY (unset: the script's default). +run_bm() { + OUT=$(cd "$FIX/repo" && PATH="$T/bin:$PATH" GITHUB_REPOSITORY=o/r BOT_MERGE_NOW=$NOW \ + GITHUB_SERVER_URL=https://github.com GITHUB_RUN_ID=777 env ${CAP:+MAX_BOT_MERGES_PER_DAY=$CAP} \ + timeout 120 bash hack/bot-merge 2>&1); RC=$? +} +merges() { wc -l < "$(G merges)"; } +merged() { grep -cE "^$1 " "$(G merges)"; } +pinned() { grep -E "^$1 " "$(G merges)" | grep -oE -- '--match-head-commit [0-9a-f]+' | cut -d' ' -f2; } +disp() { grep -cE "$1" "$(G dispatches)"; } +comment() { jq -r '.[].body' "$FIX/gh/comments/$1.json" 2>/dev/null; } +ncomments() { jq length "$FIX/gh/comments/$1.json" 2>/dev/null || echo 0; } +show() { echo "rc=$RC merges=$(merges) dispatches=[$(tr '\n' ';' < "$(G dispatches)")] out=$(tr '\n' ';' <<<"$OUT")"; } + +# 1. auto verdict, all green: one merge pinned to the head, then ci.yml dispatched on main; +# the squash subject is conventional and the body carries the Bot-merge line. +fx auto_green; mkpr 1; run_bm +main_head=$(git --git-dir="$FIX/origin.git" log -1 --format=%B main) +if [ "$RC" -eq 0 ] && [ "$(merges)" -eq 1 ] && [ "$(pinned 1)" = "$SHA" ] && grep -q -- '--squash' "$(G merges)" \ + && [ "$(disp '^ci.yml ref=main$')" -eq 1 ] && head -1 <<<"$main_head" | grep -qxE 'chore\(g1\): bump g1 to its newest patch \(#1\)' \ + && grep -qx 'Bot-merge: pr=1 group=g1 urgent=false' <<<"$main_head" \ + && comment 1 | grep -q 'Merged' && comment 1 | grep -qF 'actions/runs/'; then + ok "auto_green: merged at the head, ci.yml dispatched on main, evidence comment" +else fail "auto_green: $(show) msg=$(tr '\n' '|' <<<"$main_head")"; fi + +# 2. review verdict, no approval: no merge, a blocking comment that names the reason. +fx review_no_approval; mkpr 1; pol "$SHA" '.verdict = "review" | .reasons = ["minor update of a static-only engine"]'; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'needs review' && comment 1 | grep -q 'static-only engine' \ + && [ "$(disp ci.yml)" -eq 0 ]; then ok "review_no_approval: blocked with a comment" +else fail "review_no_approval: $(show) comment=$(comment 1)"; fi + +# 3. review verdict, approved only by a bot account: no merge. +fx review_bot_approval; mkpr 1; pol "$SHA" '.verdict = "review"' +review 1 'github-actions[bot]' Bot NONE APPROVED "$SHA"; review 1 'some-app[bot]' Bot MEMBER APPROVED "$SHA"; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'needs review'; then ok "review_bot_approval: a bot approval never counts" +else fail "review_bot_approval: $(show)"; fi + +# 4. review verdict with an OWNER approval at the head: merges. +fx review_owner_approval; mkpr 1; pol "$SHA" '.verdict = "review"'; review 1 88plug User OWNER APPROVED "$SHA"; run_bm +if [ "$(merged 1)" -eq 1 ] && [ "$(pinned 1)" = "$SHA" ] && comment 1 | grep -q '@88plug'; then + ok "review_owner_approval: merged on the human approval at the head" +else fail "review_owner_approval: $(show)"; fi + +# 4b. an approval from a user without write access (CONTRIBUTOR) does not count. +fx review_contributor; mkpr 1; pol "$SHA" '.verdict = "review"'; review 1 someone User CONTRIBUTOR APPROVED "$SHA"; run_bm +if [ "$(merges)" -eq 0 ]; then ok "review_contributor: no merge"; else fail "review_contributor: $(show)"; fi + +# 5. an approval at an older SHA does not count for the new head. +fx approval_old_sha; mkpr 1; old=$SHA; mkpr 1 pins.txt 45; pol "$SHA" '.verdict = "review"' +review 1 88plug User OWNER APPROVED "$old"; jqi pulls.json 'map(select(.head.sha != "'"$old"'"))'; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'needs review'; then ok "approval_old_sha: no merge" +else fail "approval_old_sha: $(show)"; fi + +# 5b. a human's later changes-requested review blocks even an auto verdict, and such a PR +# is not regenerated when main moves. +fx changes_requested; mkpr 1; review 1 88plug User OWNER APPROVED "$SHA" 30; review 1 88plug User OWNER CHANGES_REQUESTED "$SHA" 5 +main_commit 10 scripts/other.sh 'echo moved'; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'changes requested' && [ "$(disp freshness.yml)" -eq 0 ]; then ok "changes_requested: blocks, no regeneration" +else fail "changes_requested: $(show) comment=$(comment 1)"; fi + +# 6. a held (action_required) pull_request run newer than a green dispatch run: merges. +fx held_plus_dispatch; mkpr 1; run ci.yml pull_request "$SHA" autobump/g1 completed action_required "$(ago 5)"; run_bm +if [ "$(merged 1)" -eq 1 ]; then ok "held_plus_dispatch: the dispatch run counts, the held run is ignored" +else fail "held_plus_dispatch: $(show)"; fi + +# 7. CI still running at the head: skip. +fx ci_pending; mkpr 1; no_ci "$SHA"; run ci.yml workflow_dispatch "$SHA" autobump/g1 in_progress '' "$(ago 2)"; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'CI is pending'; then ok "ci_pending: skipped" +else fail "ci_pending: $(show) comment=$(comment 1)"; fi + +# 7b. CI failed at the head: skip. +fx ci_failed; mkpr 1; run ci.yml workflow_dispatch "$SHA" autobump/g1 completed failure "$(ago 2)"; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'CI failed'; then ok "ci_failed: skipped" +else fail "ci_failed: $(show)"; fi + +# 8. canary engines mapped but no canary run, or one still running: skip. +fx canary_missing; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]'; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -qi 'canary has not run'; then ok "canary_missing: skipped" +else fail "canary_missing: $(show) comment=$(comment 1)"; fi +fx canary_pending; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]'; canary "$SHA" autobump/g1 '{}' in_progress ''; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -qi 'canary is pending'; then ok "canary_pending: skipped" +else fail "canary_pending: $(show) comment=$(comment 1)"; fi + +# 8b. canary green for every mapped engine: merges, and the comment links the canary run. +fx canary_pass; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu","node"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"kind":"runtime","verdict":"pass","hard":true,"exact":true},"node":{"kind":"node","verdict":"pass","hard":true,"exact":true}}' +crun=$LASTRUN; run_bm +if [ "$(merged 1)" -eq 1 ] && comment 1 | grep -qF "actions/runs/$crun"; then ok "canary_pass: merged" +else fail "canary_pass: $(show)"; fi + +# 8c. a canary result for another SHA, or one lacking a mapped engine, is not green. +fx canary_wrong_sha; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":true}}' +jq '.sha = "0000000"' "$FIX/gh/artifacts/$LASTRUN/canary-result/canary-result.json" > "$T/x" && mv "$T/x" "$FIX/gh/artifacts/$LASTRUN/canary-result/canary-result.json"; run_bm +if [ "$(merges)" -eq 0 ]; then ok "canary_wrong_sha: no merge"; else fail "canary_wrong_sha: $(show)"; fi +fx canary_lacks_engine; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu","ovms-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":true}}'; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'ovms-cpu'; then ok "canary_lacks_engine: no merge"; else fail "canary_lacks_engine: $(show)"; fi +# an engine entry that is not a result object (null) is not a verdict. +fx canary_null_entry; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu","ovms-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":true},"ovms-cpu":null}'; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'ovms-cpu'; then ok "canary_null_entry: no merge"; else fail "canary_null_entry: $(show)"; fi + +# 8d. a held (action_required) pull_request canary run newer than the green dispatched one +# is not an attempt: merges. +fx canary_held_newer; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":true}}' completed success 20 +run canary.yml pull_request "$SHA" autobump/g1 completed action_required "$(ago 5)" Canary; run_bm +if [ "$(merged 1)" -eq 1 ]; then ok "canary_held_newer: the held run is ignored" +else fail "canary_held_newer: $(show) comment=$(comment 1)"; fi +# 8e. an isa-skip re-dispatch reruns only the skipped engine: each engine's newest result at +# the head decides, so the first run's pass for vllm-cpu and the re-dispatch's pass for +# ovms-cpu merge, and the comment links both runs. +fx canary_redispatch; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu","ovms-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":true},"ovms-cpu":{"verdict":"isa-skip","hard":false,"exact":false}}' completed failure 40 +c1=$LASTRUN +canary "$SHA" autobump/g1 '{"ovms-cpu":{"verdict":"pass","hard":true,"exact":true}}' completed success 15 +c2=$LASTRUN; run_bm +if [ "$(merged 1)" -eq 1 ] && comment 1 | grep -qF "actions/runs/$c1" && comment 1 | grep -qF "actions/runs/$c2"; then + ok "canary_redispatch: per-engine newest result, merged" +else fail "canary_redispatch: $(show) comment=$(comment 1)"; fi +# ...but a re-dispatch that fails blocks... +fx canary_redispatch_fail; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu","ovms-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":true},"ovms-cpu":{"verdict":"isa-skip","hard":false,"exact":false}}' completed failure 40 +canary "$SHA" autobump/g1 '{"ovms-cpu":{"verdict":"fail","hard":false,"exact":false}}' completed failure 15; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'ovms-cpu: fail'; then ok "canary_redispatch_fail: no merge" +else fail "canary_redispatch_fail: $(show) comment=$(comment 1)"; fi +# ...a newer result for the same engine supersedes an older pass... +fx canary_newer_fail; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":true}}' completed success 40 +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"fail","hard":false,"exact":false}}' completed failure 15; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'vllm-cpu: fail'; then ok "canary_newer_fail: no merge" +else fail "canary_newer_fail: $(show) comment=$(comment 1)"; fi +# ...a newer completed run without a canary-result artifact blocks: what it found is unknown... +fx canary_newest_no_artifact; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":true}}' completed success 40 +run canary.yml workflow_dispatch "$SHA" autobump/g1 completed cancelled "$(ago 15)" Canary; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'no canary-result artifact'; then ok "canary_newest_no_artifact: no merge" +else fail "canary_newest_no_artifact: $(show) comment=$(comment 1)"; fi +# ...a newest run that is red on an engine the policy does not map still blocks... +fx canary_run_red; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":true},"node":{"verdict":"fail","hard":false,"exact":false}}' completed failure; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'canary run concluded failure'; then ok "canary_run_red: no merge" +else fail "canary_run_red: $(show) comment=$(comment 1)"; fi +# ...and a canary dispatched on another branch at the same SHA is not this PR's run. +fx canary_other_branch; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' +canary "$SHA" autobump/other '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":true}}'; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -qi 'canary has not run'; then ok "canary_other_branch: no merge" +else fail "canary_other_branch: $(show) comment=$(comment 1)"; fi + +# ...nor is a pull_request run at the same SHA whose head is a fork's (a fork can push the +# bot's commit to a branch it names autobump/g1 and open a PR into any base): newer and +# exact, it does not override this PR's own drift, so the PR still needs a review... +fx canary_fork_run; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":false}}' completed success 30 +run canary.yml pull_request "$SHA" autobump/g1 completed success "$(ago 10)" Canary someone/r +mkdir -p "$FIX/gh/artifacts/$LASTRUN/canary-result" +jq -n --arg sha "$SHA" '{sha: $sha, ref: "autobump/g1", engines: {"vllm-cpu": {verdict: "pass", hard: true, exact: true}}}' \ + > "$FIX/gh/artifacts/$LASTRUN/canary-result/canary-result.json"; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'canary-drift'; then ok "canary_fork_run: a fork's run is not this PR's evidence" +else fail "canary_fork_run: $(show) comment=$(comment 1)"; fi +# ...while this PR's own pull_request run (approved, so not held) is. +fx canary_own_pr_run; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' +run canary.yml pull_request "$SHA" autobump/g1 completed success "$(ago 10)" Canary +mkdir -p "$FIX/gh/artifacts/$LASTRUN/canary-result" +jq -n --arg sha "$SHA" '{sha: $sha, ref: "autobump/g1", engines: {"vllm-cpu": {verdict: "pass", hard: true, exact: true}}}' \ + > "$FIX/gh/artifacts/$LASTRUN/canary-result/canary-result.json"; run_bm +if [ "$(merged 1)" -eq 1 ]; then ok "canary_own_pr_run: this PR's own pull_request run counts" +else fail "canary_own_pr_run: $(show) comment=$(comment 1)"; fi + +# 9. canary hard pass with exact false: label canary-drift, no merge (review)... +fx canary_drift; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":false,"detail":"golden differs"}}'; run_bm +if [ "$(merges)" -eq 0 ] && grep -qx '1 canary-drift' "$(G labels)" && comment 1 | grep -q 'canary-drift'; then + ok "canary_drift: labelled canary-drift, no merge" +else fail "canary_drift: $(show) labels=$(cat "$(G labels)")"; fi +# ...and with an OWNER approval at the head it merges (drift is a review condition). +fx canary_drift_approved; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' +canary "$SHA" autobump/g1 '{"vllm-cpu":{"verdict":"pass","hard":true,"exact":false}}' +review 1 88plug User OWNER APPROVED "$SHA"; run_bm +if [ "$(merged 1)" -eq 1 ]; then ok "canary_drift_approved: merged on the human approval" +else fail "canary_drift_approved: $(show)"; fi + +# 10. canary isa-skip (and too-big) is never green, even with an approval. +for v in isa-skip too-big; do + fx "canary_$v"; mkpr 1; pol "$SHA" '.canary = ["vllm-cpu"]' + canary "$SHA" autobump/g1 "{\"vllm-cpu\":{\"verdict\":\"$v\",\"hard\":false,\"exact\":false}}" + review 1 88plug User OWNER APPROVED "$SHA"; run_bm + if [ "$(merges)" -eq 0 ] && comment 1 | grep -q -- "$v" && ! grep -q canary-drift "$(G labels)"; then ok "canary_$v: no merge" + else fail "canary_$v: $(show)"; fi +done + +# 11. main moved under code since the head was tested: regeneration dispatched (freshness.yml +# group=g1); a second PR's group counts separately... +fx base_moved_code; mkpr 1; main_commit 10 scripts/other.sh 'echo moved'; run_bm +if [ "$(merges)" -eq 0 ] && [ "$(disp '^freshness.yml ref=main group=g1$')" -eq 1 ] && comment 1 | grep -q 'main moved'; then + ok "base_moved_code: regeneration dispatched" +else fail "base_moved_code: $(show)"; fi +# ...two of today's dispatched runs name g1 and one names g10: a third g1 regeneration goes out... +fx regen_under_cap; mkpr 1; main_commit 10 scripts/other.sh 'echo moved' +run freshness.yml workflow_dispatch "$(rgit rev-parse origin/main)" main completed success "$(ago 120)" "Freshness group=g1 reason=x" +run freshness.yml workflow_dispatch "$(rgit rev-parse origin/main)" main completed success "$(ago 100)" "Freshness group=g1 reason=x" +run freshness.yml workflow_dispatch "$(rgit rev-parse origin/main)" main completed success "$(ago 90)" "Freshness group=g10 reason=x" +run freshness.yml workflow_dispatch "$(rgit rev-parse origin/main)" main completed success "$(iso $((NOW - 15 * 3600)))" "Freshness group=g1 reason=x" +run_bm +if [ "$(disp '^freshness.yml ref=main group=g1$')" -eq 1 ]; then ok "regen_under_cap: 2 of 3 used today, dispatched" +else fail "regen_under_cap: $(show)"; fi +# ...with three today the cap holds. +fx regen_cap; mkpr 1; main_commit 10 scripts/other.sh 'echo moved' +for m in 120 100 80; do run freshness.yml workflow_dispatch "$(rgit rev-parse origin/main)" main completed success "$(ago $m)" "Freshness group=g1 reason=x"; done +run_bm +if [ "$(disp freshness.yml)" -eq 0 ] && [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'regeneration cap'; then ok "regen_cap: 3 regenerations today, no fourth" +else fail "regen_cap: $(show) comment=$(comment 1)"; fi +# 11b. a freshness run still in flight: wait for it, dispatch nothing. +fx regen_in_flight; mkpr 1; main_commit 10 scripts/other.sh 'echo moved' +run freshness.yml schedule "$(rgit rev-parse origin/main)" main in_progress '' "$(ago 5)" Freshness; run_bm +if [ "$(disp freshness.yml)" -eq 0 ] && [ "$(merges)" -eq 0 ]; then ok "regen_in_flight: no second dispatch" +else fail "regen_in_flight: $(show)"; fi + +# 12. main moved only by release commits (and the ledger): merges without regeneration. +fx base_moved_release; mkpr 1 +main_commit 20 scripts/k3d-gpu $'#!/bin/sh\nVERSION="0.1.80"\n' 88plug-bot 'release: aur-v0.1.80 [skip release]' +main_commit 15 docs/provenance.md $'ledger\n' +run ci.yml push "$(rgit rev-parse origin/main)" main completed success "$(ago 12)"; run_bm +if [ "$(merged 1)" -eq 1 ] && [ "$(disp freshness.yml)" -eq 0 ]; then ok "base_moved_release: merged, no regeneration" +else fail "base_moved_release: $(show)"; fi + +# 12b. main moved only in the ledger since the head was cut: the policy judges the PR's own +# diff (from its merge-base with main), not main's ledger lines read backwards, so an +# auto bump still merges. +fx ledger_moved_auto; mkpr 1; mb=$(rgit rev-parse origin/main) +main_commit 15 docs/provenance.md $'ledger\n' +run ci.yml push "$(rgit rev-parse origin/main)" main completed success "$(ago 12)"; run_bm +if [ "$(merged 1)" -eq 1 ] && [ "$(disp freshness.yml)" -eq 0 ] && grep -qxF "pins policy $mb $SHA" "$(G trace)"; then + ok "ledger_moved_auto: policy from the merge-base, merged" +else fail "ledger_moved_auto: $(show) trace=$(grep '^pins' "$(G trace)" | tr '\n' ';') comment=$(comment 1)"; fi +# 12c. a release commit that conflicts with the head (each release rewrites K3D_GPU_VERSION in +# packaging/install.sh, on the line above the k3d pin) passes base_ok, but GitHub would +# refuse that merge on every pass: regenerate instead, and the next eligible PR merges +# in the same pass. +fx base_conflict_release +main_commit 55 packaging/install.sh $'K3D_GPU_VERSION=0.1.79\nK3D_VERSION=v5.9.0\n' +mkpr 1 packaging/install.sh 50 $'K3D_GPU_VERSION=0.1.79\nK3D_VERSION=v5.9.1\n' +main_commit 20 packaging/install.sh $'K3D_GPU_VERSION=0.1.80\nK3D_VERSION=v5.9.0\n' 88plug-bot \ + 'release: install.sh k3d-gpu 0.1.80 [skip release]' +run ci.yml push "$(rgit rev-parse origin/main)" main completed success "$(ago 12)" +mkpr 2 other.txt 10; run_bm +if [ "$(merged 1)" -eq 0 ] && [ "$(merged 2)" -eq 1 ] && [ "$(disp '^freshness.yml ref=main group=g1$')" -eq 1 ] \ + && comment 1 | grep -q 'conflicts with main' && ! grep -q refused <<<"$OUT"; then + ok "base_conflict_release: no merge attempt, regenerated; the next PR merged" +else fail "base_conflict_release: $(show) comment=$(comment 1)"; fi + +# 13. main's ledger moved (a new SUPPORTED row) without ledger-rules.json: the merged tree's +# rules are stale, so regenerate. +fx rules_stale; mkpr 1 +main_commit 10 EXPERIMENTS.md $'## Hypotheses\n\n| H34 | on the i9-13900T | p | m | SUPPORTED |\n| H34b | on the i5-12500T | q | n | SUPPORTED |\n' +run ci.yml push "$(rgit rev-parse origin/main)" main completed success "$(ago 8)"; run_bm +if [ "$(merges)" -eq 0 ] && [ "$(disp '^freshness.yml ref=main group=g1$')" -eq 1 ] && comment 1 | grep -q 'ledger-rules.json'; then + ok "rules_stale: regeneration dispatched" +else fail "rules_stale: $(show) comment=$(comment 1)"; fi + +# 14. the latest completed ci.yml run on main failed: nothing merges, nothing regenerates. +fx main_ci_red; mkpr 1; mkpr 2; main_commit 10 scripts/other.sh 'echo moved' +run ci.yml push "$(rgit rev-parse origin/main)" main completed failure "$(ago 8)"; run_bm +if [ "$(merges)" -eq 0 ] && [ "$(disp freshness.yml)" -eq 0 ] && comment 1 | grep -q 'circuit breaker'; then ok "main_ci_red: nothing merges" +else fail "main_ci_red: $(show)"; fi +# a later green pull_request run whose head branch is named main (a fork's) does not close it. +fx main_ci_red_fork_main; mkpr 1 +run ci.yml push "$(rgit rev-parse origin/main)" main completed failure "$(ago 8)" +run ci.yml pull_request deadbeef main completed success "$(ago 4)"; run_bm +if [ "$(merges)" -eq 0 ]; then ok "main_ci_red_fork_main: still blocked"; else fail "main_ci_red_fork_main: $(show)"; fi + +# 15. an open ci-main alarm (and every other breaker key): nothing merges. +for k in ci-main ci-missing:freshness.yml release:aur-v0.1.79 red:canary; do + fx "alarm_${k//[:.]/_}"; mkpr 1; alarm "$k"; run_bm + if [ "$(merges)" -eq 0 ] && comment 1 | grep -qF "$k"; then ok "alarm $k: nothing merges" + else fail "alarm $k: $(show)"; fi +done + +# 15b. the alarm issues cannot be read: nothing merges, and the pass fails, so the sweep +# goes red instead of sitting green while it merges nothing. +fx breaker_unreadable; mkpr 1; echo 1 > "$(G issue_list_rc)"; run_bm +if [ "$(merges)" -eq 0 ] && [ "$RC" -eq 1 ] && comment 1 | grep -q 'cannot read ci-alarm issues'; then + ok "breaker_unreadable: no merge, exit 1" +else fail "breaker_unreadable: $(show) comment=$(comment 1)"; fi + +# 16. an alarm whose key is not a breaker key: merges. +fx alarm_other; mkpr 1; alarm 'blind-spot:k3s'; run_bm +if [ "$(merged 1)" -eq 1 ]; then ok "alarm_other: merged"; else fail "alarm_other: $(show)"; fi + +# 17. daily cap reached: the non-urgent PR waits, the urgent one merges. Yesterday's merge +# does not count. +fx daily_cap; botcommit $((15 * 60)) false 7; botcommit 300 false 8; botcommit 200 false 9 +mkpr 1 pins.txt 40; mkpr 2 other.txt 30; pol "$SHA" '.urgent = true' +CAP=2 run_bm +if [ "$(merged 1)" -eq 0 ] && [ "$(merged 2)" -eq 1 ] && comment 1 | grep -q 'daily cap'; then + ok "daily_cap: non-urgent skipped, urgent merged" +else fail "daily_cap: $(show) comment=$(comment 1)"; fi + +# 18. a PR touching .github/: never merged, and its comment says why. +fx touches_github; mkpr 1 .github/workflows/ci.yml; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -qF '.github/'; then ok "touches_github: never" +else fail "touches_github: $(show)"; fi +# a file renamed out of .github/ counts too. +fx renames_github; mkpr 1 ci-moved.yml +echo '[{"filename": "ci-moved.yml", "previous_filename": ".github/workflows/ci.yml", "status": "renamed"}]' > "$FIX/gh/files/1.json"; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -qF '.github/'; then ok "renames_github: never" +else fail "renames_github: $(show)"; fi + +# 19. ledger amendments, dependabot and fork PRs are never bot-merged (and get no comment). +fx not_candidates; mkpr 1; mkpr 2; mkpr 3; mkpr 4 +pr_set 1 '.head.ref = "autobump/amend-H54q" | .labels += [{name: "ledger-amendment"}]' +pr_set 2 '.labels += [{name: "ledger-amendment"}]' +pr_set 3 '.user = {login: "dependabot[bot]", type: "Bot"}' +pr_set 4 '.head.repo.full_name = "someone/r"' +mkpr 5; pr_set 5 '.labels = []' +mkpr 6; pr_set 6 '.head.repo = null' +run_bm +if [ "$(merges)" -eq 0 ] && [ "$(ncomments 1)$(ncomments 2)$(ncomments 3)$(ncomments 4)$(ncomments 5)$(ncomments 6)" = 000000 ]; then + ok "not_candidates: amendment, dependabot, fork, unlabelled: untouched" +else fail "not_candidates: $(show)"; fi + +# 20. two eligible PRs: one merge per pass, the urgent one first though it is newer... +fx two_urgent_first; mkpr 1 pins.txt 40; mkpr 2 other.txt 30; pol "$SHA" '.urgent = true'; run_bm +if [ "$(merges)" -eq 1 ] && [ "$(merged 2)" -eq 1 ] && comment 1 | grep -q 'one merge per pass'; then + ok "two_urgent_first: only the urgent PR merged" +else fail "two_urgent_first: $(show)"; fi +# ...else the oldest. +fx two_oldest_first; mkpr 2 other.txt 30; mkpr 1 pins.txt 40; run_bm +if [ "$(merges)" -eq 1 ] && [ "$(merged 1)" -eq 1 ]; then ok "two_oldest_first: the older PR merged" +else fail "two_oldest_first: $(show)"; fi + +# 21. a refused merge: GitHub's reason printed, the pass stops (no ci.yml dispatch, no +# release), the comment says so. +fx merge_refused; mkpr 1; echo 1 > "$(G merge_rc)"; run_bm +if [ "$(merges)" -eq 1 ] && [ "$(disp ci.yml)" -eq 0 ] && grep -q 'Base branch was modified' <<<"$OUT" \ + && comment 1 | grep -q 'refused'; then ok "merge_refused: reason shown, pass stopped" +else fail "merge_refused: $(show)"; fi + +# 22. a gh without --match-head-commit never merges. +fx old_gh; mkpr 1; echo ' --squash Squash the commits' > "$(G merge_help)"; run_bm +if [ "$RC" -eq 2 ] && [ "$(merges)" -eq 0 ]; then ok "old_gh: exit 2"; else fail "old_gh: $(show)"; fi + +# 23. a policy without a canary list fails closed; a failing policy blocks. +fx policy_no_engines; mkpr 1; pol "$SHA" 'del(.canary)'; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'no canary list'; then ok "policy_no_engines: blocked" +else fail "policy_no_engines: $(show)"; fi +fx policy_fails; mkpr 1; rm "$FIX/policy/$SHA.json"; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q 'policy'; then ok "policy_fails: blocked" +else fail "policy_fails: $(show)"; fi + +# 24. one comment per PR, edited in place when the reason changes, untouched when not. +fx comment_in_place; mkpr 1; no_ci "$SHA"; run ci.yml workflow_dispatch "$SHA" autobump/g1 in_progress '' "$(ago 2)"; run_bm; run_bm +patches1=$(grep -c comment-patch "$(G calls)") +jqi runs.json --argjson id "$LASTRUN" 'map(if .databaseId == $id then .status = "completed" | .conclusion = "failure" else . end)'; run_bm +if [ "$(ncomments 1)" -eq 1 ] && [ "$patches1" -eq 0 ] && [ "$(grep -c comment-patch "$(G calls)")" -eq 1 ] \ + && comment 1 | grep -q 'CI failed'; then ok "comment_in_place: one comment, edited only on change" +else fail "comment_in_place: comments=$(ncomments 1) patches1=$patches1 patches=$(grep -c comment-patch "$(G calls)")"; fi + +# 25. a draft PR is not merged. +fx draft; mkpr 1; pr_set 1 '.draft = true'; run_bm +if [ "$(merges)" -eq 0 ] && comment 1 | grep -q draft; then ok "draft: no merge"; else fail "draft: $(show)"; fi + +# --- release batching --------------------------------------------------------- +REL='^auto-release.yml ref=main mode=changed$' +# R1. an urgent bot merge since the last tag: release at once, though the queue is not empty +# (PR 1 is auto-eligible with green CI, held by the daily cap) and the tag is 1 h old. +fx rel_urgent; botcommit 20 true; mkpr 1; CAP=1 run_bm +if [ "$(merges)" -eq 0 ] && [ "$(disp "$REL")" -eq 1 ]; then ok "rel_urgent: dispatched" +else fail "rel_urgent: $(show)"; fi +# R2. queue empty: dispatch. +fx rel_queue_empty; botcommit 20 false; run_bm +if [ "$(disp "$REL")" -eq 1 ]; then ok "rel_queue_empty: dispatched"; else fail "rel_queue_empty: $(show)"; fi +# R3. queue non-empty, last tag under 6 h old: wait. +fx rel_queue_wait; botcommit 20 false; mkpr 1; CAP=1 run_bm +if [ "$(disp auto-release)" -eq 0 ] && grep -q 'release: wait' <<<"$OUT"; then ok "rel_queue_wait: waits" +else fail "rel_queue_wait: $(show)"; fi +# R3b. queue non-empty but the last tag is 7 h old: dispatch. +fx rel_six_hours 420; botcommit 20 false; mkpr 1; CAP=1 run_bm +if [ "$(merges)" -eq 0 ] && [ "$(disp "$REL")" -eq 1 ]; then ok "rel_six_hours: dispatched"; else fail "rel_six_hours: $(show)"; fi +# R4. main HEAD's ci.yml run has not finished green: wait. +fx rel_main_ci; botcommit 20 false +run ci.yml workflow_dispatch "$(rgit rev-parse origin/main)" main in_progress '' "$(ago 1)"; run_bm +if [ "$(disp auto-release)" -eq 0 ]; then ok "rel_main_ci: waits"; else fail "rel_main_ci: $(show)"; fi +# R5. a release run in progress: wait. +for wf in auto-release.yml build-and-push.yml; do + fx "rel_in_progress_${wf%%.*}"; botcommit 20 false + run "$wf" workflow_dispatch "$(rgit rev-parse origin/main)" main in_progress '' "$(ago 1)"; run_bm + if [ "$(disp auto-release)" -eq 0 ]; then ok "rel_in_progress ($wf): waits"; else fail "rel_in_progress ($wf): $(show)"; fi +done +# R6. no bot merge above the last tag: nothing to release. +fx rel_nothing; run_bm +if [ "$(disp auto-release)" -eq 0 ]; then ok "rel_nothing: no dispatch"; else fail "rel_nothing: $(show)"; fi +# R7. a release run already succeeded after the newest bot merge (it found nothing to ship): +# do not dispatch again every pass. +fx rel_already; botcommit 20 false +run auto-release.yml workflow_dispatch "$(rgit rev-parse origin/main)" main completed success "$(ago 10)"; run_bm +if [ "$(disp auto-release)" -eq 0 ]; then ok "rel_already: no repeat dispatch"; else fail "rel_already: $(show)"; fi +# R8. a merge in this pass leaves main's new head without CI: the release waits for it. +fx rel_after_merge; mkpr 1; run_bm +if [ "$(merged 1)" -eq 1 ] && [ "$(disp auto-release)" -eq 0 ]; then ok "rel_after_merge: waits for main CI" +else fail "rel_after_merge: $(show)"; fi + +echo "${PASS} passed, ${FAILS} failed" +[ "${FAILS}" -eq 0 ] diff --git a/tests/ci_wiring_test.sh b/tests/ci_wiring_test.sh index 6426f9b7..87b4b2b8 100755 --- a/tests/ci_wiring_test.sh +++ b/tests/ci_wiring_test.sh @@ -77,34 +77,72 @@ if grep -qF 'git status --porcelain -- .github/workflows/' "${BP}"; then else fail "repin-images has no 'git status --porcelain -- .github/workflows/' check" fi -# create-pull-request resets an existing digest branch onto main and force-pushes -# it; GitHub refuses that GITHUB_TOKEN push when the branch's workflow files differ -# from main's, though the digest commit touches none. A step before the action, -# under the same if:, deletes such a branch so the action cuts a fresh one off main. -# Its script then runs against a local origin over file:// (no network). +# Every create-pull-request step (a GITHUB_TOKEN PR) sits between two steps under +# its if:. Before it, hack/autobump-branch-reset deletes the branch when its +# workflow files differ from main's: create-pull-request resets the branch onto +# main and force-pushes it, and GitHub refuses that GITHUB_TOKEN push (no +# `workflows` permission) though the bump commit touches no workflow. After it, +# hack/bot-ci-dispatch dispatches ci.yml on the branch: the PR's own pull_request +# runs wait for approval, a workflow_dispatch run does not. The job grants +# actions: write for the dispatch. Each reset step's script then runs against a +# local origin over file:// (no network), for every such branch. T=$(mktemp -d "${TMPDIR:-/tmp}/ci_wiring.XXXXXX") trap 'rm -rf "$T"' EXIT -if python3 - "${BP}" "${T}/drop.sh" <<'PY' -import sys, yaml -steps = yaml.safe_load(open(sys.argv[1]))["jobs"]["repin-images"]["steps"] -cpr = next(i for i, s in enumerate(steps) - if str(s.get("uses", "")).startswith("peter-evans/create-pull-request@")) -branch = steps[cpr]["with"]["branch"] -drop = [s for s in steps[:cpr] if f"git push origin --delete {branch}" in s.get("run", "")] -if len(drop) != 1 or drop[0].get("if") != steps[cpr].get("if"): +if python3 - "${ROOT}" "${T}" > "${T}/cases" 2> "${T}/cpr.err" <<'PY' +import glob, os, sys, yaml +root, out = sys.argv[1], sys.argv[2] +errs, n = [], 0 +for path in sorted(glob.glob(os.path.join(root, ".github/workflows/*.yml"))): + jobs = (yaml.safe_load(open(path)) or {}).get("jobs") or {} + for jid, job in jobs.items(): + steps = job.get("steps") or [] + for i, s in enumerate(steps): + if not str(s.get("uses", "")).startswith("peter-evans/create-pull-request@"): + continue + n += 1 + where = f"{os.path.basename(path)} {jid}" + cond = s.get("if") + reset = [p for p in steps[:i] if "hack/autobump-branch-reset" in p.get("run", "") and p.get("if") == cond] + disp = [p for p in steps[i + 1:] if "hack/bot-ci-dispatch --branch" in p.get("run", "") and p.get("if") == cond] + if len(reset) != 1: + errs.append(f"{where}: no single step before create-pull-request, under its if:, that runs hack/autobump-branch-reset") + if len(disp) != 1 or "GH_TOKEN" not in (disp[0].get("env") or {}): + errs.append(f"{where}: no single step after create-pull-request, under its if:, that runs hack/bot-ci-dispatch --branch with GH_TOKEN") + if (job.get("permissions") or {}).get("actions") != "write": + errs.append(f"{where}: the job lacks permissions actions: write (bot-ci-dispatch dispatches)") + if len(reset) != 1: + continue + # A matrix branch (autobump/${{ matrix.component }}) runs as component "comp". + sub = lambda v: str(v).replace("${{ matrix.component }}", "comp") + branch = sub(s["with"]["branch"]) + env = {k: sub(v) for k, v in {**(job.get("env") or {}), **(reset[0].get("env") or {})}.items()} + script = sub(reset[0]["run"]) + if "${{" in branch + script + "".join(env.values()): + errs.append(f"{where}: the reset step uses an expression this test cannot evaluate") + continue + d = os.path.join(out, f"case{n}") + os.makedirs(d) + open(os.path.join(d, "run.sh"), "w").write(script) + open(os.path.join(d, "env"), "w").write("".join(f"export {k}='{v}'\n" for k, v in env.items())) + print(f"{where}\t{branch}\t{d}") +if errs or n == 0: + sys.stderr.write("\n".join(errs or ["no create-pull-request step found"]) + "\n") sys.exit(1) -open(sys.argv[2], "w").write(drop[0]["run"]) PY then - ok "repin-images deletes a stale digest branch before create-pull-request, under its if:" + ok "every create-pull-request step: autobump-branch-reset before, bot-ci-dispatch after, under its if:, actions: write" else - fail "repin-images has no step before create-pull-request, under its if:, that runs 'git push origin --delete autobump/image-digests'" + fail "create-pull-request wiring: $(tr '\n' ';' < "${T}/cpr.err")" fi -# $1: absent (no digest branch), same (main moved, no workflow change), differ -# (main changed a workflow after the branch was cut). Prints the branch's state -# on origin after the step's script ran in a depth-1 checkout of main. +for b in autobump/image-digests autobump/convert-pins autobump/comp; do + if cut -f2 "${T}/cases" | grep -qxF "$b"; then ok "a reset step covers ${b}"; else fail "no reset step covers ${b}"; fi +done +# $1: absent (no branch), same (main moved, no workflow change), differ (main +# changed a workflow after the branch was cut); $2 the branch; $3 the case dir. +# Prints the branch's state on origin after the step's script ran in a depth-1 +# checkout of main that carries hack/autobump-branch-reset. drop_case() { - local d="${T}/$1" b=autobump/image-digests + local d="$3/$1" b=$2 ( set -e export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 @@ -112,16 +150,17 @@ drop_case() { git init -q --bare -b main "$d/origin.git" git init -q -b main "$d/seed" cd "$d/seed" - mkdir -p .github/workflows + mkdir -p .github/workflows hack echo v1 > .github/workflows/ci.yml echo 1 > images.lock - git add .github/workflows/ci.yml images.lock + cp "${ROOT}/hack/autobump-branch-reset" hack/ + git add .github/workflows/ci.yml images.lock hack/autobump-branch-reset git commit -qm base git push -q "$d/origin.git" main if [ "$1" != absent ]; then git switch -qc "$b" echo 2 > images.lock - git commit -qam digests + git commit -qam bump git push -q "$d/origin.git" "$b" git switch -q main fi @@ -135,17 +174,27 @@ drop_case() { git push -q "$d/origin.git" main git clone -q --depth=1 --branch main "file://$d/origin.git" "$d/work" cd "$d/work" - bash --noprofile --norc -eo pipefail "${T}/drop.sh" >"$d/log" 2>&1 + # shellcheck disable=SC1091 + . "$3/env" + bash --noprofile --norc -eo pipefail "$3/run.sh" >"$d/log" 2>&1 ) || { echo "script-failed"; return; } if [ -n "$(git ls-remote --heads "$d/origin.git" "refs/heads/$b")" ]; then echo present; else echo gone; fi } -for c in absent:gone same:present differ:gone; do - got=$(drop_case "${c%%:*}") - if [ "${got}" = "${c#*:}" ]; then - ok "stale-branch step, ${c%%:*}: digest branch ${got}" - else - fail "stale-branch step, ${c%%:*}: digest branch ${got}, want ${c#*:}" - fi +while IFS=$'\t' read -r where b dir; do + for c in absent:gone same:present differ:gone; do + got=$(drop_case "${c%%:*}" "$b" "$dir") + if [ "${got}" = "${c#*:}" ]; then + ok "${where} reset step, ${c%%:*}: ${b} ${got}" + else + fail "${where} reset step, ${c%%:*}: ${b} ${got}, want ${c#*:}" + fi + done +done < "${T}/cases" +# autobump-branch-reset deletes only autobump/ branches (usage error, exit 2). +for b in main claude/x autobump/ ''; do + (cd "$T" && bash "${ROOT}/hack/autobump-branch-reset" "$b" >/dev/null 2>&1); rc=$? + if [ "$rc" -eq 2 ]; then ok "autobump-branch-reset refuses '${b}'" + else fail "autobump-branch-reset '${b}': exit ${rc}, want 2"; fi done # The release job builds the 24.04 variant from the Dockerfile's pin, which the # nightly moves; a literal pin in the workflow would never move. @@ -156,5 +205,143 @@ else fail "release does not read UBUNTU_2404_IMAGE from the Dockerfile for the 24.04 variant" fi +# ─── CI on bot PRs and alarm surfacing ─────────────────────────────────────── +WF="${ROOT}/.github/workflows" +# ci.yml runs on workflow_dispatch: hack/bot-ci-dispatch runs it on bot PR branches. +# (PyYAML reads the bare key `on` as True.) +if python3 - "${WF}/ci.yml" <<'PY' +import sys, yaml +wf = yaml.safe_load(open(sys.argv[1])) +on = wf.get("on", wf.get(True)) +sys.exit(0 if isinstance(on, dict) and "workflow_dispatch" in on else 1) +PY +then ok "ci.yml has a workflow_dispatch trigger"; else fail "ci.yml has no workflow_dispatch trigger"; fi +# The dependabot auto-merge workflow ran `gh pr merge --auto` on an unprotected main +# (no required checks, so nothing ever merged) and held a run on every bot PR. +if [ ! -e "${WF}/dependabot-auto-merge.yml" ]; then ok "dependabot-auto-merge.yml is gone" +else fail "dependabot-auto-merge.yml still exists"; fi +# bot-sweep.yml: hourly off the hour, never two at once, job ci-fill runs bot-ci-dispatch. +if python3 - "${WF}/bot-sweep.yml" <<'PY' +import sys, yaml +wf = yaml.safe_load(open(sys.argv[1])) +on = wf.get("on", wf.get(True)) or {} +crons = [c["cron"] for c in on.get("schedule") or []] +conc = wf.get("concurrency") or {} +job = (wf.get("jobs") or {}).get("ci-fill") or {} +perms = job.get("permissions") or {} +runs = " ".join(s.get("run", "") for s in job.get("steps") or []) +checks = [ + wf.get("name") == "Bot sweep", + crons == ["13 * * * *"], "workflow_dispatch" in on, + conc.get("group") == "bot-sweep", conc.get("cancel-in-progress") is False, + isinstance(job.get("timeout-minutes"), int), + perms == {"actions": "write", "contents": "read", "pull-requests": "read"}, + "hack/bot-ci-dispatch" in runs, + all("timeout-minutes" in j and "permissions" in j for j in wf["jobs"].values()), +] +sys.exit(0 if all(checks) else 1) +PY +then ok "bot-sweep.yml: 'Bot sweep', cron 13 * * * *, group bot-sweep, job ci-fill runs hack/bot-ci-dispatch" +else fail "bot-sweep.yml is missing or its name, schedule, concurrency or ci-fill job is wrong"; fi + +# ci.yml's last job, alarms: never fails, reads issues and actions only, and warns +# on every open ci-alarm issue and on a missing or stale Watchdog. Its script runs +# against a fake gh. +if python3 - "${WF}/ci.yml" "${T}/alarms.sh" <<'PY' +import sys, yaml +jobs = yaml.safe_load(open(sys.argv[1]))["jobs"] +job = jobs.get("alarms") or {} +ok = (list(jobs)[-1] == "alarms" and job.get("continue-on-error") is True + and isinstance(job.get("timeout-minutes"), int) + and job.get("permissions") == {"issues": "read", "actions": "read"} + and len(job.get("steps") or []) == 1) +if ok: + open(sys.argv[2], "w").write(job["steps"][0]["run"]) +sys.exit(0 if ok else 1) +PY +then ok "ci.yml ends with job alarms: continue-on-error, timeout, issues+actions read" +else fail "ci.yml's last job is not alarms with continue-on-error, timeout-minutes, issues/actions read and one step"; fi +mkdir -p "${T}/abin" +cat > "${T}/abin/gh" <<'STUB' +#!/usr/bin/env bash +A=$AFIX +case "$1 $2" in + "issue list") f=issues;; "workflow list") f=workflows;; "run list") f=runs;; *) exit 0;; +esac +[ -f "$A/$f.fail" ] && { echo "HTTP 502" >&2; exit 1; } +q=; while [ $# -gt 0 ]; do case $1 in -q|--jq) q=$2; shift 2;; *) shift;; esac; done +if [ -n "$q" ]; then jq -r "$q" "$A/$f.json"; else cat "$A/$f.json"; fi +STUB +chmod +x "${T}/abin/gh" +ALARM7='{"number":7,"title":"[ci-alarm] nightly: Auto-update red","url":"https://github.com/o/r/issues/7","labels":[{"name":"ci-alarm"}]}' +ALARM8='{"number":8,"title":"[ci-alarm] release: 100% missing","url":"https://github.com/o/r/issues/8","labels":[{"name":"ci-alarm"},{"name":"alarm-ack"}]}' +WATCHDOG='[{"name":"Watchdog","path":".github/workflows/watchdog.yml","state":"active"}]' +ago() { date -u -d "-$1 hours" +%Y-%m-%dT%H:%M:%SZ; } +# alarm_case NAME ISSUES WORKFLOWS RUNS [fail...]: runs the alarms script; sets AOUT, ARC. +alarm_case() { + AFIX="${T}/alarm-$1"; export AFIX; mkdir -p "$AFIX" + echo "$2" > "$AFIX/issues.json"; echo "$3" > "$AFIX/workflows.json"; echo "$4" > "$AFIX/runs.json" + local f; for f in "${@:5}"; do touch "$AFIX/$f.fail"; done + AOUT=$(PATH="${T}/abin:$PATH" bash --noprofile --norc -eo pipefail "${T}/alarms.sh" 2>&1); ARC=$? +} +alarm_case two "[$ALARM7,$ALARM8]" "$WATCHDOG" "[{\"updatedAt\":\"$(ago 1)\"}]" +if [ "$ARC" -eq 0 ] && [ "$(grep -c '^::warning::' <<<"$AOUT")" -eq 2 ] \ + && grep '^::warning::.*#7' <<<"$AOUT" | grep -F 'https://github.com/o/r/issues/7' | grep -qF 'gh issue edit 7 --add-label alarm-ack' \ + && grep '^::warning::.*#8' <<<"$AOUT" | grep -F 'https://github.com/o/r/issues/8' | grep -qF '100%25 missing' \ + && ! grep -q 'Watchdog' <<<"$AOUT"; then + ok "alarms: one warning per open alarm with its URL and ack state; a fresh Watchdog is quiet" +else fail "alarms two: rc=$ARC out=$(tr '\n' ';' <<<"$AOUT")"; fi +alarm_case none '[]' '[]' '[]' +if [ "$ARC" -eq 0 ] && [ "$(grep -c '^::warning::' <<<"$AOUT")" -eq 1 ] && grep -q '^::warning::.*Watchdog' <<<"$AOUT"; then + ok "alarms: a missing Watchdog workflow warns" +else fail "alarms none: rc=$ARC out=$(tr '\n' ';' <<<"$AOUT")"; fi +alarm_case stale '[]' "$WATCHDOG" "[{\"updatedAt\":\"$(ago 5)\"}]" +if [ "$ARC" -eq 0 ] && grep -q '^::warning::.*Watchdog.*3 h' <<<"$AOUT"; then ok "alarms: a Watchdog success over 3 h old warns" +else fail "alarms stale: rc=$ARC out=$(tr '\n' ';' <<<"$AOUT")"; fi +alarm_case never '[]' "$WATCHDOG" '[]' +if [ "$ARC" -eq 0 ] && grep -q '^::warning::.*Watchdog' <<<"$AOUT"; then ok "alarms: a Watchdog with no success warns" +else fail "alarms never: rc=$ARC out=$(tr '\n' ';' <<<"$AOUT")"; fi +alarm_case disabled '[]' '[{"name":"Watchdog","path":".github/workflows/watchdog.yml","state":"disabled_manually"}]' "[{\"updatedAt\":\"$(ago 1)\"}]" +if [ "$ARC" -eq 0 ] && grep -q '^::warning::.*Watchdog.*disabled_manually' <<<"$AOUT"; then ok "alarms: a disabled Watchdog warns" +else fail "alarms disabled: rc=$ARC out=$(tr '\n' ';' <<<"$AOUT")"; fi +alarm_case gh_down '[]' '[]' '[]' issues workflows runs +if [ "$ARC" -eq 0 ] && [ "$(grep -c '^::warning::' <<<"$AOUT")" -eq 2 ]; then ok "alarms: gh failing warns and still exits 0" +else fail "alarms gh_down: rc=$ARC out=$(tr '\n' ';' <<<"$AOUT")"; fi + +# bot-sweep.yml job merge runs hack/bot-merge after ci-fill. Its token can merge, +# so it checks out main only (never a PR's code), and the script never checks out, +# switches to or resets onto another ref. +BS="${ROOT}/.github/workflows/bot-sweep.yml" +if python3 - "${BS}" 2>/dev/null <<'PY' +import sys, yaml +jobs = yaml.safe_load(open(sys.argv[1]))["jobs"] +needs = jobs["merge"].get("needs", []) +sys.exit(0 if "ci-fill" in jobs and "ci-fill" in ([needs] if isinstance(needs, str) else needs) else 1) +PY +then + ok "bot-sweep.yml job merge needs ci-fill" +else + fail "bot-sweep.yml has no job merge that needs ci-fill" +fi +if python3 - "${BS}" 2>/dev/null <<'PY' +import sys, yaml +steps = yaml.safe_load(open(sys.argv[1]))["jobs"]["merge"]["steps"] +co = [s for s in steps if str(s.get("uses", "")).startswith("actions/checkout@")] +text = yaml.safe_dump(steps) +sys.exit(0 if co and all((s.get("with") or {}).get("ref") == "main" for s in co) + and not any(k in text for k in ("github.head_ref", "github.event", "refs/pull/", "pull_request")) + and any("hack/bot-merge" in s.get("run", "") for s in steps) else 1) +PY +then + ok "bot-sweep.yml job merge checks out main only and runs hack/bot-merge" +else + fail "bot-sweep.yml job merge checks out a ref other than main, reads a PR ref, or does not run hack/bot-merge" +fi +if [ -f "${ROOT}/hack/bot-merge" ] && ! grep -qE 'git (checkout|switch|worktree|reset)|gh pr checkout' "${ROOT}/hack/bot-merge"; then + ok "hack/bot-merge never checks out a ref" +else + fail "hack/bot-merge is missing or runs git checkout/switch/worktree/reset or gh pr checkout" +fi + echo "${PASS} passed, ${FAILS} failed" [ "${FAILS}" -eq 0 ] diff --git a/tests/gate_merge_test.sh b/tests/gate_merge_test.sh index b43faa27..762e8af3 100755 --- a/tests/gate_merge_test.sh +++ b/tests/gate_merge_test.sh @@ -7,8 +7,13 @@ # files changed on main and on the PR since the merge base). The stub gh answers # both `gh pr view --json statusCheckRollup` and `gh pr checks --json bucket` # from the same rollup (bucket mapping as in cli/cli pkg/cmd/pr/checks/aggregate.go). -# Cases 18-19 run the gate's sync() with real git against a bare origin (gh and +# `gh run list -e/-b` filter runs that carry event/headBranch (bare runs match +# any). Alarms: `gh issue list` answers the next line of alarms_seq (default +# none; issues_off makes it fail), and the PR's changed files are pr_paths +# (pr_paths_fail makes that fail). +# Cases 18-19 and 24-25 run the gate with real git against a bare origin (gh and # sleep still stubbed): the head the stub gh reports is the origin's feat ref. +# The ci_status and rules_stale cases source hack/gate_lib.sh directly. # # Usage: tests/gate_merge_test.sh @@ -65,12 +70,25 @@ case "$1 $2" in if [ -n "$q" ]; then jq -r "$q" <<<"$obj"; else jq "{$fields}" <<<"$obj" 2>/dev/null || echo "$obj"; fi exit 0;; "run list") - shift 2; sha= q= wf= - while [ $# -gt 0 ]; do case $1 in -c|--commit) sha=$2; shift 2;; -w|--workflow) wf=$2; shift 2;; -q) q=$2; shift 2;; *) shift;; esac; done - echo "run-list $sha ${wf:-all}" >> "$F/calls" + shift 2; sha= q= wf= ev= rb= + while [ $# -gt 0 ]; do case $1 in -c|--commit) sha=$2; shift 2;; -w|--workflow) wf=$2; shift 2;; -e|--event) ev=$2; shift 2;; -b|--branch) rb=$2; shift 2;; -q) q=$2; shift 2;; *) shift;; esac; done + echo "run-list $sha ${wf:-all} ${ev:-any}" >> "$F/calls" if [ -n "$wf" ] || [ ! -f "$F/allruns.json" ]; then out=$(cat "$F/runs.json"); else out=$(cat "$F/allruns.json"); fi + # -e and -b filter like the API on runs that carry event / headBranch; a bare run matches any. + out=$(jq --arg e "$ev" --arg b "$rb" \ + 'map(select(($e == "" or (.event // $e) == $e) and ($b == "" or (.headBranch // $b) == $b)))' <<<"$out") if [ -n "$q" ]; then jq -r "$q" <<<"$out"; else echo "$out"; fi exit 0;; + "issue list") + # open ci-alarm issues: the next line of alarms_seq (one JSON array per call), default none + [ -f "$F/issues_off" ] && { echo "the 'o/r' repository has disabled issues" >&2; exit 1; } + if [ -f "$F/alarms_seq" ]; then pop alarms_seq; else echo '[]'; fi + exit 0;; + "api --paginate") + case $* in */pulls/*/files*) + [ -f "$F/pr_paths_fail" ] && { echo "HTTP 502: Bad Gateway" >&2; exit 1; } + cat "$F/pr_paths"; exit 0;; esac + exit 0;; esac exit 0 STUB @@ -130,6 +148,7 @@ fixture() { echo 0 > "$FIX/ancestor"; echo scripts/k3d-gpu > "$FIX/base_files"; echo scripts/k3d-gpu > "$FIX/pr_files" printf '%s\n' ${6:-MERGEABLE} > "$FIX/mergeable"; echo "${7:-$GOOD_HELP}" > "$FIX/merge_help" echo OPEN > "$FIX/state"; : > "$FIX/calls"; : > "$FIX/git"; : > "$FIX/trace" + echo scripts/k3d-gpu > "$FIX/pr_paths" } run_gate() { OUT=$(PATH="$T/bin:$PATH" GATE_REPO="$FIX" GATE_WT_ROOT="$FIX/wt" GATE_POLLS=3 \ @@ -313,8 +332,12 @@ realgit_fixture() { printf '#!/bin/sh\nexit 0\n' > "$R/tests/claims_check.sh" printf '## Hypotheses\n\n| H34 | on the i9-13900T | p | m | SUPPORTED |\n' > "$R/EXPERIMENTS.md" echo base > "$R/docs/provenance.md" + mkdir -p "$R/scripts" "$R/packaging/aur" "$R/Formula" + printf '#!/usr/bin/env bash\nset -euo pipefail\n\nVERSION="0.1.0" # kept in lockstep\nCLUSTER=gpu\n' > "$R/scripts/k3d-gpu" + echo 'pkgver=0.1.0' > "$R/packaging/aur/PKGBUILD"; echo 'pkgver = 0.1.0' > "$R/packaging/aur/.SRCINFO" + echo 'VERSION=0.1.0' > "$R/packaging/install.sh"; echo 'version "0.1.0"' > "$R/Formula/k3d-gpu.rb" "$REALPY" "$R/console/model_opt.py" ledger-rules --ledger "$R/EXPERIMENTS.md" --out "$R/console/ledger-rules.json" - repo_git add console tests docs EXPERIMENTS.md && repo_git commit -q -m base \ + repo_git add console tests docs scripts packaging Formula EXPERIMENTS.md && repo_git commit -q -m base \ && repo_git remote add origin "$FIX/origin.git" && repo_git push -q origin main && repo_git checkout -q -b feat echo "$FIX/origin.git" > "$FIX/origin_git" } @@ -373,5 +396,213 @@ if [ "$RC" -eq 1 ] && ! grep -q 'synced' <<<"$OUT" && [ "$(syncs)" -eq 1 ] && [ ok "sync_merge_commit_fails: exit 1 at once, no push" else fail "sync_merge_commit_fails: rc=$RC syncs=$(syncs) merges=$(merges) origin=$before->${after:-gone} out=$(tr '\n' ';' <<<"$OUT")"; fi +# ─── CI on bot PRs: a ci.yml run dispatched on the PR branch is CI evidence ──── +# A PR that GITHUB_TOKEN opened gets pull_request runs held for approval +# (conclusion action_required); hack/bot-ci-dispatch runs ci.yml on the branch +# by workflow_dispatch instead. The stub PR's branch is feat. +# run_json EVENT BRANCH STATUS CONCLUSION CREATED: one ci.yml run for the head SHA. +run_json() { printf '{"workflowName":"CI","event":"%s","headBranch":"%s","status":"%s","conclusion":"%s","createdAt":"%s"}' "$@"; } +HELD_PR=$(run_json pull_request feat completed action_required 2026-10-20T01:00:00Z) +DISP_OK=$(run_json workflow_dispatch feat completed success 2026-10-20T01:05:00Z) +HELD_CI_CHECK=$(concl ACTION_REQUIRED) +HELD_DEP_CHECK='{"__typename":"CheckRun","name":"automerge","workflowName":"Auto-merge Dependabot PRs","status":"COMPLETED","conclusion":"ACTION_REQUIRED","startedAt":"2026-10-20T01:00:00Z"}' +HELD_DEP_RUN='{"workflowName":"Auto-merge Dependabot PRs","event":"pull_request","headBranch":"feat","status":"completed","conclusion":"action_required","createdAt":"2026-10-20T01:00:00Z"}' + +# 20. no pull_request run at all, a green ci.yml run dispatched on the PR branch: merge. +fixture dispatch_green "[$SUCCESS]" "[$DISP_OK]" abc123; run_gate +if [ "$RC" -eq 0 ] && [ "$(merges)" -eq 1 ] && grep -q -- '--match-head-commit abc123' "$FIX/merges"; then + ok "dispatch_green: a dispatched run on the branch counts as CI" +else fail "dispatch_green: rc=$RC merges=$(merges) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 21. held pull_request runs (CI and the dependabot workflow, in the rollup and the +# run list) plus a green dispatched run: the held entries do not block; merge. +fixture held_plus_dispatch "[$SUCCESS,$HELD_CI_CHECK,$HELD_DEP_CHECK]" "[$HELD_PR,$DISP_OK]" abc123 +echo "[$HELD_PR,$HELD_DEP_RUN,$DISP_OK]" > "$FIX/allruns.json"; run_gate +if [ "$RC" -eq 0 ] && [ "$(merges)" -eq 1 ]; then ok "held_plus_dispatch: held runs ignored once a dispatched run is green" +else fail "held_plus_dispatch: rc=$RC merges=$(merges) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 22. held alone still fails at once, as before: a held CheckRun in the rollup, or +# (no check runs for a held run) only the held ci.yml run. +fixture held_alone_rollup "[$SUCCESS,$HELD_CI_CHECK]" "[$HELD_PR]" abc123; run_gate +if [ "$RC" -eq 1 ] && [ "$(merges)" -eq 0 ] && grep -q FAIL <<<"$OUT"; then ok "held_alone_rollup: exit 1" +else fail "held_alone_rollup: rc=$RC merges=$(merges)"; fi +fixture held_alone_run "[$SUCCESS]" "[$HELD_PR]" abc123; run_gate +if [ "$RC" -eq 1 ] && [ "$(merges)" -eq 0 ] && grep -q FAIL <<<"$OUT"; then ok "held_alone_run: exit 1" +else fail "held_alone_run: rc=$RC merges=$(merges)"; fi + +# 23. a green run dispatched on another branch is not this PR's CI: with nothing +# else no merge (wait), and it does not excuse a held run (exit 1). +fixture dispatch_other_branch "[$SUCCESS]" "[$(run_json workflow_dispatch other completed success 2026-10-20T01:05:00Z)]" abc123; run_gate +if [ "$(merges)" -eq 0 ] && grep -q 'GAVE UP' <<<"$OUT"; then ok "dispatch_other_branch: not accepted, waits" +else fail "dispatch_other_branch: rc=$RC merges=$(merges)"; fi +fixture dispatch_other_branch_held "[$SUCCESS,$HELD_CI_CHECK]" "[$HELD_PR,$(run_json workflow_dispatch other completed success 2026-10-20T01:05:00Z)]" abc123; run_gate +if [ "$RC" -eq 1 ] && [ "$(merges)" -eq 0 ]; then ok "dispatch_other_branch_held: held run still fails" +else fail "dispatch_other_branch_held: rc=$RC merges=$(merges)"; fi + +# 23b. the latest dispatched run failed after an earlier green one: exit 1. +fixture dispatch_latest_failed "[$SUCCESS]" "[$DISP_OK,$(run_json workflow_dispatch feat completed failure 2026-10-20T02:00:00Z)]" abc123; run_gate +if [ "$RC" -eq 1 ] && [ "$(merges)" -eq 0 ]; then ok "dispatch_latest_failed: exit 1" +else fail "dispatch_latest_failed: rc=$RC merges=$(merges)"; fi + +# ─── ci_status and rules_stale, called from hack/gate_lib.sh directly ───────── +lib_ci() { # lib_ci NAME RUNS WANT: ci_status abc123 feat over the ci.yml runs RUNS + fixture "lib_$1" '[]' "$2" abc123 + local got; got=$(PATH="$T/bin:$PATH" bash -c '. "$1"; ci_status abc123 feat' _ "$ROOT/hack/gate_lib.sh" 2>&1) + if [ "$got" = "$3" ]; then ok "ci_status $1: $got"; else fail "ci_status $1: got '$got', want $3"; fi +} +lib_ci none '[]' none +lib_ci pr_green "[$(run_json pull_request feat completed success 2026-10-20T01:00:00Z)]" green +lib_ci dispatch_green "[$DISP_OK]" green +lib_ci held_only "[$HELD_PR]" failed +lib_ci held_plus_dispatch "[$HELD_PR,$DISP_OK]" green +lib_ci held_plus_running "[$HELD_PR,$(run_json workflow_dispatch feat in_progress '' 2026-10-20T01:05:00Z)]" pending +lib_ci dispatch_queued "[$(run_json workflow_dispatch feat queued '' 2026-10-20T01:05:00Z)]" pending +lib_ci other_branch "[$(run_json workflow_dispatch other completed success 2026-10-20T01:05:00Z)]" none +lib_ci latest_failed "[$DISP_OK,$(run_json workflow_dispatch feat completed failure 2026-10-20T02:00:00Z)]" failed +lib_ci failed_then_green "[$(run_json workflow_dispatch feat completed failure 2026-10-20T01:00:00Z),$DISP_OK]" green +lib_ci failed_then_running "[$(run_json workflow_dispatch feat completed failure 2026-10-20T01:00:00Z),$(run_json workflow_dispatch feat in_progress '' 2026-10-20T01:05:00Z)]" pending + +# rules_stale keeps its behaviour: stale tree -> 0, matching tree -> 1, no tree -> 1. +lib_rules() { # lib_rules NAME WANT_RC [current] + fixture "librules_$1" '[]' '[]' abc123 + mkdir -p "$FIX/show/console"; cp "$ROOT/console/model_opt.py" "$ROOT/console/convert-pins.json" "$FIX/show/console/" + printf '## Hypotheses\n\n| H34 | on the i9-13900T | p | m | SUPPORTED |\n' > "$FIX/show/EXPERIMENTS.md" + if [ "${3:-}" = current ]; then + python3 "$ROOT/console/model_opt.py" ledger-rules --ledger "$FIX/show/EXPERIMENTS.md" --out "$FIX/show/console/ledger-rules.json" + else echo '{"schema": "k3d-gpu.ledger-rules/v1", "rows": []}' > "$FIX/show/console/ledger-rules.json"; fi + [ "$1" = notree ] || echo t1 > "$FIX/tree" + PATH="$T/bin:$PATH" bash -c '. "$1"; rules_stale abc123' _ "$ROOT/hack/gate_lib.sh" >/dev/null 2>&1; local rc=$? + if [ "$rc" -eq "$2" ]; then ok "rules_stale $1: rc $rc"; else fail "rules_stale $1: rc $rc, want $2"; fi +} +lib_rules stale 0; lib_rules current 1 current; lib_rules notree 1 + +# ─── base_ok: release-only bot commits on main do not count as a moved base ──── +# release_to_main AUTHOR SUBJECT [code]: one commit on main that moves the version +# in scripts/k3d-gpu's VERSION line and the AUR, install.sh and Formula files; +# with `code` it also changes another line of scripts/k3d-gpu. +release_to_main() { + local R=$FIX/repo + repo_git checkout -q main + sed -i 's/^VERSION="0\.1\.0"/VERSION="0.1.1"/' "$R/scripts/k3d-gpu" + [ "${3:-}" = code ] && sed -i 's/^CLUSTER=gpu$/CLUSTER=cpu/' "$R/scripts/k3d-gpu" + echo 'pkgver=0.1.1' > "$R/packaging/aur/PKGBUILD"; echo 'pkgver = 0.1.1' > "$R/packaging/aur/.SRCINFO" + repo_git add scripts packaging/aur && GIT_AUTHOR_NAME=$1 "$REALGIT" -C "$R" commit -q -m "$2" && repo_git push -q origin main +} +# install_to_main AUTHOR: the install.sh and Homebrew release commits. +install_to_main() { + local R=$FIX/repo + repo_git checkout -q main + echo 'VERSION=0.1.1' > "$R/packaging/install.sh"; repo_git add packaging/install.sh + GIT_AUTHOR_NAME=$1 "$REALGIT" -C "$R" commit -q -m "release: install.sh k3d-gpu 0.1.1 [skip release]" + echo 'version "0.1.1"' > "$R/Formula/k3d-gpu.rb"; repo_git add Formula/k3d-gpu.rb + GIT_AUTHOR_NAME=$1 "$REALGIT" -C "$R" commit -q -m "release: homebrew k3d-gpu 0.1.1 [skip release]" + repo_git push -q origin main +} + +# 24. the PR changes code, and main moved only by release-only bot commits: merge, no sync. +realgit_fixture release_only_base "[$SUCCESS]" "$CI_OK" +commit_to feat scripts.txt $'pr\n' +release_to_main 88plug-bot "release: aur-v0.1.1 [skip release]"; install_to_main 'github-actions[bot]' +run_gate_real +if [ "$RC" -eq 0 ] && [ "$(merges)" -eq 1 ] && [ "$(syncs)" -eq 0 ] && ! grep -q 'main moved under code' <<<"$OUT"; then + ok "release_only_base: merged, no sync" +else fail "release_only_base: rc=$RC merges=$(merges) syncs=$(syncs) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 25. a release commit that also changes another launcher line is a code move: sync first. +realgit_fixture release_with_code "[$SUCCESS]" "$CI_OK" +commit_to feat scripts.txt $'pr\n' +release_to_main 88plug-bot "release: aur-v0.1.1 [skip release]" code +run_gate_real +if [ "$RC" -eq 0 ] && [ "$(merges)" -eq 1 ] && [ "$(syncs)" -eq 1 ] && grep -q 'main moved under code' <<<"$OUT"; then + ok "release_with_code: synced before the merge" +else fail "release_with_code: rc=$RC merges=$(merges) syncs=$(syncs) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 25b. the same release files under a release subject, but by a human: a code move. +realgit_fixture release_by_human "[$SUCCESS]" "$CI_OK" +commit_to feat scripts.txt $'pr\n' +release_to_main gate-test "release: aur-v0.1.1 [skip release]" +run_gate_real +if [ "$RC" -eq 0 ] && [ "$(syncs)" -eq 1 ] && grep -q 'main moved under code' <<<"$OUT"; then + ok "release_by_human: synced before the merge" +else fail "release_by_human: rc=$RC merges=$(merges) syncs=$(syncs) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 25c. a bot commit on release files without [skip release] in its subject: a code move. +realgit_fixture release_no_skip "[$SUCCESS]" "$CI_OK" +commit_to feat scripts.txt $'pr\n' +release_to_main 88plug-bot "release: aur-v0.1.1" +run_gate_real +if [ "$RC" -eq 0 ] && [ "$(syncs)" -eq 1 ]; then ok "release_no_skip: synced before the merge" +else fail "release_no_skip: rc=$RC merges=$(merges) syncs=$(syncs) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# ─── alarms: open ci-alarm issues are printed, and refuse shipped changes ────── +UNACKED='{"number":7,"title":"[ci-alarm] nightly: Auto-update red","url":"https://github.com/o/r/issues/7","labels":[{"name":"ci-alarm"}]}' +ACKED='{"number":8,"title":"[ci-alarm] release: 0.1.79 missing on GHCR","url":"https://github.com/o/r/issues/8","labels":[{"name":"ci-alarm"},{"name":"alarm-ack"}]}' + +# 26. an un-acked alarm and a PR that changes the launcher: exit 4, URL and ack command shown. +fixture alarm_unacked "[$SUCCESS]" "$CI_OK" abc123; echo "[$UNACKED]" > "$FIX/alarms_seq"; run_gate +if [ "$RC" -eq 4 ] && [ "$(merges)" -eq 0 ] && grep -qF 'https://github.com/o/r/issues/7' <<<"$OUT" \ + && grep -qF 'gh issue edit 7 --add-label alarm-ack' <<<"$OUT"; then + ok "alarm_unacked: refused with exit 4, URL and ack command printed" +else fail "alarm_unacked: rc=$RC merges=$(merges) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 26b. a workflow-only PR is refused too; so is a PR whose files cannot be listed. +fixture alarm_workflow "[$SUCCESS]" "$CI_OK" abc123; echo "[$UNACKED]" > "$FIX/alarms_seq" +echo .github/workflows/ci.yml > "$FIX/pr_paths"; run_gate +if [ "$RC" -eq 4 ] && [ "$(merges)" -eq 0 ]; then ok "alarm_workflow: refused" +else fail "alarm_workflow: rc=$RC merges=$(merges)"; fi +fixture alarm_files_unknown "[$SUCCESS]" "$CI_OK" abc123; echo "[$UNACKED]" > "$FIX/alarms_seq"; touch "$FIX/pr_paths_fail"; run_gate +if [ "$RC" -eq 4 ] && [ "$(merges)" -eq 0 ]; then ok "alarm_files_unknown: refused" +else fail "alarm_files_unknown: rc=$RC merges=$(merges)"; fi + +# 26c. GATE_RELEASE_PATHS adds shipped paths (hack/release-paths format: one +# pattern per line, # comments, dir/** for a tree). +fixture alarm_release_paths "[$SUCCESS]" "$CI_OK" abc123; echo "[$UNACKED]" > "$FIX/alarms_seq" +printf '# shipped by nfpm\noracle/**\n' > "$FIX/release-paths"; echo oracle/bundle.py > "$FIX/pr_paths" +OUT=$(PATH="$T/bin:$PATH" GATE_REPO="$FIX" GATE_WT_ROOT="$FIX/wt" GATE_POLLS=3 GATE_RELEASE_PATHS="$FIX/release-paths" \ + timeout 60 bash "$GATE" 1 subj body 2>&1); RC=$? +if [ "$RC" -eq 4 ] && [ "$(merges)" -eq 0 ]; then ok "alarm_release_paths: a listed path is refused" +else fail "alarm_release_paths: rc=$RC merges=$(merges) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 26d. no alarm at the start, an un-acked one by the merge: refused before merging. +fixture alarm_opens_later "[$SUCCESS]" "$CI_OK" abc123; printf '[]\n[%s]\n' "$UNACKED" > "$FIX/alarms_seq"; run_gate +if [ "$RC" -eq 4 ] && [ "$(merges)" -eq 0 ]; then ok "alarm_opens_later: re-checked and refused before the merge" +else fail "alarm_opens_later: rc=$RC merges=$(merges) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 27. every open alarm acked: listed at the start, and the PR merges. +fixture alarm_acked "[$SUCCESS]" "$CI_OK" abc123; echo "[$ACKED]" > "$FIX/alarms_seq"; run_gate +if [ "$RC" -eq 0 ] && [ "$(merges)" -eq 1 ] && grep -F '#8' <<<"$OUT" | grep -qF 'https://github.com/o/r/issues/8'; then + ok "alarm_acked: listed, merged" +else fail "alarm_acked: rc=$RC merges=$(merges) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 28. ledger, docs, bench and test changes are never refused; nor is a path outside +# the shipped set. +fixture alarm_ledger_only "[$SUCCESS]" "$CI_OK" abc123; echo "[$UNACKED,$ACKED]" > "$FIX/alarms_seq" +printf 'EXPERIMENTS.md\ndocs/provenance.md\nbench/h54/run.sh\ntests/bench_test.sh\n' > "$FIX/pr_paths"; run_gate +if [ "$RC" -eq 0 ] && [ "$(merges)" -eq 1 ] && grep -qF 'issues/7' <<<"$OUT"; then ok "alarm_ledger_only: listed, merged" +else fail "alarm_ledger_only: rc=$RC merges=$(merges) out=$(tr '\n' ';' <<<"$OUT")"; fi +fixture alarm_unshipped "[$SUCCESS]" "$CI_OK" abc123; echo "[$UNACKED]" > "$FIX/alarms_seq" +echo README.md > "$FIX/pr_paths"; run_gate +if [ "$RC" -eq 0 ] && [ "$(merges)" -eq 1 ]; then ok "alarm_unshipped: merged" +else fail "alarm_unshipped: rc=$RC merges=$(merges)"; fi + +# 30. the gate copied alone (a scratchpad shim) finds gate_lib.sh in the repository's +# hack/; with neither copy it stops before merging. +fixture solo_gate "[$SUCCESS]" "$CI_OK" abc123; mkdir -p "$FIX/solo" "$FIX/hack" +cp "$GATE" "$FIX/solo/gate_merge.sh"; cp "$ROOT/hack/gate_lib.sh" "$FIX/hack/" +OUT=$(PATH="$T/bin:$PATH" GATE_REPO="$FIX" GATE_WT_ROOT="$FIX/wt" GATE_POLLS=3 timeout 60 bash "$FIX/solo/gate_merge.sh" 1 subj body 2>&1); RC=$? +if [ "$RC" -eq 0 ] && [ "$(merges)" -eq 1 ]; then ok "solo_gate: gate_lib.sh read from the repository's hack/" +else fail "solo_gate: rc=$RC merges=$(merges) out=$(tr '\n' ';' <<<"$OUT")"; fi +rm "$FIX/hack/gate_lib.sh"; echo OPEN > "$FIX/state"; rm -f "$FIX/merges" +OUT=$(PATH="$T/bin:$PATH" GATE_REPO="$FIX" GATE_WT_ROOT="$FIX/wt" GATE_POLLS=3 timeout 60 bash "$FIX/solo/gate_merge.sh" 1 subj body 2>&1); RC=$? +if [ "$RC" -eq 1 ] && [ "$(merges)" -eq 0 ] && grep -q 'gate_lib.sh not found' <<<"$OUT"; then ok "solo_gate: no gate_lib.sh, exit 1" +else fail "solo_gate missing lib: rc=$RC merges=$(merges) out=$(tr '\n' ';' <<<"$OUT")"; fi + +# 29. issues cannot be listed (the repository has them disabled): the gate says so, +# with gh's reason, and goes on. +fixture alarm_issues_off "[$SUCCESS]" "$CI_OK" abc123; touch "$FIX/issues_off"; run_gate +if [ "$RC" -eq 0 ] && [ "$(merges)" -eq 1 ] && grep -q 'WARNING: could not list the open ci-alarm issues.*disabled issues' <<<"$OUT"; then + ok "alarm_issues_off: warned with the reason, merged" +else fail "alarm_issues_off: rc=$RC merges=$(merges) out=$(tr '\n' ';' <<<"$OUT")"; fi + echo "${PASS} passed, ${FAILS} failed" [ "${FAILS}" -eq 0 ]