diff --git a/.github/workflows/dependabot-lock-regen.yml b/.github/workflows/dependabot-lock-regen.yml index d99097aea..973adee3b 100644 --- a/.github/workflows/dependabot-lock-regen.yml +++ b/.github/workflows/dependabot-lock-regen.yml @@ -10,13 +10,20 @@ # CREDENTIAL: the estate's dedicated applier App — vars.APP_ID and # secrets.APP_PRIVATE_KEY, the same slot apply-workflow-pins.yml and # signed-push read (owner ruling STD-R-3: NOT OikosBot, which is a ruleset -# bypass actor). Needs Contents R/W, Workflows R/W, Pull requests R, and must -# be installed on BOTH owners. Until it exists the run warns and exits 0. +# bypass actor). Needs Contents R/W, Workflows R/W, Pull requests R/W (the +# default-branch repair opens PRs), and must be installed on BOTH owners. Until +# it exists the run warns and exits 0. +# +# DEFAULT-BRANCH REPAIR: the 04:41 UTC tick (or dispatch with main-repair) also +# sets REGEN_MAIN=1, which opens one lock-only PR where a stale lock already +# landed on the default branch (drift merged before this workflow could catch +# it). Every other tick only repairs open Dependabot and applier PRs. name: Dependabot lock regeneration on: schedule: - cron: '23,53 * * * *' + - cron: '41 4 * * *' workflow_dispatch: inputs: repo: @@ -27,6 +34,10 @@ on: description: 'Regenerate and report, but commit nothing' type: boolean default: false + main-repair: + description: 'Also repair a stale actions.lock on each default branch (opens a PR)' + type: boolean + default: false concurrency: group: ${{ github.workflow }} @@ -81,6 +92,7 @@ jobs: hyperpolymath=${{ steps.tok-user.outputs.token }} metadatastician=${{ steps.tok-org.outputs.token }} DRY_RUN: ${{ inputs.dry-run && '1' || '0' }} + REGEN_MAIN: ${{ (github.event.schedule == '41 4 * * *' || inputs.main-repair) && '1' || '0' }} ONLY_REPO: ${{ inputs.repo }} run: | # A token is an empty string when the App is absent or not installed diff --git a/scripts/regen-dependabot-locks.sh b/scripts/regen-dependabot-locks.sh index a9c32cb8f..d65e48b44 100755 --- a/scripts/regen-dependabot-locks.sh +++ b/scripts/regen-dependabot-locks.sh @@ -43,10 +43,20 @@ # is reported as a warning and exits 0 — the job must not pretend it swept # anything, and must not redden every schedule tick before the App exists. # +# DEFAULT-BRANCH REPAIR (REGEN_MAIN=1) +# ------------------------------------ +# Drift that already landed (a red lock check merged through automerge) is not +# on any open PR, so it is repaired separately: the default branch is cloned, +# regenerated with the same chain and the same refusals, and one lock-only PR +# is opened from $REPAIR_BRANCH. Nothing is pushed to the default branch. An +# existing $REPAIR_BRANCH is reused only when it is this script's own leftover +# (its head commit carries the repair headline) and no PR is open from it. An +# open repair PR is swept like a Dependabot PR. See repair_main. +# # Usage: regen-dependabot-locks.sh [owner/repo] (default: every repo the # installations can see). DRY_RUN=1 regenerates but never commits. -# Env: REGEN_TOKENS, DRY_RUN, GH_BIN (test stub), REGEN_TOOLS (dir holding -# the repair scripts; default: this script's directory). +# Env: REGEN_TOKENS, DRY_RUN, REGEN_MAIN, GH_BIN (test stub), REGEN_TOOLS (dir +# holding the repair scripts; default: this script's directory). set -uo pipefail GH_BIN="${GH_BIN:-gh}" @@ -102,25 +112,35 @@ regen_lock() { ) } +# Commit messages: a Dependabot/applier PR repair, and a default-branch repair. +BUMP_HEADLINE="chore(deps): regenerate actions.lock for this bump" +BUMP_BODY="Dependabot updated workflow refs without the lockfile. Regenerated with the estate repair order (scripts/regen-dependabot-locks.sh in hyperpolymath/standards) and verified with update-actions-lock.sh --verify-local." +REPAIR_HEADLINE="fix(ci): regenerate the stale actions.lock" +REPAIR_BODY="The default branch's actions.lock no longer matches its workflows' uses: refs, so the lock gate is red and GitHub refuses to start the workflows whose refs drifted. Regenerated with the estate repair order (scripts/regen-dependabot-locks.sh in hyperpolymath/standards, REGEN_MAIN=1) and verified with update-actions-lock.sh --verify-local and check-lock-sync.sh. Only actions.lock changes." + # Commit the checkout's actions.lock onto branch $2 of repo $1, only if the -# branch head is still $3. Prints the new commit oid. +# branch head is still $3, with headline $5 and body $6 (default: the +# Dependabot-bump message). Prints the new commit oid. commit_lock() { local repo="$1" branch="$2" expected="$3" dir="$4" payload + local headline="${5:-$BUMP_HEADLINE}" body="${6:-$BUMP_BODY}" payload="$(jq -n \ --arg repo "$repo" --arg branch "$branch" --arg oid "$expected" \ --arg path "$LOCK_PATH" --arg contents "$(base64 -w0 < "$dir/$LOCK_PATH")" \ + --arg headline "$headline" --arg body "$body" \ '{query: "mutation($in: CreateCommitOnBranchInput!) { createCommitOnBranch(input: $in) { commit { oid } } }", variables: {in: { branch: {repositoryNameWithOwner: $repo, branchName: $branch}, expectedHeadOid: $oid, - message: {headline: "chore(deps): regenerate actions.lock for this bump", - body: "Dependabot updated workflow refs without the lockfile. Regenerated with the estate repair order (scripts/regen-dependabot-locks.sh in hyperpolymath/standards) and verified with update-actions-lock.sh --verify-local."}, + message: {headline: $headline, body: $body}, fileChanges: {additions: [{path: $path, contents: $contents}]}}}}')" printf '%s' "$payload" | "$GH_BIN" api graphql --input - --jq '.data.createCommitOnBranch.commit.oid' } # The branch scripts/apply-workflow-pins-remote.sh opens its PRs from. APPLIER_BRANCH="${APPLIER_BRANCH:-chore/re-point-standards-workflow-pins}" +# The branch this script opens default-branch repair PRs from (REGEN_MAIN=1). +REPAIR_BRANCH="${REPAIR_BRANCH:-chore/regenerate-actions-lock}" # Read a pulls-list JSON array on stdin; print "numberhead_refhead_sha" # for each open, non-draft PR from a same-repo branch that changes workflow refs @@ -128,18 +148,93 @@ APPLIER_BRANCH="${APPLIER_BRANCH:-chore/re-point-standards-workflow-pins}" # standards pin applier (scripts/apply-workflow-pins-remote.sh). The applier # rewrites `uses:` pins only, so without this its PRs die at startup on an # unclosed lock record. It is matched on a bot author AND its fixed branch name, -# so a human's branch of the same name is never rewritten. +# so a human's branch of the same name is never rewritten. This script's own +# default-branch repair PRs are matched the same way, so a repair PR that +# drifts again before it merges is regenerated rather than left stale. select_prs() { - jq -r --arg applier "$APPLIER_BRANCH" '.[] + jq -r --arg applier "$APPLIER_BRANCH" --arg repair "$REPAIR_BRANCH" '.[] | select(((.user.login == "dependabot[bot]") - or ((.user.login | endswith("[bot]")) and .head.ref == $applier)) + or ((.user.login | endswith("[bot]")) + and (.head.ref == $applier or .head.ref == $repair))) and (.draft | not) and .head.repo.full_name == .base.repo.full_name) | [.number, .head.ref, .head.sha] | @tsv' } +# Shallow-clone branch $3 of repo $2 into $1 with the installation token in +# $GH_TOKEN. The token travels as a header from the environment: never in +# argv, and never persisted in the checkout's remote.origin.url (CWE-522). +clone_branch() { + local dest="$1" repo="$2" ref="$3" + GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=http.extraHeader \ + GIT_CONFIG_VALUE_0="Authorization: Basic $(printf 'x-access-token:%s' "$GH_TOKEN" | base64 -w0)" \ + git clone --quiet --depth 1 --branch "$ref" \ + "https://github.com/$repo.git" "$dest" 2>/dev/null +} + +# Repair a stale actions.lock on the default branch of repo $1, whose open PRs +# are the JSON array $2. Regenerates it with the same chain as a Dependabot +# PR and opens ONE lock-only PR from $REPAIR_BRANCH; prints +# "repo@branchoutcome". Dependabot repairs fix new drift; this fixes drift +# that already landed (a red lock check merged through automerge), which +# otherwise reddens every later PR and stops GitHub starting the drifted +# workflows. Never moves a branch it did not create: an existing +# $REPAIR_BRANCH is reused only when its head commit carries this script's +# repair headline and no PR is open from it. Needs Pull requests: write. +repair_main() { + local repo="$1" prs="$2" base sha work outcome oid pr msg + if printf '%s' "$prs" | jq -e --arg b "$REPAIR_BRANCH" \ + 'any(.[]; .head.ref == $b and .head.repo.full_name == .base.repo.full_name)' >/dev/null; then + printf '%s@main\trepair-pr-open\n' "$repo"; return 0 + fi + if ! base="$("$GH_BIN" api "repos/$repo" --jq .default_branch)" || [ -z "$base" ] \ + || ! sha="$("$GH_BIN" api "repos/$repo/commits/$base" --jq .sha)"; then + printf '%s@main\tapi-error\n' "$repo"; return 0 + fi + work="$(mktemp -d)" + if ! clone_branch "$work/r" "$repo" "$base"; then + printf '%s@%s\tclone-failed\n' "$repo" "$base"; rm -rf "$work"; return 0 + fi + if [ "$(git -C "$work/r" rev-parse HEAD)" != "$sha" ]; then + printf '%s@%s\tmoved\n' "$repo" "$base"; rm -rf "$work"; return 0 + fi + outcome="$(regen_lock "$work/r")" + if [ "$outcome" = changed ] && [ "${DRY_RUN:-0}" != 1 ]; then + outcome="" + if ! "$GH_BIN" api -X POST "repos/$repo/git/refs" \ + -f ref="refs/heads/$REPAIR_BRANCH" -f sha="$sha" --silent 2>/dev/null; then + # The branch exists. Reuse it only if it is a leftover of ours. + msg="$("$GH_BIN" api "repos/$repo/commits/$REPAIR_BRANCH" --jq .commit.message 2>/dev/null || true)" + if [ "${msg%%$'\n'*}" = "$REPAIR_HEADLINE" ] \ + && "$GH_BIN" api -X PATCH "repos/$repo/git/refs/heads/$REPAIR_BRANCH" \ + -f sha="$sha" -F force=true --silent 2>/dev/null; then + : + else + outcome="repair-branch-exists" + fi + fi + if [ -z "$outcome" ]; then + if oid="$(commit_lock "$repo" "$REPAIR_BRANCH" "$sha" "$work/r" "$REPAIR_HEADLINE" "$REPAIR_BODY")" \ + && printf '%s' "$oid" | grep -Eq '^[0-9a-f]{40}$'; then + if pr="$("$GH_BIN" api -X POST "repos/$repo/pulls" -f title="$REPAIR_HEADLINE" \ + -f head="$REPAIR_BRANCH" -f base="$base" -f body="$REPAIR_BODY" --jq .number)" \ + && [ -n "$pr" ]; then + outcome="pr-opened #$pr ($oid)" + else + outcome="pr-refused ($oid)" + fi + else + outcome="commit-refused" + fi + fi + fi + printf '%s@%s\t%s\n' "$repo" "$base" "$outcome" + rm -rf "$work" +} + # Process every candidate PR of repo $1 with token $2; print one -# "repo#Noutcome" line per PR examined. +# "repo#Noutcome" line per PR examined. With REGEN_MAIN=1, then repair +# the default branch's own lock (see repair_main). process_repo() { local repo="$1" token="$2" num ref sha work outcome oid err files prs export GH_TOKEN="$token" @@ -152,6 +247,7 @@ process_repo() { if ! prs="$("$GH_BIN" api --paginate "repos/$repo/pulls?state=open&per_page=100")"; then printf '%s\tapi-error\n' "$repo"; return 0 fi + prs="$(printf '%s' "$prs" | jq -s 'add // []')" while IFS=$'\t' read -r num ref sha; do [ -n "$num" ] || continue if ! files="$("$GH_BIN" api --paginate "repos/$repo/pulls/$num/files?per_page=100" --jq '.[].filename')"; then @@ -159,12 +255,7 @@ process_repo() { fi printf '%s\n' "$files" | grep -q '^\.github/workflows/' || continue work="$(mktemp -d)" - # The token travels as a header from the environment: never in argv, and - # never persisted in the checkout's remote.origin.url (CWE-522). - if ! GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=http.extraHeader \ - GIT_CONFIG_VALUE_0="Authorization: Basic $(printf 'x-access-token:%s' "$token" | base64 -w0)" \ - git clone --quiet --depth 1 --branch "$ref" \ - "https://github.com/$repo.git" "$work/r" 2>/dev/null; then + if ! clone_branch "$work/r" "$repo" "$ref"; then printf '%s#%s\tclone-failed\n' "$repo" "$num"; rm -rf "$work"; continue fi if [ "$(git -C "$work/r" rev-parse HEAD)" != "$sha" ]; then @@ -180,7 +271,10 @@ process_repo() { fi printf '%s#%s\t%s\n' "$repo" "$num" "$outcome" rm -rf "$work" - done < <(printf '%s' "$prs" | jq -s 'add // []' | select_prs) + done < <(printf '%s' "$prs" | select_prs) + if [ "${REGEN_MAIN:-0}" = 1 ]; then + repair_main "$repo" "$prs" + fi } # Enumerate every repository visible to each owner's installation token (or diff --git a/scripts/tests/regen-dependabot-locks-test.sh b/scripts/tests/regen-dependabot-locks-test.sh index 779331324..916c7f9e1 100755 --- a/scripts/tests/regen-dependabot-locks-test.sh +++ b/scripts/tests/regen-dependabot-locks-test.sh @@ -76,6 +76,20 @@ if [ "$1 $2" = "api graphql" ]; then echo 2222222222222222222222222222222222222222 exit 0 fi +# Default-branch repair (REGEN_MAIN=1): repo metadata, heads, refs and PRs. +if [ "$1" = api ] && [ "$2" = repos/o/r ] && [ "$3" = --jq ]; then echo "${DEFAULT_BRANCH:-main}"; exit 0; fi +if [ "$1" = api ] && [ "$2" = repos/o/r/commits/chore/regenerate-actions-lock ]; then + [ -n "${LEFTOVER_MSG:-}" ] || exit 1 + printf '%s\n' "$LEFTOVER_MSG"; exit 0 +fi +if [ "$1" = api ] && [[ "$2" == repos/o/r/commits/* ]]; then echo "$MAIN_SHA"; exit 0; fi +if [ "$1 $2 $3" = "api -X POST" ] && [[ "$4" == repos/*/git/refs ]]; then + echo "POST-REF $*" >> "$GH_LOG"; [ -n "${REF_EXISTS:-}" ] && exit 1; exit 0 +fi +if [ "$1 $2 $3" = "api -X PATCH" ]; then echo "PATCH-REF $*" >> "$GH_LOG"; exit 0; fi +if [ "$1 $2 $3" = "api -X POST" ] && [[ "$4" == repos/*/pulls ]]; then + echo "POST-PR $*" >> "$GH_LOG"; [ -n "${PR_FAIL:-}" ] && exit 1; echo 42; exit 0 +fi exit 1 EOF chmod +x "$WORK/gh" @@ -180,6 +194,63 @@ expect "a rate-limited lock probe is reported, not skipped" "o/r api-error" "$(C expect "a failed PR listing is reported" "o/r api-error" "$(PULLS_FAIL=1 process_repo o/r t 2>/dev/null)" expect "a repo with no open PRs prints nothing" "" "$(process_repo o/r t)" +# --- repair_main: a stale lock that already landed on the default branch ---- +# Clone the local fixture instead of github.com; the rest is the real code. +clone_branch() { git clone --quiet "$FIXTURE" "$1" 2>/dev/null; } +export GH_LOG="$WORK/gh.log" +oid=2222222222222222222222222222222222222222 +# fresh_main : a new fixture as the default branch, an empty call log. +fresh_main() { FIXTURE="$(fixture "$1")"; MAIN_SHA="$(git -C "$FIXTURE" rev-parse HEAD)"; export MAIN_SHA; : > "$GH_LOG"; rm -f "$GH_CAPTURE"; } + +fresh_main stale +out="$(TOOL_ACTION=bump repair_main o/r '[]' 2>/dev/null)" +expect "a stale default-branch lock opens one repair PR" "o/r@main pr-opened #42 ($oid)" "$out" +expect "…from the repair branch, created at the default head" 1 "$(grep -c "POST-REF .*ref=refs/heads/chore/regenerate-actions-lock .*sha=$MAIN_SHA" "$GH_LOG")" +expect "…its commit is guarded by the default head" "$MAIN_SHA" "$(jq -r .variables.in.expectedHeadOid "$GH_CAPTURE")" +expect "…lands on the repair branch" chore/regenerate-actions-lock "$(jq -r .variables.in.branch.branchName "$GH_CAPTURE")" +expect "…carries the repair headline, not the Dependabot one" "$REPAIR_HEADLINE" "$(jq -r .variables.in.message.headline "$GH_CAPTURE")" +expect "…writes only the lock" ".github/workflows/actions.lock" "$(jq -r '.variables.in.fileChanges.additions | map(.path) | join(",")' "$GH_CAPTURE")" +expect "…and the PR goes from the repair branch to the default branch" 1 "$(grep -c 'POST-PR .*head=chore/regenerate-actions-lock .*base=main' "$GH_LOG")" + +fresh_main current +expect "a current default-branch lock opens nothing" "o/r@main current" "$(TOOL_ACTION=none repair_main o/r '[]' 2>/dev/null)" +expect "…and touches no ref" 0 "$(grep -c . "$GH_LOG")" + +fresh_main dry +expect "DRY_RUN reports the change" "o/r@main changed" "$(DRY_RUN=1 TOOL_ACTION=bump repair_main o/r '[]' 2>/dev/null)" +expect "…and writes nothing" 0 "$(grep -c . "$GH_LOG")" + +fresh_main unver +expect "a lock the verifier rejects is not proposed" "o/r@main unverified" "$(VERIFY_RC=1 TOOL_ACTION=bump repair_main o/r '[]' 2>/dev/null)" +expect "…and no branch is created" 0 "$(grep -c . "$GH_LOG")" + +fresh_main moved +expect "a default branch that moved under the clone is refused" "o/r@main moved" "$(MAIN_SHA=4444444444444444444444444444444444444444 TOOL_ACTION=bump repair_main o/r '[]' 2>/dev/null)" + +fresh_main open +open='[{"number":9,"draft":false,"user":{"login":"estate-applier[bot]"},"head":{"ref":"chore/regenerate-actions-lock","sha":"hhh","repo":{"full_name":"o/r"}},"base":{"repo":{"full_name":"o/r"}}}]' +expect "an open repair PR is left to the PR sweep" "o/r@main repair-pr-open" "$(TOOL_ACTION=bump repair_main o/r "$open" 2>/dev/null)" +expect "…which selects it, because a bot opened it on the repair branch" "9 chore/regenerate-actions-lock hhh" "$(printf '%s' "$open" | select_prs)" +human='[{"number":10,"draft":false,"user":{"login":"someone"},"head":{"ref":"chore/regenerate-actions-lock","sha":"iii","repo":{"full_name":"o/r"}},"base":{"repo":{"full_name":"o/r"}}}]' +expect "…but never a human's PR on that branch name" "" "$(printf '%s' "$human" | select_prs)" + +fresh_main foreign +expect "an existing branch the script did not create is never moved" "o/r@main repair-branch-exists" "$(REF_EXISTS=1 LEFTOVER_MSG='feat: someone else' TOOL_ACTION=bump repair_main o/r '[]' 2>/dev/null)" +expect "…no PATCH, no commit" "0 no" "$(grep -c PATCH-REF "$GH_LOG") $([ -e "$GH_CAPTURE" ] && echo yes || echo no)" + +fresh_main leftover +out="$(REF_EXISTS=1 LEFTOVER_MSG="$REPAIR_HEADLINE"$'\n\n'"body" TOOL_ACTION=bump repair_main o/r '[]' 2>/dev/null)" +expect "a leftover repair branch of ours is reused" "o/r@main pr-opened #42 ($oid)" "$out" +expect "…by force-moving it to the default head" 1 "$(grep -c "PATCH-REF .*git/refs/heads/chore/regenerate-actions-lock .*sha=$MAIN_SHA .*force=true" "$GH_LOG")" + +fresh_main prfail +expect "a refused PR creation is reported with the commit" "o/r@main pr-refused ($oid)" "$(PR_FAIL=1 TOOL_ACTION=bump repair_main o/r '[]' 2>/dev/null)" + +fresh_main wired +expect "process_repo runs the default-branch repair only with REGEN_MAIN=1" "o/r@main pr-opened #42 ($oid)" "$(REGEN_MAIN=1 TOOL_ACTION=bump process_repo o/r t 2>/dev/null)" +fresh_main unwired +expect "…and not without it" "" "$(TOOL_ACTION=bump process_repo o/r t 2>/dev/null)" + # --- missing credential ---------------------------------------------------- out="$(REGEN_TOKENS='' main 2>&1)"; rc=$? expect "no token: exits 0" 0 "$rc"