GitLab Status: recognize and retry builder runner failures

.github/scripts/track-build.sh

@@ -12,7 +12,9 @@
 #
 # Exits non-zero when no matching job starts, when it does not succeed, or when
 # a testing/stable package does not reach https://repo.biglinux.com.br: the
-# server drops some uploads without any error in the job.
+# server drops some uploads without any error in the job. Exit status 3 means
+# the builder's own infrastructure failed (GitLab runner), so nothing was built
+# and requesting the build again is safe.
 set -euo pipefail
 
 : "${PACKAGE:?}" "${ORG:?}" "${BRANCH:?}" "${SINCE:?}"
@@ -30,6 +32,15 @@ get() {
     curl -fsS --retry 3 --retry-delay 5 --retry-all-errors --max-time 30 "$@"
 }
 
+# GitLab failure reasons that come from the runners, not from the package
+infra_failure='^(runner_system_failure|scheduler_failure|api_failure|data_integrity_failure)$'
+
+infra_error() {
+    echo "::error::The BigLinux builder's infrastructure failed ($1): $2. Nothing was built; request the build again."
+    printf '### %s: builder infrastructure failure\n%s: [%s](%s)\n' "$PACKAGE" "$1" "$2" "$2" >>"$summary"
+    exit 3
+}
+
 # One of the header lines the build job prints ("org: biglinux", ...)
 header() {
     sed -nE "s/^[^ ]+ [0-9]+[OE]\+? ?$1: (.*)$/\1/p" <<<"$2" | head -n1
@@ -53,10 +64,25 @@ echo "Looking for the $PACKAGE build of $url (branch $BRANCH) requested at $SINC
 #    has one job named after the package.
 job='' child='' rejected=" " deadline=$((SECONDS + START_TIMEOUT))
 while [ -z "$job" ]; do
-    while read -r pid created; do
-        [ "$(date -u -d "$created" +%s)" -ge $((since - 30)) ] || continue
+    while read -r pid created status; do
+        t=$(date -u -d "$created" +%s)
+        [ "$t" -ge $((since - 30)) ] || continue
         child=$(get "$api/pipelines/$pid/bridges" | jq -r '.[0].downstream_pipeline.id // empty')
-        [ -n "$child" ] || continue
+        if [ -z "$child" ]; then
+            # The request failed before the build job existed. Only the
+            # generating job's log names the package, so also require the
+            # pipeline to start right after the request (the builder needs 1-3 s).
+            if [[ "$status" =~ ^(failed|canceled)$ ]] && [ "$t" -le $((since + 15)) ]; then
+                read -r gid reason < <(get "$api/pipelines/$pid/jobs" |
+                    jq -r '.[] | select(.name == "generate-PackageBuild") | "\(.id) \(.failure_reason // "none")"')
+                if [ -n "${gid:-}" ] && get "$web/-/jobs/$gid/raw" | grep -qE "[0-9]+[OE]\+? ?package_name=$PACKAGE$"; then
+                    [[ "$reason" =~ $infra_failure ]] && infra_error "$reason" "$web/-/jobs/$gid"
+                    echo "::error::The builder pipeline $web/-/pipelines/$pid failed before building ($reason)"
+                    exit 1
+                fi
+            fi
+            continue
+        fi
         id=$(get "$api/pipelines/$child/jobs" | jq -r --arg n "$PACKAGE" '.[] | select(.name == $n) | .id')
         if [ -z "$id" ] || [[ "$rejected" == *" $id "* ]]; then continue; fi
         log=$(get "$web/-/jobs/$id/raw" || true)
@@ -69,7 +95,7 @@ while [ -z "$job" ]; do
         fi
         rejected+="$id "
     done < <(get "$api/pipelines?source=trigger&updated_after=$after&order_by=id&sort=asc&per_page=100" |
-        jq -r '.[] | "\(.id) \(.created_at)"')
+        jq -r '.[] | "\(.id) \(.created_at) \(.status)"')
 
     if [ -z "$job" ]; then
         if [ "$SECONDS" -ge "$deadline" ]; then
@@ -85,11 +111,16 @@ done
 # 2. Wait for it to finish
 deadline=$((SECONDS + BUILD_TIMEOUT)) last=
 while :; do
-    status=$(get "$api/pipelines/$child/jobs" | jq -r --argjson id "$job" '.[] | select(.id == $id) | .status')
+    read -r status reason < <(get "$api/pipelines/$child/jobs" |
+        jq -r --argjson id "$job" '.[] | select(.id == $id) | "\(.status) \(.failure_reason // "none")"')
     [ "$status" = "$last" ] || echo "Job status: $status"
     last=$status
     case "$status" in
-    success | failed | canceled | skipped) break ;;
+    failed)
+        if [[ "$reason" =~ $infra_failure ]]; then infra_error "$reason" "$web/-/jobs/$job"; fi
+        break
+        ;;
+    success | canceled | skipped) break ;;
     esac
     if [ "$SECONDS" -ge "$deadline" ]; then
         echo "::error::Builder job $web/-/jobs/$job still '$status' after ${BUILD_TIMEOUT}s"

.github/workflows/gitlab-status.yml

@@ -2,7 +2,9 @@ name: GitLab Status
 
 # Follows the job the BigLinux builder starts after a successful
 # "Build Package" run, and fails when it does not start or does not succeed.
-# Read only: it uses the public GitLab API and no secret. The name avoids the
+# It uses the public GitLab API and no secret. When the builder's own runners
+# fail before building, it re-runs the "build" job of that run (at most twice),
+# which makes the organization request the build again. The name avoids the
 # words "Build" and "Package", so this run never requests a build itself.
 
 on:
@@ -20,6 +22,7 @@ on:
 
 permissions:
   contents: read
+  actions: write # re-run the build job after a builder infrastructure failure
 
 jobs:
   track:
@@ -41,4 +44,27 @@ jobs:
           ORG: biglinux
           BRANCH: ${{ github.event.workflow_run.head_branch || inputs.branch }}
           SINCE: ${{ github.event.workflow_run.updated_at || inputs.since }}
-        run: .github/scripts/track-build.sh
+          # Only the tracker of a run's first attempt re-runs it
+          RUN_ID: ${{ github.event.workflow_run.run_attempt == 1 && github.event.workflow_run.id || '' }}
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          for retry in 0 1 2; do
+            status=0
+            .github/scripts/track-build.sh || status=$?
+            if [ "$status" -ne 3 ] || [ -z "$RUN_ID" ] || [ "$retry" -eq 2 ]; then
+              exit "$status"
+            fi
+            build=$(gh run view "$RUN_ID" -R "$GITHUB_REPOSITORY" --json jobs               --jq '.jobs[] | select(.name == "build") | .databaseId')
+            echo "::warning::Builder infrastructure failure; re-running the build job of run $RUN_ID"
+            attempt=$(gh run view "$RUN_ID" -R "$GITHUB_REPOSITORY" --json attempt --jq .attempt)
+            gh run rerun "$RUN_ID" -R "$GITHUB_REPOSITORY" --job "$build"
+            # Wait for the new attempt to finish
+            until gh run view "$RUN_ID" -R "$GITHUB_REPOSITORY" --json attempt,status \
+              --jq ".attempt > $attempt and .status == \"completed\"" | grep -qx true; do
+              sleep 15
+            done
+            read -r conclusion SINCE < <(gh run view "$RUN_ID" -R "$GITHUB_REPOSITORY" \
+              --json conclusion,updatedAt --jq '"\(.conclusion) \(.updatedAt)"')
+            [ "$conclusion" = success ] || { echo "::error::The re-run ended '$conclusion'"; exit 1; }
+            export SINCE
+          done

README.md

@@ -263,7 +263,7 @@ The suite checks the page configuration, translations, version consistency and t
 |---|---|---|
 | Tests | pull requests, manual | tests, lint, translations, package build/install/remove |
 | Build Package | pushes to `main`, `testing-*`, `stable-*`, manual | the tests above, then the `build` job |
-| GitLab Status | after Build Package (biglinux only), manual | follows the builder job; fails if it fails or if testing/stable never publish the file |
+| GitLab Status | after Build Package (biglinux only), manual | follows the builder job, retries runner failures; fails if the build fails or if testing/stable never publish the file |
 
 The BigLinux builder ([packagebuilder](https://gitlab.bigib.org/builder/packagebuilder)) starts by itself when a **Build Package** run in `biglinux/biglinux-welcome` succeeds; no secret is needed. `main` is built for the unstable repository, a `testing-*` branch for testing and a `stable-*` branch for stable. Failed tests mean no build. Details, evidence and the release checklist: [docs/ci-packaging.md](docs/ci-packaging.md).
 

docs/ci-packaging.md

@@ -19,9 +19,18 @@ push to main / testing-* / stable-*          (biglinux/biglinux-welcome)
   └─ GitLab Status workflow (workflow_run) follows that job through the public
      GitLab API and fails if it does not start within 10 min or does not succeed;
      for testing/stable it then waits up to 1 h for
-     repo.biglinux.com.br/<repo>/x86_64/<file> and fails if it never appears
+     repo.biglinux.com.br/<repo>/x86_64/<file> and fails if it never appears.
+     If the builder's runners fail (GitLab failure reason runner_system_failure,
+     scheduler_failure, api_failure, data_integrity_failure), nothing was
+     built: it re-runs the "build" job of that run, at most twice, and follows
+     the new request
 ```
 
+Runner failures are not rare. On 2026-10-11 the request for
+`testing-2026-10-11` reached the builder 1 s after the run (pipeline 37669),
+but its first job died with `runner_system_failure` (Kubernetes
+`502 Bad Gateway`), as did two other pipelines that day.
+
 Destination repository, decided by the builder from `git_branch`
 (`package-build` in packagebuilder):
 
@@ -158,8 +167,9 @@ None is needed for builds. Recommended, by a `biglinux` admin:
 4. To publish to testing: create branch `testing-YYYY-MM-DD` from `main`
    (`git push origin main:refs/heads/testing-$(date +%F)`). Same for stable
    with `stable-YYYY-MM-DD_HH-MM`. Check **GitLab Status** again.
-5. If the builder job failed, open the job link, fix, and rerun
-   **Build Package** on that branch (Actions → Run workflow).
+5. If the builder job failed, open the job link. Runner failures are retried
+   automatically; for anything else, fix it and re-run the `build` job of the
+   **Build Package** run for that branch (or Actions → Run workflow).
 
 Each branch gets one build per successful run; rerunning the workflow
 requests another build of the same branch.