From 21ebbb36835c139474e1703b00c1ccc8b5066078 Mon Sep 17 00:00:00 2001 From: Austin Burdine Date: Fri, 9 Oct 2026 17:35:38 -0400 Subject: [PATCH] Kept CI's Docker Hub pulls under the anonymous rate limit Test jobs failed intermittently with Docker Hub's "toomanyrequests: You have reached your unauthenticated pull rate limit" while building the manager image from its Docker Hub bases (run 37993666771). No workflow logged in to Docker Hub. Every Linux job that pulls now has the daemon pull through mirror.gcr.io and, when the secrets reach it, logs in with a read-only token (DOCKERHUB_USERNAME, DOCKERHUB_TOKEN). The mirror is for every job, not only those without the token: the manager runs Compose in its own container with the daemon's socket and none of the host's credentials, so its pulls of Ghost, MySQL and Caddy are anonymous whatever the host logged in with. Pull requests from forks and Dependabot get no secrets and run with the mirror alone, rather than failing on a missing login. - .github/actions/docker-hub/action.yml: the mirror and the login, shared by every job, skipping the login when there is no token. - .github/workflows/test.yml: used by each Linux job that runs Docker. - .github/workflows/image.yml: used before the build; the buildx builder runs in its own container, which ignores the daemon's mirror, so it is given the same one. Takes the secrets when called. - .github/workflows/launcher.yml: used by the job that installs with the served launcher. Takes the secrets when called. - .github/workflows/release.yml: passes the secrets to both. --- .github/actions/docker-hub/action.yml | 45 +++++++++++++++++++++++++++ .github/workflows/image.yml | 15 ++++++++- .github/workflows/launcher.yml | 10 ++++++ .github/workflows/release.yml | 6 ++++ .github/workflows/test.yml | 40 ++++++++++++++++++++++++ 5 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 .github/actions/docker-hub/action.yml diff --git a/.github/actions/docker-hub/action.yml b/.github/actions/docker-hub/action.yml new file mode 100644 index 00000000..bd1a2057 --- /dev/null +++ b/.github/actions/docker-hub/action.yml @@ -0,0 +1,45 @@ +name: Docker Hub +description: >- + Keeps a job's Docker Hub pulls under the anonymous rate limit: the daemon + pulls through mirror.gcr.io, and the client logs in with a read-only token + when the repository's secrets reach the job. + +# The mirror is for every pull, not only those without the token: the manager +# runs Compose in its own container with the daemon's socket and none of the +# host's credentials, so its pulls of Ghost, MySQL and Caddy are anonymous +# whatever the host logged in with. The daemon falls back to Docker Hub for an +# image the mirror does not have. The login covers the host's own builds and +# pulls, and whatever falls back. Pull requests from forks, and Dependabot's, +# get no secrets, so they run with the mirror alone. +inputs: + username: + description: The Docker Hub user the token belongs to + required: false + token: + description: A read-only Docker Hub access token + required: false + +runs: + using: composite + steps: + - name: Pull Docker Hub images through mirror.gcr.io + shell: bash + run: | + set -euo pipefail + config=/etc/docker/daemon.json + current=$(sudo cat "$config" 2>/dev/null || printf '{}') + jq '.["registry-mirrors"] = ["https://mirror.gcr.io"]' <<<"$current" | sudo tee "$config" >/dev/null + sudo systemctl restart docker + echo "Registry mirrors: $(docker info --format '{{json .RegistryConfig.Mirrors}}')" + + - name: Log in to Docker Hub + if: inputs.username != '' && inputs.token != '' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + username: ${{ inputs.username }} + password: ${{ inputs.token }} + + - name: No Docker Hub token + if: inputs.username == '' || inputs.token == '' + shell: bash + run: echo "::notice::No Docker Hub token reached this job, so what the mirror does not have is pulled anonymously" diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index e88d3056..4dcbeda7 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -22,6 +22,11 @@ on: description: The release tag to publish type: string required: true + secrets: + DOCKERHUB_USERNAME: + required: false + DOCKERHUB_TOKEN: + required: false permissions: contents: read @@ -45,6 +50,11 @@ jobs: # Every release tag, to tell whether this one is the newest. fetch-depth: 0 + - uses: ./.github/actions/docker-hub + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + token: ${{ secrets.DOCKERHUB_TOKEN }} + - uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0 with: node-version: "26" @@ -77,7 +87,10 @@ jobs: # arm64 is built under emulation. docker run --privileged --rm tonistiigi/binfmt --install arm64 >/dev/null - docker buildx create --use >/dev/null + # The builder runs in a container of its own, which does not use the + # daemon's mirror, so it is given the same one. + printf '[registry."docker.io"]\n mirrors = ["mirror.gcr.io"]\n' >"$RUNNER_TEMP/buildkitd.toml" + docker buildx create --use --buildkitd-config "$RUNNER_TEMP/buildkitd.toml" >/dev/null docker buildx build \ --platform linux/amd64,linux/arm64 \ diff --git a/.github/workflows/launcher.yml b/.github/workflows/launcher.yml index fcd875d3..d239758b 100644 --- a/.github/workflows/launcher.yml +++ b/.github/workflows/launcher.yml @@ -18,6 +18,11 @@ on: tag: type: string required: true + secrets: + DOCKERHUB_USERNAME: + required: false + DOCKERHUB_TOKEN: + required: false workflow_dispatch: inputs: tag: @@ -109,6 +114,11 @@ jobs: with: ref: ${{ inputs.tag }} + - uses: ./.github/actions/docker-hub + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + token: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Wait until the release's launcher is served run: | set -euo pipefail diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cc024615..eb8ea3d9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -99,6 +99,9 @@ jobs: uses: ./.github/workflows/image.yml with: tag: ${{ needs.tag.outputs.tag }} + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} notes: name: GitHub release @@ -143,3 +146,6 @@ jobs: uses: ./.github/workflows/launcher.yml with: tag: ${{ needs.tag.outputs.tag }} + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 18b7be76..60004aae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -76,6 +76,11 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: ./.github/actions/docker-hub + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + token: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Integration tests run: manager/test/integration/run.sh @@ -88,6 +93,11 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: ./.github/actions/docker-hub + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + token: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Confirm the daemon answers as this user run: docker info --format '{{.ServerVersion}}' && docker compose version @@ -105,6 +115,11 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: ./.github/actions/docker-hub + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + token: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Install end to end run: tests/e2e/install.sh @@ -118,6 +133,11 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: ./.github/actions/docker-hub + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + token: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Self-update end to end run: tests/e2e/self-update.sh @@ -131,6 +151,11 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: ./.github/actions/docker-hub + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + token: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Migration end to end run: tests/e2e/migrate-main.sh @@ -144,6 +169,11 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: ./.github/actions/docker-hub + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + token: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Backup and restore end to end run: tests/e2e/backup.sh @@ -157,6 +187,11 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: ./.github/actions/docker-hub + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + token: ${{ secrets.DOCKERHUB_TOKEN }} + - uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0 with: node-version: "22" @@ -175,6 +210,11 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: ./.github/actions/docker-hub + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + token: ${{ secrets.DOCKERHUB_TOKEN }} + - uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0 with: node-version: "22"