From 5e89949845e9613fc4dc23e4d34ca3a9e7f460da Mon Sep 17 00:00:00 2001 From: Dipesh Babu Date: Wed, 18 Nov 2026 19:51:12 -0500 Subject: [PATCH] Fall back to official image mirrors when ECR Public throttles CI Retrying ECR Public alone was not enough: it returned 429 for every attempt over three minutes. The deployment smoke now tries the same official Python and Postgres tags from ECR Public, mirror.gcr.io and Docker Hub in each round, backing off between rounds, and tags the Postgres image under the name the CI Compose override expects. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 41 ++++++++++++++++++++++++++++++++-------- docs/CI.md | 5 ++++- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ba40c6..708fdf4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -511,19 +511,44 @@ jobs: run: | # Change only the registry for CI; retain the source Python tag and all # application/build instructions. Validate that exactly one base exists. - mirror_file="$RUNNER_TEMP/Dockerfile.agentloop-ci" + # ECR Public can throttle hosted runners (429) for minutes, so each round + # tries the same official image from ECR Public, Google's Docker Hub + # mirror and Docker Hub before backing off. test "$(grep -c '^FROM python:' Dockerfile)" -eq 1 - sed 's|^FROM python:|FROM public.ecr.aws/docker/library/python:|' Dockerfile > "$mirror_file" - # The public mirror occasionally answers 429 Too Many Requests; retry - # with backoff before failing the job. - for delay in 20 60 120 0; do - if docker build --file "$mirror_file" --tag agentloop:local .; then - exit 0 + mirror_file="$RUNNER_TEMP/Dockerfile.agentloop-ci" + for delay in 30 90 0; do + for registry in public.ecr.aws/docker/library mirror.gcr.io/library docker.io/library; do + sed "s|^FROM python:|FROM $registry/python:|" Dockerfile > "$mirror_file" + if docker build --file "$mirror_file" --tag agentloop:local .; then + exit 0 + fi + echo "::warning::Image build from $registry failed" + done + if [ "$delay" -eq 0 ]; then + exit 1 fi + echo "::warning::Retrying image build in ${delay}s" + sleep "$delay" + done + + - name: Pull the Postgres image used by Compose + shell: bash + run: | + # Compose uses the ECR Public name from .github/docker-compose.ci.yml. + # Fall back to identical official mirrors and tag the result locally. + target=public.ecr.aws/docker/library/postgres:16-alpine + for delay in 30 90 0; do + for registry in public.ecr.aws/docker/library mirror.gcr.io/library docker.io/library; do + if docker pull "$registry/postgres:16-alpine"; then + docker tag "$registry/postgres:16-alpine" "$target" + exit 0 + fi + echo "::warning::Postgres pull from $registry failed" + done if [ "$delay" -eq 0 ]; then exit 1 fi - echo "::warning::Image build failed; retrying in ${delay}s" + echo "::warning::Retrying Postgres pull in ${delay}s" sleep "$delay" done diff --git a/docs/CI.md b/docs/CI.md index 4c5ddce..d154570 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -22,7 +22,10 @@ Dockerfile/Compose defaults, health checks, database contracts and application validation remain unchanged. Docker lists Python/Postgres in its -[official ECR publisher gallery](https://gallery.ecr.aws/docker/). These mirrors +[official ECR publisher gallery](https://gallery.ecr.aws/docker/). ECR Public can +also throttle hosted runners, so the deployment smoke tries the same official +tags from ECR Public, Google's `mirror.gcr.io` Docker Hub mirror and Docker Hub, +with backoff between rounds. These mirrors do not replace checks: pull failures still fail CI, and the full interpreter, Postgres and deployment tests must run before a PR can merge.