From b966f63479ee96cfd5a18503b25423ad0568f513 Mon Sep 17 00:00:00 2001 From: James Ray Date: Tue, 6 Oct 2026 01:16:33 -0700 Subject: [PATCH 1/2] Wait for the mission architecture deploy slot The architecture project runs one build at a time and CodeBuild rejects a StartBuild over that limit instead of queueing it. A base-image rebuild starts every Lambda build for a mission at once, so all but the first hand-off were dropped and those Lambdas kept their old images. Retry throttled StartBuild calls every 30 seconds, for up to 55 minutes from the build's start, and fail immediately on any other error. Co-Authored-By: Claude Opus 5.5 --- buildspec.yml | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 8d6ca83..e84270f 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -243,9 +243,18 @@ phases: docker push "${FULL_ECR_REPO}:latest" docker push "${FULL_ECR_REPO}:${VERSION}" + # The mission architecture project runs one deploy at a time, and + # CodeBuild rejects a StartBuild over that limit instead of queueing + # it. A base-image rebuild starts every Lambda build for the mission at + # once, so wait for the deploy slot. The deadline stays inside this + # project's 60-minute build timeout (set in sdc_aws_architecture). + # Only throttling and limit errors are retried; anything else fails + # the build immediately. ARCHITECTURE_PROJECT_NAME="build_${MISSION_NAME}_sdc_aws_pipeline_architecture" + BUILD_START_SECONDS=$(( ${CODEBUILD_START_TIME:-$(( $(date +%s) * 1000 ))} / 1000 )) + DEPLOY_START_DEADLINE=$(( BUILD_START_SECONDS + 55 * 60 )) echo "Starting ${ARCHITECTURE_PROJECT_NAME} for ${CDK_ENVIRONMENT}" - aws codebuild start-build \ + until DEPLOY_BUILD=$(aws codebuild start-build \ --project-name "${ARCHITECTURE_PROJECT_NAME}" \ --region "${REGION}" \ --source-version main \ @@ -253,7 +262,23 @@ phases: name=CDK_ENVIRONMENT,value="${CDK_ENVIRONMENT}",type=PLAINTEXT \ name=MISSION_NAME,value="${MISSION_NAME}",type=PLAINTEXT \ name=LAMBDA_PIPELINE,value="${LAMBDA_PIPELINE}",type=PLAINTEXT \ - name=TAG,value="${VERSION}",type=PLAINTEXT + name=TAG,value="${VERSION}",type=PLAINTEXT \ + --query 'build.id' --output text 2>&1); do + case "${DEPLOY_BUILD}" in + *AccountLimitExceededException*|*ThrottlingException*|*TooManyRequestsException*|*RequestLimitExceeded*|*"Rate exceeded"*) ;; + *) + echo "StartBuild for ${ARCHITECTURE_PROJECT_NAME} failed and will not be retried: ${DEPLOY_BUILD}" + exit 1 + ;; + esac + if [ "$(date +%s)" -ge "${DEPLOY_START_DEADLINE}" ]; then + echo "Gave up waiting for ${ARCHITECTURE_PROJECT_NAME}: ${DEPLOY_BUILD}" + exit 1 + fi + echo "${ARCHITECTURE_PROJECT_NAME} is busy, retrying in 30s: ${DEPLOY_BUILD}" + sleep 30 + done + echo "Started ${DEPLOY_BUILD}" post_build: commands: From 3c3e71c590ab9d0638213dea17bc5693e82cc72c Mon Sep 17 00:00:00 2001 From: James Ray Date: Wed, 7 Oct 2026 10:12:26 -0700 Subject: [PATCH 2/2] Never retry the architecture StartBuild past the deadline The deadline was checked only before the fixed 30-second sleep, so a busy response just before the 55-minute cutoff still led to one more StartBuild after it. Cap each wait at the time remaining and check the deadline again before every retry. Co-Authored-By: Claude Opus 5.5 --- buildspec.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index e84270f..2846e6f 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -246,7 +246,8 @@ phases: # The mission architecture project runs one deploy at a time, and # CodeBuild rejects a StartBuild over that limit instead of queueing # it. A base-image rebuild starts every Lambda build for the mission at - # once, so wait for the deploy slot. The deadline stays inside this + # once, so wait for the deploy slot. The first attempt always runs; + # retries stop 55 minutes after this build started, inside the # project's 60-minute build timeout (set in sdc_aws_architecture). # Only throttling and limit errors are retried; anything else fails # the build immediately. @@ -271,12 +272,20 @@ phases: exit 1 ;; esac + # Never sleep past the deadline, and check it again before the + # next attempt so no StartBuild goes out after the cutoff. + RETRY_WAIT=$(( DEPLOY_START_DEADLINE - $(date +%s) )) + if [ "${RETRY_WAIT}" -gt 30 ]; then + RETRY_WAIT=30 + fi + if [ "${RETRY_WAIT}" -gt 0 ]; then + echo "${ARCHITECTURE_PROJECT_NAME} is busy, retrying in ${RETRY_WAIT}s: ${DEPLOY_BUILD}" + sleep "${RETRY_WAIT}" + fi if [ "$(date +%s)" -ge "${DEPLOY_START_DEADLINE}" ]; then echo "Gave up waiting for ${ARCHITECTURE_PROJECT_NAME}: ${DEPLOY_BUILD}" exit 1 fi - echo "${ARCHITECTURE_PROJECT_NAME} is busy, retrying in 30s: ${DEPLOY_BUILD}" - sleep 30 done echo "Started ${DEPLOY_BUILD}"