From 27f588f4cec675ca087bc8097e6af2f5d432d011 Mon Sep 17 00:00:00 2001 From: Igor Pecovnik Date: Thu, 20 Aug 2026 17:11:13 +0200 Subject: [PATCH 1/4] repository update: ingest base-files into the package storage armbian/ci builds base-files in their own pipeline and uploads them to incoming/base-files/, decoupled from the package builds so they publish even when nothing else ran (armbian/build#9476). Nothing consumed that directory: target 'base-files/' fell through to the no-op *) branch of the Copy operations case, so the debs accumulated in incoming and never reached a repository. Copy incoming/base-files/{debs,debs-beta} into STORAGE_PATH, which is where repo.sh reads its input from, unconditionally and regardless of which target triggered the run - the same treatment external packages already get. Copied rather than moved: unlike incoming/cron these are small and versioned, rsync skips what is already identical, and keeping the source means a failure later in the pipeline cannot lose them. Signed-off-by: Igor Pecovnik --- .../infrastructure-repository-update.yml | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.github/workflows/infrastructure-repository-update.yml b/.github/workflows/infrastructure-repository-update.yml index 15b8b117a0..ffeadfa3e2 100644 --- a/.github/workflows/infrastructure-repository-update.yml +++ b/.github/workflows/infrastructure-repository-update.yml @@ -178,6 +178,35 @@ jobs: ;; esac + # Always sync base-files, whatever the target was. + # + # armbian/ci builds these in their own pipeline (build-base-files.yml, + # uploaded to incoming/base-files/) because they must publish even when + # a package build did not run - see armbian/build#9476. Nothing used to + # consume that directory: "base-files/" fell through to the no-op *) + # branch, so the debs piled up in incoming and never reached a + # repository. Copy them into STORAGE_PATH, which is what repo.sh reads + # (INPUT_DIR="${STORAGE_PATH}/${REPO_NAME}"). + # + # Copied, not moved: unlike incoming/cron these are small, versioned, + # and cheap to re-send, and keeping them means a failed run later in the + # pipeline cannot lose them. rsync skips what is already identical. + for BF_REPO in debs debs-beta; do + if [ -d "${INCOMING_PATH}/base-files/${BF_REPO}" ]; then + echo "## Copy base-files ${BF_REPO}" >> "$GITHUB_STEP_SUMMARY" + rsync -av \ + --include='*/' \ + --include='*.deb' \ + --exclude='*' \ + --omit-dir-times --no-perms --no-group \ + "${INCOMING_PATH}/base-files/${BF_REPO}/" "${STORAGE_PATH}/${BF_REPO}/" \ + 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" || \ + echo "Warning: Some files/attrs were not transferred (code 23)" >> "$GITHUB_STEP_SUMMARY" + else + echo "## No base-files/${BF_REPO} in incoming, skipping" >> "$GITHUB_STEP_SUMMARY" + fi + done + # Always sync external if [ -d "${INCOMING_PATH}/external/debs" ]; then echo "## Copy external debs " From 87bc857d7aac0d3cb0e0e1ad6120eec470cd09c2 Mon Sep 17 00:00:00 2001 From: Igor Pecovnik Date: Thu, 20 Aug 2026 17:19:47 +0200 Subject: [PATCH 2/4] repository update: fail the base-files copy on a real rsync error The copy was written as 'rsync ... | tee ... || echo warning', which cannot work: with the pipe, $? is tee's status, and the step sets no pipefail, so the fallback branch is effectively unreachable and every rsync failure is silent. Copying would go green, prepare-repos through update-repository would run, and repo.sh would publish a storage tree missing those packages - to every mirror. Read PIPESTATUS[0] instead and branch on it: 0 continues, 23 warns (expected - --no-perms/--no-group on the shared-group storage means some attributes really do not transfer), anything else is an error that stops the run before the partial copy can be published. Signed-off-by: Igor Pecovnik --- .../infrastructure-repository-update.yml | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/infrastructure-repository-update.yml b/.github/workflows/infrastructure-repository-update.yml index ffeadfa3e2..3525174f58 100644 --- a/.github/workflows/infrastructure-repository-update.yml +++ b/.github/workflows/infrastructure-repository-update.yml @@ -200,8 +200,25 @@ jobs: --exclude='*' \ --omit-dir-times --no-perms --no-group \ "${INCOMING_PATH}/base-files/${BF_REPO}/" "${STORAGE_PATH}/${BF_REPO}/" \ - 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" || \ - echo "Warning: Some files/attrs were not transferred (code 23)" >> "$GITHUB_STEP_SUMMARY" + 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" + # PIPESTATUS, not $?: the pipe through tee would otherwise report + # tee's status and hide every rsync failure. 23 is expected here - + # --no-perms/--no-group on the shared-group storage means some + # attributes legitimately do not transfer. Anything else is real, + # and must stop the run: update-repository would happily publish a + # storage tree that is missing these packages. + BF_RC=${PIPESTATUS[0]} + case "${BF_RC}" in + 0) ;; + 23) + echo "Warning: some files/attrs were not transferred (rsync 23)" >> "$GITHUB_STEP_SUMMARY" + ;; + *) + echo "::error::rsync of base-files/${BF_REPO} failed (rsync exit ${BF_RC})" + echo "**rsync of base-files/${BF_REPO} failed (exit ${BF_RC})**" >> "$GITHUB_STEP_SUMMARY" + exit 1 + ;; + esac else echo "## No base-files/${BF_REPO} in incoming, skipping" >> "$GITHUB_STEP_SUMMARY" fi From e4e3f2f06b697c2a0651fd9312ea846c4a2bb9ed Mon Sep 17 00:00:00 2001 From: Igor Pecovnik Date: Thu, 20 Aug 2026 17:21:53 +0200 Subject: [PATCH 3/4] repository update: move base-files out of incoming instead of copying incoming is a drop box, so what has been taken into storage should leave it - same as incoming/cron. Done with rsync --remove-source-files rather than a following rm -rf, so rsync deletes exactly the files it confirmed on the receiving side: a partial transfer (exit 23) leaves whatever did not land in place for the next run instead of dropping it. --remove-source-files only removes files, so the emptied directories are pruned afterwards, and the base-files drop box itself with rmdir - which is a no-op while anything is still in there, keeping a partial transfer visible. Signed-off-by: Igor Pecovnik --- .../infrastructure-repository-update.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/infrastructure-repository-update.yml b/.github/workflows/infrastructure-repository-update.yml index 3525174f58..8838b40459 100644 --- a/.github/workflows/infrastructure-repository-update.yml +++ b/.github/workflows/infrastructure-repository-update.yml @@ -188,13 +188,16 @@ jobs: # repository. Copy them into STORAGE_PATH, which is what repo.sh reads # (INPUT_DIR="${STORAGE_PATH}/${REPO_NAME}"). # - # Copied, not moved: unlike incoming/cron these are small, versioned, - # and cheap to re-send, and keeping them means a failed run later in the - # pipeline cannot lose them. rsync skips what is already identical. + # Moved, not copied: incoming is a drop box, so what has been taken + # into storage is removed, the same way incoming/cron is. The move is + # done with --remove-source-files rather than a following rm -rf, so + # rsync deletes exactly the files it confirmed on the receiving side - + # a partial transfer (exit 23) leaves whatever did not land in place + # for the next run instead of dropping it. for BF_REPO in debs debs-beta; do if [ -d "${INCOMING_PATH}/base-files/${BF_REPO}" ]; then echo "## Copy base-files ${BF_REPO}" >> "$GITHUB_STEP_SUMMARY" - rsync -av \ + rsync -av --remove-source-files \ --include='*/' \ --include='*.deb' \ --exclude='*' \ @@ -219,10 +222,16 @@ jobs: exit 1 ;; esac + # --remove-source-files only removes files; clear the directories + # it emptied, and the per-repo dir itself once it is empty. + find "${INCOMING_PATH}/base-files/${BF_REPO}" -type d -empty -delete else echo "## No base-files/${BF_REPO} in incoming, skipping" >> "$GITHUB_STEP_SUMMARY" fi done + # Drop the now-empty drop box. Left alone if anything remains, so a + # partial transfer stays visible for the next run. + rmdir "${INCOMING_PATH}/base-files" 2>/dev/null || true # Always sync external if [ -d "${INCOMING_PATH}/external/debs" ]; then From 23bf9f4e1f36686f452738d2d73947f26238f11f Mon Sep 17 00:00:00 2001 From: Igor Pecovnik Date: Thu, 20 Aug 2026 17:30:52 +0200 Subject: [PATCH 4/4] repository update: ingest all release-generic packages, not just base-files The incoming/base-files drop box carries everything the general packages build produces, not only base-files: verified against apt.armbian.com, that is base-files (in -utils), armbian-firmware, armbian-firmware-full, armbian-plymouth-theme, armbian-zsh, fake-ubuntu-advantage-tools and armbian-bsp-cli-. All of them belong in the repository. The move already matched on *.deb rather than package names, so it covered them in fact; this makes that explicit, drops the base-files-only naming, and turns the drop box into a list so adding another is one word. kernel, linux-dtb, linux-libc-dev and u-boot stay out on purpose. There is one of each per (linuxfamily, branch) and per (board, branch), so choosing which belong in the repository is a selection rather than a sweep - scripts/copy-kernel-packages.sh already implements it and will be driven from a config file. Signed-off-by: Igor Pecovnik --- .../infrastructure-repository-update.yml | 112 +++++++++++------- 1 file changed, 67 insertions(+), 45 deletions(-) diff --git a/.github/workflows/infrastructure-repository-update.yml b/.github/workflows/infrastructure-repository-update.yml index 8838b40459..875fd3fcf5 100644 --- a/.github/workflows/infrastructure-repository-update.yml +++ b/.github/workflows/infrastructure-repository-update.yml @@ -178,15 +178,22 @@ jobs: ;; esac - # Always sync base-files, whatever the target was. + # Always ingest the release-generic packages, whatever the target was. # - # armbian/ci builds these in their own pipeline (build-base-files.yml, - # uploaded to incoming/base-files/) because they must publish even when - # a package build did not run - see armbian/build#9476. Nothing used to - # consume that directory: "base-files/" fell through to the no-op *) - # branch, so the debs piled up in incoming and never reached a - # repository. Copy them into STORAGE_PATH, which is what repo.sh reads - # (INPUT_DIR="${STORAGE_PATH}/${REPO_NAME}"). + # armbian/ci builds these in their own pipeline and uploads them to + # incoming/base-files/ - decoupled so they publish even when a package + # build did not run (armbian/build#9476). Nothing used to consume that + # directory: the target fell through to the no-op *) branch, so the + # debs piled up in incoming and never reached a repository. + # + # The drop box carries more than base-files. Everything the general + # packages build produces lands here and all of it belongs in the + # repository - published today as base-files, armbian-firmware, + # armbian-firmware-full, armbian-plymouth-theme, armbian-zsh, + # fake-ubuntu-advantage-tools and armbian-bsp-cli-. So this + # matches on *.deb only: no package list to keep in sync, and a new + # generic package needs no change here. repo.sh decides which + # component each one belongs to. # # Moved, not copied: incoming is a drop box, so what has been taken # into storage is removed, the same way incoming/cron is. The move is @@ -194,44 +201,59 @@ jobs: # rsync deletes exactly the files it confirmed on the receiving side - # a partial transfer (exit 23) leaves whatever did not land in place # for the next run instead of dropping it. - for BF_REPO in debs debs-beta; do - if [ -d "${INCOMING_PATH}/base-files/${BF_REPO}" ]; then - echo "## Copy base-files ${BF_REPO}" >> "$GITHUB_STEP_SUMMARY" - rsync -av --remove-source-files \ - --include='*/' \ - --include='*.deb' \ - --exclude='*' \ - --omit-dir-times --no-perms --no-group \ - "${INCOMING_PATH}/base-files/${BF_REPO}/" "${STORAGE_PATH}/${BF_REPO}/" \ - 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" - # PIPESTATUS, not $?: the pipe through tee would otherwise report - # tee's status and hide every rsync failure. 23 is expected here - - # --no-perms/--no-group on the shared-group storage means some - # attributes legitimately do not transfer. Anything else is real, - # and must stop the run: update-repository would happily publish a - # storage tree that is missing these packages. - BF_RC=${PIPESTATUS[0]} - case "${BF_RC}" in - 0) ;; - 23) - echo "Warning: some files/attrs were not transferred (rsync 23)" >> "$GITHUB_STEP_SUMMARY" - ;; - *) - echo "::error::rsync of base-files/${BF_REPO} failed (rsync exit ${BF_RC})" - echo "**rsync of base-files/${BF_REPO} failed (exit ${BF_RC})**" >> "$GITHUB_STEP_SUMMARY" - exit 1 - ;; - esac - # --remove-source-files only removes files; clear the directories - # it emptied, and the per-repo dir itself once it is empty. - find "${INCOMING_PATH}/base-files/${BF_REPO}" -type d -empty -delete - else - echo "## No base-files/${BF_REPO} in incoming, skipping" >> "$GITHUB_STEP_SUMMARY" - fi + # + # Only the generic packages move wholesale. kernel, linux-dtb, + # linux-libc-dev and u-boot are deliberately NOT in here: there is one + # of each per (linuxfamily, branch) and per (board, branch), so which + # of them belong in the repository is a selection, not a sweep. + # scripts/copy-kernel-packages.sh already does that selection + # (SELECT / UBOOT_SELECT / INCLUDE_LIBC_DEV / COPY_UBOOT, deduplicating + # against image-info.json); driving it from a config file is the next + # step, and belongs in the per-target branches above. + # + # Add another drop box by naming it here. + GENERIC_INCOMING="base-files" + + for GP_DIR in ${GENERIC_INCOMING}; do + for GP_REPO in debs debs-beta; do + if [ -d "${INCOMING_PATH}/${GP_DIR}/${GP_REPO}" ]; then + echo "## Move ${GP_DIR}/${GP_REPO} into storage" >> "$GITHUB_STEP_SUMMARY" + rsync -av --remove-source-files \ + --include='*/' \ + --include='*.deb' \ + --exclude='*' \ + --omit-dir-times --no-perms --no-group \ + "${INCOMING_PATH}/${GP_DIR}/${GP_REPO}/" "${STORAGE_PATH}/${GP_REPO}/" \ + 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" + # PIPESTATUS, not $?: the pipe through tee would otherwise report + # tee's status and hide every rsync failure. 23 is expected here - + # --no-perms/--no-group on the shared-group storage means some + # attributes legitimately do not transfer. Anything else is real, + # and must stop the run: update-repository would happily publish + # a storage tree that is missing these packages. + GP_RC=${PIPESTATUS[0]} + case "${GP_RC}" in + 0) ;; + 23) + echo "Warning: some files/attrs were not transferred (rsync 23)" >> "$GITHUB_STEP_SUMMARY" + ;; + *) + echo "::error::rsync of ${GP_DIR}/${GP_REPO} failed (rsync exit ${GP_RC})" + echo "**rsync of ${GP_DIR}/${GP_REPO} failed (exit ${GP_RC})**" >> "$GITHUB_STEP_SUMMARY" + exit 1 + ;; + esac + # --remove-source-files only removes files; clear the directories + # it emptied, and the per-repo dir itself once it is empty. + find "${INCOMING_PATH}/${GP_DIR}/${GP_REPO}" -type d -empty -delete + else + echo "## No ${GP_DIR}/${GP_REPO} in incoming, skipping" >> "$GITHUB_STEP_SUMMARY" + fi + done + # Drop the now-empty drop box. Left alone if anything remains, so a + # partial transfer stays visible for the next run. + rmdir "${INCOMING_PATH}/${GP_DIR}" 2>/dev/null || true done - # Drop the now-empty drop box. Left alone if anything remains, so a - # partial transfer stays visible for the next run. - rmdir "${INCOMING_PATH}/base-files" 2>/dev/null || true # Always sync external if [ -d "${INCOMING_PATH}/external/debs" ]; then