diff --git a/.github/workflows/infrastructure-repository-update.yml b/.github/workflows/infrastructure-repository-update.yml index 875fd3fcf5..a4d91cd4bd 100644 --- a/.github/workflows/infrastructure-repository-update.yml +++ b/.github/workflows/infrastructure-repository-update.yml @@ -135,18 +135,156 @@ jobs: TARGET="cron/" fi + # Ingest the curated kernel selection out of incoming/stable. + # + # Kernels are a selection rather than a sweep - there is one + # linux-image per (linuxfamily, branch) - so which families the + # repository carries comes from config/repository-kernel-selection.conf, + # and copy-kernel-packages.sh resolves what exists from image-info.json. + # + # Called from both the stable/ and base-files/ targets: the weekly + # chain builds the packages into incoming/stable/ and then runs + # base-files, and it is base-files that fires the repository update - + # so base-files/ is the signal that the chain finished. Gating it there + # rather than running unconditionally also keeps the eight daily cron + # updates from re-copying the same debs, since this copies rather than + # moves. + ingest_stable_kernels() { + [ -d "${INCOMING_PATH}/stable" ] || { + echo "## No stable/ in incoming, skipping kernel selection" >> "$GITHUB_STEP_SUMMARY" + return 0 + } + # shellcheck source=config/repository-kernel-selection.conf + . "${GITHUB_WORKSPACE}/config/repository-kernel-selection.conf" + echo "## Stable kernel selection: ${KERNEL_SELECT}" >> "$GITHUB_STEP_SUMMARY" + local ST_REPO ST_SRC ST_RC + for ST_REPO in debs debs-beta; do + ST_SRC="${INCOMING_PATH}/stable/${ST_REPO}" + if [ ! -d "${ST_SRC}" ]; then + echo "## No stable/${ST_REPO} in incoming, skipping" >> "$GITHUB_STEP_SUMMARY" + continue + fi + SELECT="${KERNEL_SELECT}" \ + UBOOT_SELECT="${UBOOT_SELECT}" \ + COPY_UBOOT="${COPY_UBOOT}" \ + INCLUDE_LIBC_DEV="${INCLUDE_LIBC_DEV}" \ + DRY_RUN=false \ + SRC_DIR="${ST_SRC}" \ + DST_DIR="${STORAGE_PATH}/${ST_REPO}" \ + scripts/copy-kernel-packages.sh 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" + ST_RC=${PIPESTATUS[0]} + if [ "${ST_RC}" -ne 0 ]; then + echo "::error::copy-kernel-packages.sh failed for stable/${ST_REPO} (exit ${ST_RC})" + return 1 + fi + done + + # Drain the drop box: what was wanted has been copied, the rest is + # every other family's kernel and every other board's u-boot, which + # this repository does not carry. Leaving them means incoming grows + # without bound and each weekly run re-copies over the last. + # + # Only reached when every selection above succeeded, so nothing is + # dropped on the strength of a failed copy. The count is reported + # first: widening KERNEL_SELECT means next week's build supplies + # them again, but this week's are gone. + local ST_TOTAL + ST_TOTAL=$(find "${INCOMING_PATH}/stable" -type f -name '*.deb' 2>/dev/null | wc -l) + echo "## Dropping incoming/stable (${ST_TOTAL} .deb, selected ones already copied)" >> "$GITHUB_STEP_SUMMARY" + case "${INCOMING_PATH}" in + *"/armbian/openssh-server/storage"*) + rm -rf "${INCOMING_PATH}/stable" + ;; + *) + echo "::error::refusing to remove ${INCOMING_PATH}/stable - not under the storage path" + return 1 + ;; + esac + } + + # Ingest the release-generic packages out of incoming/base-files. + # + # armbian/ci builds these in their own pipeline and uploads them there, + # decoupled so they publish even when a package build did not run + # (armbian/build#9476). The drop box carries more than base-files: + # everything the general packages build produces lands in it and all of + # it belongs in the repository - published today as base-files, + # armbian-firmware, armbian-firmware-full, armbian-plymouth-theme, + # armbian-zsh, fake-ubuntu-advantage-tools and armbian-bsp-cli-. + # So this matches on *.deb only: no package list to keep in sync, and + # repo.sh decides which component each one belongs to. + # + # Moved, not copied, with --remove-source-files rather than a following + # rm -rf, so rsync deletes exactly the files it confirmed on the + # receiving side - a partial transfer (exit 23) leaves whatever did not + # land in place for the next run instead of dropping it. + # + # Called only from the stable-side targets. cron/ builds with + # nightlybuild=yes and feeds debs-beta: the nightly repository has no + # business pulling in stable packages. + # + # Add another drop box by naming it here. + GENERIC_INCOMING="base-files" + + ingest_generic_packages() { + local GP_DIR GP_REPO GP_RC + for GP_DIR in ${GENERIC_INCOMING}; do + for GP_REPO in debs debs-beta; do + if [ -d "${INCOMING_PATH}/${GP_DIR}/${GP_REPO}" ]; then + echo "## Move ${GP_DIR}/${GP_REPO} into storage" >> "$GITHUB_STEP_SUMMARY" + rsync -av --remove-source-files \ + --include='*/' \ + --include='*.deb' \ + --exclude='*' \ + --omit-dir-times --no-perms --no-group \ + "${INCOMING_PATH}/${GP_DIR}/${GP_REPO}/" "${STORAGE_PATH}/${GP_REPO}/" \ + 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" + # PIPESTATUS, not $?: the pipe through tee would otherwise + # report tee's status and hide every rsync failure. 23 is + # expected here - --no-perms/--no-group on the shared-group + # storage means some attributes legitimately do not transfer. + # Anything else is real, and must stop the run: + # update-repository would happily publish a storage tree that + # is missing these packages. + GP_RC=${PIPESTATUS[0]} + case "${GP_RC}" in + 0) ;; + 23) + echo "Warning: some files/attrs were not transferred (rsync 23)" >> "$GITHUB_STEP_SUMMARY" + ;; + *) + echo "::error::rsync of ${GP_DIR}/${GP_REPO} failed (rsync exit ${GP_RC})" + echo "**rsync of ${GP_DIR}/${GP_REPO} failed (exit ${GP_RC})**" >> "$GITHUB_STEP_SUMMARY" + return 1 + ;; + esac + # --remove-source-files only removes files; clear the + # directories it emptied, and the per-repo dir once it is empty. + find "${INCOMING_PATH}/${GP_DIR}/${GP_REPO}" -type d -empty -delete + else + echo "## No ${GP_DIR}/${GP_REPO} in incoming, skipping" >> "$GITHUB_STEP_SUMMARY" + fi + done + # Drop the now-empty drop box. Left alone if anything remains, so a + # partial transfer stays visible for the next run. + rmdir "${INCOMING_PATH}/${GP_DIR}" 2>/dev/null || true + done + } + case "$TARGET" in "stable/") - if [ -d "${INCOMING_PATH}/stable" ]; then - COPY_UBOOT=false \ - DRY_RUN=true \ - SELECT=':edge' \ - SRC_DIR="${{ env.INCOMING_PATH }}"/nightly/debs-beta/ \ - DST_DIR=/tmp/x \ - scripts/copy-kernel-packages.sh 2>&1 | tee -a $GITHUB_STEP_SUMMARY - else - echo "## Source folder INCOMING/stable does not exist, skipping" >> $GITHUB_STEP_SUMMARY - fi + # A stable build dispatching for itself (manual, or if the chain + # is ever unpicked). + ingest_stable_kernels || exit 1 + ingest_generic_packages || exit 1 + ;; + + "base-files/") + # The weekly chain's completion signal: build-all-stable uploads + # to incoming/stable/, then runs base-files, and base-files fires + # this update. So this is where the stable drop box is drained. + ingest_stable_kernels || exit 1 + ingest_generic_packages || exit 1 ;; "cron/") # move what is inside incoming @@ -178,83 +316,6 @@ jobs: ;; esac - # Always ingest the release-generic packages, whatever the target was. - # - # armbian/ci builds these in their own pipeline and uploads them to - # incoming/base-files/ - decoupled so they publish even when a package - # build did not run (armbian/build#9476). Nothing used to consume that - # directory: the target fell through to the no-op *) branch, so the - # debs piled up in incoming and never reached a repository. - # - # The drop box carries more than base-files. Everything the general - # packages build produces lands here and all of it belongs in the - # repository - published today as base-files, armbian-firmware, - # armbian-firmware-full, armbian-plymouth-theme, armbian-zsh, - # fake-ubuntu-advantage-tools and armbian-bsp-cli-. So this - # matches on *.deb only: no package list to keep in sync, and a new - # generic package needs no change here. repo.sh decides which - # component each one belongs to. - # - # Moved, not copied: incoming is a drop box, so what has been taken - # into storage is removed, the same way incoming/cron is. The move is - # done with --remove-source-files rather than a following rm -rf, so - # rsync deletes exactly the files it confirmed on the receiving side - - # a partial transfer (exit 23) leaves whatever did not land in place - # for the next run instead of dropping it. - # - # Only the generic packages move wholesale. kernel, linux-dtb, - # linux-libc-dev and u-boot are deliberately NOT in here: there is one - # of each per (linuxfamily, branch) and per (board, branch), so which - # of them belong in the repository is a selection, not a sweep. - # scripts/copy-kernel-packages.sh already does that selection - # (SELECT / UBOOT_SELECT / INCLUDE_LIBC_DEV / COPY_UBOOT, deduplicating - # against image-info.json); driving it from a config file is the next - # step, and belongs in the per-target branches above. - # - # Add another drop box by naming it here. - GENERIC_INCOMING="base-files" - - for GP_DIR in ${GENERIC_INCOMING}; do - for GP_REPO in debs debs-beta; do - if [ -d "${INCOMING_PATH}/${GP_DIR}/${GP_REPO}" ]; then - echo "## Move ${GP_DIR}/${GP_REPO} into storage" >> "$GITHUB_STEP_SUMMARY" - rsync -av --remove-source-files \ - --include='*/' \ - --include='*.deb' \ - --exclude='*' \ - --omit-dir-times --no-perms --no-group \ - "${INCOMING_PATH}/${GP_DIR}/${GP_REPO}/" "${STORAGE_PATH}/${GP_REPO}/" \ - 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" - # PIPESTATUS, not $?: the pipe through tee would otherwise report - # tee's status and hide every rsync failure. 23 is expected here - - # --no-perms/--no-group on the shared-group storage means some - # attributes legitimately do not transfer. Anything else is real, - # and must stop the run: update-repository would happily publish - # a storage tree that is missing these packages. - GP_RC=${PIPESTATUS[0]} - case "${GP_RC}" in - 0) ;; - 23) - echo "Warning: some files/attrs were not transferred (rsync 23)" >> "$GITHUB_STEP_SUMMARY" - ;; - *) - echo "::error::rsync of ${GP_DIR}/${GP_REPO} failed (rsync exit ${GP_RC})" - echo "**rsync of ${GP_DIR}/${GP_REPO} failed (exit ${GP_RC})**" >> "$GITHUB_STEP_SUMMARY" - exit 1 - ;; - esac - # --remove-source-files only removes files; clear the directories - # it emptied, and the per-repo dir itself once it is empty. - find "${INCOMING_PATH}/${GP_DIR}/${GP_REPO}" -type d -empty -delete - else - echo "## No ${GP_DIR}/${GP_REPO} in incoming, skipping" >> "$GITHUB_STEP_SUMMARY" - fi - done - # Drop the now-empty drop box. Left alone if anything remains, so a - # partial transfer stays visible for the next run. - rmdir "${INCOMING_PATH}/${GP_DIR}" 2>/dev/null || true - done - # Always sync external if [ -d "${INCOMING_PATH}/external/debs" ]; then echo "## Copy external debs " diff --git a/config/repository-kernel-selection.conf b/config/repository-kernel-selection.conf new file mode 100644 index 0000000000..507b1e3a00 --- /dev/null +++ b/config/repository-kernel-selection.conf @@ -0,0 +1,51 @@ +# +# Which kernel packages reach the apt repositories, for the targets that publish +# a curated selection rather than everything they built. +# +# Sourced by the "stable/" branch of Infrastructure: APT repositories update and +# handed to scripts/copy-kernel-packages.sh, which resolves what exists from +# image-info.json and copies the matching debs out of the incoming drop box. +# +# KERNEL_SELECT tokens are linuxfamily[:branch] , space separated, globs +# allowed. A bare family means every branch that family builds: +# +# x86 -> all branches of linuxfamily x86 (uefi-x86) +# x86:current -> only the current branch +# bcm27* -> every family whose name starts bcm27 +# +# The family is the LINUXFAMILY, which is not always the board or the +# BOARDFAMILY name - uefi-x86 builds linuxfamily "x86", uefi-arm64 builds +# "arm64", and the Raspberry Pi boards build "bcm2711". +# +# Copied for each selected (family, branch): linux-image, linux-dtb, +# linux-headers, plus linux-libc-dev when INCLUDE_LIBC_DEV is true. +# + +# Weekly stable update: the generic UEFI kernels, the Raspberry Pi one, and imx6 +# for udoo. The first three move slowly enough to be worth carrying in the +# repository between stable releases; udoo is here to give the weekly update one +# ordinary, non-critical board that gets the complete set - kernel and u-boot - +# so a regression in the full path shows up on something harmless. +KERNEL_SELECT="x86 arm64 bcm2711 imx6" + +# linux-libc-dev-- alongside the kernel. +INCLUDE_LIBC_DEV="true" + +# U-Boot is per BOARD *and* per branch - the package is +# linux-u-boot-- - so it needs its own list, and a board building +# current and edge produces two of them. Tokens are board[:branch]: +# +# udoo -> every branch udoo builds (current and edge) +# udoo:current -> only current +# +# A bare board name is what makes a board completely updatable: the kernel it +# gets from KERNEL_SELECT covers each branch, and the bootloader has to match. +# +# CAREFUL: an empty UBOOT_SELECT means EVERY board, not none - the filter is +# skipped when it is empty. With COPY_UBOOT true it must always name boards. +# The UEFI targets boot via firmware and GRUB and the Pi through its own +# bootloader, so none of them have a u-boot package to carry. udoo and cubox-i +# do, and both are imx6 - so the imx6 kernel above and these two bootloaders +# together give two complete, updatable boards. +COPY_UBOOT="true" +UBOOT_SELECT="udoo cubox-i"