From 8d4bff35f5a07781642883a0e9804a4e6b221832 Mon Sep 17 00:00:00 2001 From: "Jeremy [KK7GWY]" Date: Sat, 3 Oct 2026 10:42:15 -0700 Subject: [PATCH 1/3] ci(sanitizers): link the ASan lane against a shared core so it fits on disk. Principle VIII. The weekly ASan + UBSan lane has failed at Build for three weeks running with "final link failed: No space left on device", so it has tested nothing. Each test statically links the whole core: a Debug ASan test averages ~200 MB, and even with -g1 and compressed debug sections the tree measured 127 GB locally against ~83 GB free on the runner. New option AETHER_SHARED_CORE (default OFF; not on Windows) builds libaethercore shared with position-independent code. The ASan entry turns it on: the full local Debug ASan tree is 17 GB, all 591 tests link, and test executables average 13 MB. Running the whole suite locally under ASan gave the same failures with the shared core as with static linking, so results are unchanged. On the runner (run 37141490698) the build used 19 GB of disk and the full suite ran: 618 of 623 tests passed. TSan, release and ordinary builds keep the static core. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/sanitizers.yml | 11 ++++++++++- CMakeLists.txt | 19 ++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml index 8042b446f8..95b6933eff 100644 --- a/.github/workflows/sanitizers.yml +++ b/.github/workflows/sanitizers.yml @@ -36,6 +36,10 @@ jobs: label: ASan + UBSan image: ghcr.io/aethersdr/aethersdr-ci:latest build_type: Debug + # A Debug ASan tree with the core linked statically into ~600 + # tests outgrew the runner's disk ("No space left on device", + # three weeks running). One shared core keeps it near 17 GB. + shared_core: "ON" # ON here too, not only under TSan: the churn test's documented # failure class is a heap use-after-free on the receiver vector, # and a sequential UAF produces no TSan report — catching it is @@ -63,6 +67,7 @@ jobs: # TSan cannot coexist with ASan, so build RelWithDebInfo and # inject the TSan flags ourselves. build_type: RelWithDebInfo + shared_core: "OFF" receiver_churn: "ON" extra_cxx: "-fsanitize=thread -g3 -fno-omit-frame-pointer -O1" extra_ld: "-fsanitize=thread" @@ -122,6 +127,7 @@ jobs: # that it is out of the default graph. cmake -B build -G Ninja \ -DCMAKE_BUILD_TYPE=${{ matrix.sanitizer.build_type }} \ + -DAETHER_SHARED_CORE=${{ matrix.sanitizer.shared_core }} \ -DAETHER_ENABLE_HL2_RECEIVER_CHURN_TEST=${{ matrix.sanitizer.receiver_churn }} \ -DAETHER_ENABLE_HL2_TX_LOOPBACK_TEST=ON @@ -140,7 +146,10 @@ jobs: CFLAGS: ${{ matrix.sanitizer.extra_cxx }} CXXFLAGS: ${{ matrix.sanitizer.extra_cxx }} LDFLAGS: ${{ matrix.sanitizer.extra_ld }} - run: cmake --build build --parallel + run: | + df -h / + cmake --build build --parallel + df -h / - name: Run tests under sanitizer id: ctest diff --git a/CMakeLists.txt b/CMakeLists.txt index 871e3defc6..7bcec19516 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -143,6 +143,23 @@ if(NOT AETHERSDR_SANITIZER STREQUAL "none") message(STATUS "AetherSDR: sanitizer = ${AETHERSDR_SANITIZER} (whole tree)") unset(_aether_san_list) endif() +# Build libaethercore as one shared library instead of a static archive. Every +# test links the whole core, so a static Debug ASan build carries ~600 full +# copies (~200 MB each) and exhausts a hosted runner's disk; shared, the test +# executables average ~13 MB. For the sanitizer lane only, never for releases: +# the shipped app links the core statically. +option(AETHER_SHARED_CORE "Build libaethercore as a shared library (sanitizer CI only)" OFF) +if(AETHER_SHARED_CORE) + if(WIN32) + message(FATAL_ERROR "AETHER_SHARED_CORE is not supported on Windows") + endif() + # Vendored static libraries end up inside the shared core. + set(CMAKE_POSITION_INDEPENDENT_CODE ON) + set(_aether_core_library_type SHARED) + message(STATUS "AetherSDR: libaethercore built SHARED (AETHER_SHARED_CORE)") +else() + set(_aether_core_library_type STATIC) +endif() option(ENABLE_RTL "Enable the experimental receive-only RTL-SDR backend" ON) # Which SSB modulator the Hermes-Lite 2 transmit path is BUILT with. There is # no runtime switch and there must not be one: two silent transmit paths are @@ -1574,7 +1591,7 @@ add_subdirectory(third_party/wdsp EXCLUDE_FROM_ALL) # their feature blocks). Qt resources (resources.qrc, DFNR model qrc) stay on # the executable: rcc registration runs from the exe's static initializers and # is process-global, so core code reading ":/..." keeps working. -add_library(aethercore STATIC +add_library(aethercore ${_aether_core_library_type} ${CORE_SOURCES} ${MODEL_SOURCES} ${RNNOISE_SOURCES} From fde204849164c4d4fdee9164bcbb7d94d49897ae Mon Sep 17 00:00:00 2001 From: "Jeremy [KK7GWY]" Date: Sat, 3 Oct 2026 19:56:36 -0700 Subject: [PATCH 2/3] ci(sanitizers): count the core's consumers, keep the failure-time df, check the option per PR. Principle VIII. Review on #6136: - The comments said ~600 tests link the core. About 250 executables do; ~600 is the test count. - The after-build df ran only when the build succeeded. The step now keeps the build status and exits with it after the second df, so a disk-full failure still logs it. - Record that WDSP symbols don't reach the shared core's consumers. - static-checks' 'Sanitizer option configures' job gains a fifth case: ON must generate libaethercore.so and the default must stay .a. Without it the option's first exercise is the weekly cron. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/sanitizers.yml | 10 ++++++---- .github/workflows/static-checks.yml | 19 +++++++++++++++++++ CMakeLists.txt | 14 +++++++++----- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml index 95b6933eff..548d49737a 100644 --- a/.github/workflows/sanitizers.yml +++ b/.github/workflows/sanitizers.yml @@ -36,9 +36,9 @@ jobs: label: ASan + UBSan image: ghcr.io/aethersdr/aethersdr-ci:latest build_type: Debug - # A Debug ASan tree with the core linked statically into ~600 - # tests outgrew the runner's disk ("No space left on device", - # three weeks running). One shared core keeps it near 17 GB. + # A Debug ASan tree with the core linked statically into ~250 + # test executables outgrew the runner's disk ("No space left on + # device", three weeks running). One shared core keeps it near 17 GB. shared_core: "ON" # ON here too, not only under TSan: the churn test's documented # failure class is a heap use-after-free on the receiver vector, @@ -147,9 +147,11 @@ jobs: CXXFLAGS: ${{ matrix.sanitizer.extra_cxx }} LDFLAGS: ${{ matrix.sanitizer.extra_ld }} run: | + # The trap keeps the after-build reading when the build fails, which + # is when it matters. + trap 'df -h /' EXIT df -h / cmake --build build --parallel - df -h / - name: Run tests under sanitizer id: ctest diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml index bf83c5c5ae..ef85ffb881 100644 --- a/.github/workflows/static-checks.yml +++ b/.github/workflows/static-checks.yml @@ -141,6 +141,25 @@ jobs: fail=1 fi + # 5. AETHER_SHARED_CORE=ON (the ASan lane's link mode) must produce a + # shared core, and the default must stay static. The sanitizer + # lane is weekly, so this is the only per-PR check of the option. + # Asserted on the generated ninja file, like case 2. + if cmake -B /tmp/cfg-shared -S . -G Ninja \ + -DAETHER_SHARED_CORE=ON > /tmp/shared.log 2>&1; then + grep -q '^build libaethercore\.so' /tmp/cfg-shared/build.ninja \ + || { echo "::error::AETHER_SHARED_CORE=ON configured, but no libaethercore.so rule"; fail=1; } + else + echo "::error::-DAETHER_SHARED_CORE=ON failed to configure" + tail -30 /tmp/shared.log + fail=1 + fi + if [ -f /tmp/cfg-thread/build.ninja ] \ + && ! grep -q '^build libaethercore\.a' /tmp/cfg-thread/build.ninja; then + echo "::error::the default configure no longer builds a static libaethercore.a" + fail=1 + fi + exit "$fail" static-checks: diff --git a/CMakeLists.txt b/CMakeLists.txt index 7bcec19516..1b212df90c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -143,17 +143,21 @@ if(NOT AETHERSDR_SANITIZER STREQUAL "none") message(STATUS "AetherSDR: sanitizer = ${AETHERSDR_SANITIZER} (whole tree)") unset(_aether_san_list) endif() -# Build libaethercore as one shared library instead of a static archive. Every -# test links the whole core, so a static Debug ASan build carries ~600 full -# copies (~200 MB each) and exhausts a hosted runner's disk; shared, the test -# executables average ~13 MB. For the sanitizer lane only, never for releases: +# Build libaethercore as one shared library instead of a static archive. The +# ~250 test executables that link the core each absorb a full static copy +# (~200 MB in a Debug ASan build), which exhausts a hosted runner's disk; +# shared, they average ~13 MB. For the sanitizer lane only, never for releases: # the shipped app links the core statically. option(AETHER_SHARED_CORE "Build libaethercore as a shared library (sanitizer CI only)" OFF) if(AETHER_SHARED_CORE) if(WIN32) message(FATAL_ERROR "AETHER_SHARED_CORE is not supported on Windows") endif() - # Vendored static libraries end up inside the shared core. + # Vendored static libraries end up inside the shared core. Their symbols + # do not reach the core's consumers: a PRIVATE dependency stops at a + # shared library, and aether_wdsp is built with hidden visibility. A test + # that calls WDSP directly while linking only aethercore links against + # the static core and fails here; go through core/dsp/WdspChannel.h. set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(_aether_core_library_type SHARED) message(STATUS "AetherSDR: libaethercore built SHARED (AETHER_SHARED_CORE)") From e9255efb8ea2c92c596e6ef1df06773a60b12739 Mon Sep 17 00:00:00 2001 From: "Jeremy [KK7GWY]" Date: Sat, 3 Oct 2026 20:03:25 -0700 Subject: [PATCH 3/3] ci(sanitizers): file the tracking issue when the lane fails to build. Principle VIII. Every report step was gated on ctest's exit code, so a Configure or Build failure skipped them all: no artifact, no sticky issue. That is how the ASan lane went dark for three weeks. A new step runs only when Configure or Build failed. It writes a report naming the failed step, with a stable per-step fingerprint, and the artifact and tracking-issue steps now also run on that path. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/sanitizers.yml | 38 ++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml index 548d49737a..5af5795390 100644 --- a/.github/workflows/sanitizers.yml +++ b/.github/workflows/sanitizers.yml @@ -107,6 +107,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - name: Configure + id: configure env: # CFLAGS as well as CXXFLAGS. The tree is not all C++: WDSP and its # Win32 port, liquid-dsp, rnnoise and the other vendored C libraries @@ -132,6 +133,7 @@ jobs: -DAETHER_ENABLE_HL2_TX_LOOPBACK_TEST=ON - name: Build + id: build # The SAME sanitizer flags as Configure, because this step also runs # configure: vendored Opus is an ExternalProject_Add, so its child # CMake project configures during the parent's BUILD, when a Configure- @@ -147,11 +149,33 @@ jobs: CXXFLAGS: ${{ matrix.sanitizer.extra_cxx }} LDFLAGS: ${{ matrix.sanitizer.extra_ld }} run: | - # The trap keeps the after-build reading when the build fails, which - # is when it matters. - trap 'df -h /' EXIT + # The after-build reading must survive a failed build, which is + # when it matters. Explicit status rather than an EXIT trap: this + # step runs under the container's sh, not bash. df -h / - cmake --build build --parallel + rc=0 + cmake --build build --parallel || rc=$? + df -h / + exit "$rc" + + # A Configure or Build failure skips every step below, ctest's report + # chain included, so the lane used to go dark with no artifact and no + # tracking issue (three weeks of "No space left on device"). This step + # leaves that chain a report, and a stable per-step fingerprint so the + # sticky issue collects repeats rather than opening one per week. + - name: Report a lane that did not build + id: nobuild + if: failure() && (steps.configure.outcome == 'failure' || steps.build.outcome == 'failure') + env: + FAILED_STEP: ${{ steps.configure.outcome == 'failure' && 'Configure' || 'Build' }} + run: | + { + echo "The lane did NOT run its tests: the $FAILED_STEP step failed." + echo "No result in this run is meaningful; the step log in the run says why." + } > report.txt + cp report.txt sanitizer.log + fp=$(printf 'nobuild:%s' "$FAILED_STEP" | sha256sum | cut -c1-12) + echo "fingerprint=${fp}" >> "$GITHUB_OUTPUT" - name: Run tests under sanitizer id: ctest @@ -301,7 +325,7 @@ jobs: echo "fingerprint=${fp}" >> "$GITHUB_OUTPUT" - name: Upload sanitizer artifacts - if: steps.ctest.outputs.exit != '0' + if: (success() && steps.ctest.outputs.exit != '0') || (failure() && steps.nobuild.outcome == 'success') uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: ${{ matrix.sanitizer.id }}-${{ github.run_id }} @@ -311,12 +335,12 @@ jobs: retention-days: 30 - name: Open or update tracking issue - if: steps.ctest.outputs.exit != '0' + if: (success() && steps.ctest.outputs.exit != '0') || (failure() && steps.nobuild.outcome == 'success') uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 env: SAN_ID: ${{ matrix.sanitizer.id }} SAN_LABEL: ${{ matrix.sanitizer.label }} - FP: ${{ steps.extract.outputs.fingerprint }} + FP: ${{ steps.extract.outputs.fingerprint || steps.nobuild.outputs.fingerprint }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} with: script: |