diff --git a/.github/workflows/os-native-test-binaries.yml b/.github/workflows/os-native-test-binaries.yml new file mode 100644 index 00000000..2ecf0c8c --- /dev/null +++ b/.github/workflows/os-native-test-binaries.yml @@ -0,0 +1,296 @@ +# OS-native test binaries — manual, cost-gated evidence job. +# +# Why this exists: two credential tests are `#[ignore]`d AND env-gated, and the +# regular CI matrix never passes `-- --ignored` nor sets their env vars, so they +# have never executed anywhere: +# +# credentials::tests::os_keychain_smoke_save_import_delete_tombstone_and_redaction +# gate: AUDIO_GRAPH_RUN_OS_KEYCHAIN_SMOKE=1 (needs a real OS keychain) +# credentials::filesystem_policy::windows::tests::native_metadata_smoke_uses_only_closed_observations +# gate: AUDIO_GRAPH_WINDOWS_FILESYSTEM_SMOKE_DIR (needs real Win32 + NTFS) +# +# This builds the `--lib` test binary for ONE OS, uploads it as an artifact, and +# also attempts the test on the runner. Two independent signals per run: +# +# 1. The runner attempt shows how the test behaves on virtualized CI storage / +# a CI keychain. Non-blocking — a failure here is data, not a defect. +# 2. The downloaded binary re-runs unlimited times on real hardware (real +# internal fixed NTFS, a real login keychain) at zero further cost. +# +# (2) matters for the Windows test specifically: it asserts `internal_fixed`, +# `identity_stable`, and `access_controls_enforced` — storage-CLASS properties. +# A runner's virtual disk may legitimately classify differently from a real +# internal SSD, so green on CI is not the same claim as green on hardware. +# +# One OS per dispatch, deliberately: you pay for exactly the platform you asked +# for, and the two tests currently live on different refs (see the `ref` input). +# Dispatch twice to cover both. +# +# NEVER a PR gate. `workflow_dispatch` only, so it costs money only when asked. +# +# Third-party actions are SHA-pinned with a trailing `# vN` comment, matching +# ci.yml, so Dependabot can bump them without losing reproducibility. + +name: OS-native test binaries + +on: + workflow_dispatch: + inputs: + os: + description: Which OS to build the test binary for + type: choice + options: [windows, macos] + required: true + ref: + description: >- + Ref to build. NOTE: the Windows filesystem test lives only on the + credential-v2 branches (work/audio-graph-cred-v2-integration, + work/audio-graph-12c4-windows-credential-manager, ...). master has the + keychain test but NOT filesystem_policy/windows.rs. + type: string + default: master + test_filter: + description: >- + Override the ignored-test filter. Blank uses the per-OS default + (native_metadata_smoke on Windows, os_keychain_smoke on macOS). + type: string + default: "" + run_on_runner: + description: Also attempt the test on the CI runner (non-blocking) + type: boolean + default: true + +permissions: + contents: read + +# Serialize dispatches per OS so two runs cannot race the same cache, and let +# the two OSes proceed independently. Never cancel: a run in flight is spending +# paid runner minutes to produce evidence. +concurrency: + group: os-native-test-binaries-${{ inputs.os }} + cancel-in-progress: false + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + +jobs: + test-binary: + name: Test binary (${{ inputs.os }} @ ${{ inputs.ref }}) + # Paid runners: a hung cargo build or a hung ignored test would otherwise + # burn to the 6-hour default. 90m leaves room for a cold Windows build. + timeout-minutes: 90 + runs-on: >- + ${{ inputs.os == 'windows' + && 'blacksmith-4vcpu-windows-2025' + || 'blacksmith-6vcpu-macos-15' }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v5 + with: + ref: ${{ inputs.ref }} + # No step here pushes, and later steps run build output from an + # arbitrary ref, so do not leave GITHUB_TOKEN in .git/config. + persist-credentials: false + + # `ref` may be a mutable branch, so pin the exact commit into the + # artifact. Otherwise a downloaded binary cannot be tied to the + # implementation it tested once the branch advances. + - name: Record resolved source commit + shell: bash + env: + BUILD_REF: ${{ inputs.ref }} + run: | + sha=$(git rev-parse HEAD) + echo "SOURCE_SHA=$sha" >> "$GITHUB_ENV" + echo "Resolved $BUILD_REF -> $sha" + + # rsac is a SHA-pinned git dependency on every current branch, so unlike + # ci.yml's older jobs this needs no sibling checkout. + - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 + with: + workspaces: src-tauri + + - name: Install macOS system dependencies + if: inputs.os == 'macos' + run: brew list cmake >/dev/null 2>&1 || brew install cmake + + - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1 + if: inputs.os == 'windows' + + # Inputs reach `run:` through env, never `${{ }}` interpolation, so a + # crafted filter cannot inject shell. See + # https://github.blog/security/vulnerability-research/how-to-catch-github-actions-workflow-injections-before-attackers-do/ + - name: Resolve per-OS gates and filter + shell: bash + env: + TARGET_OS: ${{ inputs.os }} + INPUT_FILTER: ${{ inputs.test_filter }} + run: | + case "$TARGET_OS" in + windows) + default_filter=native_metadata_smoke + smoke_dir='C:\ag-smoke' + keychain_gate="" + run_env="AUDIO_GRAPH_WINDOWS_FILESYSTEM_SMOKE_DIR='C:\\ag-smoke'" + run_note="Create the directory first (mkdir C:\\ag-smoke) on an internal fixed NTFS volume. Never inside a OneDrive-synced tree - the detector then reports os_managed_cloud_root and the test correctly fails." + ;; + macos) + default_filter=os_keychain_smoke + smoke_dir="" + keychain_gate=1 + run_env="AUDIO_GRAPH_RUN_OS_KEYCHAIN_SMOKE=1" + run_note="Artifact upload drops the exec bit and browser downloads are quarantined, so first run chmod +x and xattr -d com.apple.quarantine. Needs a real unlocked login keychain (a GUI session, not SSH). The test scopes itself to a UUID service name and cleans up via ScopedOsKeychainCleanup." + ;; + *) + echo "::error::unsupported os: $TARGET_OS"; exit 1 ;; + esac + + filter="$INPUT_FILTER" + [ -n "$filter" ] || filter="$default_filter" + # A cargo test filter never needs more than a test-path token. + case "$filter" in + *[!A-Za-z0-9_:-]*) + echo "::error::test_filter must match [A-Za-z0-9_:-]+ (got: $filter)" + exit 1 ;; + esac + + { + echo "TEST_FILTER=$filter" + echo "SMOKE_DIR=$smoke_dir" + echo "KEYCHAIN_GATE=$keychain_gate" + echo "RUN_ENV=$run_env" + echo "RUN_NOTE=$run_note" + } >> "$GITHUB_ENV" + echo "Using ignored-test filter: $filter" + + - name: Build lib test binary + shell: bash + working-directory: src-tauri + env: + # build.rs reads this at BUILD time and emits /MANIFEST:EMBED plus + # /MANIFESTINPUT link args (src-tauri/build.rs:64). Without it an + # unmanifested debug MSVC test harness can abort during process + # loading, so the binary would fail to list its tests and never get + # staged. It is debug-profile-only and build.rs panics on release, + # which is why it can be set unconditionally for the windows build. + AUDIOGRAPH_EMBED_WINDOWS_TEST_MANIFEST: ${{ inputs.os == 'windows' && '1' || '' }} + run: | + # json-render-diagnostics keeps compiler errors readable on stderr + # while the JSON artifact records still land on stdout. + cargo test --locked --no-default-features --features cloud \ + --lib --no-run --message-format=json-render-diagnostics \ + > cargo-test-build.json + + # The unittest binary is the lib-kind artifact built in test profile. + bin=$(jq -r 'select(.reason=="compiler-artifact") + | select(.profile.test==true) + | select(.target.kind[]=="lib") + | .executable' cargo-test-build.json | tail -1) + [ -n "$bin" ] && [ "$bin" != "null" ] \ + || { echo "::error::could not resolve the lib test binary from cargo JSON"; exit 1; } + + # cargo emits native paths; git bash needs forward slashes on Windows. + bin="${bin//\\//}" + [ -f "$bin" ] || { echo "::error::resolved binary does not exist: $bin"; exit 1; } + echo "TEST_BIN=$bin" >> "$GITHUB_ENV" + echo "Built: $bin" + + # The whole point of this job is that these tests run nowhere today. A + # filter matching zero tests would reproduce that silence while reporting + # green, so fail loudly instead. + - name: Assert the filter matches an ignored test + shell: bash + working-directory: src-tauri + env: + BUILD_REF: ${{ inputs.ref }} + TARGET_OS: ${{ inputs.os }} + run: | + # Do not mask the exit status: if the binary cannot start (a missing + # sidecar library, for instance) grep would find nothing and the + # message below would blame the ref and OS for the wrong reason. + list_status=0 + "$TEST_BIN" --ignored --list > ignored-tests.txt 2>&1 || list_status=$? + if [ "$list_status" -ne 0 ]; then + echo "::error::the test binary failed to list its tests (exit $list_status) — it likely could not start" + cat ignored-tests.txt + exit 1 + fi + if grep -F "$TEST_FILTER" ignored-tests.txt; then + echo "--- all ignored tests on this ref/OS ---" + cat ignored-tests.txt + else + echo "::error::no ignored test matches '$TEST_FILTER' on ref '$BUILD_REF' ($TARGET_OS)." + echo "::error::the Windows filesystem test exists only on the credential-v2 branches." + echo "--- ignored tests actually present ---" + cat ignored-tests.txt + exit 1 + fi + + - name: Stage binary, sidecar libraries, and run instructions + shell: bash + working-directory: src-tauri + env: + BUILD_REF: ${{ inputs.ref }} + TARGET_OS: ${{ inputs.os }} + run: | + mkdir -p dist + cp "$TEST_BIN" dist/ + # A bare test exe can fail to load if the crate links sidecar libs + # (WebView2Loader.dll and friends). Ship them alongside it. + for f in target/debug/*.dll target/debug/*.dylib \ + target/debug/deps/*.dll target/debug/deps/*.dylib; do + [ -e "$f" ] && cp "$f" dist/ + done + cp ignored-tests.txt dist/ + + binname=$(basename "$TEST_BIN") + { + echo "# $binname" + echo + echo "ref \`$BUILD_REF\` | os \`$TARGET_OS\` | filter \`$TEST_FILTER\`" + echo + echo "Built from commit \`$SOURCE_SHA\`. Cite that SHA, not the branch" + echo "name, when recording this run as evidence — the branch moves." + echo + echo "Keep the sidecar libraries in this directory next to the binary." + echo "\`ignored-tests.txt\` lists every ignored test this binary contains." + echo + echo '## Run' + echo + echo '```bash' + echo "$RUN_ENV \\" + echo " ./$binname --ignored --nocapture --test-threads=1 $TEST_FILTER" + echo '```' + echo + echo "$RUN_NOTE" + } > dist/RUN.md + + cat dist/RUN.md + ls -la dist/ + + - name: Upload test binary + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: os-native-test-binary-${{ inputs.os }}-${{ env.SOURCE_SHA }} + if-no-files-found: error + retention-days: 14 + path: src-tauri/dist/ + + # Non-blocking. On Windows this runs against the runner's VIRTUAL disk, so + # a storage-class assertion failure is expected-and-informative rather than + # a defect: it is exactly why the artifact above exists. + - name: Attempt the test on the runner (non-blocking) + if: inputs.run_on_runner + continue-on-error: true + shell: bash + working-directory: src-tauri + env: + AUDIO_GRAPH_RUN_OS_KEYCHAIN_SMOKE: ${{ env.KEYCHAIN_GATE }} + AUDIO_GRAPH_WINDOWS_FILESYSTEM_SMOKE_DIR: ${{ env.SMOKE_DIR }} + run: | + if [ -n "$AUDIO_GRAPH_WINDOWS_FILESYSTEM_SMOKE_DIR" ]; then + mkdir -p "$(cygpath -u "$AUDIO_GRAPH_WINDOWS_FILESYSTEM_SMOKE_DIR")" + echo "Runner smoke dir: $AUDIO_GRAPH_WINDOWS_FILESYSTEM_SMOKE_DIR (virtualized CI storage)" + fi + "$TEST_BIN" --ignored --nocapture --test-threads=1 "$TEST_FILTER"