tests.yml

name: CI

on:
  push:
    # train/** branches are the CI-gated landing path: a train is pushed
    # there, CI runs on all platforms, and master fast-forwards only to a sha
    # that already carries a green check.
    branches: ["train/**"]
  # The push gate asks whether a change is good. This run asks whether what
  # already landed is still good on the current runner images and toolchains.
  schedule:
    - cron: "0 6 * * *"
  pull_request:
  workflow_dispatch:

# Per-ref group: a repush to the same train cancels its superseded run, while
# a train push never cancels an in-flight master run (a shared group would).
concurrency:
  group: ci-${{ github.ref }}
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always
  LLAMA_BUILD_GATE_BUDGET_SECONDS: "900"
  LLAMA_BUILD_EVIDENCE_CONTRACT: "nonmac-build-gates-v1"
  LLAMA_CUDA_TOOLKIT_PACKAGE: "cuda-toolkit-12-6"
  LLAMA_CUDA_TOOLKIT_DEB_VERSION: "12.6.1-1"
  LLAMA_CUDA_KEYRING_SHA256: "d2a6b11c096396d868758b86dab1823b25e14d70333f1dfa74da5ddaf6a06dba"
  LLAMA_CUDA_REPO: "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64"
  LLAMA_VULKAN_SDK_VERSION: "1.4.357.0"
  LLAMA_VULKAN_SDK_ARCHIVE_URL: "https://sdk.lunarg.com/sdk/download/1.4.357.0/windows/vulkansdk-windows-X64-1.4.357.0.exe"
  LLAMA_VULKAN_SDK_ARCHIVE_SHA256: "81f474711e9042f4cd22b31b2f7a8870db2e428b21586fb43dd80150be97310d"
  # GPU build evidence is a readiness-matrix contract: GPU jobs run only on
  # workflow_dispatch, retain toolchain/package and binary digests plus output,
  # and mark skipped execution as not a pass. CPU gates remain push-triggered.
  # Production crates gated by CI. Bench lanes are excluded on purpose:
  # heavy engine deps (burn, candle, python venvs) with no gate value.
  # Every workspace member that compiles on a runner without Metal or CUDA. The
  # list is explicit rather than --workspace because the bench lanes and the
  # macOS- and CUDA-only engines cannot build here, but anything portable left
  # out is simply unbuilt and untested: the operator CLI shipped for weeks with
  # its tests never once running in CI.
  SYNAPSE_CRATES: -p synapse-core -p synapse-module -p synapse-worker-llama -p owned-decode-worker -p synapse-worker-decode -p synapse-opctl -p synapse-engine-cuda -p synapse-worker-cuda

jobs:
  test:
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: linux
            runner: ubuntu-24.04
          - name: windows
            runner: windows-2025
          # A macOS Metal/ANE lane requires Apple Silicon with real GPU/ANE
          # access (hosted macOS runners virtualize the GPU), so that coverage
          # runs on developer machines. Restore by registering a self-hosted
          # Apple Silicon runner labeled `macos-metal` and adding:
          # - name: macos-metal
          #   runner: [self-hosted, macos-metal]
    name: ${{ matrix.name }}
    runs-on: ${{ matrix.runner }}
    steps:
      - name: Checkout synapse
        uses: actions/checkout@v4
        with:
          path: synapse

      # Validate the workflow before toolchain setup spends time.
      - name: Check train CI preconditions
        if: matrix.name == 'linux'
        shell: bash
        run: |
          bash synapse/scripts/check-train-preconditions.sh
          python3 synapse/scripts/test-ci-e2e-output.py

      - name: Check release-candidate guards
        if: matrix.name == 'linux'
        shell: bash
        run: python3 synapse/scripts/check-release-candidate.py --self-test

      - name: Check tag release gates
        if: matrix.name == 'linux'
        working-directory: synapse
        run: python3 -m unittest discover -s crates/synapse-release-checks -v

      # The release benchmark producer must keep emitting reports the tag
      # validator accepts; its fixtures run without any hardware.
      - name: Release benchmark producer tests
        if: matrix.name == 'linux'
        working-directory: synapse
        run: python3 -m unittest discover -s bench/harness -p 'test_release_matrix.py' -v

      - uses: dtolnay/rust-toolchain@stable
        with:
          components: clippy, rustfmt

      # bench/parity is its own workspace (kept out of the root lockfile), so the
      # workspace test run never reaches it. Its evaluator and manifest
      # validator gate releases, so test them here.
      - name: Restore parity Rust cache
        if: matrix.name == 'linux'
        uses: Swatinem/rust-cache@v2
        with:
          workspaces: synapse/bench/parity -> target
          prefix-key: parity-v1

      - name: Parity manifest and evaluator tests
        if: matrix.name == 'linux'
        working-directory: synapse/bench/parity
        timeout-minutes: 20
        run: cargo test --locked

      # Direct prebuilt-binary install instead of taiki-e/install-action: the
      # action's bootstrap script is broken by a runner-image bash regression
      # (actions/partner-runner-images#169) and took every lane down with it.
      # get.nexte.st serves the same prebuilt archives without a bootstrap layer.
      - name: Install cargo-nextest (linux)
        if: matrix.name == 'linux'
        shell: bash
        timeout-minutes: 10
        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"

      - name: Install cargo-nextest (windows)
        if: matrix.name == 'windows'
        shell: pwsh
        # Windows curl uses schannel, which by default fails the whole TLS
        # handshake when the certificate's revocation server can't be reached
        # (CRYPT_E_REVOCATION_OFFLINE); a scheduled run failed that way.
        # --ssl-revoke-best-effort still checks revocation when the server
        # answers, and the retries cover a transient network failure. The exit
        # check stops a failed download from surfacing later as a missing-file
        # error from tar.
        run: |
          curl.exe --fail --location --silent --show-error --ssl-revoke-best-effort `
            --retry 3 --retry-delay 10 --retry-all-errors `
            --output nextest.tar.gz https://get.nexte.st/latest/windows-tar
          if ($LASTEXITCODE -ne 0) { throw "cargo-nextest download failed (curl exit $LASTEXITCODE)" }
          tar zxf nextest.tar.gz -C "$env:USERPROFILE\.cargo\bin"
          Remove-Item nextest.tar.gz

      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: synapse
          # v2: CRT flavor lives in cmake's cached configure (CMakeCache.txt
          # inside OUT_DIR) — a cache from the MD era keeps producing MD
          # objects even after LLAMA_STATIC_CRT changes, because cmake reruns
          # against its cached configuration. Bump to force clean configure.
          prefix-key: v2-static-crt

      # llama-cpp-sys-2 needs cmake everywhere and libclang (bindgen) on
      # linux/windows. GitHub/Blacksmith images ship LLVM + CMake; these
      # steps only pin the env vars the build scripts read.
      - name: Windows build env (libclang for bindgen)
        if: matrix.name == 'windows'
        shell: pwsh
        run: |
          # GitHub-hosted windows images ship LLVM, so the existence guard
          # usually skips this entirely; the retrying install stays for image
          # drift. choco's CDN has failed with a bare exit 1 before — retry so
          # a red here means the failure survived three attempts, not one bad
          # CDN draw.
          if (!(Test-Path "C:\Program Files\LLVM\bin\libclang.dll")) {
            $attempts = 0
            while ($true) {
              $attempts++
              choco install llvm -y --no-progress | Out-Null
              if ($LASTEXITCODE -eq 0) { break }
              if ($attempts -ge 3) { throw "choco install llvm failed after $attempts attempts" }
              Start-Sleep -Seconds 20
            }
          }
          echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" >> $env:GITHUB_ENV

      - name: MiniLM e2e fixture cache
        id: fixture-cache
        uses: actions/cache@v4
        with:
          path: ~/.cache/huggingface/hub/models--Qdrant--all-MiniLM-L6-v2-onnx
          key: minilm-onnx-fixture-v1

      - name: Download MiniLM e2e fixture
        if: steps.fixture-cache.outputs.cache-hit != 'true'
        shell: bash
        timeout-minutes: 10
        run: |
          snap="$HOME/.cache/huggingface/hub/models--Qdrant--all-MiniLM-L6-v2-onnx/snapshots/main"
          mkdir -p "$snap"
          base="https://huggingface.co/Qdrant/all-MiniLM-L6-v2-onnx/resolve/main"
          for f in config.json model.onnx special_tokens_map.json tokenizer.json tokenizer_config.json vocab.txt; do
            curl -sSfL "$base/$f" -o "$snap/$f"
          done

      # The repository is public, so a tracked home path, hostname or credential
      # marker is a disclosure rather than untidiness. This ran only from a
      # developer's shell until a leaked worktree path reached master, so it is
      # a gate here: one lane is enough because the check reads tracked files.
      - name: public banlist
        if: matrix.name == 'linux'
        working-directory: synapse
        run: bash scripts/public-banlist-check.sh

      # Crates from other CortexKit repos come from crates.io or a pinned git
      # commit, never a path into a sibling checkout, whose releases would
      # leave Cargo.lock stale. The self-test proves the check can refuse.
      - name: no external path dependencies
        if: matrix.name == 'linux'
        working-directory: synapse
        run: |
          bash scripts/check-no-external-path-deps.sh --self-test
          bash scripts/check-no-external-path-deps.sh

      # The ANE campaign harness pins the probe's comparator, gate and row set by
      # digest, and refuses every candidate when they drift. A spike merge once
      # changed that code without re-pinning and nothing noticed until a later
      # measurement run. The self-test needs only Python and the tracked files.
      - name: ANE campaign harness self-test
        if: matrix.name == 'linux'
        working-directory: synapse
        run: bash bench/campaign/ane-direct-embed-harness.sh --self-test .

      - name: fmt
        working-directory: synapse
        shell: bash
        run: |
          set -euo pipefail
          # `cargo fmt --all` follows path dependencies OUT of the workspace, so it
          # formatted-checked ../commons and ../subconscious from here and could
          # fail this job on a sibling's mid-edit file. Enumerate our own members
          # and pass them explicitly. cargo tree is used rather than jq over
          # cargo metadata because jq is not present on the windows runner and
          # this step must behave identically on every leg.
          members=$(cargo tree --workspace --depth 0 --prefix none | awk 'NF { print $1 }' | sort -u)
          packages=()
          while IFS= read -r member; do
            if [[ -n "$member" ]]; then
              packages+=(-p "$member")
            fi
          done <<< "$members"
          if (( ${#packages[@]} == 0 )); then
            echo "error: cargo tree returned an empty workspace member list; refusing vacuous fmt check" >&2
            exit 1
          fi
          echo "Formatting $((${#packages[@]} / 2)) workspace members"
          cargo fmt --check "${packages[@]}"

      - name: clippy
        working-directory: synapse
        shell: bash
        run: |
          crates="$SYNAPSE_CRATES"
          cargo clippy $crates --all-targets -- -D warnings

      # Windows tests run --release: rust links the release CRT even in debug
      # builds, while cc-built C++ deps (esaxx, onig) follow the cargo profile
      # — release everywhere is the only combination where every object agrees
      # on MD_DynamicRelease (llama.cpp's cmake default matches cc's release
      # default and rust's msvcrt default).
      - name: nextest
        working-directory: synapse
        shell: bash
        run: |
          set -euo pipefail
          crates="$SYNAPSE_CRATES"
          profile=""
          if [ "${{ matrix.name }}" = "windows" ]; then
            profile="--release"
          fi
          # Partition the suite so optional worker checkpoints cannot be mistaken
          # for missing mandatory e2e fixtures. Every test still runs only once.
          cargo nextest run $crates $profile --retries 0 -E 'not binary(skeleton_e2e)'
          cargo nextest run $crates $profile --retries 0 -E 'binary(skeleton_e2e)' --no-capture 2>&1 | tee skeleton-e2e.output

      - name: Linux llama CPU build gate
        if: matrix.name == 'linux'
        working-directory: synapse
        shell: bash
        run: |
          set -o pipefail
          mkdir -p build-gates
          command='cargo build -p synapse-worker-llama --no-default-features --features cpu --release'
          log='build-gates/linux-x86_64-gnu-llama-cpu.output'
          set +e
          bash -c "$command" 2>&1 | tee "$log"
          status=${PIPESTATUS[0]}
          set -e
          binary='target/release/ck-synapse-worker-llama'
          digest='unavailable'
          if [ -f "$binary" ]; then digest=$(sha256sum "$binary" | cut -d ' ' -f1); fi
          {
            echo "target_triple=x86_64-unknown-linux-gnu"
            echo "command=$command"
            echo "rustc=$(rustc --version)"
            echo "cargo=$(cargo --version)"
            echo "machine_image=${ImageOS:-unknown}"
            echo "budget_seconds=$LLAMA_BUILD_GATE_BUDGET_SECONDS"
            echo "evidence_contract=$LLAMA_BUILD_EVIDENCE_CONTRACT"
            echo "binary=$binary"
            echo "sha256=$digest"
            echo "execution_status=$([ "$status" -eq 0 ] && echo executed_successful || echo failed)"
            echo "successful_output=$log"
            echo "readiness_matrix_trigger=${{ github.event_name }}"
            echo "skipped_is_not_pass=true"
          } > build-gates/linux-x86_64-gnu-llama-cpu.txt
          if [ "$status" -ne 0 ]; then exit "$status"; fi

      - name: Windows llama CPU build gate
        if: matrix.name == 'windows'
        working-directory: synapse
        shell: pwsh
        run: |
          New-Item -ItemType Directory -Force build-gates | Out-Null
          $command = 'cargo build -p synapse-worker-llama --no-default-features --features cpu --release'
          $log = 'build-gates/windows-x86_64-msvc-llama-cpu.output'
          & cargo build -p synapse-worker-llama --no-default-features --features cpu --release 2>&1 | Tee-Object -FilePath $log
          $status = $LASTEXITCODE
          $binary = 'target/release/ck-synapse-worker-llama.exe'
          $digest = if (Test-Path $binary) { (Get-FileHash $binary -Algorithm SHA256).Hash.ToLowerInvariant() } else { 'unavailable' }
          $execution = if ($status -eq 0) { 'executed_successful' } else { 'failed' }
          @"
          target_triple=x86_64-pc-windows-msvc
          command=$command
          rustc=$(rustc --version)
          cargo=$(cargo --version)
          machine_image=$env:ImageOS
          budget_seconds=$env:LLAMA_BUILD_GATE_BUDGET_SECONDS
          evidence_contract=$env:LLAMA_BUILD_EVIDENCE_CONTRACT
          binary=$binary
          sha256=$digest
          execution_status=$execution
          successful_output=$log
          readiness_matrix_trigger=$env:GITHUB_EVENT_NAME
          skipped_is_not_pass=true
          "@ | Set-Content build-gates/windows-x86_64-msvc-llama-cpu.txt
          if ($status -ne 0) { exit $status }

      - name: Retain llama build-gate evidence
        if: always() && (matrix.name == 'linux' || matrix.name == 'windows')
        uses: actions/upload-artifact@v4
        with:
          name: llama-build-gates-${{ matrix.name }}-${{ github.run_id }}
          path: synapse/build-gates
          if-no-files-found: ignore

      # Hollow-green guard: fixture e2e "pass" with a skip message when the
      # snapshot is missing; fail the lane loudly instead of shipping air.
      - name: Assert no silently-skipped e2e
        working-directory: synapse
        shell: bash
        run: |
          # Read the first run's transcript instead of loading every fixture again.
          test -s skeleton-e2e.output
          out=$(<skeleton-e2e.output)
          # Not `echo "$out" | grep -q`: this shell runs with pipefail, grep -q
          # closes the pipe on its first match, and a producer still writing a
          # multi-megabyte nextest transcript then dies with SIGPIPE (141), so
          # the pipeline reads FALSE exactly when the needle is present — the
          # guard would pass on the one output it exists to catch. A here-string
          # has no writer to signal. The exit code is captured and stated, not
          # branched on blind, so an unexpected code fails loudly too. The
          # `|| skip_rc=$?` form is load-bearing: this shell also runs with
          # errexit, which would abort the step on grep's exit 1 (needle
          # absent, the green case) before the code was ever read.
          skip_rc=0
          grep -q "skipping" <<<"$out" || skip_rc=$?
          case "$skip_rc" in
            0)
              grep "skipping" <<<"$out"
              echo "::error::fixture e2e silently skipped — fixture staging is broken on this runner"
              exit 1
              ;;
            1) ;;
            *)
              echo "::error::skip guard could not read the nextest output (grep exit $skip_rc)"
              exit 1
              ;;
          esac

      # When a self-hosted `macos-metal` runner returns, restore the
      # checkpoint-gated owned-decode battery steps for that lane: build
      # synapse-worker-decode with the Xcode toolchain, assert the pinned
      # Qwen3/LFM2 checkpoints exist in the runner's HF cache, gate on
      # loadavg, then run the owned_decode_parity, worker_transport, and
      # skeleton_e2e ignored-only batteries.

  linux-llama-cuda-manual:
    if: github.event_name == 'workflow_dispatch'
    name: linux-llama-cuda-manual-gate
    # ubuntu-24.04 pinned (not -latest): the CUDA repo below is the ubuntu2404
    # channel and must move together with the image.
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout synapse
        uses: actions/checkout@v4
        with:
          path: synapse

      - uses: dtolnay/rust-toolchain@stable
        with:
          components: rustfmt

      - name: Provision pinned NVIDIA CUDA toolkit
        shell: bash
        timeout-minutes: 20
        run: |
          set -euo pipefail
          keyring="$RUNNER_TEMP/cuda-keyring_1.1-1_all.deb"
          curl -fsSL "$LLAMA_CUDA_REPO/cuda-keyring_1.1-1_all.deb" -o "$keyring"
          keyring_digest=$(sha256sum "$keyring" | cut -d ' ' -f1)
          test "$keyring_digest" = "$LLAMA_CUDA_KEYRING_SHA256"
          sudo dpkg -i "$keyring"
          sudo apt-get update
          sudo apt-get install --download-only --reinstall -y --no-install-recommends \
            "$LLAMA_CUDA_TOOLKIT_PACKAGE=$LLAMA_CUDA_TOOLKIT_DEB_VERSION"
          mkdir -p "$GITHUB_WORKSPACE/synapse/build-gates"
          sha256sum "$keyring" > "$GITHUB_WORKSPACE/synapse/build-gates/linux-cuda-keyring.sha256"
          # -maxdepth 1 is load-bearing: apt's archives/partial subdirectory is
          # root-only, so an unprivileged descent into it exits find non-zero on
          # EACCES and pipefail kills this step after every package has already
          # downloaded. The .deb files land directly in archives/; partial/ only
          # ever holds in-flight downloads.
          find /var/cache/apt/archives -maxdepth 1 -type f -name '*.deb' -print0 \
            | sort -z \
            | xargs -0 -r sha256sum \
            > "$GITHUB_WORKSPACE/synapse/build-gates/linux-cuda-packages.sha256"
          sudo apt-get install -y --no-install-recommends \
            "$LLAMA_CUDA_TOOLKIT_PACKAGE=$LLAMA_CUDA_TOOLKIT_DEB_VERSION"
          export PATH=/usr/local/cuda-12.6/bin:$PATH
          export CUDAToolkit_ROOT=/usr/local/cuda-12.6
          echo "PATH=$PATH" >> "$GITHUB_ENV"
          echo "CUDAToolkit_ROOT=$CUDAToolkit_ROOT" >> "$GITHUB_ENV"
          nvcc --version

      - name: Linux llama CUDA manual build gate
        working-directory: synapse
        shell: bash
        run: |
          set -o pipefail
          command='cargo build -p synapse-worker-llama --no-default-features --features cuda --release'
          log='build-gates/linux-x86_64-gnu-llama-cuda.output'
          set +e
          bash -c "$command" 2>&1 | tee "$log"
          status=${PIPESTATUS[0]}
          set -e
          binary='target/release/ck-synapse-worker-llama'
          digest='unavailable'
          if [ -f "$binary" ]; then digest=$(sha256sum "$binary" | cut -d ' ' -f1); fi
          {
            echo "target_triple=x86_64-unknown-linux-gnu"
            echo "command=$command"
            echo "rustc=$(rustc --version)"
            echo "cargo=$(cargo --version)"
            echo "cuda_repo=$LLAMA_CUDA_REPO"
            echo "cuda_package=$LLAMA_CUDA_TOOLKIT_PACKAGE=$LLAMA_CUDA_TOOLKIT_DEB_VERSION"
            echo "cuda_keyring_sha256=$LLAMA_CUDA_KEYRING_SHA256"
            echo "cuda_toolkit=$(nvcc --version 2>&1)"
            echo "machine_image=${ImageOS:-unknown}"
            echo "budget_seconds=$LLAMA_BUILD_GATE_BUDGET_SECONDS"
            echo "evidence_contract=$LLAMA_BUILD_EVIDENCE_CONTRACT"
            echo "binary=$binary"
            echo "sha256=$digest"
            echo "execution_status=$([ "$status" -eq 0 ] && echo executed_successful || echo failed)"
            echo "successful_output=$log"
            echo "readiness_matrix_trigger=workflow_dispatch"
            echo "skipped_is_not_pass=true"
            echo "manual_gate=mandatory"
            echo "package_digests=build-gates/linux-cuda-packages.sha256"
            echo "keyring_digest=build-gates/linux-cuda-keyring.sha256"
          } > build-gates/linux-x86_64-gnu-llama-cuda.txt
          if [ "$status" -ne 0 ]; then exit "$status"; fi

      - name: Retain Linux CUDA gate evidence
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: llama-build-gates-linux-cuda-manual-${{ github.run_id }}
          path: synapse/build-gates
          if-no-files-found: error

  windows-llama-vulkan-manual:
    if: github.event_name == 'workflow_dispatch'
    name: windows-llama-vulkan-manual-gate
    runs-on: windows-2025
    steps:
      - name: Checkout synapse
        uses: actions/checkout@v4
        with:
          path: synapse

      - uses: dtolnay/rust-toolchain@stable
        with:
          components: rustfmt

      # A stalled download here previously ran for 2h26m with no output and no
      # failure, because Invoke-WebRequest has no default timeout and the step
      # had no bound. The timeout converts a stall into a fast, named failure;
      # the retry covers the transient case. The SDK host sits behind a CDN that
      # can stall a datacenter client indefinitely, which is invisible from a
      # developer machine where the same URL serves fine.
      - name: Provision pinned Vulkan SDK archive
        shell: pwsh
        timeout-minutes: 20
        run: |
          $ErrorActionPreference = 'Stop'
          $ProgressPreference = 'SilentlyContinue'
          $installer = Join-Path $env:RUNNER_TEMP "VulkanSDK-$env:LLAMA_VULKAN_SDK_VERSION-Installer.exe"
          # curl.exe rather than Invoke-WebRequest: the same URL serves this
          # 175 MB installer fine from a developer machine and from NVIDIA's
          # host in the sibling gate, but stalls indefinitely here. The SDK is
          # served through a CDN that treats an unadorned PowerShell client from
          # a datacenter range differently, so this sends a conventional
          # user-agent and bounds every phase of the transfer. --fail turns an
          # HTML challenge page into a non-zero exit instead of a file that then
          # fails the digest check with a misleading message.
          curl.exe --fail --location --silent --show-error --ssl-revoke-best-effort `
            --connect-timeout 30 --max-time 600 `
            --retry 3 --retry-delay 15 --retry-all-errors `
            --user-agent 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)' `
            --output $installer $env:LLAMA_VULKAN_SDK_ARCHIVE_URL
          if ($LASTEXITCODE -ne 0) { throw "Vulkan SDK download failed (curl exit $LASTEXITCODE)" }
          $size = (Get-Item $installer).Length
          Write-Host "downloaded $size bytes"
          $digest = (Get-FileHash $installer -Algorithm SHA256).Hash.ToLowerInvariant()
          if ($digest -ne $env:LLAMA_VULKAN_SDK_ARCHIVE_SHA256) {
            throw "Vulkan SDK installer digest $digest does not match pinned $env:LLAMA_VULKAN_SDK_ARCHIVE_SHA256"
          }
          # /S and /D= are NSIS flags. LunarG moved to the Qt installer framework
          # in SDK 1.2.189.1 and documents those flags as replaced; a 1.3.x
          # installer does not recognise them, so it falls back to opening its
          # GUI and waits for input that never comes on a runner. That is the
          # real reason this gate consumed 2h26m without output, and it is why
          # the earlier read of "the download stalls" was wrong: with the
          # transfer now reporting its byte count, the log shows all 175 MB
          # arriving and the time going to the installer instead.
          # No copy_only=1: llama.cpp's own Vulkan CI, wgpu's install action and
          # every other reference invocation install normally, and this gate has
          # no reason to differ from the recipe its own dependency is built with
          # upstream. The runner is ephemeral, so registry and PATH writes cost
          # nothing here.
          $install = Join-Path $env:RUNNER_TEMP "vulkan-sdk-$env:LLAMA_VULKAN_SDK_VERSION"
          $process = Start-Process -FilePath $installer -Wait -PassThru -ArgumentList @(
            '--root', $install,
            '--accept-licenses',
            '--default-answer',
            '--confirm-command', 'install'
          )
          if ($process.ExitCode -ne 0) { throw "Vulkan SDK installer failed with exit code $($process.ExitCode)" }
          $sdk = @((Get-Item $install), (Get-ChildItem $install -Directory -Recurse)) |
            Where-Object { Test-Path (Join-Path $_.FullName 'Include\vulkan\vulkan.h') } |
            Select-Object -First 1
          if (-not $sdk) { throw 'Vulkan SDK installer did not provide pinned headers' }
          $sdkPath = $sdk.FullName
          "VULKAN_SDK=$sdkPath" >> $env:GITHUB_ENV
          "LLAMA_VULKAN_SDK_ARCHIVE_SHA256=$digest" >> $env:GITHUB_ENV
          "PATH=$sdkPath\Bin;$env:PATH" >> $env:GITHUB_ENV
          New-Item -ItemType Directory -Force "$env:GITHUB_WORKSPACE\synapse\build-gates" | Out-Null
          "installer=$installer`nsdk=$sdkPath`nsha256=$digest`nrevision=$env:LLAMA_VULKAN_SDK_VERSION" |
            Set-Content "$env:GITHUB_WORKSPACE\synapse\build-gates\windows-vulkan-sdk.txt"
          & "$sdkPath\Bin\glslc.exe" --version

      # Ninja is not on the hosted Windows image by default, and the build step
      # below selects it as the CMake generator. Retried because the Chocolatey
      # CDN has flaked on this repository before.
      - name: Install Ninja
        shell: pwsh
        timeout-minutes: 10
        run: |
          if (Get-Command ninja -ErrorAction SilentlyContinue) {
            Write-Host "ninja already present: $((Get-Command ninja).Source)"
          } else {
            $attempts = 0
            while ($true) {
              $attempts++
              choco install ninja -y --no-progress | Out-Null
              if ($LASTEXITCODE -eq 0) { break }
              if ($attempts -ge 3) { throw "choco install ninja failed after $attempts attempts" }
              Start-Sleep -Seconds 20
            }
          }
          ninja --version

      - name: Windows llama Vulkan manual build gate
        working-directory: synapse
        shell: pwsh
        env:
          # ggml-vulkan builds its shader generator as a CMake ExternalProject.
          # Under the default Visual Studio generator MSBuild runs those steps
          # as parallel projects, and the install step reached cmake_install.cmake
          # before the configure step had written it — the log interleaves
          # "4>Performing install step" with "3>Performing configure step" and
          # then fails with "Not a file". Ninja orders the sub-build correctly,
          # and is what llama.cpp's own Vulkan workflow uses on Windows for the
          # same build.
          CMAKE_GENERATOR: Ninja
          # ggml-vulkan nests a shader-generator ExternalProject deep inside the
          # cargo build directory, and from the default workspace path that
          # reaches 239 characters against CMake's 250-character budget for a
          # full object path (CMAKE_OBJECT_PATH_MAX), so object files cannot be
          # placed and the build fails. This is NOT the 260-character Windows
          # MAX_PATH; it is CMake's own shorter limit, which is why measuring a
          # single file against 260 was the wrong check. A short target root
          # buys back about sixty characters of prefix.
          CARGO_TARGET_DIR: D:\t
        run: |
          New-Item -ItemType Directory -Force build-gates | Out-Null
          $command = 'cargo build -p synapse-worker-llama --no-default-features --features vulkan --release'
          $log = 'build-gates/windows-x86_64-msvc-llama-vulkan.output'
          & cargo build -p synapse-worker-llama --no-default-features --features vulkan --release 2>&1 | Tee-Object -FilePath $log
          $status = $LASTEXITCODE
          $binary = Join-Path $env:CARGO_TARGET_DIR 'release/ck-synapse-worker-llama.exe'
          $digest = if (Test-Path $binary) { (Get-FileHash $binary -Algorithm SHA256).Hash.ToLowerInvariant() } else { 'unavailable' }
          $execution = if ($status -eq 0) { 'executed_successful' } else { 'failed' }
          @"
          target_triple=x86_64-pc-windows-msvc
          command=$command
          rustc=$(rustc --version)
          cargo=$(cargo --version)
          vulkan_sdk=$env:VULKAN_SDK
          vulkan_sdk_revision=$env:LLAMA_VULKAN_SDK_VERSION
          vulkan_sdk_archive=$env:LLAMA_VULKAN_SDK_ARCHIVE_URL
          vulkan_sdk_archive_sha256=$env:LLAMA_VULKAN_SDK_ARCHIVE_SHA256
          vulkan_sdk_archive_digest_recorded=true
          machine_image=$env:ImageOS
          budget_seconds=$env:LLAMA_BUILD_GATE_BUDGET_SECONDS
          evidence_contract=$env:LLAMA_BUILD_EVIDENCE_CONTRACT
          binary=$binary
          sha256=$digest
          execution_status=$execution
          successful_output=$log
          readiness_matrix_trigger=workflow_dispatch
          skipped_is_not_pass=true
          manual_gate=mandatory
          "@ | Set-Content build-gates/windows-x86_64-msvc-llama-vulkan.txt
          if ($status -ne 0) { exit $status }

      - name: Retain Windows Vulkan gate evidence
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: llama-build-gates-windows-vulkan-manual-${{ github.run_id }}
          path: synapse/build-gates
          if-no-files-found: error

  windows-owned-cuda-manual:
    if: github.event_name == 'workflow_dispatch'
    name: windows-owned-cuda-manual-gate
    # The owned-CUDA engine (crates/synapse-engine-cuda) is the VRAM-resident
    # embedding lane: safetensors weights held on the GPU, CUDA Graphs, PTX
    # targeting virtual arch compute_75. It has never had a Windows build
    # gate, so nothing in CI proved its build script could even find nvcc on
    # MSVC — nvcc needs the .exe suffix there, and -Xcompiler=-fPIC is a
    # cl-fatal unknown option. Both are pinned by this lane.
    #
    # The toolkit is assembled from NVIDIA's redist archives (per-component,
    # SHA256-pinned, no installer, no driver) merged into one root — the same
    # recipe llama.cpp's windows-setup-cuda action uses. ubuntu-24.04's apt
    # approach has no Windows analogue, and the network-bound CI runner is
    # where this belongs rather than a developer machine.
    runs-on: windows-2025
    env:
      OWNED_CUDA_REDIST_BASE: https://developer.download.nvidia.com/compute/cuda/redist
      # CUDA 13.2.1 redist. 13.x is the line whose runtime DLLs
      # (cudart64_13 / cublas64_13 / cublasLt64_13) llama.cpp's Windows CUDA
      # builds already ship, so a worker built against it drops into an
      # existing CUDA-13 deployment without dragging a second runtime.
      # The engine floor (driver API >= 12040, virtual arch compute_75) is
      # satisfied: 13.x still targets compute_75 as its minimum live arch.
      OWNED_CUDA_VERSION: 13.2.1
      # component version + windows-x86_64 sha256 from redistrib_13.2.1.json.
      # nvcc/crt/nvvm/cccl are the compile-time set (the crt and cccl headers
      # are included by crt/math_functions.h chains); cudart/libcublas supply
      # the import libraries the link flags name and the runtime DLLs.
      OWNED_CUDA_NVCC_SHA256: da33f46a1a907a12abd0c192bbe907057b1e2269fdccfc778101fb02161e1c59
      OWNED_CUDA_CRT_SHA256: 0e19f9d23451d77e32794d53bb110a4eecb26d9542391dbb3f997a688c9ddecc
      OWNED_CUDA_NVVM_SHA256: 2cbd83a3d8bd594cb53cb1b7d0e129b0931f21915aa61c71c7491374b23e9b62
      OWNED_CUDA_CCCL_SHA256: 4c799e2c502ccc9a712d90baa102d9a2f6c50d6303f9e07a227ef4756c4d0a6a
      OWNED_CUDA_CUDART_SHA256: 8c7f187543545cefdbf55f66ecb8c990e159e38a78694fda50e251fcb2f4fe5f
      OWNED_CUDA_CUBLAS_SHA256: af2de4aab12ce773f934529877ae444452ddf9a5b1156aa2d7549f0ca39693ea
    steps:
      - name: Checkout synapse
        uses: actions/checkout@v4
        with:
          path: synapse

      - uses: dtolnay/rust-toolchain@stable
        with:
          components: rustfmt

      - name: Assemble pinned CUDA toolkit from redist archives
        shell: pwsh
        timeout-minutes: 25
        run: |
          $ErrorActionPreference = 'Stop'
          $root = Join-Path $env:RUNNER_TEMP 'cuda-owned'
          New-Item -ItemType Directory -Force $root | Out-Null
          $components = @(
            @{ name = 'cuda_nvcc';    version = '13.2.78';   sha256 = $env:OWNED_CUDA_NVCC_SHA256 },
            @{ name = 'cuda_crt';     version = '13.2.78';   sha256 = $env:OWNED_CUDA_CRT_SHA256 },
            @{ name = 'libnvvm';      version = '13.2.78';   sha256 = $env:OWNED_CUDA_NVVM_SHA256 },
            @{ name = 'cuda_cccl';    version = '13.2.75';   sha256 = $env:OWNED_CUDA_CCCL_SHA256 },
            @{ name = 'cuda_cudart';  version = '13.2.75';   sha256 = $env:OWNED_CUDA_CUDART_SHA256 },
            @{ name = 'libcublas';    version = '13.4.0.1';  sha256 = $env:OWNED_CUDA_CUBLAS_SHA256 }
          )
          $digests = @()
          foreach ($c in $components) {
            $archive = "$($c.name)-windows-x86_64-$($c.version)-archive.zip"
            $url = "$env:OWNED_CUDA_REDIST_BASE/$($c.name)/windows-x86_64/$archive"
            $zip = Join-Path $env:RUNNER_TEMP $archive
            Invoke-WebRequest -Uri $url -OutFile $zip
            $actual = (Get-FileHash $zip -Algorithm SHA256).Hash.ToLowerInvariant()
            if ($actual -ne $c.sha256) {
              throw "$($c.name) digest $actual does not match pinned $($c.sha256)"
            }
            $expanded = Join-Path $env:RUNNER_TEMP "x-$($c.name)"
            Expand-Archive $zip -DestinationPath $expanded
            # Each archive extracts to a single nested package directory; the
            # toolkit root must hold bin/, include/, lib/ directly, so copy the
            # contents of that directory rather than the directory itself.
            Copy-Item (Join-Path $expanded '*\*') $root -Recurse -Force
            $digests += "$($c.name)=$($c.version) sha256=$actual"
          }
          "CUDA_PATH=$root" >> $env:GITHUB_ENV
          "CUDA_HOME=$root" >> $env:GITHUB_ENV
          # bin holds the nvcc drivers; bin\x64 holds CUDA 13's runtime DLLs.
          "PATH=$root\bin\x64;$root\bin;$env:PATH" >> $env:GITHUB_ENV
          New-Item -ItemType Directory -Force "$env:GITHUB_WORKSPACE\synapse\build-gates" | Out-Null
          $digests -join "`n" |
            Set-Content "$env:GITHUB_WORKSPACE\synapse\build-gates\windows-owned-cuda-toolkit.txt"
          & (Join-Path $root 'bin\nvcc.exe') --version

      - name: Windows owned-CUDA manual build gate
        working-directory: synapse
        shell: pwsh
        run: |
          # nvcc drives the host C++ compiler (cl.exe) by name and it is not on
          # PATH on a clean runner; cc-rs then fails before compiling anything.
          # Locally reproduced: `nvcc fatal: Cannot find compiler 'cl.exe' in
          # PATH`. Enter the MSVC developer environment for this shell only —
          # a per-step import, so no later step inherits it unexpectedly.
          $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe"
          $install = & $vswhere -latest -products * `
            -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 `
            -property installationPath
          if (-not $install) { throw 'no MSVC with VC.Tools.x86.x64 on this runner' }
          Import-Module (Join-Path $install 'Common7\Tools\Microsoft.VisualStudio.DevShell.dll')
          Enter-VsDevShell -VsInstallPath $install -SkipAutomaticLocation `
            -DevCmdArguments '-arch=x64 -host_arch=x64' | Out-Null
          New-Item -ItemType Directory -Force build-gates | Out-Null
          $command = 'cargo build -p synapse-worker-cuda --no-default-features --features cuda --release'
          $log = 'build-gates/windows-x86_64-msvc-owned-cuda.output'
          & cargo build -p synapse-worker-cuda --no-default-features --features cuda --release 2>&1 | Tee-Object -FilePath $log
          $status = $LASTEXITCODE
          $binary = 'target/release/ck-synapse-worker-cuda.exe'
          if ($status -ne 0) {
            $digest = 'unavailable'
          } else {
            $digest = (Get-FileHash $binary -Algorithm SHA256).Hash.ToLowerInvariant()
          }
          $execution = if ($status -eq 0) { 'executed_successful' } else { 'failed' }
          @"
          target_triple=x86_64-pc-windows-msvc
          command=$command
          rustc=$(rustc --version)
          cargo=$(cargo --version)
          cuda_version=$env:OWNED_CUDA_VERSION
          cuda_toolkit=$(nvcc --version 2>&1 | Select-Object -Last 1)
          cuda_component_digests=build-gates/windows-owned-cuda-toolkit.txt
          machine_image=$env:ImageOS
          evidence_contract=$env:LLAMA_BUILD_EVIDENCE_CONTRACT
          binary=$binary
          sha256=$digest
          execution_status=$execution
          successful_output=$log
          readiness_matrix_trigger=workflow_dispatch
          skipped_is_not_pass=true
          manual_gate=mandatory
          "@ | Set-Content build-gates/windows-x86_64-msvc-owned-cuda.txt
          if ($status -ne 0) { exit $status }

      # Derive sidecars from the same verified runtime components used to build.
      # No independently maintained DLL manifest; nvcc and driver files are not
      # redistribution inputs. The packager carries component licenses/hashes.
      - name: Package and verify Windows CUDA sidecars
        working-directory: synapse
        shell: pwsh
        run: |
          ./scripts/package-owned-cuda.ps1 `
            -Worker target/release/ck-synapse-worker-cuda.exe `
            -RuntimeComponents @("$env:RUNNER_TEMP/x-cuda_cudart", "$env:RUNNER_TEMP/x-libcublas") `
            -Output build-gates/owned-cuda-windows-x64.zip
          ./scripts/test-owned-cuda-package.ps1 -Archive build-gates/owned-cuda-windows-x64.zip
          'cuda_package_probe=passed; GPU execution requires a GPU runner' >> $env:GITHUB_STEP_SUMMARY

      - name: Retain Windows owned-CUDA gate evidence
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: owned-cuda-build-gates-windows-manual-${{ github.run_id }}
          path: |
            synapse/build-gates
            synapse/target/release/ck-synapse-worker-cuda.exe
          if-no-files-found: error