diff --git a/.github/actions/ensure-psmux/action.yml b/.github/actions/ensure-psmux/action.yml new file mode 100644 index 00000000..22903e27 --- /dev/null +++ b/.github/actions/ensure-psmux/action.yml @@ -0,0 +1,52 @@ +name: Ensure psmux +description: >- + Download one psmux release's windows-x64 archive, put psmux.exe on PATH, and + assert that bmad-loop admits it. Needs `uv sync` to have run first. +inputs: + version: + description: A psmux release tag (for example `v3.3.8`), or `latest`. + required: true + github-token: + description: Token for the release lookup. + required: true +runs: + using: composite + steps: + - name: Install psmux + shell: pwsh + env: + GH_TOKEN: ${{ inputs.github-token }} + PSMUX_VERSION: ${{ inputs.version }} + run: | + $ErrorActionPreference = 'Stop' + # The release archive, not `choco install`. Chocolatey's community feed + # answered the V2 package lookup with 503 on BOTH matrix legs of the gate's + # first run, while the package itself sat published and healthy. + # Chocolatey document that feed as neither supported nor guaranteed and + # rate-limited per IP behind Cloudflare, and hosted runners share egress + # addresses — other projects track the same feed-503 flake. This is one + # GET against the host the job already depends on. + $release = if ($env:PSMUX_VERSION -eq 'latest') { 'releases/latest' } else { "releases/tags/$env:PSMUX_VERSION" } + $url = gh api "repos/psmux/psmux/$release" --jq '.assets[] | select(.name | endswith("windows-x64.zip")) | .browser_download_url' + if (-not $url) { throw "no windows-x64 asset on psmux $env:PSMUX_VERSION" } + $zip = Join-Path $env:RUNNER_TEMP 'psmux.zip' + $dir = Join-Path $env:RUNNER_TEMP 'psmux' + Invoke-WebRequest -Uri $url -OutFile $zip -MaximumRetryCount 3 -RetryIntervalSec 5 + Expand-Archive -LiteralPath $zip -DestinationPath $dir -Force + $exe = Get-ChildItem -Path $dir -Recurse -Filter psmux.exe | Select-Object -First 1 + if (-not $exe) { throw "no psmux.exe in $url" } + # The archive ships tmux.exe beside psmux.exe, which looks alarming on a + # PATH but is inert: test_generic_tmux's HAVE_TMUX gates on + # `sys.platform != "win32"` before it ever looks for the binary. + # PATH for this step, GITHUB_PATH for the steps after it. + $env:PATH = "$($exe.Directory.FullName);$env:PATH" + Add-Content -Path $env:GITHUB_PATH -Value $exe.Directory.FullName + psmux -V + # `psmux -V` is not the assertion. test_psmux_live.py skips itself on + # PsmuxMultiplexer.available(), so an install at or below the version + # floor would leave the job green with the entire live gate never run + # — the one failure a "did it install?" check cannot see. Calling + # available() also keeps the floor in a single place (it reads + # `_LAST_UNSUPPORTED`, so a floor bump needs no edit here) and covers + # `pwsh`, which every parked window needs. + uv run python -c "import sys; from bmad_loop.adapters.psmux_backend import PsmuxMultiplexer as M; m = M(); sys.exit(0 if m.available() else f'psmux is not an admitted version: {m.version()!r}')" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6798fcac..ed0cc9aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -250,48 +250,21 @@ jobs: # Bounded, for the reason the Linux `Ensure tmux` step above documents at # length: an unbounded fetch wedged that job for 14 minutes with no exit # status for `|| true` to discard. Here the retries are bounded by - # Invoke-WebRequest and everything else by the step timeout. + # Invoke-WebRequest and everything else by the step timeout. After + # `Install the project`, because the action's floor assertion runs + # `uv run`; the aggregator's `verify-executed` then refuses any skip + # inside the gate itself. timeout-minutes: 6 - shell: pwsh - env: - GH_TOKEN: ${{ github.token }} - # After `Install the project`, because the assertion below runs `uv run`. - run: | - $ErrorActionPreference = 'Stop' - # The release archive, not `choco install`. Chocolatey's community feed - # answered the V2 package lookup with 503 on BOTH matrix legs of this - # job's first run, while the package itself sat published and healthy. - # Chocolatey document that feed as neither supported nor guaranteed and - # rate-limited per IP behind Cloudflare, and hosted runners share egress - # addresses — other projects track the same feed-503 flake. This is one - # GET against the host the job already depends on. - # - # `releases/latest`, not a pinned tag: the floor assertion below stays - # the only version gate, so a psmux release needs no edit here. - $url = gh api repos/psmux/psmux/releases/latest --jq '.assets[] | select(.name | endswith("windows-x64.zip")) | .browser_download_url' - if (-not $url) { throw 'no windows-x64 asset on the latest psmux release' } - $zip = Join-Path $env:RUNNER_TEMP 'psmux.zip' - $dir = Join-Path $env:RUNNER_TEMP 'psmux' - Invoke-WebRequest -Uri $url -OutFile $zip -MaximumRetryCount 3 -RetryIntervalSec 5 - Expand-Archive -LiteralPath $zip -DestinationPath $dir -Force - $exe = Get-ChildItem -Path $dir -Recurse -Filter psmux.exe | Select-Object -First 1 - if (-not $exe) { throw "no psmux.exe in $url" } - # The archive ships tmux.exe beside psmux.exe, which looks alarming on a - # PATH but is inert: test_generic_tmux's HAVE_TMUX gates on - # `sys.platform != "win32"` before it ever looks for the binary. - # PATH for this step, GITHUB_PATH for the gate step. - $env:PATH = "$($exe.Directory.FullName);$env:PATH" - Add-Content -Path $env:GITHUB_PATH -Value $exe.Directory.FullName - psmux -V - # `psmux -V` is not the assertion. test_psmux_live.py skips itself on - # PsmuxMultiplexer.available(), so an install at or below the version - # floor would leave this job green with the entire live gate never run - # — the one failure a "did it install?" check cannot see. Calling - # available() also keeps the floor in a single place (it reads - # `_LAST_UNSUPPORTED`, so a floor bump needs no edit here) and covers - # `pwsh`, which every parked window needs. The aggregator's - # `verify-executed` then refuses any skip inside the gate itself. - uv run python -c "import sys; from bmad_loop.adapters.psmux_backend import PsmuxMultiplexer as M; m = M(); sys.exit(0 if m.available() else f'psmux is not an admitted version: {m.version()!r}')" + uses: ./.github/actions/ensure-psmux + with: + # A pinned release, not `releases/latest`: the gate's `test_premise_*` + # rows turn red by design when a psmux release fixes what they pin, so + # an unpinned install turned every PR red the day psmux 3.4.0 shipped. + # Moving this pin is a deliberate PR that also moves the floor and + # retires the workarounds; psmux-canary.yml runs the same gate against + # `latest` to say when that PR is due. + version: v3.3.8 + github-token: ${{ github.token }} - name: Run the live psmux gate # Serial, and out of the parallel shards: these 15 tests drive real psmux diff --git a/.github/workflows/psmux-canary.yml b/.github/workflows/psmux-canary.yml new file mode 100644 index 00000000..fd54e7df --- /dev/null +++ b/.github/workflows/psmux-canary.yml @@ -0,0 +1,63 @@ +name: psmux canary + +# The live psmux gate against the LATEST psmux release, outside the required +# checks. ci.yml pins the gate to one psmux release so an upstream release can +# never turn a PR red. This job is how that release still gets noticed: a red +# `test_premise_*` row here means psmux fixed a defect bmad-loop works around, +# and the next pin bump can retire it (its message names the workaround). Any +# other red row is a psmux regression to report upstream before moving the pin. +on: + schedule: + - cron: 17 6 * * 1 + workflow_dispatch: + +permissions: + contents: read + +jobs: + canary: + name: psmux canary (windows, latest) + runs-on: windows-latest + timeout-minutes: 30 + env: + PYTHONUTF8: '1' + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@v10.2.0 + with: + enable-cache: false + python-version: '3.14' + + - name: Install the project + run: uv sync --locked --all-extras + + - name: Ensure psmux (latest release) + timeout-minutes: 6 + uses: ./.github/actions/ensure-psmux + with: + version: latest + github-token: ${{ github.token }} + + - name: Run the live psmux gate + # The blocking gate's runner and timeout reserve, so the metrics below + # exist even when the gate is stopped at the job's deadline. + env: + TMP: ${{ runner.temp }} + TEMP: ${{ runner.temp }} + run: >- + uv run python scripts/run_test_benchmark.py + --metrics-dir "$env:RUNNER_TEMP/test-metrics/live" + --timeout 1500 + -- -q tests/test_psmux_live.py --durations=15 + "--junitxml=$env:RUNNER_TEMP/test-metrics/live/junit.xml" + + - name: Verify the live gate ran every test + # As in ci.yml's aggregator: pytest exits 0 when a live row skips, but a + # skip in the live gate means that behaviour was never exercised, so a + # green canary would vouch for a release it did not test. + if: always() + run: uv run python tests/perf_report.py verify-executed "$env:RUNNER_TEMP/test-metrics/live" diff --git a/docs/testing.md b/docs/testing.md index 16feab8c..f89faed5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -296,8 +296,12 @@ The Windows jobs split one leg across machines without dropping a test: never split. Units are assigned by a SHA-256 digest of the unit's nodeid, never Python's randomized `hash()` or a file list, so a new test is assigned automatically and existing tests do not move. The digest balances test counts, not seconds. -- **test-windows-live** installs psmux and runs the L5 gate once per leg, serially, on a - machine of its own. +- **test-windows-live** installs one pinned psmux release and runs the L5 gate once per leg, + serially, on a machine of its own. The pin, not `latest`: the gate's `test_premise_*` rows + turn red by design when a psmux release fixes what they pin, so an upstream release must + never reach a PR's checks. Moving the pin is its own PR, which also moves the version floor + and retires the workarounds. The non-required **psmux canary** workflow (weekly, or on + demand) runs the same gate against the latest release and goes red when that PR is due. - **test-windows** keeps the check name `test (windows, py3.x)` and is the leg's verdict. `perf_report.py verify-shards --count 2` requires both shards present and green, their selections disjoint, and their union equal to the collected set, with no function family