From faa54b4c6b6abb54b8f18325983aa6fe79061842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Davor=20Raci=C4=87?= Date: Tue, 13 Oct 2026 18:25:06 +0200 Subject: [PATCH 1/2] ci: pin the blocking psmux gate to one release and add a latest-release canary The live gate's test_premise_* rows turn red by design when a psmux release fixes what they pin. With the gate installing releases/latest, psmux 3.4.0 turned every PR red on the day it shipped. The blocking gate now installs a pinned release (v3.3.8) through a shared composite action, and a non-required weekly psmux-canary workflow runs the same gate against the latest release, so an upstream release is noticed without blocking anyone. Moving the pin is a deliberate PR that also moves the version floor and retires the workarounds. --- .github/actions/ensure-psmux/action.yml | 52 +++++++++++++++++++++++ .github/workflows/ci.yml | 55 +++++++------------------ .github/workflows/psmux-canary.yml | 49 ++++++++++++++++++++++ docs/testing.md | 8 +++- 4 files changed, 121 insertions(+), 43 deletions(-) create mode 100644 .github/actions/ensure-psmux/action.yml create mode 100644 .github/workflows/psmux-canary.yml diff --git a/.github/actions/ensure-psmux/action.yml b/.github/actions/ensure-psmux/action.yml new file mode 100644 index 00000000..22903e27 --- /dev/null +++ b/.github/actions/ensure-psmux/action.yml @@ -0,0 +1,52 @@ +name: Ensure psmux +description: >- + Download one psmux release's windows-x64 archive, put psmux.exe on PATH, and + assert that bmad-loop admits it. Needs `uv sync` to have run first. +inputs: + version: + description: A psmux release tag (for example `v3.3.8`), or `latest`. + required: true + github-token: + description: Token for the release lookup. + required: true +runs: + using: composite + steps: + - name: Install psmux + shell: pwsh + env: + GH_TOKEN: ${{ inputs.github-token }} + PSMUX_VERSION: ${{ inputs.version }} + run: | + $ErrorActionPreference = 'Stop' + # The release archive, not `choco install`. Chocolatey's community feed + # answered the V2 package lookup with 503 on BOTH matrix legs of the gate's + # first run, while the package itself sat published and healthy. + # Chocolatey document that feed as neither supported nor guaranteed and + # rate-limited per IP behind Cloudflare, and hosted runners share egress + # addresses — other projects track the same feed-503 flake. This is one + # GET against the host the job already depends on. + $release = if ($env:PSMUX_VERSION -eq 'latest') { 'releases/latest' } else { "releases/tags/$env:PSMUX_VERSION" } + $url = gh api "repos/psmux/psmux/$release" --jq '.assets[] | select(.name | endswith("windows-x64.zip")) | .browser_download_url' + if (-not $url) { throw "no windows-x64 asset on psmux $env:PSMUX_VERSION" } + $zip = Join-Path $env:RUNNER_TEMP 'psmux.zip' + $dir = Join-Path $env:RUNNER_TEMP 'psmux' + Invoke-WebRequest -Uri $url -OutFile $zip -MaximumRetryCount 3 -RetryIntervalSec 5 + Expand-Archive -LiteralPath $zip -DestinationPath $dir -Force + $exe = Get-ChildItem -Path $dir -Recurse -Filter psmux.exe | Select-Object -First 1 + if (-not $exe) { throw "no psmux.exe in $url" } + # The archive ships tmux.exe beside psmux.exe, which looks alarming on a + # PATH but is inert: test_generic_tmux's HAVE_TMUX gates on + # `sys.platform != "win32"` before it ever looks for the binary. + # PATH for this step, GITHUB_PATH for the steps after it. + $env:PATH = "$($exe.Directory.FullName);$env:PATH" + Add-Content -Path $env:GITHUB_PATH -Value $exe.Directory.FullName + psmux -V + # `psmux -V` is not the assertion. test_psmux_live.py skips itself on + # PsmuxMultiplexer.available(), so an install at or below the version + # floor would leave the job green with the entire live gate never run + # — the one failure a "did it install?" check cannot see. Calling + # available() also keeps the floor in a single place (it reads + # `_LAST_UNSUPPORTED`, so a floor bump needs no edit here) and covers + # `pwsh`, which every parked window needs. + uv run python -c "import sys; from bmad_loop.adapters.psmux_backend import PsmuxMultiplexer as M; m = M(); sys.exit(0 if m.available() else f'psmux is not an admitted version: {m.version()!r}')" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6798fcac..ed0cc9aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -250,48 +250,21 @@ jobs: # Bounded, for the reason the Linux `Ensure tmux` step above documents at # length: an unbounded fetch wedged that job for 14 minutes with no exit # status for `|| true` to discard. Here the retries are bounded by - # Invoke-WebRequest and everything else by the step timeout. + # Invoke-WebRequest and everything else by the step timeout. After + # `Install the project`, because the action's floor assertion runs + # `uv run`; the aggregator's `verify-executed` then refuses any skip + # inside the gate itself. timeout-minutes: 6 - shell: pwsh - env: - GH_TOKEN: ${{ github.token }} - # After `Install the project`, because the assertion below runs `uv run`. - run: | - $ErrorActionPreference = 'Stop' - # The release archive, not `choco install`. Chocolatey's community feed - # answered the V2 package lookup with 503 on BOTH matrix legs of this - # job's first run, while the package itself sat published and healthy. - # Chocolatey document that feed as neither supported nor guaranteed and - # rate-limited per IP behind Cloudflare, and hosted runners share egress - # addresses — other projects track the same feed-503 flake. This is one - # GET against the host the job already depends on. - # - # `releases/latest`, not a pinned tag: the floor assertion below stays - # the only version gate, so a psmux release needs no edit here. - $url = gh api repos/psmux/psmux/releases/latest --jq '.assets[] | select(.name | endswith("windows-x64.zip")) | .browser_download_url' - if (-not $url) { throw 'no windows-x64 asset on the latest psmux release' } - $zip = Join-Path $env:RUNNER_TEMP 'psmux.zip' - $dir = Join-Path $env:RUNNER_TEMP 'psmux' - Invoke-WebRequest -Uri $url -OutFile $zip -MaximumRetryCount 3 -RetryIntervalSec 5 - Expand-Archive -LiteralPath $zip -DestinationPath $dir -Force - $exe = Get-ChildItem -Path $dir -Recurse -Filter psmux.exe | Select-Object -First 1 - if (-not $exe) { throw "no psmux.exe in $url" } - # The archive ships tmux.exe beside psmux.exe, which looks alarming on a - # PATH but is inert: test_generic_tmux's HAVE_TMUX gates on - # `sys.platform != "win32"` before it ever looks for the binary. - # PATH for this step, GITHUB_PATH for the gate step. - $env:PATH = "$($exe.Directory.FullName);$env:PATH" - Add-Content -Path $env:GITHUB_PATH -Value $exe.Directory.FullName - psmux -V - # `psmux -V` is not the assertion. test_psmux_live.py skips itself on - # PsmuxMultiplexer.available(), so an install at or below the version - # floor would leave this job green with the entire live gate never run - # — the one failure a "did it install?" check cannot see. Calling - # available() also keeps the floor in a single place (it reads - # `_LAST_UNSUPPORTED`, so a floor bump needs no edit here) and covers - # `pwsh`, which every parked window needs. The aggregator's - # `verify-executed` then refuses any skip inside the gate itself. - uv run python -c "import sys; from bmad_loop.adapters.psmux_backend import PsmuxMultiplexer as M; m = M(); sys.exit(0 if m.available() else f'psmux is not an admitted version: {m.version()!r}')" + uses: ./.github/actions/ensure-psmux + with: + # A pinned release, not `releases/latest`: the gate's `test_premise_*` + # rows turn red by design when a psmux release fixes what they pin, so + # an unpinned install turned every PR red the day psmux 3.4.0 shipped. + # Moving this pin is a deliberate PR that also moves the floor and + # retires the workarounds; psmux-canary.yml runs the same gate against + # `latest` to say when that PR is due. + version: v3.3.8 + github-token: ${{ github.token }} - name: Run the live psmux gate # Serial, and out of the parallel shards: these 15 tests drive real psmux diff --git a/.github/workflows/psmux-canary.yml b/.github/workflows/psmux-canary.yml new file mode 100644 index 00000000..c3a56db1 --- /dev/null +++ b/.github/workflows/psmux-canary.yml @@ -0,0 +1,49 @@ +name: psmux canary + +# The live psmux gate against the LATEST psmux release, outside the required +# checks. ci.yml pins the gate to one psmux release so an upstream release can +# never turn a PR red. This job is how that release still gets noticed: a red +# `test_premise_*` row here means psmux fixed a defect bmad-loop works around, +# and the next pin bump can retire it (its message names the workaround). Any +# other red row is a psmux regression to report upstream before moving the pin. +on: + schedule: + - cron: 17 6 * * 1 + workflow_dispatch: + +permissions: + contents: read + +jobs: + canary: + name: psmux canary (windows, latest) + runs-on: windows-latest + timeout-minutes: 30 + env: + PYTHONUTF8: '1' + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@v10.2.0 + with: + enable-cache: false + python-version: '3.14' + + - name: Install the project + run: uv sync --locked --all-extras + + - name: Ensure psmux (latest release) + timeout-minutes: 6 + uses: ./.github/actions/ensure-psmux + with: + version: latest + github-token: ${{ github.token }} + + - name: Run the live psmux gate + env: + TMP: ${{ runner.temp }} + TEMP: ${{ runner.temp }} + run: uv run pytest -q tests/test_psmux_live.py --durations=15 diff --git a/docs/testing.md b/docs/testing.md index 16feab8c..f89faed5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -296,8 +296,12 @@ The Windows jobs split one leg across machines without dropping a test: never split. Units are assigned by a SHA-256 digest of the unit's nodeid, never Python's randomized `hash()` or a file list, so a new test is assigned automatically and existing tests do not move. The digest balances test counts, not seconds. -- **test-windows-live** installs psmux and runs the L5 gate once per leg, serially, on a - machine of its own. +- **test-windows-live** installs one pinned psmux release and runs the L5 gate once per leg, + serially, on a machine of its own. The pin, not `latest`: the gate's `test_premise_*` rows + turn red by design when a psmux release fixes what they pin, so an upstream release must + never reach a PR's checks. Moving the pin is its own PR, which also moves the version floor + and retires the workarounds. The non-required **psmux canary** workflow (weekly, or on + demand) runs the same gate against the latest release and goes red when that PR is due. - **test-windows** keeps the check name `test (windows, py3.x)` and is the leg's verdict. `perf_report.py verify-shards --count 2` requires both shards present and green, their selections disjoint, and their union equal to the collected set, with no function family From 6c8542885d6dc338c2f50e1f17278187031449b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Davor=20Raci=C4=87?= Date: Tue, 13 Oct 2026 18:40:57 +0200 Subject: [PATCH 2/2] ci: fail the psmux canary when a live row skips pytest exits 0 when a live-gate row skips, but a skip means that behaviour was never exercised. Run the canary through the blocking gate's benchmark runner and check its metrics with perf_report.py verify-executed, as ci.yml's aggregator does. --- .github/workflows/psmux-canary.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/psmux-canary.yml b/.github/workflows/psmux-canary.yml index c3a56db1..fd54e7df 100644 --- a/.github/workflows/psmux-canary.yml +++ b/.github/workflows/psmux-canary.yml @@ -43,7 +43,21 @@ jobs: github-token: ${{ github.token }} - name: Run the live psmux gate + # The blocking gate's runner and timeout reserve, so the metrics below + # exist even when the gate is stopped at the job's deadline. env: TMP: ${{ runner.temp }} TEMP: ${{ runner.temp }} - run: uv run pytest -q tests/test_psmux_live.py --durations=15 + run: >- + uv run python scripts/run_test_benchmark.py + --metrics-dir "$env:RUNNER_TEMP/test-metrics/live" + --timeout 1500 + -- -q tests/test_psmux_live.py --durations=15 + "--junitxml=$env:RUNNER_TEMP/test-metrics/live/junit.xml" + + - name: Verify the live gate ran every test + # As in ci.yml's aggregator: pytest exits 0 when a live row skips, but a + # skip in the live gate means that behaviour was never exercised, so a + # green canary would vouch for a release it did not test. + if: always() + run: uv run python tests/perf_report.py verify-executed "$env:RUNNER_TEMP/test-metrics/live"