ci: let a cancelled job's test harness stop its VMs and remove its scratch

.github/actions/check-quality/action.yml

@@ -51,6 +51,7 @@ runs:
           scripts/setup/setup-linux-mshv.sh \
           scripts/setup/setup-linux-runner.sh
         ${SHELLCHECK} --shell=bash \
+          .github/actions/run-benchmark/forward-signals.sh \
           .github/specula/setup-runner.sh guest/ubuntu/nvx-bashrc \
           scripts/setup/azure/linux/bootstrap-runner.sh \
           scripts/setup/azure/linux/block-runner-metadata.sh \
@@ -72,6 +73,7 @@ runs:
           scripts/setup/setup-linux-mshv.sh \
           scripts/setup/setup-linux-runner.sh
         ${SHFMT} -d -ln bash -i 4 -ci \
+          .github/actions/run-benchmark/forward-signals.sh \
           .github/specula/setup-runner.sh guest/ubuntu/nvx-bashrc \
           scripts/setup/azure/linux/bootstrap-runner.sh \
           scripts/setup/azure/linux/block-runner-metadata.sh \

.github/actions/run-benchmark/action.yml

@@ -51,6 +51,10 @@ runs:
           --output "data/runs/${{ inputs.platform }}/${{ inputs.platform }}-host.json" \
           --summary "${GITHUB_STEP_SUMMARY}"
 
+    # The runner signals only a step's shell when the job is cancelled, so the
+    # Linux steps run the test harness with run_harness, which passes SIGINT
+    # and SIGTERM on to it as SIGTERM, or exec it, so that it can stop OpenVMM
+    # and remove its scratch directories (doc/ci.md#cancellation).
     - name: Run Linux acceptance test
       if: runner.os != 'Windows'
       shell: bash
@@ -60,6 +64,7 @@ runs:
         NVX_BENCHMARK_MARKER_TIMEOUT: ${{ inputs.marker-timeout }}
       run: |
         set -euo pipefail
+        source "${GITHUB_ACTION_PATH}/forward-signals.sh"
         cpu_args=(--host-cpu-reserve "${NVX_BENCHMARK_HOST_CPU_RESERVE}")
         if [[ -n "${NVX_BENCHMARK_CPUS}" ]]; then
           cpu_args+=(--cpus "${NVX_BENCHMARK_CPUS}")
@@ -71,7 +76,7 @@ runs:
           "${run_dir}"/acceptance-attempt-*.json
         for attempt in 1 2; do
           attempt_output="${run_dir}/acceptance-attempt-${attempt}.json"
-          python3 scripts/nvx.py benchmark \
+          run_harness python3 scripts/nvx.py benchmark \
             --suite e2e \
             --backend "${{ inputs.backend }}" \
             --platform "${{ inputs.platform }}" \
@@ -108,6 +113,7 @@ runs:
         NVX_BENCHMARK_MARKER_TIMEOUT: ${{ inputs.marker-timeout }}
       run: |
         set -euo pipefail
+        source "${GITHUB_ACTION_PATH}/forward-signals.sh"
         cpu_args=(--host-cpu-reserve "${NVX_BENCHMARK_HOST_CPU_RESERVE}")
         if [[ -n "${NVX_BENCHMARK_CPUS}" ]]; then
           cpu_args+=(--cpus "${NVX_BENCHMARK_CPUS}")
@@ -118,7 +124,7 @@ runs:
         rm -rf "${run_dir}"/performance-attempt-*
         for attempt in 1 2; do
           attempt_dir="${run_dir}/performance-attempt-${attempt}"
-          python3 scripts/nvx.py benchmark \
+          run_harness python3 scripts/nvx.py benchmark \
             --suite performance \
             --backend "${{ inputs.backend }}" \
             --platform "${{ inputs.platform }}" \
@@ -154,6 +160,7 @@ runs:
         NVX_BENCHMARK_MARKER_TIMEOUT: ${{ inputs.marker-timeout }}
       run: |
         set -euo pipefail
+        source "${GITHUB_ACTION_PATH}/forward-signals.sh"
         cpu_args=(--host-cpu-reserve "${NVX_BENCHMARK_HOST_CPU_RESERVE}")
         if [[ -n "${NVX_BENCHMARK_CPUS}" ]]; then
           cpu_args+=(--cpus "${NVX_BENCHMARK_CPUS}")
@@ -165,7 +172,7 @@ runs:
           rm -rf "${run_dir}"/shell-snapshot-restore-attempt-*
           for attempt in 1 2; do
             attempt_dir="${run_dir}/shell-snapshot-restore-attempt-${attempt}"
-            python3 scripts/nvx.py benchmark \
+            run_harness python3 scripts/nvx.py benchmark \
               --suite shell-snapshot-restore \
               --backend "${{ inputs.backend }}" \
               --platform "${{ inputs.platform }}" \
@@ -207,7 +214,7 @@ runs:
           cpu_args+=(--cpus "${NVX_BENCHMARK_CPUS}")
         fi
         run_dir="data/runs/${{ inputs.platform }}/microvm-v2/1vcpu/device-io"
-        python3 scripts/nvx.py benchmark \
+        exec python3 scripts/nvx.py benchmark \
           --suite device-io \
           --backend "${{ inputs.backend }}" \
           --platform "${{ inputs.platform }}" \

.github/actions/run-benchmark/forward-signals.sh

@@ -0,0 +1,64 @@
+# shellcheck shell=bash
+# Sourced by the Linux steps of run-benchmark, which run the test harness
+# among other commands (see doc/ci.md#cancellation).
+#
+# When a job is cancelled, the runner sends SIGINT, and then SIGTERM, to the
+# step's shell alone, and Bash runs a trap only once its foreground command
+# ends. So sourcing this file traps both signals: one that arrives while
+# run_harness runs a command reaches that command as SIGTERM, which nvx.py
+# handles as SIGINT, so that it stops its OpenVMM processes and removes its
+# scratch directories, and ends the step once the command exits; one that
+# arrives at any other time ends the step at once.
+
+harness_pid=
+starting=
+deferred=
+cancelled=
+
+on_cancellation() {
+    cancelled=$1
+    if [[ -n "${starting}" ]]; then
+        # The command may have started, but run_harness doesn't know its PID
+        # yet, so it passes the signal on once it does.
+        deferred=1
+    elif [[ -n "${harness_pid}" ]]; then
+        # The command may have exited already, so that the signal reaches nothing.
+        kill -TERM "${harness_pid}" 2>/dev/null || true
+    else
+        exit "$1"
+    fi
+}
+
+trap 'on_cancellation 130' INT
+trap 'on_cancellation 143' TERM
+
+# Runs a command in the background, where Bash would ignore SIGINT unless the
+# command restored it, and returns its exit status once it exits. A
+# cancellation reaches the command as SIGTERM, which it never ignores, not
+# even before it has restored SIGINT, as it has not when a cancellation that
+# arrived as it started reaches it.
+run_harness() {
+    starting=1
+    (
+        trap - INT
+        exec "$@"
+    ) &
+    harness_pid=$!
+    starting=
+    if [[ -n "${deferred}" ]]; then
+        deferred=
+        kill -TERM "${harness_pid}" 2>/dev/null || true
+    fi
+    local status=0
+    wait "${harness_pid}" || status=$?
+    # A trapped signal ends the wait before the command exits.
+    while kill -0 "${harness_pid}" 2>/dev/null; do
+        status=0
+        wait "${harness_pid}" || status=$?
+    done
+    harness_pid=
+    if [[ -n "${cancelled}" ]]; then
+        exit "${cancelled}"
+    fi
+    return "${status}"
+}

.github/actions/validate-runner/action.yml

@@ -211,6 +211,29 @@ runs:
           exit 1
         fi
 
+    # A harness that something killed outright, as the runner's orphan
+    # cleanup killed those of cancelled jobs, leaves its scenarios' scratch
+    # directories in the temporary directory (#946). As with OpenVMM, no step
+    # before this one runs the harness, so the account's harness directories
+    # there are left from earlier jobs.
+    - name: Remove scratch left by earlier jobs
+      if: runner.os != 'Windows'
+      shell: bash
+      run: |
+        set -euo pipefail
+        temporary=$(python3 -c 'import tempfile; print(tempfile.gettempdir())')
+        find "${temporary}" -mindepth 1 -maxdepth 1 -type d -user "$(id -u)" \
+          \( -name 'nvx-*' -o -name 'openvmm-*' \) -print0 |
+          while IFS= read -r -d '' directory; do
+            modified=$(stat -c %y "${directory}" || true)
+            echo "::notice::Removing ${directory}, last modified ${modified:-at an unknown time}, which an earlier job left behind"
+            # A scenario can leave a directory that even its owner can't write.
+            chmod -R u+rwX -- "${directory}" 2>/dev/null || true
+            if ! rm -rf --one-file-system -- "${directory}"; then
+              echo "::warning::Could not remove ${directory}, which an earlier job left behind"
+            fi
+          done
+
     # Host time. The time ABI host qualification (doc/ci.md,
     # "Host qualification") checks the backend, the CPU fingerprint and
     # generation, and the TSC rate stability on its short CI schedule, and,

.github/workflows/run-nvx-microvm-tests.yml

@@ -179,11 +179,14 @@ jobs:
           backend: ${{ inputs.backend }}
           verify-openvmm: "true"
 
+      # The runner signals only a step's shell when the job is cancelled, so
+      # the Linux steps exec the test harness, which can then stop OpenVMM and
+      # remove its scratch directories (doc/ci.md#cancellation).
       - name: Run NVX microVM tests on Linux
         if: runner.os != 'Windows' && !inputs.guests-and-sandboxes
         shell: bash
         run: >-
-          python3 scripts/nvx.py test-microvm
+          exec python3 scripts/nvx.py test-microvm
           --backend "${{ inputs.backend }}"
           ${{ inputs.debug-kernel && '--debug-kernel' || '' }}
           ${{ inputs.shard && format('--shard {0}/{1}', inputs.shard, inputs.shard-count) || '' }}
@@ -203,7 +206,7 @@ jobs:
         if: runner.os != 'Windows' && inputs.guests-and-sandboxes && !inputs.debug-kernel
         shell: bash
         run: >-
-          python3 scripts/nvx.py test-microvm
+          exec python3 scripts/nvx.py test-microvm
           --backend "${{ inputs.backend }}"
           --scenario managed-exec-config
           --scenario sandbox-lifecycle
@@ -225,7 +228,7 @@ jobs:
         if: runner.os != 'Windows' && inputs.guests-and-sandboxes && !inputs.debug-kernel
         shell: bash
         run: >-
-          python3 scripts/nvx.py test-microvm
+          exec python3 scripts/nvx.py test-microvm
           --backend "${{ inputs.backend }}"
           --guest ubuntu
           --processors 1
@@ -257,7 +260,7 @@ jobs:
         if: runner.os != 'Windows' && inputs.guests-and-sandboxes && !inputs.debug-kernel
         shell: bash
         run: >-
-          python3 scripts/nvx.py test-microvm
+          exec python3 scripts/nvx.py test-microvm
           --backend "${{ inputs.backend }}"
           --guest azurelinux
           --processors 1
@@ -291,7 +294,7 @@ jobs:
         if: inputs.backend == 'kvm' && inputs.guests-and-sandboxes && !inputs.debug-kernel
         shell: bash
         run: >-
-          python3 scripts/nvx.py test-microvm
+          exec python3 scripts/nvx.py test-microvm
           --backend kvm
           --guest ubuntu
           --scenario managed-lifecycle

doc/benchmarks.md

@@ -871,7 +871,11 @@ After the merge, `dev` pushes restart the series' ten-point warmup.
 
 The lifecycle benchmark uses the optimized 128 MiB shell-ready guest as one baseline across KVM,
 MSHV, and WHP. Host timing starts immediately before `Popen`, so cold-start and restore values
-include OpenVMM process startup and VM construction. Snapshot generation uses OpenVMM's monotonic
+include OpenVMM process startup and VM construction. On Linux, that startup includes the execs of
+`setpriv` and of a shell that checks that the coordinator still runs, which make the coordinator's
+exit kill OpenVMM (see [Cancellation](ci.md#cancellation)). On a Xeon Silver 4114 host, they added
+2.1 ms to the time until OpenVMM ran with Dash as `/bin/sh`, and 3.5 ms with Bash, which the Azure
+Linux runners use. Snapshot generation uses OpenVMM's monotonic
 process clock from the beginning of input gating, after the guest requests the snapshot, through
 atomic publication. It does not include delivery of `nvx-snapshot` over the polling console.
 Host request-to-publication and first publication observation (polled every 1 ms) remain diagnostics.

doc/ci.md

@@ -270,11 +270,18 @@ Every job that uses the `validate-runner` action first ends, on Linux, any
 OpenVMM process that its runner's account still runs, with a warning that
 names it, and fails if one survives SIGKILL or the processes can't be listed.
 A runner runs one job at a time, so such a process is left from an
-earlier job: when a job is cancelled, the runner signals only the step's shell,
-so the test harness keeps running, and it can start a VM after the runner's
-orphan cleanup has listed the job's processes; the cleanup then kills the
-harness, and the VM runs on (#768). The action then reports the Linux
-runner's kernel, CPU, clocksource, and TSC flags, and qualifies the runner
+earlier job. Before a cancellation reached the test harness (see
+[Cancellation](#cancellation)), the harness of a cancelled job kept running
+and could start a VM after the runner's orphan cleanup had listed the job's
+processes; the cleanup then killed the harness, and the VM ran on for 8 days
+(#768). It still ends a VM that escapes both the harness's cleanup and its
+parent-death signal. The action next removes the account's `nvx-*` and
+`openvmm-*` directories in Python's temporary directory, where the test
+harness runs its scenarios, with a notice that names each one, and fails if it
+can't list them: a harness that something killed outright left them behind,
+and on October 28, 2026, the ten Linux runners held 64 of them (#946). It then
+reports the Linux runner's kernel, CPU, clocksource, and TSC flags, and
+qualifies the runner
 for the time ABI with `nvx.py doctor --checks H1 H2 H4 --ci-schedule` (see
 [Host qualification](#host-qualification)): the backend, the CPU fingerprint
 and generation, and the TSC rate stability on its short schedule. The microVM
@@ -984,6 +991,54 @@ SHA after applying the results to the current `dev` history: it drops rows
 already present for the same platform, ABI, processor count, commit, and
 metric, but otherwise appends them and pushes a baseline update.
 
+## Cancellation
+
+When a run is cancelled, as a push to a pull request cancels the run of its
+previous head, the runner
+[cancels each running step](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-cancellation):
+it sends SIGINT to the step's shell, SIGTERM if the shell still runs 7.5 s
+later, and kills the shell's process tree another 2.5 s later. After the
+job's last step, the runner's orphan cleanup kills every process that the job
+started and that still runs. `nvx.py` handles SIGTERM as it handles SIGINT:
+either one interrupts its command, which then stops the OpenVMM processes and
+removes the scratch directories that it started and created, and exits with
+status 130 (see [Commands](usage.md#commands)).
+
+The runner signals only the step's shell, though, and Bash runs a trap only
+once the command in its foreground ends, so a harness that a Linux step ran in
+the foreground never learned of the cancellation: it ran on until the orphan
+cleanup killed it, which left the scratch directories of its scenario behind.
+The Linux steps that run `test-microvm` therefore `exec` it, so that it is the
+step's process. The Linux steps of `run-benchmark`, which run the harness among
+other commands, `exec` it as their last command or source
+`.github/actions/run-benchmark/forward-signals.sh`, whose `run_harness` runs
+the harness in the background, with the SIGINT that Bash ignores there
+restored, and passes SIGINT and SIGTERM on to it as SIGTERM, which, unlike
+SIGINT, the harness never ignores, not even before it has restored SIGINT. A
+signal that arrives as `run_harness` starts the harness, before it knows the
+harness's PID, reaches the harness once it does. The step then ends once the
+harness exits, even if the signal came too late to stop it, and a signal that
+arrives between two harness runs ends the step at once.
+
+On Linux, OpenVMM also ends with `nvx.py` when something kills `nvx.py`
+before its cleanup runs, as the orphan cleanup does. Where the host has
+util-linux's `setpriv`, as the runners do, `nvx.py` starts OpenVMM through
+`setpriv --pdeathsig KILL`, so that the kernel kills OpenVMM when `nvx.py`
+exits, and through a shell that starts OpenVMM only if `nvx.py` is still its
+parent, since an `nvx.py` that died before `setpriv` armed the signal sent
+none. A contained process tree, such as `nvx-rs run` and the
+OpenVMM that it starts, runs under a supervisor that the kernel sends SIGTERM
+when `nvx.py` exits, and which then kills the whole tree. The supervisor runs
+without the job's `RUNNER_TRACKING_ID`, which it gives back to the command,
+because the orphan cleanup kills the processes whose environment holds it and
+could kill the supervisor before it ended a tree that the cleanup didn't list,
+such as an OpenVMM that started after the listing. The kernel sends
+these signals when the thread that started the process exits, so only a
+process that the main thread starts gets them, and `nvx.py` starts its VMs
+from its main thread. The `validate-runner` action of the runner's next job
+ends the OpenVMM processes and removes the scratch directories that still
+escape all of this.
+
 ## OpenVMM VMM tests
 
 OpenVMM's `vmm-tests-run` builds the tests that a filter selects and then runs

doc/usage.md

@@ -80,8 +80,8 @@ match codes, and stable fragments of messages, such as the names of options,
 fields, and files, never whole messages. A code keeps its meaning and is never
 reused, and a later release may add codes, which callers treat as failures of
 the command. The usage errors that `nvx.py` itself reports, as `argparse` does,
-carry no code, and neither does an interruption, which prints `Interrupted`
-and exits with status 130. The
+carry no code, and neither does an interruption by SIGINT or, outside
+Windows, SIGTERM, which prints `Interrupted` and exits with status 130. The
 [`nvx-rs` README](../nvx-rs/README.md#error-codes) gives the `ErrorKind` of
 each code for callers of the `nvx` library.
 
@@ -448,6 +448,14 @@ is authorized for the organization when it enforces single sign-on.
 | `archive-release` | Create a deterministic archive from a staged distribution. |
 | `verify` | Verify source and submodule inputs. |
 
+Ctrl-C, that is SIGINT, interrupts a command, and so does SIGTERM outside
+Windows, which is how CI ends a cancelled step (see
+[Cancellation](ci.md#cancellation)). The command then runs its cleanup, as
+`test-microvm` and `benchmark` stop their OpenVMM processes and remove their
+scratch directories, prints `Interrupted`, and exits with status 130. On
+Linux, the OpenVMM processes that they start, and the process trees that they
+contain, end with `nvx.py` even when something kills it outright.
+
 ## Initialization and verification
 
 ### `init`
@@ -1166,7 +1174,8 @@ follow one another without gaps, so they add up to the run's time:
   outcome.
 
 Every span has its `started_at`, its `seconds`, rounded to the millisecond, its
-`outcome`, `passed`, `failed`, or `interrupted`, for example by `SIGINT`, and
+`outcome`, `passed`, `failed`, or `interrupted`, for example by `SIGINT` or
+`SIGTERM`, and
 an `error`: for a failure, the exception's type and message, cut to 1,000
 characters, and otherwise `null`. A `scenario` record also has the
 `scenario`, the `guest`, `backend`, and `kernel`, and its `processors`: the