From 04ddc4f3f30ea36dbcb320ffdebd16760ca74a49 Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Wed, 16 Sep 2026 20:09:45 +0200 Subject: [PATCH 1/9] ci(anvil): limit mutation concurrency Run one diff mutant at a time to test whether automatic parallelism causes hosted runner shutdowns on allocator-heavy changes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- justfiles/anvil/checks/mutants-diff.just | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfiles/anvil/checks/mutants-diff.just b/justfiles/anvil/checks/mutants-diff.just index e4efc812f..f41a2c117 100644 --- a/justfiles/anvil/checks/mutants-diff.just +++ b/justfiles/anvil/checks/mutants-diff.just @@ -50,7 +50,7 @@ anvil-mutants-diff: anvil-mutants-diff-validate-prereqs anvil-impact # not a behaviour change for it. git diff "$base" --output=$diff_path if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - & cargo {{_anvil_stable_toolchain_args}} mutants --in-diff $diff_path --no-shuffle --jobs 0 + & cargo {{_anvil_stable_toolchain_args}} mutants --in-diff $diff_path --no-shuffle --jobs 1 if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # --- pr-mutants members --- From c1ff0c11501f72857712a1134255c6183c9a7930 Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Wed, 16 Sep 2026 20:27:11 +0200 Subject: [PATCH 2/9] test(anvil): exercise parent mutation diff Point only the diagnostic mutation group at PR 764's original base so it runs the same allocator mutants with single-job concurrency. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/anvil-pr-impl.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/anvil-pr-impl.yml b/.github/workflows/anvil-pr-impl.yml index b00fbf555..e7a36c261 100644 --- a/.github/workflows/anvil-pr-impl.yml +++ b/.github/workflows/anvil-pr-impl.yml @@ -316,7 +316,10 @@ jobs: publish_commit_statuses: ${{ inputs.publish_commit_statuses }} impact_mode: consume env: - BASE_REF: ${{ inputs.base_ref }} + # Diagnostic PR #765 targets PR #764's head. Diff mutation testing + # against #764's original base so this run exercises the same 1,025 + # mutants as the cancelled jobs, now with --jobs 1. + BASE_REF: ec1f6bc2d289f73202665952aa8a1492d13c40c4 required-checks: # This display name is the repository-ruleset context. Keep it stable From 8b08b3d3bdd93a44852b3d73c5be47c81ff9772f Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Thu, 17 Sep 2026 10:43:51 +0200 Subject: [PATCH 3/9] test(anvil): trace suspect allocator mutant Run only the next deterministic Linux allocator mutant and stream cgroup and process memory telemetry so evidence survives runner termination. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/anvil-pr-impl.yml | 4 +-- justfiles/anvil/checks/mutants-diff.just | 41 ++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/.github/workflows/anvil-pr-impl.yml b/.github/workflows/anvil-pr-impl.yml index e7a36c261..be362fdc1 100644 --- a/.github/workflows/anvil-pr-impl.yml +++ b/.github/workflows/anvil-pr-impl.yml @@ -317,8 +317,8 @@ jobs: impact_mode: consume env: # Diagnostic PR #765 targets PR #764's head. Diff mutation testing - # against #764's original base so this run exercises the same 1,025 - # mutants as the cancelled jobs, now with --jobs 1. + # against #764's original base so this run can isolate the suspected + # Linux allocator mutant with streamed resource telemetry. BASE_REF: ec1f6bc2d289f73202665952aa8a1492d13c40c4 required-checks: diff --git a/justfiles/anvil/checks/mutants-diff.just b/justfiles/anvil/checks/mutants-diff.just index f41a2c117..64aa5006e 100644 --- a/justfiles/anvil/checks/mutants-diff.just +++ b/justfiles/anvil/checks/mutants-diff.just @@ -50,8 +50,45 @@ anvil-mutants-diff: anvil-mutants-diff-validate-prereqs anvil-impact # not a behaviour change for it. git diff "$base" --output=$diff_path if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - & cargo {{_anvil_stable_toolchain_args}} mutants --in-diff $diff_path --no-shuffle --jobs 1 - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $monitor = $null + $monitor_path = Join-Path $tmp_dir 'anvil-mutants-memory-monitor.sh' + if ($IsLinux) { + $monitor_script = @' + #!/usr/bin/env bash + while true; do + printf 'ANVIL_MUTANT_TELEMETRY timestamp=%s\n' "$(date --utc --iso-8601=seconds)" + for file in memory.current memory.max memory.events; do + path="/sys/fs/cgroup/$file" + if [[ -r "$path" ]]; then + printf 'ANVIL_MUTANT_TELEMETRY %s=' "$file" + tr '\n' ' ' <"$path" + printf '\n' + fi + done + free -m | sed 's/^/ANVIL_MUTANT_TELEMETRY /' + ps -eo pid,ppid,rss,vsz,comm --sort=-rss | + head -n 12 | + sed 's/^/ANVIL_MUTANT_TELEMETRY process /' + sleep 2 + done + '@ + [System.IO.File]::WriteAllText($monitor_path, $monitor_script, [System.Text.UTF8Encoding]::new($false)) + $monitor = Start-Process -FilePath '/bin/bash' -ArgumentList $monitor_path -NoNewWindow -PassThru + } + try { + & cargo {{_anvil_stable_toolchain_args}} mutants --in-diff $diff_path --no-shuffle --jobs 1 ` + --re '^crates/rallocator/src/allocator\.rs:1259:57: replace \* with \+ in Rallocator::allocate_medium$' ` + --exclude 'crates/rallocator/src/hal/win64.rs' ` + --exclude 'crates/seismograph_cli/**' + $status = $LASTEXITCODE + } finally { + if ($null -ne $monitor -and -not $monitor.HasExited) { + Stop-Process -Id $monitor.Id + $monitor.WaitForExit() + } + Remove-Item -LiteralPath $monitor_path -Force -ErrorAction SilentlyContinue + } + if ($status -ne 0) { exit $status } # --- pr-mutants members --- From b8d68443b63182527af468433cc1988a435b34b2 Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Thu, 17 Sep 2026 10:49:11 +0200 Subject: [PATCH 4/9] test(anvil): trace under-commit mutant Advance the isolated Linux diagnostic to the next deterministic allocate_medium mutation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- justfiles/anvil/checks/mutants-diff.just | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfiles/anvil/checks/mutants-diff.just b/justfiles/anvil/checks/mutants-diff.just index 64aa5006e..f33453dec 100644 --- a/justfiles/anvil/checks/mutants-diff.just +++ b/justfiles/anvil/checks/mutants-diff.just @@ -77,7 +77,7 @@ anvil-mutants-diff: anvil-mutants-diff-validate-prereqs anvil-impact } try { & cargo {{_anvil_stable_toolchain_args}} mutants --in-diff $diff_path --no-shuffle --jobs 1 ` - --re '^crates/rallocator/src/allocator\.rs:1259:57: replace \* with \+ in Rallocator::allocate_medium$' ` + --re '^crates/rallocator/src/allocator\.rs:1259:57: replace \* with / in Rallocator::allocate_medium$' ` --exclude 'crates/rallocator/src/hal/win64.rs' ` --exclude 'crates/seismograph_cli/**' $status = $LASTEXITCODE From cdb14ba749e785b6baf15e7fa6182966dc81cdbb Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Thu, 17 Sep 2026 10:58:43 +0200 Subject: [PATCH 5/9] test(anvil): trace next allocator mutants Run the next eight deterministic allocate_medium mutants as a bounded telemetry batch. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- justfiles/anvil/checks/mutants-diff.just | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfiles/anvil/checks/mutants-diff.just b/justfiles/anvil/checks/mutants-diff.just index f33453dec..2fbfe07ad 100644 --- a/justfiles/anvil/checks/mutants-diff.just +++ b/justfiles/anvil/checks/mutants-diff.just @@ -77,7 +77,7 @@ anvil-mutants-diff: anvil-mutants-diff-validate-prereqs anvil-impact } try { & cargo {{_anvil_stable_toolchain_args}} mutants --in-diff $diff_path --no-shuffle --jobs 1 ` - --re '^crates/rallocator/src/allocator\.rs:1259:57: replace \* with / in Rallocator::allocate_medium$' ` + --re '^crates/rallocator/src/allocator\.rs:(1260:70|1267:53|1269:43|1270:44):' ` --exclude 'crates/rallocator/src/hal/win64.rs' ` --exclude 'crates/seismograph_cli/**' $status = $LASTEXITCODE From 8329b9744637a8d5f5371390b6aba3602376c72c Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Thu, 17 Sep 2026 11:04:55 +0200 Subject: [PATCH 6/9] test(anvil): trace telemetry iterator mutants Advance the Linux shutdown diagnostic to the next deterministic mutation block. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- justfiles/anvil/checks/mutants-diff.just | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfiles/anvil/checks/mutants-diff.just b/justfiles/anvil/checks/mutants-diff.just index 2fbfe07ad..4970ac86c 100644 --- a/justfiles/anvil/checks/mutants-diff.just +++ b/justfiles/anvil/checks/mutants-diff.just @@ -77,7 +77,7 @@ anvil-mutants-diff: anvil-mutants-diff-validate-prereqs anvil-impact } try { & cargo {{_anvil_stable_toolchain_args}} mutants --in-diff $diff_path --no-shuffle --jobs 1 ` - --re '^crates/rallocator/src/allocator\.rs:(1260:70|1267:53|1269:43|1270:44):' ` + --re '^crates/rallocator/src/allocator\.rs:(1948|1951|1954|1955|1962):' ` --exclude 'crates/rallocator/src/hal/win64.rs' ` --exclude 'crates/seismograph_cli/**' $status = $LASTEXITCODE From 2e3756c0060d5a50b99b990ac43ee17398561f03 Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Thu, 17 Sep 2026 11:14:55 +0200 Subject: [PATCH 7/9] test(anvil): isolate empty telemetry iterator Narrow the reproduced OOM batch to its first deterministic mutant. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- justfiles/anvil/checks/mutants-diff.just | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfiles/anvil/checks/mutants-diff.just b/justfiles/anvil/checks/mutants-diff.just index 4970ac86c..6fe902218 100644 --- a/justfiles/anvil/checks/mutants-diff.just +++ b/justfiles/anvil/checks/mutants-diff.just @@ -77,7 +77,7 @@ anvil-mutants-diff: anvil-mutants-diff-validate-prereqs anvil-impact } try { & cargo {{_anvil_stable_toolchain_args}} mutants --in-diff $diff_path --no-shuffle --jobs 1 ` - --re '^crates/rallocator/src/allocator\.rs:(1948|1951|1954|1955|1962):' ` + --re '^crates/rallocator/src/allocator\.rs:1948:5: replace telemetry_used_slice_indices -> .*empty\(\)$' ` --exclude 'crates/rallocator/src/hal/win64.rs' ` --exclude 'crates/seismograph_cli/**' $status = $LASTEXITCODE From c12b3030cbb0e90fe12256d9165b4c8a06e10db4 Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Thu, 17 Sep 2026 11:22:58 +0200 Subject: [PATCH 8/9] test(anvil): isolate non-clearing bitmap mutant Test the mutation that turns the finite telemetry bitmap iterator into an unbounded stream. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- justfiles/anvil/checks/mutants-diff.just | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfiles/anvil/checks/mutants-diff.just b/justfiles/anvil/checks/mutants-diff.just index 6fe902218..6334ccfdc 100644 --- a/justfiles/anvil/checks/mutants-diff.just +++ b/justfiles/anvil/checks/mutants-diff.just @@ -77,7 +77,7 @@ anvil-mutants-diff: anvil-mutants-diff-validate-prereqs anvil-impact } try { & cargo {{_anvil_stable_toolchain_args}} mutants --in-diff $diff_path --no-shuffle --jobs 1 ` - --re '^crates/rallocator/src/allocator\.rs:1948:5: replace telemetry_used_slice_indices -> .*empty\(\)$' ` + --re '^crates/rallocator/src/allocator\.rs:1955:23: replace &= with \|= in telemetry_used_slice_indices$' ` --exclude 'crates/rallocator/src/hal/win64.rs' ` --exclude 'crates/seismograph_cli/**' $status = $LASTEXITCODE From 14a40a6de16e8589be43d6c3cde2c8aca682b5a7 Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Thu, 17 Sep 2026 13:09:13 +0200 Subject: [PATCH 9/9] test(rallocator): skip runaway bitmap mutants Exclude the finite telemetry bitmap iterator from mutation generation because bit-clearing mutants create unbounded snapshot streams that OOM Linux runners. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- crates/rallocator/Cargo.toml | 4 +-- crates/rallocator/src/allocator.rs | 1 + justfiles/anvil/checks/mutants-diff.just | 41 ++---------------------- 3 files changed, 4 insertions(+), 42 deletions(-) diff --git a/crates/rallocator/Cargo.toml b/crates/rallocator/Cargo.toml index 2c11127c5..69981704a 100644 --- a/crates/rallocator/Cargo.toml +++ b/crates/rallocator/Cargo.toml @@ -20,9 +20,6 @@ repository = "https://github.com/microsoft/oxidizer/tree/main/crates/rallocator" [target.'cfg(target_os = "linux")'.dependencies] libc.workspace = true -[target.'cfg(target_os = "linux")'.dev-dependencies] -mutants.workspace = true - [target.'cfg(loom)'.dev-dependencies] loom.workspace = true @@ -52,6 +49,7 @@ criterion.workspace = true hashbrown = { workspace = true, features = ["default-hasher"] } mimalloc.workspace = true multitude = { path = "../multitude", default-features = false, features = ["hashbrown"] } +mutants.workspace = true performables = { path = "../performables", features = ["seismograph"] } seismograph = { path = "../seismograph", features = ["monitor"] } seismograph_runtime = { path = "../seismograph_runtime" } diff --git a/crates/rallocator/src/allocator.rs b/crates/rallocator/src/allocator.rs index bcbb3250c..6e285ec60 100644 --- a/crates/rallocator/src/allocator.rs +++ b/crates/rallocator/src/allocator.rs @@ -1944,6 +1944,7 @@ pub(crate) fn flush_thread_aggregate_batch() { unsafe { flush_aggregate_batch(state) }; } +#[cfg_attr(test, mutants::skip)] // Bit-clearing mutants create an unbounded iterator and OOM Linux runners. fn telemetry_used_slice_indices(bitmap: &[u64]) -> impl Iterator + '_ { bitmap.iter().enumerate().flat_map(|(word_index, &word)| { let mut remaining = word; diff --git a/justfiles/anvil/checks/mutants-diff.just b/justfiles/anvil/checks/mutants-diff.just index 6334ccfdc..e4efc812f 100644 --- a/justfiles/anvil/checks/mutants-diff.just +++ b/justfiles/anvil/checks/mutants-diff.just @@ -50,45 +50,8 @@ anvil-mutants-diff: anvil-mutants-diff-validate-prereqs anvil-impact # not a behaviour change for it. git diff "$base" --output=$diff_path if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $monitor = $null - $monitor_path = Join-Path $tmp_dir 'anvil-mutants-memory-monitor.sh' - if ($IsLinux) { - $monitor_script = @' - #!/usr/bin/env bash - while true; do - printf 'ANVIL_MUTANT_TELEMETRY timestamp=%s\n' "$(date --utc --iso-8601=seconds)" - for file in memory.current memory.max memory.events; do - path="/sys/fs/cgroup/$file" - if [[ -r "$path" ]]; then - printf 'ANVIL_MUTANT_TELEMETRY %s=' "$file" - tr '\n' ' ' <"$path" - printf '\n' - fi - done - free -m | sed 's/^/ANVIL_MUTANT_TELEMETRY /' - ps -eo pid,ppid,rss,vsz,comm --sort=-rss | - head -n 12 | - sed 's/^/ANVIL_MUTANT_TELEMETRY process /' - sleep 2 - done - '@ - [System.IO.File]::WriteAllText($monitor_path, $monitor_script, [System.Text.UTF8Encoding]::new($false)) - $monitor = Start-Process -FilePath '/bin/bash' -ArgumentList $monitor_path -NoNewWindow -PassThru - } - try { - & cargo {{_anvil_stable_toolchain_args}} mutants --in-diff $diff_path --no-shuffle --jobs 1 ` - --re '^crates/rallocator/src/allocator\.rs:1955:23: replace &= with \|= in telemetry_used_slice_indices$' ` - --exclude 'crates/rallocator/src/hal/win64.rs' ` - --exclude 'crates/seismograph_cli/**' - $status = $LASTEXITCODE - } finally { - if ($null -ne $monitor -and -not $monitor.HasExited) { - Stop-Process -Id $monitor.Id - $monitor.WaitForExit() - } - Remove-Item -LiteralPath $monitor_path -Force -ErrorAction SilentlyContinue - } - if ($status -ne 0) { exit $status } + & cargo {{_anvil_stable_toolchain_args}} mutants --in-diff $diff_path --no-shuffle --jobs 0 + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # --- pr-mutants members ---