From e42137ba18f3d1121ccdc9e59bc96584bce1f4f3 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 29 Oct 2026 14:49:34 -0700 Subject: [PATCH] scripts: test the mutants merge as #952 changed it #952 changed the summarize job of nvx-rs-mutants.yml: each shard now stages its cargo-mutants output under its index before the upload, so that one shard's artifact, like several, downloads to shards//; the merge counts a shard as reported when its outcomes.json is there; and the merge step first fails unless the mutants job succeeded, which it reads from needs.mutants.result as MUTANTS_RESULT. Its tests weren't updated, and it merged without CI, which a fork's pull request runs only once approved. Since then, two of MutantsWorkflowStepTests fail on every Linux host, and so the NVX CLI job of every CI run: test_the_outcomes_of_every_shard_merge and test_shards_that_report_nothing_fail_the_merge run the step without MUTANTS_RESULT, which set -u rejects ("MUTANTS_RESULT: unbound variable"), and write each shard's lists under the old artifact name, which the step no longer reads. _merge now passes the mutants job's result, success unless a test says otherwise, and the tests write each shard's lists with its outcomes.json under its index, as the staging step uploads them. The test of shards that report nothing adds lists without an outcomes.json, which the merge must neither count nor merge, and a new test checks that a mutants job that failed or was skipped fails the merge before it merges or summarizes anything. The tests skip on Windows, which runs no bash steps. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/test_nvx_rs_mutants.py | 40 +++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/scripts/test_nvx_rs_mutants.py b/scripts/test_nvx_rs_mutants.py index ba250eb03..78cbaeabf 100644 --- a/scripts/test_nvx_rs_mutants.py +++ b/scripts/test_nvx_rs_mutants.py @@ -277,9 +277,13 @@ def test_missed_mutants_and_timeouts_are_outcomes_and_failures_fail(self): self.assertEqual(result.returncode, expected) def _merge( - self, shards: Mapping[str, Mapping[str, str]], count: int + self, + shards: Mapping[str, Mapping[str, str]], + count: int, + mutants: str = "success", ) -> tuple[subprocess.CompletedProcess[str], dict[str, str], str]: - """Merges the outcome lists of `shards` of `count` jobs, and returns the + """Merges the outcome lists of `shards`, keyed by shard index, of `count` + jobs whose matrix ended with the result `mutants`, and returns the result, the merged lists, and the summary.""" with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -294,6 +298,7 @@ def _merge( "TARGETS": "", "OS": "ubuntu-latest", "SHARDS": str(count), + "MUTANTS_RESULT": mutants, }, root, ) @@ -304,15 +309,17 @@ def _merge( return result, merged, summary def test_the_outcomes_of_every_shard_merge(self): + # Each shard's artifact holds its cargo-mutants output under its index. result, merged, summary = self._merge( { - "nvx-rs-mutants-shard-0-7": { + "0": { + "outcomes.json": "{}", "caught.txt": "src/a.rs:2:1: b\nsrc/a.rs:1:1: a\n", "missed.txt": "src/a.rs:9:1: m\n", "timeout.txt": "", "unviable.txt": "", }, - "nvx-rs-mutants-shard-1-7": {"caught.txt": "src/a.rs:3:1: c\n"}, + "1": {"outcomes.json": "{}", "caught.txt": "src/a.rs:3:1: c\n"}, }, 2, ) @@ -332,14 +339,37 @@ def test_the_outcomes_of_every_shard_merge(self): self.assertIn("src/a.rs:9:1: m\n", summary) def test_shards_that_report_nothing_fail_the_merge(self): + # A shard reports with its outcomes.json; lists without one don't count. result, merged, summary = self._merge( - {"nvx-rs-mutants-shard-0-7": {"caught.txt": "src/a.rs:1:1: a\n"}}, 3 + { + "0": {"outcomes.json": "{}", "caught.txt": "src/a.rs:1:1: a\n"}, + "2": {"caught.txt": "src/a.rs:5:1: e\n"}, + }, + 3, ) self.assertEqual(result.returncode, 1) self.assertIn("::error::Only 1 of 3 shards reported", result.stdout) self.assertEqual(merged["caught.txt"], "src/a.rs:1:1: a\n") self.assertNotIn("Missed mutants", summary) + def test_a_mutants_job_that_did_not_succeed_fails_the_merge(self): + # The merge runs whatever the shards' result, and fails before it merges + # anything unless every shard succeeded. + for mutants in ("failure", "skipped"): + with self.subTest(mutants=mutants): + result, merged, summary = self._merge( + {"0": {"outcomes.json": "{}", "caught.txt": "src/a.rs:1:1: a\n"}}, + 1, + mutants, + ) + self.assertEqual(result.returncode, 1) + self.assertIn( + f"::error::The mutants job did not succeed ({mutants}).", + result.stdout, + ) + self.assertEqual(merged, {}) + self.assertEqual(summary, "") + if __name__ == "__main__": unittest.main()