From 2fc4d74b26cac74d8b28532b05e8f0eea036637e Mon Sep 17 00:00:00 2001
From: Pedro Henrique Penna <ppenna@microsoft.com>
Date: Thu, 29 Oct 2026 14:44:42 -0700
Subject: [PATCH] scripts: test the mutants merge against the shards' indexed
 outcomes

#952 changed how the summarize job of nvx-rs-mutants.yml finds the
outcomes of the shards. Each shard now uploads its outcomes under its
index, the download merges the artifacts of all the shards into
shards/, and the merge counts a shard as reported when
shards/INDEX/outcomes.json exists. The merge step also fails before it
merges anything when the mutants job did not succeed, which it reads
from MUTANTS_RESULT under set -u.

MutantsWorkflowStepTests still ran the merge step with each shard's
lists in a directory named after its artifact, and without
MUTANTS_RESULT. On Linux, where these tests run,
test_the_outcomes_of_every_shard_merge failed with "MUTANTS_RESULT:
unbound variable", and test_shards_that_report_nothing_fail_the_merge
did not find its "Only 1 of 3 shards reported" error. Both fail at
b558d34bd, which merged #952, and pass at its parent, 29c68547b; they
failed NVX CLI / Linux in CI run 40958853538 of #1021.

The tests now lay the outcomes out as the download does, under the
index of each shard and with outcomes.json, and pass the mutants job's
result. The test of missing shards also gives one shard lists without
outcomes.json, which the merge must leave out of both the lists and
the count. A new test requires that a mutants job that failed, was
cancelled, or was skipped fail the merge before it merges or
summarizes anything.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 scripts/test_nvx_rs_mutants.py | 46 ++++++++++++++++++++++++++++------
 1 file changed, 38 insertions(+), 8 deletions(-)

diff --git a/scripts/test_nvx_rs_mutants.py b/scripts/test_nvx_rs_mutants.py
index ba250eb03..e3d792095 100644
--- a/scripts/test_nvx_rs_mutants.py
+++ b/scripts/test_nvx_rs_mutants.py
@@ -277,14 +277,22 @@ def test_missed_mutants_and_timeouts_are_outcomes_and_failures_fail(self):
                 self.assertEqual(result.returncode, expected)
 
     def _merge(
-        self, shards: Mapping[str, Mapping[str, str]], count: int
+        self,
+        shards: Mapping[str, Mapping[str, str]],
+        count: int,
+        mutants_result: str = "success",
     ) -> tuple[subprocess.CompletedProcess[str], dict[str, str], str]:
-        """Merges the outcome lists of `shards` of `count` jobs, and returns the
-        result, the merged lists, and the summary."""
+        """Merges the outcome files of `shards`, which maps the index of each
+        shard to its files, for `count` shards whose mutants job ended with
+        `mutants_result`, and returns the result, the merged lists, and the
+        summary.
+
+        The download merges the artifacts of the shards into shards/, where
+        each artifact holds its shard's outcomes under the shard's index."""
         with tempfile.TemporaryDirectory() as directory:
             root = Path(directory)
-            for shard, lists in shards.items():
-                _write(root / "shards" / shard, lists)
+            for index, files in shards.items():
+                _write(root / "shards" / index, files)
             result, _, summary = self._run(
                 "summarize",
                 "Merge the outcomes of the shards",
@@ -294,6 +302,7 @@ def _merge(
                     "TARGETS": "",
                     "OS": "ubuntu-latest",
                     "SHARDS": str(count),
+                    "MUTANTS_RESULT": mutants_result,
                 },
                 root,
             )
@@ -306,13 +315,14 @@ def _merge(
     def test_the_outcomes_of_every_shard_merge(self):
         result, merged, summary = self._merge(
             {
-                "nvx-rs-mutants-shard-0-7": {
+                "0": {
+                    "outcomes.json": "{}\n",
                     "caught.txt": "src/a.rs:2:1: b\nsrc/a.rs:1:1: a\n",
                     "missed.txt": "src/a.rs:9:1: m\n",
                     "timeout.txt": "",
                     "unviable.txt": "",
                 },
-                "nvx-rs-mutants-shard-1-7": {"caught.txt": "src/a.rs:3:1: c\n"},
+                "1": {"outcomes.json": "{}\n", "caught.txt": "src/a.rs:3:1: c\n"},
             },
             2,
         )
@@ -333,13 +343,33 @@ def test_the_outcomes_of_every_shard_merge(self):
 
     def test_shards_that_report_nothing_fail_the_merge(self):
         result, merged, summary = self._merge(
-            {"nvx-rs-mutants-shard-0-7": {"caught.txt": "src/a.rs:1:1: a\n"}}, 3
+            {
+                "0": {"outcomes.json": "{}\n", "caught.txt": "src/a.rs:1:1: a\n"},
+                # A shard without outcomes.json reported nothing, and the merge
+                # leaves its lists out.
+                "1": {"caught.txt": "src/a.rs:2:1: b\n"},
+            },
+            3,
         )
         self.assertEqual(result.returncode, 1)
         self.assertIn("::error::Only 1 of 3 shards reported", result.stdout)
         self.assertEqual(merged["caught.txt"], "src/a.rs:1:1: a\n")
+        self.assertIn("on ubuntu-latest: 1 of 3 shards reported.", summary)
         self.assertNotIn("Missed mutants", summary)
 
+    def test_a_mutants_job_that_did_not_succeed_fails_the_merge(self):
+        shards = {"0": {"outcomes.json": "{}\n", "caught.txt": "src/a.rs:1:1: a\n"}}
+        for mutants_result in ("failure", "cancelled", "skipped"):
+            with self.subTest(mutants_result=mutants_result):
+                result, merged, summary = self._merge(shards, 1, mutants_result)
+                self.assertEqual(result.returncode, 1)
+                self.assertIn(
+                    f"::error::The mutants job did not succeed ({mutants_result}).",
+                    result.stdout,
+                )
+                # The merge stops before it merges or summarizes anything.
+                self.assertEqual((merged, summary), ({}, ""))
+
 
 if __name__ == "__main__":
     unittest.main()