From 166475ee8ac77f379281f1a2eba399699f43caea Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 29 Oct 2026 14:29:28 -0700 Subject: [PATCH 1/2] scripts: keep the egress probes' results in TMPDIR l3-l4-egress-protocols.sh.in, l3-l4-egress-port-ranges.sh.in, and ipv6-egress-policy.sh.in start their probes in the background, and each probe writes its result to a fixed path, /tmp/nvx-egress-PROBE, which the script reads once they all end. Before it probes, the script removes every such file with rm -f /tmp/nvx-egress-*. A guest's /tmp is private, but scripts/test_microvm_tests.py also runs these scripts in the host's shell, where /tmp is shared, so that runs at once on one host remove and overwrite each other's results (#1007). Each script now keeps its results in TMPDIR, and in /tmp when TMPDIR is unset or empty. The runners stage the scripts with a quoted here-document, so the guest's shell expands TMPDIR only when it runs a script, and nothing in NVX's guests sets it: the scenarios keep using the guest's /tmp, while a test can give each run of a script a directory of its own. The three scenarios pass with this change on KVM, MSHV, and WHP, on test VMs and on bare metal, and their guests report every probe as before. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../ipv6-egress-policy.sh.in | 17 +++++++++++------ .../l3-l4-egress-port-ranges.sh.in | 15 ++++++++++----- .../l3-l4-egress-protocols.sh.in | 17 +++++++++++------ 3 files changed, 32 insertions(+), 17 deletions(-) diff --git a/scripts/nvx_tools/microvm_test_scripts/ipv6-egress-policy.sh.in b/scripts/nvx_tools/microvm_test_scripts/ipv6-egress-policy.sh.in index 900cc74ea..6f6f093e4 100644 --- a/scripts/nvx_tools/microvm_test_scripts/ipv6-egress-policy.sh.in +++ b/scripts/nvx_tools/microvm_test_scripts/ipv6-egress-policy.sh.in @@ -20,6 +20,11 @@ if [ -n "$dns" ]; then fi echo "NVX-IPV6-EGRESS-DNS ${dns:-none}" +# The probes keep their results in TMPDIR, or else in the guest's private +# /tmp. The host's /tmp is shared, so the tests that run this script in the +# host's shell give each run a TMPDIR of its own (#1007). +probe_dir=${TMPDIR:-/tmp} + probe_tcp() { if response=$(wget -T 2 -qO- "$2" 2>/dev/null); then if [ "$response" = "NVX-EGRESS-PROBE-$1" ]; then @@ -29,7 +34,7 @@ probe_tcp() { fi else echo blocked - fi >"/tmp/nvx-egress-$1" + fi >"$probe_dir/nvx-egress-$1" } # Only the host can observe UDP delivery. A denied flow may return 1; sender @@ -41,7 +46,7 @@ probe_udp() { case "$status" in 0 | 1) echo sent ;; *) echo "failed-$status" ;; - esac >"/tmp/nvx-egress-$1" + esac >"$probe_dir/nvx-egress-$1" } probe_icmp() { @@ -49,10 +54,10 @@ probe_icmp() { echo reached else echo blocked - fi >"/tmp/nvx-egress-$1" + fi >"$probe_dir/nvx-egress-$1" } -rm -f /tmp/nvx-egress-* +rm -f "$probe_dir"/nvx-egress-* probe_tcp tcp6-named 'http://[@GATEWAY_IPV6@]:@NAMED_PORT@/tcp6-named' & probe_tcp tcp6-other 'http://[@GATEWAY_IPV6@]:@OTHER_PORT@/tcp6-other' & probe_udp udp6-named @GATEWAY_IPV6@ @NAMED_PORT@ & @@ -60,8 +65,8 @@ probe_icmp icmp6 @GATEWAY_IPV6@ & probe_tcp tcp4-named 'http://@GATEWAY_IPV4@:@NAMED_PORT@/tcp4-named' & wait for probe in tcp6-named tcp6-other udp6-named icmp6 tcp4-named; do - [ -s "/tmp/nvx-egress-$probe" ] || fail 70 - echo "NVX-L3-L4-PROBE $probe=$(cat "/tmp/nvx-egress-$probe")" + [ -s "$probe_dir/nvx-egress-$probe" ] || fail 70 + echo "NVX-L3-L4-PROBE $probe=$(cat "$probe_dir/nvx-egress-$probe")" done echo NVX-IPV6-EGRESS-OK diff --git a/scripts/nvx_tools/microvm_test_scripts/l3-l4-egress-port-ranges.sh.in b/scripts/nvx_tools/microvm_test_scripts/l3-l4-egress-port-ranges.sh.in index 699a4ecf1..a5f1e752f 100644 --- a/scripts/nvx_tools/microvm_test_scripts/l3-l4-egress-port-ranges.sh.in +++ b/scripts/nvx_tools/microvm_test_scripts/l3-l4-egress-port-ranges.sh.in @@ -6,6 +6,11 @@ fail() { exit "$code" } +# The probes keep their results in TMPDIR, or else in the guest's private +# /tmp. The host's /tmp is shared, so the tests that run this script in the +# host's shell give each run a TMPDIR of its own (#1007). +probe_dir=${TMPDIR:-/tmp} + probe_tcp() { if response=$(wget -T 2 -qO- "http://@GATEWAY_IPV4@:$2/$1" 2>/dev/null); then if [ "$response" = "NVX-EGRESS-PROBE-$1" ]; then @@ -15,7 +20,7 @@ probe_tcp() { fi else echo blocked - fi >"/tmp/nvx-egress-$1" + fi >"$probe_dir/nvx-egress-$1" } # Only the host can observe UDP delivery. A denied flow may return 1; sender @@ -27,10 +32,10 @@ probe_udp() { case "$status" in 0 | 1) echo sent ;; *) echo "failed-$status" ;; - esac >"/tmp/nvx-egress-$1" + esac >"$probe_dir/nvx-egress-$1" } -rm -f /tmp/nvx-egress-* +rm -f "$probe_dir"/nvx-egress-* probe_tcp tcp-below @BELOW_PORT@ & probe_tcp tcp-first @FIRST_PORT@ & probe_tcp tcp-inside @INSIDE_PORT@ & @@ -44,8 +49,8 @@ probe_udp udp-above @ABOVE_PORT@ & wait for probe in tcp-below tcp-first tcp-inside tcp-last tcp-above \ udp-below udp-first udp-inside udp-last udp-above; do - [ -s "/tmp/nvx-egress-$probe" ] || fail 70 - echo "NVX-L3-L4-PROBE $probe=$(cat "/tmp/nvx-egress-$probe")" + [ -s "$probe_dir/nvx-egress-$probe" ] || fail 70 + echo "NVX-L3-L4-PROBE $probe=$(cat "$probe_dir/nvx-egress-$probe")" done echo NVX-L3-L4-EGRESS-PORT-RANGES-OK diff --git a/scripts/nvx_tools/microvm_test_scripts/l3-l4-egress-protocols.sh.in b/scripts/nvx_tools/microvm_test_scripts/l3-l4-egress-protocols.sh.in index c86728117..5611eff55 100644 --- a/scripts/nvx_tools/microvm_test_scripts/l3-l4-egress-protocols.sh.in +++ b/scripts/nvx_tools/microvm_test_scripts/l3-l4-egress-protocols.sh.in @@ -6,6 +6,11 @@ fail() { exit "$code" } +# The probes keep their results in TMPDIR, or else in the guest's private +# /tmp. The host's /tmp is shared, so the tests that run this script in the +# host's shell give each run a TMPDIR of its own (#1007). +probe_dir=${TMPDIR:-/tmp} + probe_tcp() { if response=$(wget -T 2 -qO- "http://@GATEWAY_IPV4@:$2/$1" 2>/dev/null); then if [ "$response" = "NVX-EGRESS-PROBE-$1" ]; then @@ -15,7 +20,7 @@ probe_tcp() { fi else echo blocked - fi >"/tmp/nvx-egress-$1" + fi >"$probe_dir/nvx-egress-$1" } # Only the host can observe UDP delivery. A denied flow may return 1; sender @@ -27,7 +32,7 @@ probe_udp() { case "$status" in 0 | 1) echo sent ;; *) echo "failed-$status" ;; - esac >"/tmp/nvx-egress-$1" + esac >"$probe_dir/nvx-egress-$1" } probe_icmp() { @@ -35,10 +40,10 @@ probe_icmp() { echo reached else echo blocked - fi >/tmp/nvx-egress-icmp + fi >"$probe_dir/nvx-egress-icmp" } -rm -f /tmp/nvx-egress-* +rm -f "$probe_dir"/nvx-egress-* probe_tcp tcp-named @NAMED_PORT@ & probe_tcp tcp-other @OTHER_PORT@ & probe_udp udp-named @NAMED_PORT@ & @@ -46,8 +51,8 @@ probe_udp udp-other @OTHER_PORT@ & probe_icmp & wait for probe in tcp-named tcp-other udp-named udp-other icmp; do - [ -s "/tmp/nvx-egress-$probe" ] || fail 70 - echo "NVX-L3-L4-PROBE $probe=$(cat "/tmp/nvx-egress-$probe")" + [ -s "$probe_dir/nvx-egress-$probe" ] || fail 70 + echo "NVX-L3-L4-PROBE $probe=$(cat "$probe_dir/nvx-egress-$probe")" done echo NVX-L3-L4-EGRESS-PROTOCOLS-OK From a79ca92ed8acc4c7660283b324543231701e2298 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 29 Oct 2026 14:29:29 -0700 Subject: [PATCH 2/2] scripts: give each host run of an egress script a TMPDIR of its own Several validate-nvx runs at once on one host failed the tests that run the egress scripts in the host's shell, with 70 != 0 and with "cat: /tmp/nvx-egress-tcp-inside: No such file or directory", on Linux and on Windows, where Git for Windows' sh maps /tmp to the user's temporary directory (#1007): one run's rm -f /tmp/nvx-egress-* removed the results of another between its probes and its report. On Windows, removing a result that another run holds open can also fail, and the script then exits with "rm: cannot remove '/tmp/nvx-egress-icmp': Permission denied". The scripts now keep their results in TMPDIR, which _probe_environment sets to a directory of each run's own: a temporary directory for the L3/L4 scripts, through the new _run_egress_script, and for ipv6-egress-policy the tmp of the fake guest root in which _run_ipv6_guest_script already gives the script its /proc and resolver. Each test also leaves a stale result in that directory and requires that the run replace it with exactly its probes' results, so a probe or a cleanup that went back to a shared /tmp fails the test even when it runs alone, as the scripts did before the previous commit. Before this change, every one of eight processes that each ran the three tests 15 times at once failed on Linux KVM and MSHV hosts, and every one of six processes that each ran them 10 times failed on Windows hosts. Four whole runs of scripts/test_microvm_tests.py at once failed the egress tests in most runs on the Linux hosts, and in two of three runs on a Windows host. With this change, the egress tests passed in every one of these runs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/test_microvm_tests.py | 95 +++++++++++++++++++++++++++++------ 1 file changed, 79 insertions(+), 16 deletions(-) diff --git a/scripts/test_microvm_tests.py b/scripts/test_microvm_tests.py index 11e3dd354..489324441 100644 --- a/scripts/test_microvm_tests.py +++ b/scripts/test_microvm_tests.py @@ -81,6 +81,16 @@ def _posix_shell() -> str | None: return shell +def _probe_environment(probe_dir: Path) -> dict[str, str]: + """Return the environment of a run of an egress script in the host's + shell, which then keeps its probes' results in `probe_dir`. + + The scripts keep the results in TMPDIR, or else in /tmp, which is private + to a guest but shared on the host: runs there at once would delete and + overwrite each other's results (#1007).""" + return {**os.environ, "TMPDIR": str(probe_dir)} + + def _bdi_stats(dirty: int, io: int = 0, more_io: int = 0) -> str: """Return a backing device's debugfs statistics whose writeback lists hold `dirty`, `io`, and `more_io` inodes, in the kernel's layout.""" @@ -8441,6 +8451,26 @@ def test_l3_l4_egress_protocols_reject_leaks_and_missed_traffic(self): (output_dir / "l3-l4-egress-protocols-results.json").exists() ) + def _run_egress_script( + self, shell: str, script: str + ) -> tuple[subprocess.CompletedProcess[str], list[str]]: + """Run an L3/L4 egress script in the host's shell with a TMPDIR of its + own, which holds a stale result, and return the shell's result and the + names of the files that the run left in that TMPDIR.""" + with tempfile.TemporaryDirectory() as temporary: + probe_dir = Path(temporary) + (probe_dir / "nvx-egress-stale").write_text("stale\n", encoding="utf-8") + result = subprocess.run( + [shell], + input=script, + text=True, + capture_output=True, + timeout=10, + check=False, + env=_probe_environment(probe_dir), + ) + return result, sorted(path.name for path in probe_dir.iterdir()) + def test_l3_l4_egress_protocol_script_reports_each_probe(self): shell = _posix_shell() if shell is None: @@ -8468,14 +8498,7 @@ def test_l3_l4_egress_protocol_script_reports_each_probe(self): "}\n" "ping() { return 1; }\n" ) - result = subprocess.run( - [shell], - input=stubs + script, - text=True, - capture_output=True, - timeout=10, - check=False, - ) + result, probes = self._run_egress_script(shell, stubs + script) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual( @@ -8490,6 +8513,17 @@ def test_l3_l4_egress_protocol_script_reports_each_probe(self): "NVX-EXIT 0", ], ) + # The run replaced the stale result in its TMPDIR with its probes'. + self.assertEqual( + probes, + [ + "nvx-egress-icmp", + "nvx-egress-tcp-named", + "nvx-egress-tcp-other", + "nvx-egress-udp-named", + "nvx-egress-udp-other", + ], + ) def test_egress_probe_server_records_only_matching_probes(self): tcp, udp = microvm_tests._bind_dual_protocol_ports(2, "127.0.0.1") @@ -9085,14 +9119,7 @@ def test_l3_l4_egress_port_range_script_reports_each_probe(self): " esac\n" "}\n" ) - result = subprocess.run( - [shell], - input=stubs + script, - text=True, - capture_output=True, - timeout=10, - check=False, - ) + result, probes = self._run_egress_script(shell, stubs + script) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual( @@ -9112,6 +9139,22 @@ def test_l3_l4_egress_port_range_script_reports_each_probe(self): "NVX-EXIT 0", ], ) + # The run replaced the stale result in its TMPDIR with its probes'. + self.assertEqual( + probes, + [ + "nvx-egress-tcp-above", + "nvx-egress-tcp-below", + "nvx-egress-tcp-first", + "nvx-egress-tcp-inside", + "nvx-egress-tcp-last", + "nvx-egress-udp-above", + "nvx-egress-udp-below", + "nvx-egress-udp-first", + "nvx-egress-udp-inside", + "nvx-egress-udp-last", + ], + ) def test_runner_dispatches_l3_l4_egress_port_ranges(self): def require(path: Path, _description: str) -> Path: @@ -9638,6 +9681,9 @@ def _run_ipv6_guest_script( .replace("/proc/", f"{root.as_posix()}/proc/") .replace("/etc/resolv.conf", f"{root.as_posix()}/etc/resolv.conf") ) + # The root's tmp stands for the guest's /tmp, where the probes keep + # their results. + (root / "tmp").mkdir(exist_ok=True) return subprocess.run( [shell], input="record_exit() { printf 'NVX-EXIT %s\\n' \"$1\"; }\n" @@ -9647,6 +9693,7 @@ def _run_ipv6_guest_script( capture_output=True, timeout=10, check=False, + env=_probe_environment(root / "tmp"), ) def test_ipv6_egress_script_checks_the_identity_and_reports_each_probe(self): @@ -9705,6 +9752,10 @@ def test_ipv6_egress_script_checks_the_identity_and_reports_each_probe(self): root = Path(temporary) (root / "proc" / "net").mkdir(parents=True) (root / "etc").mkdir() + (root / "tmp").mkdir() + (root / "tmp" / "nvx-egress-stale").write_text( + "stale\n", encoding="utf-8" + ) (root / "proc" / "cmdline").write_text( "quiet virtnet_ip=192.0.2.2 virtnet_mask=255.255.255.0 " f"virtnet_gw=192.0.2.1{dns} virtnet_ip6=fd00::c000:202/120 " @@ -9741,6 +9792,18 @@ def test_ipv6_egress_script_checks_the_identity_and_reports_each_probe(self): "NVX-EXIT 0", ], ) + # The run replaced the stale result in the root's tmp with its + # probes'. + self.assertEqual( + sorted(path.name for path in (root / "tmp").iterdir()), + [ + "nvx-egress-icmp6", + "nvx-egress-tcp4-named", + "nvx-egress-tcp6-named", + "nvx-egress-tcp6-other", + "nvx-egress-udp6-named", + ], + ) def test_ipv6_egress_snapshot_script_probes_across_the_snapshot(self): shell = _posix_shell()