From b6f5fbc44b0c2211d3560f5be94ca4579b5ee634 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sat, 17 Oct 2026 16:37:09 -0700 Subject: [PATCH 1/3] Tell a Timed-Out Node Probe From an Engine Disagreement in test_spec_validate The corpus probe ran node under a 60 second timeout, and a cold start on a slow hosted Windows runner raised TimeoutExpired, failing the suite as though the schema patterns disagreed. A timeout now skips the test with that reason, as a missing node already does. A pattern that backtracks without end still fails rather than skipping, since Python matches the same corpus before node starts. Fixes #3182 Co-Authored-By: Claude Opus 5.5 --- tests/test_spec_validate.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/test_spec_validate.py b/tests/test_spec_validate.py index 96627095..69a73421 100755 --- a/tests/test_spec_validate.py +++ b/tests/test_spec_validate.py @@ -1481,6 +1481,9 @@ def test_both_engines_agree_on_every_pattern_over_a_corpus(self) -> None: Node is what an editor's JSON language service uses, so it is the engine the schema actually meets. A host without node skips this rather than failing, since the shape check above still binds everywhere. + + A probe that times out skips too, since a cold start on a slow runner is no verdict on the patterns. A + pattern that backtracks without end cannot hide behind that skip, since Python matches the corpus first. """ node = shutil.which("node") if node is None: @@ -1539,14 +1542,17 @@ def test_both_engines_agree_on_every_pattern_over_a_corpus(self) -> None: "console.log(JSON.stringify(Object.fromEntries(" "pats.map(p => [p, cases.map(c => new RegExp(p).test(c))]))));" ) - result = subprocess.run( - [node, "-e", probe, json.dumps([patterns, corpus])], - capture_output=True, - text=True, - encoding="utf-8", - timeout=60, - check=True, - ) + try: + result = subprocess.run( + [node, "-e", probe, json.dumps([patterns, corpus])], + capture_output=True, + text=True, + encoding="utf-8", + timeout=60, + check=True, + ) + except subprocess.TimeoutExpired: + self.skipTest("node timed out, which is no verdict on the patterns") self.assertEqual(json.loads(result.stdout), want) def test_the_description_pattern_never_refuses_what_the_validator_accepts(self) -> None: From a7a2daf70684f49deaab9d6197043a0cb99c8c04 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sat, 17 Oct 2026 16:43:42 -0700 Subject: [PATCH 2/3] Test the Timed-Out Corpus Probe Skip in test_spec_validate The skip was reachable only through a real 60 second timeout, so nothing protected it from being removed or mis-scoped. The new test makes the probe raise TimeoutExpired and asserts the corpus test skips with that reason rather than failing. Co-Authored-By: Claude Opus 5.5 --- tests/test_spec_validate.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_spec_validate.py b/tests/test_spec_validate.py index 69a73421..d3ec91ff 100755 --- a/tests/test_spec_validate.py +++ b/tests/test_spec_validate.py @@ -1555,6 +1555,19 @@ def test_both_engines_agree_on_every_pattern_over_a_corpus(self) -> None: self.skipTest("node timed out, which is no verdict on the patterns") self.assertEqual(json.loads(result.stdout), want) + def test_a_timed_out_corpus_probe_skips_rather_than_failing(self) -> None: + """A real timeout takes a minute, so the probe is made to raise one rather than to be slow.""" + timeout = subprocess.TimeoutExpired("node", 60) + with ( + unittest.mock.patch.object(shutil, "which", return_value="node"), + unittest.mock.patch.object(subprocess, "run", side_effect=timeout), + ): + result = unittest.TestResult() + type(self)("test_both_engines_agree_on_every_pattern_over_a_corpus").run(result) + self.assertEqual([], result.failures + result.errors) + self.assertEqual(1, len(result.skipped)) + self.assertIn("node timed out", result.skipped[0][1]) + def test_the_description_pattern_never_refuses_what_the_validator_accepts(self) -> None: """The schema's own statement rather than a copy, so the direction is what has to hold. From 1eba44ada19ab7e67c7522bdb25ea56e6ac15dec Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sat, 17 Oct 2026 16:44:57 -0700 Subject: [PATCH 3/3] Fail the Skip Test When the Timeout Handler Widens in test_spec_validate The skip test fed in only TimeoutExpired, so an except widened to every subprocess error would still pass it while turning node refusing a pattern into a skip. It now also feeds in CalledProcessError and asserts that one errors rather than skipping. Co-Authored-By: Claude Opus 5.5 --- tests/test_spec_validate.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/tests/test_spec_validate.py b/tests/test_spec_validate.py index d3ec91ff..c725d697 100755 --- a/tests/test_spec_validate.py +++ b/tests/test_spec_validate.py @@ -1556,17 +1556,27 @@ def test_both_engines_agree_on_every_pattern_over_a_corpus(self) -> None: self.assertEqual(json.loads(result.stdout), want) def test_a_timed_out_corpus_probe_skips_rather_than_failing(self) -> None: - """A real timeout takes a minute, so the probe is made to raise one rather than to be slow.""" - timeout = subprocess.TimeoutExpired("node", 60) - with ( - unittest.mock.patch.object(shutil, "which", return_value="node"), - unittest.mock.patch.object(subprocess, "run", side_effect=timeout), - ): + """A real timeout takes a minute, so the probe is made to raise one rather than to be slow. + + Node refusing a pattern exits non-zero, and that still fails, so a widened skip is caught too. + """ + + def probe(side_effect: BaseException) -> unittest.TestResult: result = unittest.TestResult() - type(self)("test_both_engines_agree_on_every_pattern_over_a_corpus").run(result) - self.assertEqual([], result.failures + result.errors) - self.assertEqual(1, len(result.skipped)) - self.assertIn("node timed out", result.skipped[0][1]) + with ( + unittest.mock.patch.object(shutil, "which", return_value="node"), + unittest.mock.patch.object(subprocess, "run", side_effect=side_effect), + ): + type(self)("test_both_engines_agree_on_every_pattern_over_a_corpus").run(result) + return result + + timed_out = probe(subprocess.TimeoutExpired("node", 60)) + self.assertEqual([], timed_out.failures + timed_out.errors) + self.assertEqual(1, len(timed_out.skipped)) + self.assertIn("node timed out", timed_out.skipped[0][1]) + refused = probe(subprocess.CalledProcessError(1, "node")) + self.assertEqual([], refused.skipped) + self.assertEqual(1, len(refused.errors)) def test_the_description_pattern_never_refuses_what_the_validator_accepts(self) -> None: """The schema's own statement rather than a copy, so the direction is what has to hold.