From e748c7dd9ba901aa91b9c7a6cf001cc8d94e132a Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Wed, 28 Oct 2026 16:59:05 -0700 Subject: [PATCH 1/2] doc: state the thresholds that CI's performance gate uses #400 replaced the thresholds in doc/benchmarks.md, 50% and 10 ms, with the defaults of performance gate, 40% and 5 ms. That paragraph describes the CI gate, and ci.yml has passed 50% and 10 ms since the first commit, so the guide described a stricter gate than CI runs. Restore CI's thresholds, name ci.yml as their source, and point to the command's defaults in the CLI reference. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- doc/benchmarks.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/benchmarks.md b/doc/benchmarks.md index ed54bd4c0..fd0fcc3ec 100644 --- a/doc/benchmarks.md +++ b/doc/benchmarks.md @@ -799,9 +799,12 @@ series, so a `dev` run that finishes after the reset cannot refill it. The regression gate compares the target p50 with the median of the latest 10 p50 values on the pull request's base branch and requires all 10 -matching history points. A metric regresses only when it is more than 40% -worse. Lower-is-better millisecond metrics must also be more than 5 ms +matching history points. A metric regresses only when it is more than 50% +worse. Lower-is-better millisecond metrics must also be more than 10 ms slower; higher-is-better metrics use the percentage comparison alone. +[`ci.yml`](../.github/workflows/ci.yml) sets these thresholds; +[`performance gate`](usage.md#performance-gate) defaults to 40% and 5 ms when +it runs without them. Missing or insufficient history is a warmup, not a failure. Successful `dev` builds append collected results to topology-specific files in `data/`. Every new ABI/count series begins as a warmup baseline before its regression gate From ef38ef14f9969c4569552b2ba4cd53887b54b6e4 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Wed, 28 Oct 2026 16:59:05 -0700 Subject: [PATCH 2/2] scripts: check that the guides state the performance gate's thresholds The benchmark guide states the thresholds that ci.yml passes to the regression gate, and both guides state the defaults of performance gate. Read both from their sources, so that a change to either, or an edit like #400's, fails the tests until the guides follow. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/test_performance.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/scripts/test_performance.py b/scripts/test_performance.py index 2f7096491..897923d3c 100644 --- a/scripts/test_performance.py +++ b/scripts/test_performance.py @@ -634,6 +634,38 @@ def test_documented_metric_tables_match_collectors(self): | performance.DEVICE_IO_METRIC_NAMES, ) + def test_documented_gate_thresholds_match_ci_and_cli(self): + repository = Path(__file__).parents[1] + workflow = (repository / ".github" / "workflows" / "ci.yml").read_text( + encoding="utf-8" + ) + threshold, tolerance = ( + re.findall(rf"^[ \t]*{name}:[ \t]*(\d+)[ \t]*$", workflow, re.MULTILINE) + for name in ( + "PERFORMANCE_REGRESSION_THRESHOLD", + "PERFORMANCE_REGRESSION_ABSOLUTE_TOLERANCE_MS", + ) + ) + benchmarks, usage = ( + " ".join((repository / "doc" / name).read_text(encoding="utf-8").split()) + for name in ("benchmarks.md", "usage.md") + ) + defaults = nvx.parse_args( + ["performance", "gate", "--baseline-dir", "base", "--target-dir", "target"] + ) + threshold_default = f"{defaults.threshold:g}" + tolerance_default = f"{defaults.absolute_tolerance_ms:g}" + + self.assertEqual(len(threshold), 1) + self.assertEqual(len(tolerance), 1) + self.assertIn(f"more than {threshold[0]}% worse", benchmarks) + self.assertIn(f"more than {tolerance[0]} ms slower", benchmarks) + self.assertIn( + f"defaults to {threshold_default}% and {tolerance_default} ms", benchmarks + ) + self.assertIn(f"`--threshold` to `{threshold_default}`", usage) + self.assertIn(f"`--absolute-tolerance-ms` to `{tolerance_default}`", usage) + def test_collect_cli_accepts_lifecycle_input(self): args = nvx.parse_args( [