diff --git a/doc/benchmarks.md b/doc/benchmarks.md index ed54bd4c0..fd0fcc3ec 100644 --- a/doc/benchmarks.md +++ b/doc/benchmarks.md @@ -799,9 +799,12 @@ series, so a `dev` run that finishes after the reset cannot refill it. The regression gate compares the target p50 with the median of the latest 10 p50 values on the pull request's base branch and requires all 10 -matching history points. A metric regresses only when it is more than 40% -worse. Lower-is-better millisecond metrics must also be more than 5 ms +matching history points. A metric regresses only when it is more than 50% +worse. Lower-is-better millisecond metrics must also be more than 10 ms slower; higher-is-better metrics use the percentage comparison alone. +[`ci.yml`](../.github/workflows/ci.yml) sets these thresholds; +[`performance gate`](usage.md#performance-gate) defaults to 40% and 5 ms when +it runs without them. Missing or insufficient history is a warmup, not a failure. Successful `dev` builds append collected results to topology-specific files in `data/`. Every new ABI/count series begins as a warmup baseline before its regression gate diff --git a/scripts/test_performance.py b/scripts/test_performance.py index 2f7096491..897923d3c 100644 --- a/scripts/test_performance.py +++ b/scripts/test_performance.py @@ -634,6 +634,38 @@ def test_documented_metric_tables_match_collectors(self): | performance.DEVICE_IO_METRIC_NAMES, ) + def test_documented_gate_thresholds_match_ci_and_cli(self): + repository = Path(__file__).parents[1] + workflow = (repository / ".github" / "workflows" / "ci.yml").read_text( + encoding="utf-8" + ) + threshold, tolerance = ( + re.findall(rf"^[ \t]*{name}:[ \t]*(\d+)[ \t]*$", workflow, re.MULTILINE) + for name in ( + "PERFORMANCE_REGRESSION_THRESHOLD", + "PERFORMANCE_REGRESSION_ABSOLUTE_TOLERANCE_MS", + ) + ) + benchmarks, usage = ( + " ".join((repository / "doc" / name).read_text(encoding="utf-8").split()) + for name in ("benchmarks.md", "usage.md") + ) + defaults = nvx.parse_args( + ["performance", "gate", "--baseline-dir", "base", "--target-dir", "target"] + ) + threshold_default = f"{defaults.threshold:g}" + tolerance_default = f"{defaults.absolute_tolerance_ms:g}" + + self.assertEqual(len(threshold), 1) + self.assertEqual(len(tolerance), 1) + self.assertIn(f"more than {threshold[0]}% worse", benchmarks) + self.assertIn(f"more than {tolerance[0]} ms slower", benchmarks) + self.assertIn( + f"defaults to {threshold_default}% and {tolerance_default} ms", benchmarks + ) + self.assertIn(f"`--threshold` to `{threshold_default}`", usage) + self.assertIn(f"`--absolute-tolerance-ms` to `{tolerance_default}`", usage) + def test_collect_cli_accepts_lifecycle_input(self): args = nvx.parse_args( [