Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions doc/benchmarks.md
Original file line number Diff line number Diff line change
Expand Up @@ -799,9 +799,12 @@ series, so a `dev` run that finishes after the reset cannot refill it.

The regression gate compares the target p50 with the median of the latest 10
p50 values on the pull request's base branch and requires all 10
matching history points. A metric regresses only when it is more than 40%
worse. Lower-is-better millisecond metrics must also be more than 5 ms
matching history points. A metric regresses only when it is more than 50%
worse. Lower-is-better millisecond metrics must also be more than 10 ms
slower; higher-is-better metrics use the percentage comparison alone.
[`ci.yml`](../.github/workflows/ci.yml) sets these thresholds;
[`performance gate`](usage.md#performance-gate) defaults to 40% and 5 ms when
it runs without them.
Missing or insufficient history is a warmup, not a failure. Successful `dev`
builds append collected results to topology-specific files in `data/`. Every
new ABI/count series begins as a warmup baseline before its regression gate
Expand Down
32 changes: 32 additions & 0 deletions scripts/test_performance.py
Original file line number Diff line number Diff line change
Expand Up @@ -634,6 +634,38 @@ def test_documented_metric_tables_match_collectors(self):
| performance.DEVICE_IO_METRIC_NAMES,
)

def test_documented_gate_thresholds_match_ci_and_cli(self):
repository = Path(__file__).parents[1]
workflow = (repository / ".github" / "workflows" / "ci.yml").read_text(
encoding="utf-8"
)
threshold, tolerance = (
re.findall(rf"^[ \t]*{name}:[ \t]*(\d+)[ \t]*$", workflow, re.MULTILINE)
for name in (
"PERFORMANCE_REGRESSION_THRESHOLD",
"PERFORMANCE_REGRESSION_ABSOLUTE_TOLERANCE_MS",
)
)
benchmarks, usage = (
" ".join((repository / "doc" / name).read_text(encoding="utf-8").split())
for name in ("benchmarks.md", "usage.md")
)
defaults = nvx.parse_args(
["performance", "gate", "--baseline-dir", "base", "--target-dir", "target"]
)
threshold_default = f"{defaults.threshold:g}"
tolerance_default = f"{defaults.absolute_tolerance_ms:g}"

self.assertEqual(len(threshold), 1)
self.assertEqual(len(tolerance), 1)
self.assertIn(f"more than {threshold[0]}% worse", benchmarks)
self.assertIn(f"more than {tolerance[0]} ms slower", benchmarks)
self.assertIn(
f"defaults to {threshold_default}% and {tolerance_default} ms", benchmarks
)
self.assertIn(f"`--threshold` to `{threshold_default}`", usage)
self.assertIn(f"`--absolute-tolerance-ms` to `{tolerance_default}`", usage)

def test_collect_cli_accepts_lifecycle_input(self):
args = nvx.parse_args(
[
Expand Down
Loading