ci: the daily performance run reports regressions, improvements and a ratchet

.github/workflows/perf-daily.yaml

@@ -11,6 +11,12 @@ name: Perf daily
 # PR's days included): no new run, no new or updated pull request. A manual
 # dispatch always runs.
 #
+# The run compares each set's newest history line with the previous one and
+# reports regressions, improvements and the ratchet (bench/ab/history/ratchet.json,
+# rows as good as Loki) at the top of the daily pull request and in the job
+# summary. A regression or a lost ratchet row turns the run red, which is the
+# notification; nothing else is opened or labelled.
+#
 # A pull request that edits this pipeline runs a short validation instead
 # (control set, 1h, 3 runs): it writes the files on the runner and shows the
 # trend report in the job summary, but opens no pull request.
@@ -80,6 +86,8 @@ jobs:
       - name: Publish trend
         if: always()
         run: |
+          # Alerts are shown here and never fail a pull request's validation.
+          if [ -f "$OUT/alerts.md" ]; then cat "$OUT/alerts.md" >> "$GITHUB_STEP_SUMMARY"; fi
           if [ -f bench/ab/history/trend.md ]; then cat bench/ab/history/trend.md >> "$GITHUB_STEP_SUMMARY"; fi
           git status --short
 
@@ -91,6 +99,7 @@ jobs:
           path: |
             ${{ env.OUT }}/*.json
             ${{ env.OUT }}/*.log
+            ${{ env.OUT }}/*.md
             bench/ab/results/daily-*.json
             bench/ab/history/
           if-no-files-found: ignore
@@ -179,6 +188,7 @@ jobs:
           path: |
             ${{ env.OUT }}/*.json
             ${{ env.OUT }}/*.log
+            ${{ env.OUT }}/*.md
             bench/ab/results/daily-*.json
             bench/ab/history/
           if-no-files-found: ignore
@@ -196,8 +206,14 @@ jobs:
           # config, and the checkout above did not keep one.
           HEADER="$(printf 'x-access-token:%s' "${PUSH_TOKEN}" | base64 | tr -d '\n')"
           echo "::add-mask::${HEADER}"
+          # bench/ab/history includes trend.md and ratchet.json.
           FILES=(bench/ab/results bench/ab/history conformance/registry/generated/perf-evidence.json
                  conformance/reports/performance.md conformance/reports/gaps.md)
+          ALERTS=""
+          if [ -f "$OUT/alerts.md" ]; then
+            ALERTS="$(cat "$OUT/alerts.md")"
+            echo "$ALERTS" >> "$GITHUB_STEP_SUMMARY"
+          fi
           if [ -z "$(git status --porcelain -- "${FILES[@]}")" ]; then
             echo "nothing changed"
             exit 0
@@ -210,9 +226,10 @@ jobs:
           git commit -m "bench: daily performance report ${DATE}"
           git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic ${HEADER}" push origin "$BRANCH" --force
           TITLE="bench: daily performance report ${DATE}"
-          BODY="$(printf '%s\n\n%s\n\n%s\n' \
+          BODY="$(printf '%s\n\n%s\n\n%s\n\n%s\n' \
+            "$ALERTS" \
             "Daily A/B run from \`main\` ($(git rev-parse --short HEAD^1)): the last release vs \`main\` vs Loki direct on a fresh stack, every shape set over its ranges up to 24h." \
-            "Updates \`bench/ab/results/daily-*.json\`, appends \`bench/ab/history/*.jsonl\`, regenerates \`bench/ab/history/trend.md\` and the registry performance evidence. Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
+            "Updates \`bench/ab/results/daily-*.json\`, appends \`bench/ab/history/*.jsonl\`, regenerates \`bench/ab/history/trend.md\`, updates \`bench/ab/history/ratchet.json\` and the registry performance evidence. Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
             "Merging keeps the history continuous; an unmerged day is carried into the next day's PR.")"
           PR_NUMBER="$(gh pr list --repo "$GITHUB_REPOSITORY" --head "$BRANCH" --base main --state open --json number --jq '.[0].number // empty')"
           if [ -n "$PR_NUMBER" ]; then
@@ -241,3 +258,19 @@ jobs:
           if ! gh pr merge "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" --auto --squash; then
             echo "::warning::could not arm auto-merge on pull request #${PR_NUMBER}; enable it by hand"
           fi
+
+      - name: Fail on regressions
+        if: steps.measured.outputs.run != 'false'
+        run: |
+          # Red run = the notification. Regressions are transitions since the
+          # previous run; a ratchet loss stays red until fixed or the row is
+          # removed from ratchet.json by hand.
+          [ -f "$OUT/alerts.json" ] || exit 0
+          python3 - "$OUT/alerts.json" <<'PY'
+          import json, sys
+          a = json.load(open(sys.argv[1]))
+          for key in ("regressions", "ratchet_lost"):
+              for r in a[key]:
+                  print(f"::error title=perf {key}::{r['set']} / {r['shape']} / {r['range']}: {r['detail']} ({r['commits']})")
+          sys.exit(1 if a["regressions"] or a["ratchet_lost"] else 0)
+          PY

CHANGELOG.md

@@ -27,6 +27,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   measurement of every current shape set was taken at (7 runs, every range up
   to 24h; an open daily pull request's days included). A manual dispatch
   always runs, and a history that cannot be read runs the measurement.
+- **The daily performance run reports regressions, improvements and a ratchet
+  of the rows that are as good as Loki, and turns red on a regression.** The
+  daily run only recorded numbers, so a result that started differing from
+  Loki or a query that started failing was found by whoever read the trend
+  next. `bench/ab/history.py alerts` now compares each set's newest history
+  line with the previous valid one and reports transitions, not state (many
+  rows already differ from Loki and must not alert every day): a result that
+  went `same` to `differs`, a status that went 200 to an error Loki does not
+  give, and a warm p50 slower beyond the 25% and 50 ms noise rule that held for
+  two runs or that the same run's verdict against the release also calls
+  `slower`; and the improvements (the reverse of the first two, or faster beyond
+  the noise rule in one run). Each alert names its commit range to bisect. A new
+  history column, `loki_cold_computed`, records from Loki's cache counters
+  whether Loki computed its cold run or its results cache answered it (null on
+  older lines, which each carry their own `cols`). `bench/ab/history/ratchet.json`
+  lists the rows whose newest line has the same result as Loki, answers 200 and
+  has a cold run no slower than a Loki cold run that was computed (never warm
+  p50, which for Loki is its results cache); the run only adds to it, rows leave
+  it by hand, and a ratcheted row that differs or fails, or whose cold run is
+  slower than Loki's computed cold run in two runs in a row, is a ratchet
+  loss. The alerts go at the top of the daily pull request and into the job
+  summary (`alerts.md` is written next to the run's files, not committed), and
+  a final `Fail on regressions` step fails the job on a regression or a ratchet
+  loss, so GitHub notifies the owner; no issue is opened and no label added.
+  The pull request validation shows the alerts and does not fail on them.
 
 ### Documentation
 

bench/ab/README.md

@@ -353,8 +353,9 @@ release build's 24h ranges are capped at 3 runs. It writes:
 | path | content |
 |---|---|
 | `results/daily-<set>.json` | the day's summary per set (`report.py` format), overwritten daily; git history keeps every day, and `perf_evidence.py` reads it as the latest evidence |
-| `history/<set>.jsonl` | one line per set per day, append-only: date, commit, baseline, runs, validity, and one row per shape × range in the column order its `cols` names (proxy status, warm p50, cold, Loki p50, cold and status, result vs Loki, release p50, verdict vs release) |
+| `history/<set>.jsonl` | one line per set per day, append-only: date, commit, baseline, runs, validity, and one row per shape × range in the column order its `cols` names (proxy status, warm p50, cold, Loki p50, cold and status, result vs Loki, release p50, verdict vs release, and whether Loki computed its cold run or its results cache answered it) |
 | `history/trend.md` | per set: proxy p50, day-over-day and week-over-week change (🟢/🔴 beyond 25% and 50 ms), Loki p50, proxy ÷ Loki warm and cold, result vs Loki and a 14-day sparkline |
+| `history/ratchet.json` | the rows that were as good as Loki, one entry per set × shape × range; the run only adds to it |
 | `conformance/registry/generated/perf-evidence.json`, `conformance/reports/performance.md`, `gaps.md` | regenerated from the new results |
 
 The files reach `main` through a bot pull request on the rolling branch
@@ -377,6 +378,32 @@ stack's `test/e2e-compat/docker-compose.yml`, `loki-local-config.yaml` and
 `log-generator.py`, and the workflow itself. A manual dispatch always runs, and
 a history that cannot be read runs the measurement.
 
+Each run also compares every set's newest history line with the previous
+valid one (`history.py alerts`) and reports transitions, not the state many rows
+are already in (rows that differ from Loki today do not alert again). A
+regression is a result that went `same` → `differs`, a status that went 200 →
+an error Loki does not give, or a warm p50 slower beyond 25% and 50 ms that held
+for two runs (both worse than the run before them) or that the same run's
+verdict against the release also calls `slower`. An improvement is the reverse
+of the first two, or a warm p50 faster beyond the same margin (one run is
+enough). Each alert names the commit range to bisect, `<previous line's
+commit>..<newest commit>` (a slowdown that held for two runs starts at the line
+before it began). A row enters `history/ratchet.json` when its newest line has
+the same result as Loki, answers 200 and its cold run is no slower than Loki's
+cold run, and only when Loki computed that cold run (`loki_cold_computed`, from
+Loki's cache counters): never on warm p50, which for Loki is its results cache,
+and never on lines written before the column existed. A ratchet loss is a
+ratcheted row whose newest line differs from Loki or fails (at once), or whose
+cold run is slower than Loki's computed cold run beyond the margin in the newest
+line and the previous valid one (a cold run is one sample); it repeats every run
+until fixed.
+Rows leave the list only by hand. The run writes the alerts to
+`<out>/alerts.md` and `alerts.json` (not committed: they describe one run), puts
+them at the top of the daily pull request and in the job summary, and a final
+step fails the job when there is a regression or a ratchet loss, so the red run
+is the notification. A pull request that edits the pipeline shows them and never
+fails on them.
+
 For docs, read `history/<set>.jsonl` (one JSON object per line; zip `cols`
 with each row) or reuse `history/trend.md` as is. A pull request that edits
 the daily pipeline runs a short version of it (control set, 1h, 3 runs) that

bench/ab/daily.py

@@ -9,6 +9,11 @@
                                       history keeps every day)
   bench/ab/history/<set>.jsonl        one appended line per set per day
   bench/ab/history/trend.md           day-over-day and week-over-week report
+  bench/ab/history/ratchet.json       rows as good as Loki (only ever added to)
+  <out>/alerts.md, alerts.json        regressions, improvements and ratchet
+                                      losses of this run's lines (not committed:
+                                      they describe one run; the workflow puts
+                                      them in the pull request and fails on them)
   conformance/registry/generated/perf-evidence.json, conformance/reports/
   performance.md and gaps.md          regenerated from the new results
 
@@ -117,12 +122,20 @@ def main():
         # of results and history, fail the job so the next day retries.
         stack.log("run invalid (VictoriaLogs restarted); nothing written")
         return 2
+    written = []
     for summary in summaries:
         with open(os.path.join(RESULTS, f"daily-{summary['set']}.json"), "w") as f:
             json.dump(summary, f, indent=1)
-        history.append(summary, commit=head)
+        if history.append(summary, commit=head)[1]:
+            written.append(summary["set"])
     with open(os.path.join(history.HISTORY, "trend.md"), "w") as f:
         f.write(history.trend())
+    # Only the sets this run wrote: a manual run of one set must not report the others' old lines again.
+    found = history.write_alerts(history.HISTORY, written, os.path.join(out, "alerts.json"),
+                                 os.path.join(out, "alerts.md"))
+    stack.log(f"alerts: {len(found['regressions'])} regressions, {len(found['improvements'])} improvements, "
+              f"{len(found['ratchet_lost'])} ratchet losses, ratchet {len(found['ratchet'])} rows "
+              f"(+{len(found['ratchet_added'])})")
     for script in ("perf_evidence.py", "gaps.py"):
         subprocess.run([PY, os.path.join(ROOT, "conformance/scripts", script)], cwd=ROOT, check=True)
     stack.log(f"daily run written in {(time.time() - t0) / 60:.1f} min")