measure(ci): cibench re-baseline on the landing branch (P7-CI-46)

.github/workflows/ci-bench.yml

@@ -1,13 +1,17 @@
 name: ci-bench
 
+# Local overhead of the nself ci V15 engine against the v1.4 gate (P7-CI-46).
+# Runs cibench on a hosted runner, then the cli's own `perfbench check` against the
+# per-platform budget (absolute p95) and `cibench -ratio-check` (V15 median against
+# the v1.4 median). Two controls prove the gates can fire: a result ten times over
+# every budget must fail both checks, and a result inside the relative budget must
+# pass. Budgets and baselines: hq plan/p7/evidence/CI-46-perf.md.
+
 on:
   pull_request:
     paths:
-      - "free/ci/internal/sched/**"
-      - "free/ci/internal/dispatch/**"
-      - "free/ci/internal/world/**"
-      - "free/ci/bench/**"
-      - ".github/workflows/ci-bench.yml"
+      - 'free/ci/**'
+      - '.github/workflows/ci-bench.yml'
   workflow_dispatch:
 
 permissions:
@@ -17,43 +21,98 @@ concurrency:
   group: ci-bench-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  CGO_ENABLED: '0'
+  # nself-org/cli revision whose tools/perfbench reads the result and budget files.
+  CLI_SHA: '529e8917244bffe4ab1aa9d42b081dd3efe5ee73'
+
 jobs:
-  schedbench:
-    if: github.repository == 'nself-org/plugins'
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
+  bench:
+    name: bench (${{ matrix.name }})
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: linux-amd64
+            os: ubuntu-latest
+            budget: perf-budget.json
+            services: true
+          - name: macos-arm64
+            os: macos-14
+            budget: perf-budget-darwin-arm64.json
+            services: false
     env:
-      CGO_ENABLED: "0"
-      GOFLAGS: -mod=vendor
+      BUDGET: ${{ github.workspace }}/free/ci/bench/${{ matrix.budget }}
+      SLOW: ${{ github.workspace }}/free/ci/bench/testdata/slow-result.json
+      OK: ${{ github.workspace }}/free/ci/bench/testdata/ratio-ok-result.json
     steps:
-      - name: Check out the repository
-        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
         with:
           persist-credentials: false
-      - name: Set up Go
-        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
+
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
         with:
-          go-version-file: free/ci/go.mod
+          repository: nself-org/cli
+          ref: ${{ env.CLI_SHA }}
+          path: cli
+          persist-credentials: false
+
+      # The cli module needs a newer Go than the plugin pins, and setup-go sets
+      # GOTOOLCHAIN=local, so one toolchain (the newer) builds both.
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
+        with:
+          go-version-file: cli/go.mod
           cache: false
-      - name: Measure scheduler and admission on linux-amd64
-        shell: bash
+
+      - name: Harness tests
+        working-directory: free/ci
+        run: go test -mod=vendor -count=1 ./bench/...
+
+      - name: Measure
+        working-directory: free/ci
+        run: go run -mod=vendor ./bench/cmd/cibench -runs 20 -warmup 2 -out "$RUNNER_TEMP/ci-bench.json"
+
+      - name: Absolute p95 budget (cli perfbench check)
+        working-directory: cli
+        run: go run -mod=vendor ./tools/perfbench check -budget "$BUDGET" -in "$RUNNER_TEMP/ci-bench.json"
+
+      - name: Relative budget (V15 against v1.4)
+        working-directory: free/ci
+        run: go run -mod=vendor ./bench/cmd/cibench -ratio-check -in "$RUNNER_TEMP/ci-bench.json"
+
+      - name: Control - the gates fail an injected slowdown and pass a good result
+        if: ${{ !cancelled() }}
         run: |
-          set -euo pipefail
-          test "$(go env GOOS)/$(go env GOARCH)" = linux/amd64
-          mkdir -p "$RUNNER_TEMP/schedbench"
-          {
-            echo "revision=$GITHUB_SHA"
-            uname -a
-            lscpu
-            go version
-            echo "command=CGO_ENABLED=0 go -C free/ci test -mod=vendor -run ^$ -bench Benchmark(Place|Admit)$ -benchmem -count=10 ./internal/sched/ ./internal/dispatch/"
-          } > "$RUNNER_TEMP/schedbench/machine.txt"
-          go -C free/ci test -mod=vendor -run '^$' -bench 'Benchmark(Place|Admit)$' -count=2 ./internal/sched/ ./internal/dispatch/ > "$RUNNER_TEMP/schedbench/warmups.txt"
-          go -C free/ci test -mod=vendor -run '^$' -bench 'Benchmark(Place|Admit)$' -benchmem -count=10 ./internal/sched/ ./internal/dispatch/ | tee "$RUNNER_TEMP/schedbench/samples.txt"
-      - name: Upload raw measurements
+          set -u
+          rc=0
+          (cd cli && go run -mod=vendor ./tools/perfbench check -budget "$BUDGET" -in "$SLOW") || rc=$?
+          if [ "$rc" -ne 1 ]; then
+            echo "::error::perfbench check exited $rc on a result ten times over budget; want 1 (a violation)"; exit 1
+          fi
+          cd free/ci
+          rc=0
+          go run -mod=vendor ./bench/cmd/cibench -ratio-check -in "$SLOW" || rc=$?
+          if [ "$rc" -ne 1 ]; then
+            echo "::error::the ratio check exited $rc on a result ten times over budget; want 1 (a violation)"; exit 1
+          fi
+          go run -mod=vendor ./bench/cmd/cibench -ratio-check -in "$OK"
+
+      - name: Service start (warm and cold)
+        if: ${{ !cancelled() && matrix.services }}
+        run: |
+          set -eu
+          (cd cli && go build -mod=vendor -o "$RUNNER_TEMP/nself" ./cmd/nself)
+          cd free/ci
+          go run -mod=vendor ./bench/cmd/cibench -runs 5 -warmup 0 -skip-medium -services -cold -cold-runs 3 \
+            -nself "$RUNNER_TEMP/nself" -out "$RUNNER_TEMP/ci-bench-services.json"
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: always()
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: schedbench-linux-amd64
-          path: ${{ runner.temp }}/schedbench/
-          if-no-files-found: error
+          name: ci-bench-${{ matrix.name }}
+          path: |
+            ${{ runner.temp }}/ci-bench.json
+            ${{ runner.temp }}/ci-bench-services.json
+          if-no-files-found: ignore

free/ci/architecture/layers.yaml

@@ -174,6 +174,8 @@ layers:
     - tools/deps
     - internal/storetest
     - tools/schemagen
+    - bench
+    - bench/cmd/cibench
     - bench/sched
     - bench/cmd/schedbench
     - bench/cache

free/ci/architecture/layers_test.go

@@ -90,6 +90,9 @@ func checkImport(m layerMap, pkg, file, imported string) error {
 	}
 	local := strings.TrimPrefix(imported, module)
 	isLocal := imported != local
+	if imported == "os/exec" && pkg == "bench" && file == "spawn.go" && kind == "supporting" {
+		return nil // the timing harness starts the processes it times, outside the engine's funnel (P7-CI-46)
+	}
 	if imported == "os/exec" && pkg != "internal/exec" && pkg != "internal/legacy" {
 		// These frozen pre-fabric sites are tracked by the single-exec-site test.
 		for _, old := range []struct{ pkg, file string }{{"internal/forgejo", "forgejo.go"}, {"internal/nodes/provision", "exec.go"}, {"internal/serve", "serve_job.go"}, {"internal/serve", "serve_job_report.go"}} {

free/ci/bench/bench_test.go

@@ -0,0 +1,79 @@
+package bench
+
+// Purpose: the Go benchmarks of the in-process scenarios (`go test -bench`).
+// They call the same functions cibench times, so a benchmark number and a
+// result-JSON number measure the same code.
+
+import (
+	"context"
+	"testing"
+)
+
+func benchInproc(b *testing.B) *Inproc {
+	b.Helper()
+	dir := b.TempDir()
+	if err := Materialize(context.Background(), Tiny, dir+"/repo"); err != nil {
+		b.Fatal(err)
+	}
+	p, err := PrepareInproc(dir+"/repo", dir+"/scratch")
+	if err != nil {
+		b.Fatal(err)
+	}
+	return p
+}
+
+func BenchmarkDetect(b *testing.B) {
+	p := benchInproc(b)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := p.Detect(); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkPlan(b *testing.B) {
+	p := benchInproc(b)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := p.Plan(); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkStoreOpen(b *testing.B) {
+	p := benchInproc(b)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := p.StoreOpen(); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkExecOverhead(b *testing.B) {
+	p := benchInproc(b)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if _, err := p.ExecOverhead(context.Background()); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkRawSpawn(b *testing.B) {
+	p := benchInproc(b)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if _, err := p.RawSpawn(context.Background()); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkCapacityProbe(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		CapacityProbe()
+	}
+}

free/ci/bench/budget.go

@@ -0,0 +1,100 @@
+package bench
+
+// Purpose: the perf-budget.json document (perfbench-budget/v1) and its
+// derivation from a recorded baseline.
+// Inputs:  a baseline Result, a headroom factor, a metric filter.
+// Outputs: a Budget whose absolute p95 ceilings are the baseline p95 plus the
+// headroom, rounded up to a whole millisecond, never below MinCeilingMS.
+// Constraints: the field names are the ones `perfbench check` decodes with
+// DisallowUnknownFields; a metric that is not in the baseline is never given a
+// ceiling (an invented number is the one thing this Ticket must not ship).
+
+import (
+	"fmt"
+	"math"
+	"sort"
+)
+
+// BudgetSchemaV1 is the value of the "schema" field of the budget file.
+const BudgetSchemaV1 = "perfbench-budget/v1"
+
+// Headroom is the allowed growth over the baseline p95 (Ticket constraint: +25%).
+const Headroom = 1.25
+
+// MinCeilingMS is the smallest ceiling a budget gets. Below it a timing is
+// scheduler noise on a shared runner (a 0.3 ms planner run can take 3 ms), so a
+// 25 percent rule on such a baseline would fail on weather.
+const MinCeilingMS = 5.0
+
+// Budget is the budget document. P95MS maps a metric name to its p95 ceiling in ms.
+type Budget struct {
+	Note     string             `json:"_note"`
+	Schema   string             `json:"schema"`
+	Platform string             `json:"platform"`
+	P95MS    map[string]float64 `json:"p95_ms"`
+}
+
+// WorstOf folds independent baseline runs of one platform into one result whose
+// p50, p95 and max are, per metric, the highest seen in any run. One run of 20
+// samples under-states the p95 of a noisy machine; the worst of several is the
+// baseline of record. The runs must come from one goos/goarch.
+func WorstOf(runs []Result) (Result, error) {
+	if len(runs) == 0 {
+		return Result{}, fmt.Errorf("no baseline runs")
+	}
+	out := runs[0]
+	out.Metrics = append([]Metric(nil), runs[0].Metrics...)
+	for _, r := range runs[1:] {
+		if r.GOOS != out.GOOS || r.GOARCH != out.GOARCH {
+			return Result{}, fmt.Errorf("baseline runs mix %s/%s and %s/%s", out.GOOS, out.GOARCH, r.GOOS, r.GOARCH)
+		}
+		if len(r.Metrics) != len(out.Metrics) {
+			return Result{}, fmt.Errorf("baseline runs carry different metrics (%d and %d)", len(out.Metrics), len(r.Metrics))
+		}
+		for _, m := range r.Metrics {
+			base, ok := out.Find(m.Name)
+			if !ok {
+				return Result{}, fmt.Errorf("metric %s is missing from the first baseline run", m.Name)
+			}
+			for i := range out.Metrics {
+				if out.Metrics[i].Name == base.Name {
+					out.Metrics[i].P50 = math.Max(base.P50, m.P50)
+					out.Metrics[i].P95 = math.Max(base.P95, m.P95)
+					out.Metrics[i].Max = math.Max(base.Max, m.Max)
+					out.Metrics[i].N = min(base.N, m.N)
+				}
+			}
+		}
+	}
+	return out, nil
+}
+
+// BudgetFromResult derives a budget from a baseline. keep selects the metrics
+// that get a ceiling (nil keeps every metric measured in milliseconds).
+func BudgetFromResult(base Result, note string, keep func(name string) bool) (Budget, error) {
+	b := Budget{Note: note, Schema: BudgetSchemaV1,
+		Platform: fmt.Sprintf("%s/%s", base.GOOS, base.GOARCH), P95MS: map[string]float64{}}
+	for _, m := range base.Metrics {
+		if m.Unit != "ms" || (keep != nil && !keep(m.Name)) {
+			continue
+		}
+		if math.IsNaN(m.P95) || m.P95 <= 0 {
+			return b, fmt.Errorf("metric %s: baseline p95 %v is not a usable timing", m.Name, m.P95)
+		}
+		b.P95MS[m.Name] = math.Max(MinCeilingMS, math.Ceil(m.P95*Headroom))
+	}
+	if len(b.P95MS) == 0 {
+		return b, fmt.Errorf("baseline has no metric to budget")
+	}
+	return b, nil
+}
+
+// Names lists the budgeted metric names, sorted.
+func (b Budget) Names() []string {
+	out := make([]string, 0, len(b.P95MS))
+	for n := range b.P95MS {
+		out = append(out, n)
+	}
+	sort.Strings(out)
+	return out
+}

free/ci/bench/budget_test.go

@@ -0,0 +1,33 @@
+package bench
+
+import "testing"
+
+func TestWorstOf(t *testing.T) {
+	a := NewResult("", 20, []Metric{{Name: "x", Unit: "ms", P50: 1, P95: 5, Max: 6, N: 20}, {Name: "y", Unit: "ms", P95: 2, N: 20}})
+	b := NewResult("", 20, []Metric{{Name: "x", Unit: "ms", P50: 2, P95: 4, Max: 9, N: 20}, {Name: "y", Unit: "ms", P95: 3, N: 10}})
+	got, err := WorstOf([]Result{a, b})
+	if err != nil {
+		t.Fatal(err)
+	}
+	x, _ := got.Find("x")
+	y, _ := got.Find("y")
+	if x.P50 != 2 || x.P95 != 5 || x.Max != 9 || y.P95 != 3 || y.N != 10 {
+		t.Fatalf("%+v %+v", x, y)
+	}
+	if a.Metrics[0].P50 != 1 {
+		t.Fatal("WorstOf changed its input")
+	}
+	other := b
+	other.GOARCH = "riscv64"
+	if _, err := WorstOf([]Result{a, other}); err == nil {
+		t.Fatal("runs from two platforms were folded")
+	}
+	short := NewResult("", 20, []Metric{{Name: "x", Unit: "ms", P95: 1, N: 20}})
+	short.GOOS, short.GOARCH = a.GOOS, a.GOARCH
+	if _, err := WorstOf([]Result{a, short}); err == nil {
+		t.Fatal("a run with a missing metric was folded")
+	}
+	if _, err := WorstOf(nil); err == nil {
+		t.Fatal("no runs accepted")
+	}
+}