fix: h09-plugins-stale-prs

.github/workflows/ci-bench.yml

@@ -0,0 +1,118 @@
+name: ci-bench
+
+# Local overhead of the nself ci V15 engine against the v1.4 gate (P7-CI-46).
+# Runs cibench on a hosted runner, then the cli's own `perfbench check` against the
+# per-platform budget (absolute p95) and `cibench -ratio-check` (V15 median against
+# the v1.4 median). Two controls prove the gates can fire: a result ten times over
+# every budget must fail both checks, and a result inside the relative budget must
+# pass. Budgets and baselines: hq plan/p7/evidence/CI-46-perf.md.
+
+on:
+  pull_request:
+    paths:
+      - 'free/ci/**'
+      - '.github/workflows/ci-bench.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-bench-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  CGO_ENABLED: '0'
+  # nself-org/cli revision whose tools/perfbench reads the result and budget files.
+  CLI_SHA: '529e8917244bffe4ab1aa9d42b081dd3efe5ee73'
+
+jobs:
+  bench:
+    name: bench (${{ matrix.name }})
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: linux-amd64
+            os: ubuntu-latest
+            budget: perf-budget.json
+            services: true
+          - name: macos-arm64
+            os: macos-14
+            budget: perf-budget-darwin-arm64.json
+            services: false
+    env:
+      BUDGET: ${{ github.workspace }}/free/ci/bench/${{ matrix.budget }}
+      SLOW: ${{ github.workspace }}/free/ci/bench/testdata/slow-result.json
+      OK: ${{ github.workspace }}/free/ci/bench/testdata/ratio-ok-result.json
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
+
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          repository: nself-org/cli
+          ref: ${{ env.CLI_SHA }}
+          path: cli
+          persist-credentials: false
+
+      # The cli module needs a newer Go than the plugin pins, and setup-go sets
+      # GOTOOLCHAIN=local, so one toolchain (the newer) builds both.
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
+        with:
+          go-version-file: cli/go.mod
+          cache: false
+
+      - name: Harness tests
+        working-directory: free/ci
+        run: go test -mod=vendor -count=1 ./bench/...
+
+      - name: Measure
+        working-directory: free/ci
+        run: go run -mod=vendor ./bench/cmd/cibench -runs 20 -warmup 2 -out "$RUNNER_TEMP/ci-bench.json"
+
+      - name: Absolute p95 budget (cli perfbench check)
+        working-directory: cli
+        run: go run -mod=vendor ./tools/perfbench check -budget "$BUDGET" -in "$RUNNER_TEMP/ci-bench.json"
+
+      - name: Relative budget (V15 against v1.4)
+        working-directory: free/ci
+        run: go run -mod=vendor ./bench/cmd/cibench -ratio-check -in "$RUNNER_TEMP/ci-bench.json"
+
+      - name: Control - the gates fail an injected slowdown and pass a good result
+        if: ${{ !cancelled() }}
+        run: |
+          set -u
+          rc=0
+          (cd cli && go run -mod=vendor ./tools/perfbench check -budget "$BUDGET" -in "$SLOW") || rc=$?
+          if [ "$rc" -ne 1 ]; then
+            echo "::error::perfbench check exited $rc on a result ten times over budget; want 1 (a violation)"; exit 1
+          fi
+          cd free/ci
+          rc=0
+          go run -mod=vendor ./bench/cmd/cibench -ratio-check -in "$SLOW" || rc=$?
+          if [ "$rc" -ne 1 ]; then
+            echo "::error::the ratio check exited $rc on a result ten times over budget; want 1 (a violation)"; exit 1
+          fi
+          go run -mod=vendor ./bench/cmd/cibench -ratio-check -in "$OK"
+
+      - name: Service start (warm and cold)
+        if: ${{ !cancelled() && matrix.services }}
+        run: |
+          set -eu
+          (cd cli && go build -mod=vendor -o "$RUNNER_TEMP/nself" ./cmd/nself)
+          cd free/ci
+          go run -mod=vendor ./bench/cmd/cibench -runs 5 -warmup 0 -skip-medium -services -cold -cold-runs 3 \
+            -nself "$RUNNER_TEMP/nself" -out "$RUNNER_TEMP/ci-bench-services.json"
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        if: always()
+        with:
+          name: ci-bench-${{ matrix.name }}
+          path: |
+            ${{ runner.temp }}/ci-bench.json
+            ${{ runner.temp }}/ci-bench-services.json
+          if-no-files-found: ignore

free/ci/architecture/layers.yaml

@@ -173,6 +173,8 @@ layers:
     - tools/deps
     - internal/storetest
     - tools/schemagen
+    - bench
+    - bench/cmd/cibench
     - bench/sched
     - bench/cmd/schedbench
     - bench/cache

free/ci/architecture/layers_test.go

@@ -90,6 +90,9 @@ func checkImport(m layerMap, pkg, file, imported string) error {
 	}
 	local := strings.TrimPrefix(imported, module)
 	isLocal := imported != local
+	if imported == "os/exec" && pkg == "bench" && file == "spawn.go" && kind == "supporting" {
+		return nil // the timing harness starts the processes it times, outside the engine's funnel (P7-CI-46)
+	}
 	if imported == "os/exec" && pkg != "internal/exec" && pkg != "internal/legacy" {
 		// These frozen pre-fabric sites are tracked by the single-exec-site test.
 		for _, old := range []struct{ pkg, file string }{{"internal/forgejo", "forgejo.go"}, {"internal/nodes/provision", "exec.go"}, {"internal/serve", "serve_job.go"}, {"internal/serve", "serve_job_report.go"}} {

free/ci/bench/bench_test.go

@@ -0,0 +1,79 @@
+package bench
+
+// Purpose: the Go benchmarks of the in-process scenarios (`go test -bench`).
+// They call the same functions cibench times, so a benchmark number and a
+// result-JSON number measure the same code.
+
+import (
+	"context"
+	"testing"
+)
+
+func benchInproc(b *testing.B) *Inproc {
+	b.Helper()
+	dir := b.TempDir()
+	if err := Materialize(context.Background(), Tiny, dir+"/repo"); err != nil {
+		b.Fatal(err)
+	}
+	p, err := PrepareInproc(dir+"/repo", dir+"/scratch")
+	if err != nil {
+		b.Fatal(err)
+	}
+	return p
+}
+
+func BenchmarkDetect(b *testing.B) {
+	p := benchInproc(b)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := p.Detect(); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkPlan(b *testing.B) {
+	p := benchInproc(b)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := p.Plan(); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkStoreOpen(b *testing.B) {
+	p := benchInproc(b)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := p.StoreOpen(); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkExecOverhead(b *testing.B) {
+	p := benchInproc(b)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if _, err := p.ExecOverhead(context.Background()); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkRawSpawn(b *testing.B) {
+	p := benchInproc(b)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if _, err := p.RawSpawn(context.Background()); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkCapacityProbe(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		CapacityProbe()
+	}
+}

free/ci/bench/budget.go

@@ -0,0 +1,100 @@
+package bench
+
+// Purpose: the perf-budget.json document (perfbench-budget/v1) and its
+// derivation from a recorded baseline.
+// Inputs:  a baseline Result, a headroom factor, a metric filter.
+// Outputs: a Budget whose absolute p95 ceilings are the baseline p95 plus the
+// headroom, rounded up to a whole millisecond, never below MinCeilingMS.
+// Constraints: the field names are the ones `perfbench check` decodes with
+// DisallowUnknownFields; a metric that is not in the baseline is never given a
+// ceiling (an invented number is the one thing this Ticket must not ship).
+
+import (
+	"fmt"
+	"math"
+	"sort"
+)
+
+// BudgetSchemaV1 is the value of the "schema" field of the budget file.
+const BudgetSchemaV1 = "perfbench-budget/v1"
+
+// Headroom is the allowed growth over the baseline p95 (Ticket constraint: +25%).
+const Headroom = 1.25
+
+// MinCeilingMS is the smallest ceiling a budget gets. Below it a timing is
+// scheduler noise on a shared runner (a 0.3 ms planner run can take 3 ms), so a
+// 25 percent rule on such a baseline would fail on weather.
+const MinCeilingMS = 5.0
+
+// Budget is the budget document. P95MS maps a metric name to its p95 ceiling in ms.
+type Budget struct {
+	Note     string             `json:"_note"`
+	Schema   string             `json:"schema"`
+	Platform string             `json:"platform"`
+	P95MS    map[string]float64 `json:"p95_ms"`
+}
+
+// WorstOf folds independent baseline runs of one platform into one result whose
+// p50, p95 and max are, per metric, the highest seen in any run. One run of 20
+// samples under-states the p95 of a noisy machine; the worst of several is the
+// baseline of record. The runs must come from one goos/goarch.
+func WorstOf(runs []Result) (Result, error) {
+	if len(runs) == 0 {
+		return Result{}, fmt.Errorf("no baseline runs")
+	}
+	out := runs[0]
+	out.Metrics = append([]Metric(nil), runs[0].Metrics...)
+	for _, r := range runs[1:] {
+		if r.GOOS != out.GOOS || r.GOARCH != out.GOARCH {
+			return Result{}, fmt.Errorf("baseline runs mix %s/%s and %s/%s", out.GOOS, out.GOARCH, r.GOOS, r.GOARCH)
+		}
+		if len(r.Metrics) != len(out.Metrics) {
+			return Result{}, fmt.Errorf("baseline runs carry different metrics (%d and %d)", len(out.Metrics), len(r.Metrics))
+		}
+		for _, m := range r.Metrics {
+			base, ok := out.Find(m.Name)
+			if !ok {
+				return Result{}, fmt.Errorf("metric %s is missing from the first baseline run", m.Name)
+			}
+			for i := range out.Metrics {
+				if out.Metrics[i].Name == base.Name {
+					out.Metrics[i].P50 = math.Max(base.P50, m.P50)
+					out.Metrics[i].P95 = math.Max(base.P95, m.P95)
+					out.Metrics[i].Max = math.Max(base.Max, m.Max)
+					out.Metrics[i].N = min(base.N, m.N)
+				}
+			}
+		}
+	}
+	return out, nil
+}
+
+// BudgetFromResult derives a budget from a baseline. keep selects the metrics
+// that get a ceiling (nil keeps every metric measured in milliseconds).
+func BudgetFromResult(base Result, note string, keep func(name string) bool) (Budget, error) {
+	b := Budget{Note: note, Schema: BudgetSchemaV1,
+		Platform: fmt.Sprintf("%s/%s", base.GOOS, base.GOARCH), P95MS: map[string]float64{}}
+	for _, m := range base.Metrics {
+		if m.Unit != "ms" || (keep != nil && !keep(m.Name)) {
+			continue
+		}
+		if math.IsNaN(m.P95) || m.P95 <= 0 {
+			return b, fmt.Errorf("metric %s: baseline p95 %v is not a usable timing", m.Name, m.P95)
+		}
+		b.P95MS[m.Name] = math.Max(MinCeilingMS, math.Ceil(m.P95*Headroom))
+	}
+	if len(b.P95MS) == 0 {
+		return b, fmt.Errorf("baseline has no metric to budget")
+	}
+	return b, nil
+}
+
+// Names lists the budgeted metric names, sorted.
+func (b Budget) Names() []string {
+	out := make([]string, 0, len(b.P95MS))
+	for n := range b.P95MS {
+		out = append(out, n)
+	}
+	sort.Strings(out)
+	return out
+}

free/ci/bench/budget_test.go

@@ -0,0 +1,33 @@
+package bench
+
+import "testing"
+
+func TestWorstOf(t *testing.T) {
+	a := NewResult("", 20, []Metric{{Name: "x", Unit: "ms", P50: 1, P95: 5, Max: 6, N: 20}, {Name: "y", Unit: "ms", P95: 2, N: 20}})
+	b := NewResult("", 20, []Metric{{Name: "x", Unit: "ms", P50: 2, P95: 4, Max: 9, N: 20}, {Name: "y", Unit: "ms", P95: 3, N: 10}})
+	got, err := WorstOf([]Result{a, b})
+	if err != nil {
+		t.Fatal(err)
+	}
+	x, _ := got.Find("x")
+	y, _ := got.Find("y")
+	if x.P50 != 2 || x.P95 != 5 || x.Max != 9 || y.P95 != 3 || y.N != 10 {
+		t.Fatalf("%+v %+v", x, y)
+	}
+	if a.Metrics[0].P50 != 1 {
+		t.Fatal("WorstOf changed its input")
+	}
+	other := b
+	other.GOARCH = "riscv64"
+	if _, err := WorstOf([]Result{a, other}); err == nil {
+		t.Fatal("runs from two platforms were folded")
+	}
+	short := NewResult("", 20, []Metric{{Name: "x", Unit: "ms", P95: 1, N: 20}})
+	short.GOOS, short.GOARCH = a.GOOS, a.GOARCH
+	if _, err := WorstOf([]Result{a, short}); err == nil {
+		t.Fatal("a run with a missing metric was folded")
+	}
+	if _, err := WorstOf(nil); err == nil {
+		t.Fatal("no runs accepted")
+	}
+}

free/ci/bench/build.go

@@ -0,0 +1,46 @@
+package bench
+
+// Purpose: build the binary under test and name the revision measured.
+// Inputs:  the free/ci module root and an output path.
+// Outputs: the binary path; the git HEAD of the tree, with -dirty appended when
+// tracked files changed.
+// Constraints: CGO_ENABLED=0 and -mod=vendor, as every build of this module.
+
+import (
+	"context"
+	"fmt"
+	"strings"
+)
+
+// BuildBinary builds ./cmd of the module at root into out.
+func BuildBinary(ctx context.Context, root, out string) (string, error) {
+	env := append(baseEnv(), "CGO_ENABLED=0")
+	if _, err := must(ctx, []string{"go", "build", "-mod=vendor", "-o", out, "./cmd"}, root, env); err != nil {
+		return "", fmt.Errorf("build nself-ci: %w", err)
+	}
+	return out, nil
+}
+
+// TreeSHA is the git HEAD of dir, "-dirty" when tracked files are modified, or
+// "" when dir is not a checkout.
+func TreeSHA(ctx context.Context, dir string) string {
+	r, err := must(ctx, []string{"git", "rev-parse", "HEAD"}, dir, baseEnv())
+	if err != nil {
+		return ""
+	}
+	sha := strings.TrimSpace(r.Tail)
+	if st, err := must(ctx, []string{"git", "status", "--porcelain", "--untracked-files=no"}, dir, baseEnv()); err == nil && st.Tail != "" {
+		sha += "-dirty"
+	}
+	return sha
+}
+
+// Format renders a result as aligned text lines (stderr summary).
+func Format(r Result) string {
+	var b strings.Builder
+	fmt.Fprintf(&b, "%s/%s runs=%d sha=%s\n", r.GOOS, r.GOARCH, r.Runs, r.SHA)
+	for _, m := range r.Metrics {
+		fmt.Fprintf(&b, "%-24s p50 %8.1f %s  p95 %8.1f %s  max %8.1f %s  n=%d\n", m.Name, m.P50, m.Unit, m.P95, m.Unit, m.Max, m.Unit, m.N)
+	}
+	return strings.TrimRight(b.String(), "\n")
+}