test(ci): selftest, alert and tidy tests and the audit pipeline template (P7-AUTO-20)

free/ci/internal/selftest/audit_template_test.go

@@ -0,0 +1,105 @@
+package selftest_test
+
+// Purpose: the audit pipeline template (P7-AUTO-20) is a valid pipeline config
+// whose only trigger is `schedule` (so it runs in the background band), and the
+// watcher's idle tick, which starts it, starts nothing on a host that is not ready.
+// Inputs: free/ci/templates/audit-pipeline.yml; a watcher with stub ports and no
+// watched repositories. Outputs: pass or fail. Constraints: no network, no clock.
+
+import (
+	"context"
+	"os"
+	"slices"
+	"testing"
+	"time"
+
+	"github.com/nself-org/plugins/free/ci/internal/background"
+	"github.com/nself-org/plugins/free/ci/internal/config"
+	"github.com/nself-org/plugins/free/ci/internal/report"
+	"github.com/nself-org/plugins/free/ci/internal/watch"
+)
+
+const auditTemplate = "../../templates/audit-pipeline.yml"
+
+// stubs satisfy the watcher's required ports; a pass with no repositories never calls them.
+type stubs struct{ t *testing.T }
+
+func (s stubs) Scan(context.Context, string) (watch.Scan, error) {
+	s.t.Fatal("no repository is watched, nothing may be scanned")
+	return watch.Scan{}, nil
+}
+func (s stubs) PostClaim(context.Context, report.Status) (time.Time, error) {
+	s.t.Fatal("an idle pass must not claim")
+	return time.Time{}, nil
+}
+func (s stubs) ListStatuses(context.Context, string, string, string) ([]report.LedgerStatus, time.Time, error) {
+	return nil, time.Time{}, nil
+}
+func (s stubs) Post(context.Context, report.Status) error {
+	s.t.Fatal("an idle pass must not post a status")
+	return nil
+}
+func (s stubs) Run(context.Context, watch.Job) (watch.Outcome, error) {
+	s.t.Fatal("an idle pass must not run a commit")
+	return watch.Outcome{}, nil
+}
+
+func TestAuditTemplateValid(t *testing.T) {
+	data, err := os.ReadFile(auditTemplate)
+	if err != nil {
+		t.Fatal(err)
+	}
+	loaded, err := config.Parse(auditTemplate, data)
+	if err != nil {
+		t.Fatalf("the template must pass the pipeline-config schema: %v", err)
+	}
+	p, ok := loaded.Config.Pipelines["audit"]
+	if !ok {
+		t.Fatal("the template must define the audit pipeline")
+	}
+	if len(p.Triggers) != 1 || string(p.Triggers[0]) != "schedule" || p.Schedule == "" || p.MaxAge == "" {
+		t.Fatalf("audit must be scheduled only, with a schedule and a max_age: %+v", p)
+	}
+	if !background.IsBackgroundKind(string(p.Triggers[0])) {
+		t.Fatal("a scheduled pipeline must run in the background band")
+	}
+	want := map[string]string{"audit-govulncheck": "govulncheck", "audit-pnpm": "pnpm", "audit-race": "go"}
+	if len(p.Jobs) != len(want) {
+		t.Fatalf("audit jobs = %v", p.Jobs)
+	}
+	for _, id := range p.Jobs {
+		j, ok := loaded.Config.Jobs[id]
+		if !ok || len(j.Run) == 0 || j.Run[0] != want[id] {
+			t.Fatalf("job %s must run %s: %+v", id, want[id], j)
+		}
+	}
+	if !slices.Contains(loaded.Config.Jobs["audit-race"].Run, "-race") {
+		t.Fatal("the race job must pass -race")
+	}
+
+	// The drill: the idle tick is what starts a due background pipeline; a host that
+	// is not ready never reaches it, a ready host with nothing to run does.
+	for _, ready := range []bool{false, true} {
+		started := 0
+		s := stubs{t}
+		w, err := watch.New(watch.Config{Host: "mini1"}, watch.Ports{Scanner: s, Ledger: s, Reporter: s, Runner: s,
+			Ready: func(context.Context) (bool, []string) {
+				if ready {
+					return true, nil
+				}
+				return false, []string{"disk.low"}
+			},
+			Idle: func(context.Context) (bool, error) { started++; return true, nil },
+			Now:  func() time.Time { return time.Date(2026, 10, 20, 2, 41, 0, 0, time.UTC) }})
+		if err != nil {
+			t.Fatal(err)
+		}
+		rep := w.Pass(context.Background())
+		if ready && (started != 1 || !rep.Idle) {
+			t.Fatalf("a ready idle host must start one background pipeline: started=%d %+v", started, rep)
+		}
+		if !ready && (started != 0 || rep.Idle) {
+			t.Fatalf("a host that is not ready must start nothing: started=%d %+v", started, rep)
+		}
+	}
+}

free/ci/internal/selftest/selftest_test.go

@@ -0,0 +1,233 @@
+package selftest
+
+// Purpose: P7-AUTO-20 acceptance for the pure half of the selftest: planted
+// faults flip the matching check, the alert goes out once per failing set per
+// UTC day, and a tidy is report-only unless applied.
+// Inputs: hand-built Facts, a fake notifier, fake tidy steps, a temp state file.
+// Outputs: pass or fail. Constraints: no clock reads, no real disk or network.
+
+import (
+	"context"
+	"errors"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/nself-org/plugins/free/ci/internal/doctor"
+	"github.com/nself-org/plugins/free/ci/internal/notify"
+)
+
+var day1 = time.Date(2026, 10, 20, 9, 0, 0, 0, time.UTC)
+
+// healthy is a host with nothing wrong: every selftest check passes.
+func healthy() Facts {
+	return Facts{
+		Now:      day1,
+		Tools:    []Tool{{Name: "go", Found: true, Required: true}, {Name: "gitleaks", Found: true}},
+		Provider: Provider{LoggedIn: true, Reachable: true},
+		Repo:     Probe{OK: true},
+		Disk:     Disk{FreeMB: 100000},
+		Sinks:    Sinks{List: []Sink{{Name: "ntfy", Delivered: 3}}},
+		Watcher:  Watcher{Present: true, Age: time.Minute, Interval: time.Minute},
+		Claims:   Claims{Known: true, RunLimit: time.Hour, List: []Claim{{Ref: "o/r@aaa", Age: time.Minute}}},
+		Infra:    Infra{Total: 10},
+	}
+}
+
+func statusOf(checks []doctor.Check) map[string]doctor.Status {
+	out := map[string]doctor.Status{}
+	for _, c := range checks {
+		out[c.ID] = c.Status
+	}
+	return out
+}
+
+func TestSelftestChecksFlip(t *testing.T) {
+	l := DefaultLimits()
+	base := Evaluate(healthy(), l)
+	if len(base) != 9 {
+		t.Fatalf("want nine selftest checks, got %d", len(base))
+	}
+	for _, c := range base {
+		if c.Status != doctor.Pass {
+			t.Fatalf("a healthy host must pass every check, %s is %s: %s", c.ID, c.Status, c.Message)
+		}
+	}
+	plants := []struct {
+		id    string
+		plant func(*Facts)
+	}{
+		{IDClaims, func(f *Facts) {
+			f.Claims.List = append(f.Claims.List, Claim{Ref: "o/r@bbb", Age: time.Hour + 11*time.Minute})
+		}},
+		{IDOrphans, func(f *Facts) { f.Orphans.List = []string{"job-root-123"} }},
+		{IDInfra, func(f *Facts) { f.Infra.Count = l.InfraFail }},
+		{IDWatcher, func(f *Facts) { f.Watcher.Age = 4 * time.Minute }},
+		{IDDisk, func(f *Facts) { f.Disk.FreeMB = l.MinFreeMB - 1 }},
+		{IDTools, func(f *Facts) { f.Tools[0].Found = false }},
+	}
+	for _, p := range plants {
+		t.Run(p.id, func(t *testing.T) {
+			f := healthy()
+			p.plant(&f)
+			got := statusOf(Evaluate(f, l))
+			for id, st := range got {
+				want := doctor.Pass
+				if id == p.id {
+					want = doctor.Fail
+				}
+				if st != want {
+					t.Fatalf("planting %s: %s is %s, want %s", p.id, id, st, want)
+				}
+			}
+		})
+	}
+	// A fact that could not be read warns with the cause; it is never a pass.
+	f := healthy()
+	f.Orphans.Err = "permission denied"
+	if got := statusOf(Evaluate(f, l))[IDOrphans]; got != doctor.Warn {
+		t.Fatalf("an unreadable fact must warn, got %s", got)
+	}
+}
+
+// fakeSink records what it was sent; fail makes it refuse.
+type fakeSink struct {
+	got  []notify.Event
+	fail bool
+}
+
+func (s *fakeSink) Name() string { return "fake" }
+func (s *fakeSink) Notify(_ context.Context, ev notify.Event) error {
+	if s.fail {
+		return errors.New("refused")
+	}
+	s.got = append(s.got, ev)
+	return nil
+}
+
+func failing(ids ...string) []doctor.Check {
+	out := []doctor.Check{{ID: IDTools, Name: "gate tools", Status: doctor.Pass}}
+	for _, id := range ids {
+		out = append(out, doctor.Check{ID: id, Name: "check " + id, Status: doctor.Fail})
+	}
+	return out
+}
+
+func TestSelftestAlertOncePerDay(t *testing.T) {
+	ctx := context.Background()
+	sink := &fakeSink{}
+	now := day1
+	a := Alerter{Sinks: []notify.Notifier{sink}, StatePath: filepath.Join(t.TempDir(), "alert.json"), Host: "mini1", Now: func() time.Time { return now }}
+	steps := []struct {
+		name   string
+		at     time.Time
+		checks []doctor.Check
+		sent   bool
+	}{
+		{"first failing set is sent", day1, failing(IDClaims), true},
+		{"the same set later that day is not", day1.Add(10 * time.Hour), failing(IDClaims), false},
+		{"one more failing check is a new set", day1.Add(11 * time.Hour), failing(IDClaims, IDOrphans), true},
+		{"a clean report sends nothing", day1.Add(12 * time.Hour), failing(), false},
+		{"the next UTC day sends the set again", day1.Add(24 * time.Hour), failing(IDClaims), true},
+	}
+	want := 0
+	for _, s := range steps {
+		now = s.at
+		res, err := a.Alert(ctx, s.checks)
+		if err != nil {
+			t.Fatalf("%s: %v", s.name, err)
+		}
+		if res.Sent != s.sent {
+			t.Fatalf("%s: sent=%v (%+v)", s.name, res.Sent, res)
+		}
+		if s.sent {
+			want++
+		}
+		if len(sink.got) != want {
+			t.Fatalf("%s: %d message(s) sent, want %d", s.name, len(sink.got), want)
+		}
+	}
+	if msg := sink.got[1].Message; !strings.Contains(msg, IDClaims) || !strings.Contains(msg, IDOrphans) || !strings.Contains(msg, "mini1") {
+		t.Fatalf("the message must name the host and every failing id: %q", msg)
+	}
+	// A set no sink took is not recorded, so the next run tries again.
+	refusing := &fakeSink{fail: true}
+	b := Alerter{Sinks: []notify.Notifier{refusing}, StatePath: filepath.Join(t.TempDir(), "alert.json"), Now: func() time.Time { return day1 }}
+	if res, _ := b.Alert(ctx, failing(IDInfra)); res.Sent {
+		t.Fatal("a refused message must not count as sent")
+	}
+	b.Sinks = []notify.Notifier{sink}
+	if res, err := b.Alert(ctx, failing(IDInfra)); err != nil || !res.Sent {
+		t.Fatalf("a set no sink took must be tried again: %+v %v", res, err)
+	}
+}
+
+// tidyRig is a disk whose free space grows by what the steps remove.
+type tidyRig struct {
+	free    int64
+	applied map[string][]Item
+}
+
+func (r *tidyRig) step(name string, items ...Item) Step {
+	return Step{Name: name,
+		Plan: func(context.Context) ([]Item, error) { return items, nil },
+		Apply: func(_ context.Context, got []Item) error {
+			r.applied[name] = got
+			for _, it := range got {
+				r.free += it.Bytes >> 20
+			}
+			return nil
+		}}
+}
+
+func (r *tidyRig) tidy(target int64) Tidy {
+	return Tidy{TargetMB: target, FreeMB: func() (int64, error) { return r.free, nil }, Steps: []Step{
+		r.step("history", Item{Step: "history", ID: "run_1", Bytes: 300 << 20}),
+		r.step("cache", Item{Step: "cache", ID: "entry_a", Bytes: 500 << 20}, Item{Step: "cache", ID: "entry_b", Bytes: 200 << 20}),
+		r.step("orphans", Item{Step: "orphans", ID: "job-root-9", Bytes: 100 << 20}),
+	}}
+}
+
+func TestTidyReportOnlyByDefault(t *testing.T) {
+	r := &tidyRig{free: 1000, applied: map[string][]Item{}}
+	rep, err := r.tidy(1800).Run(context.Background(), false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(r.applied) != 0 || r.free != 1000 {
+		t.Fatalf("a report-only tidy removed something: %+v", r.applied)
+	}
+	if rep.Apply || len(rep.Items) != 4 || rep.FreeAfterMB != 1000 || !strings.Contains(rep.Why, "--apply would remove 4 item(s)") {
+		t.Fatalf("the report must list exactly what --apply would remove: %+v", rep)
+	}
+}
+
+func TestTidyApplyReachesTarget(t *testing.T) {
+	r := &tidyRig{free: 1000, applied: map[string][]Item{}}
+	plan, _ := r.tidy(1800).Run(context.Background(), false)
+	rep, err := r.tidy(1800).Run(context.Background(), true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var applied []Item
+	for _, name := range []string{"history", "cache", "orphans"} {
+		applied = append(applied, r.applied[name]...)
+	}
+	if !reflect.DeepEqual(applied, plan.Items) {
+		t.Fatalf("apply must remove exactly the planned items:\n got %+v\nwant %+v", applied, plan.Items)
+	}
+	if !rep.Reached || rep.FreeBeforeMB != 1000 || rep.FreeAfterMB != 2100 || rep.Why != "" {
+		t.Fatalf("the tidy must reach the target: %+v", rep)
+	}
+	// A target out of reach is reported with why, never claimed.
+	r2 := &tidyRig{free: 1000, applied: map[string][]Item{}}
+	far, err := r2.tidy(5000).Run(context.Background(), true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if far.Reached || !strings.Contains(far.Why, "below the 5000 MB target") {
+		t.Fatalf("an unreachable target must say why: %+v", far)
+	}
+}

free/ci/templates/audit-pipeline.yml

@@ -0,0 +1,35 @@
+# Audit pipeline for an unattended CI host.
+#
+# Purpose: a scheduled pipeline that runs the dependency and race audits
+#   (govulncheck, pnpm audit, go test -race) when the host has nothing else to
+#   do. `audit` is not a command: merge the `jobs` and `pipelines` blocks below
+#   into the project's .nself/ci.yaml.
+# How it starts: the only trigger is `schedule`, so its jobs run in the
+#   background band, behind every interactive and normal job. On a host that runs
+#   `nself ci serve watch`, a pass that ran no commit starts one due background
+#   pipeline, and only when the host is ready; a host that is not ready starts
+#   nothing. `max_age` makes an overdue audit show in `nself ci evidence
+#   --pipeline audit`.
+# Constraints: needs govulncheck, pnpm and go on the PATH the service runs with
+#   (`nself ci doctor --selftest` checks it); drop the pnpm job in a project
+#   without a pnpm lockfile. The race job needs cgo on the host.
+
+version: 1
+
+jobs:
+  audit-govulncheck:
+    kind: quality
+    run: [govulncheck, ./...]
+  audit-pnpm:
+    kind: quality
+    run: [pnpm, audit, --audit-level, high]
+  audit-race:
+    kind: test
+    run: [go, test, -race, ./...]
+
+pipelines:
+  audit:
+    triggers: [schedule]
+    schedule: "41 2 * * *"
+    max_age: 48h
+    jobs: [audit-govulncheck, audit-pnpm, audit-race]