From b0f4f80c1fdecdc0b415107920b53d50835ea08b Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Mon, 26 Oct 2026 12:05:49 -0400 Subject: [PATCH 1/4] feat(p7-ci-66): doc-command check: plugins:.github/scripts/check-doc-com Ticket: p7-ci-66 Swarm-Run: p7f-plugins-r1 --- .github/scripts/check-doc-commands.sh | 28 +++ free/ci/tools/doccmds/check.go | 85 +++++++++ free/ci/tools/doccmds/check_test.go | 177 ++++++++++++++++++ free/ci/tools/doccmds/extract.go | 83 ++++++++ free/ci/tools/doccmds/extract_test.go | 87 +++++++++ free/ci/tools/doccmds/main.go | 44 +++++ free/ci/tools/doccmds/runner.go | 71 +++++++ free/ci/tools/doccmds/testdata/expect-fail.md | 20 ++ free/ci/tools/doccmds/testdata/fail.md | 15 ++ free/ci/tools/doccmds/testdata/no-blocks.md | 7 + free/ci/tools/doccmds/testdata/pass.md | 15 ++ 11 files changed, 632 insertions(+) create mode 100755 .github/scripts/check-doc-commands.sh create mode 100644 free/ci/tools/doccmds/check.go create mode 100644 free/ci/tools/doccmds/check_test.go create mode 100644 free/ci/tools/doccmds/extract.go create mode 100644 free/ci/tools/doccmds/extract_test.go create mode 100644 free/ci/tools/doccmds/main.go create mode 100644 free/ci/tools/doccmds/runner.go create mode 100644 free/ci/tools/doccmds/testdata/expect-fail.md create mode 100644 free/ci/tools/doccmds/testdata/fail.md create mode 100644 free/ci/tools/doccmds/testdata/no-blocks.md create mode 100644 free/ci/tools/doccmds/testdata/pass.md diff --git a/.github/scripts/check-doc-commands.sh b/.github/scripts/check-doc-commands.sh new file mode 100755 index 000000000..2b7dc5d05 --- /dev/null +++ b/.github/scripts/check-doc-commands.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# check-doc-commands.sh — run the bash blocks of doc pages in clean fixture projects and check their exit codes. +# +# Usage: bash .github/scripts/check-doc-commands.sh ... +# A bash block must exit 0. A block directly under must exit non-zero. +# Each block runs in a fresh temp project with a git repo and NSELF_V15=1. Nothing in the working tree changes. +# All logic lives in free/ci/tools/doccmds. This script only finds free/ci and execs it. +set -euo pipefail + +if [ "$#" -eq 0 ]; then + echo "usage: check-doc-commands.sh ..." >&2 + exit 2 +fi + +script_dir="$(cd "$(dirname "$0")" && pwd)" +ci_dir="$(cd "$script_dir/../../free/ci" && pwd)" + +# go -C changes directory, so pages given relative to the caller's directory are made absolute first. +pages=() +for page in "$@"; do + case "$page" in + /*) pages+=("$page") ;; + *) pages+=("$PWD/$page") ;; + esac +done + +export CGO_ENABLED=0 +exec go -C "$ci_dir" run -mod=vendor ./tools/doccmds "${pages[@]}" diff --git a/free/ci/tools/doccmds/check.go b/free/ci/tools/doccmds/check.go new file mode 100644 index 000000000..3465d2e9e --- /dev/null +++ b/free/ci/tools/doccmds/check.go @@ -0,0 +1,85 @@ +package main + +import ( + "fmt" + "os" +) + +// Runner runs one doc block with its working directory set to dir and returns the block's exit code. +// A non-nil error means the block could not be run at all. +type Runner interface { + Run(dir, script string) (int, error) +} + +// Fixture creates a clean project directory for one block. It returns the directory and a function +// that removes it. +type Fixture func() (dir string, cleanup func(), err error) + +// Options holds the injected pieces CheckPage uses, so tests run without spawning processes. +type Options struct { + Runner Runner + Fixture Fixture +} + +// Failure reports one bash block that did not behave as its marker says it should. +type Failure struct { + // Page is the doc page path, as given on the command line. + Page string + // Block is the 1-based number of the block among the page's bash blocks. + Block int + // ExitCode is the block's exit code, or -1 when the block or its fixture could not run. + ExitCode int + // Message says what went wrong, for example "exit 3". + Message string +} + +// Error formats the failure as "page: block N: message". +func (f Failure) Error() string { + return fmt.Sprintf("%s: block %d: %s", f.Page, f.Block, f.Message) +} + +// CheckPage runs every bash block of the page at path, each in its own fresh fixture, and returns the +// failures in page order. A page without bash blocks returns no failures and runs nothing. +func CheckPage(path string, opts Options) ([]Failure, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + blocks, err := ExtractBlocks(string(data)) + if err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + var failures []Failure + for _, b := range blocks { + if f, failed := checkBlock(path, b, opts); failed { + failures = append(failures, f) + } + } + return failures, nil +} + +// checkBlock runs one block in a fresh fixture and reports whether its exit code breaks the marker. +func checkBlock(page string, b Block, opts Options) (Failure, bool) { + fail := func(code int, msg string) (Failure, bool) { + return Failure{Page: page, Block: b.Number, ExitCode: code, Message: msg}, true + } + dir, cleanup, err := opts.Fixture() + if err != nil { + return fail(-1, fmt.Sprintf("fixture: %v", err)) + } + code, err := opts.Runner.Run(dir, b.Script) + cleanup() + if err != nil { + return fail(-1, fmt.Sprintf("could not run: %v", err)) + } + if b.ExpectFail { + if code == 0 { + return fail(0, "exit 0 (expected non-zero: marked expect-fail)") + } + return Failure{}, false + } + if code != 0 { + return fail(code, fmt.Sprintf("exit %d", code)) + } + return Failure{}, false +} diff --git a/free/ci/tools/doccmds/check_test.go b/free/ci/tools/doccmds/check_test.go new file mode 100644 index 000000000..74d378285 --- /dev/null +++ b/free/ci/tools/doccmds/check_test.go @@ -0,0 +1,177 @@ +package main + +import ( + "bytes" + "errors" + "reflect" + "strings" + "testing" +) + +// fakeRunner returns a canned exit code per script and records every call. It runs nothing. +type fakeRunner struct { + codes map[string]int + err error + dirs []string + calls []string +} + +// Run records the call and returns the canned code for script, or 0 when none is set. +func (f *fakeRunner) Run(dir, script string) (int, error) { + f.dirs = append(f.dirs, dir) + f.calls = append(f.calls, script) + if f.err != nil { + return -1, f.err + } + return f.codes[script], nil +} + +// fakeFixture hands out a fixed directory and counts creations and cleanups. +type fakeFixture struct { + made int + cleaned int + err error +} + +// fixture returns the fake Fixture function bound to f. +func (f *fakeFixture) fixture() (string, func(), error) { + if f.err != nil { + return "", nil, f.err + } + f.made++ + return "fixture-dir", func() { f.cleaned++ }, nil +} + +// checkWith runs CheckPage on a testdata page with the fake runner and fixture. +func checkWith(t *testing.T, page string, r *fakeRunner, fx *fakeFixture) []Failure { + t.Helper() + failures, err := CheckPage("testdata/"+page, Options{Runner: r, Fixture: fx.fixture}) + if err != nil { + t.Fatalf("CheckPage(%s): %v", page, err) + } + return failures +} + +func TestCheckPagePasses(t *testing.T) { + r := &fakeRunner{} + fx := &fakeFixture{} + if got := checkWith(t, "pass.md", r, fx); len(got) != 0 { + t.Fatalf("failures = %v, want none", got) + } + if want := []string{"echo ok", "true"}; !reflect.DeepEqual(r.calls, want) { + t.Errorf("ran %q, want %q (the sh block must be skipped)", r.calls, want) + } + if fx.made != 2 || fx.cleaned != 2 { + t.Errorf("fixtures made=%d cleaned=%d, want 2 and 2 (one fresh fixture per block)", fx.made, fx.cleaned) + } + for _, d := range r.dirs { + if d != "fixture-dir" { + t.Errorf("block ran in %q, want the fixture dir", d) + } + } +} + +func TestCheckPageReportsNonZeroExit(t *testing.T) { + r := &fakeRunner{codes: map[string]int{"exit 3": 3, "exit 7": 7}} + got := checkWith(t, "fail.md", r, &fakeFixture{}) + if len(got) != 2 { + t.Fatalf("got %d failures, want 2: %v", len(got), got) + } + want := []Failure{ + {Page: "testdata/fail.md", Block: 2, ExitCode: 3, Message: "exit 3"}, + {Page: "testdata/fail.md", Block: 3, ExitCode: 7, Message: "exit 7"}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("failures = %+v, want %+v", got, want) + } + if msg := got[0].Error(); msg != "testdata/fail.md: block 2: exit 3" { + t.Errorf("Error() = %q", msg) + } + if msg := got[1].Error(); msg != "testdata/fail.md: block 3: exit 7" { + t.Errorf("Error() = %q", msg) + } +} + +func TestCheckPageExpectFailMarker(t *testing.T) { + r := &fakeRunner{codes: map[string]int{"exit 1": 1, "exit 5": 5}} + got := checkWith(t, "expect-fail.md", r, &fakeFixture{}) + if len(got) != 2 { + t.Fatalf("got %d failures, want 2: %v", len(got), got) + } + if got[0].Block != 2 || got[0].ExitCode != 0 || got[0].Message != "exit 0 (expected non-zero: marked expect-fail)" { + t.Errorf("marked zero-exit failure = %+v", got[0]) + } + if msg := got[0].Error(); msg != "testdata/expect-fail.md: block 2: exit 0 (expected non-zero: marked expect-fail)" { + t.Errorf("Error() = %q", msg) + } + if got[1].Block != 3 || got[1].ExitCode != 5 || got[1].Message != "exit 5" { + t.Errorf("unmarked block failure = %+v, want block 3 exit 5", got[1]) + } +} + +func TestCheckPageWithoutBashBlocksRunsNothing(t *testing.T) { + r := &fakeRunner{} + fx := &fakeFixture{} + if got := checkWith(t, "no-blocks.md", r, fx); len(got) != 0 { + t.Fatalf("failures = %v, want none", got) + } + if len(r.calls) != 0 || fx.made != 0 { + t.Errorf("runner calls=%d fixtures=%d, want 0 and 0", len(r.calls), fx.made) + } +} + +func TestCheckPageRunnerError(t *testing.T) { + r := &fakeRunner{err: errors.New("boom")} + got := checkWith(t, "pass.md", r, &fakeFixture{}) + if len(got) != 2 { + t.Fatalf("got %d failures, want 2: %v", len(got), got) + } + if got[0].ExitCode != -1 || got[0].Message != "could not run: boom" { + t.Errorf("runner error failure = %+v", got[0]) + } +} + +func TestCheckPageFixtureError(t *testing.T) { + r := &fakeRunner{} + got := checkWith(t, "pass.md", r, &fakeFixture{err: errors.New("no disk")}) + if len(got) != 2 || got[0].Message != "fixture: no disk" { + t.Fatalf("fixture failures = %v", got) + } + if len(r.calls) != 0 { + t.Errorf("runner ran %d blocks without a fixture", len(r.calls)) + } +} + +func TestCheckPageMissingFile(t *testing.T) { + _, err := CheckPage("testdata/missing.md", Options{Runner: &fakeRunner{}, Fixture: (&fakeFixture{}).fixture}) + if err == nil { + t.Fatal("CheckPage on a missing page returned no error") + } +} + +func TestRunUsage(t *testing.T) { + var out, errOut bytes.Buffer + if code := run(nil, &out, &errOut); code != 2 { + t.Errorf("run() = %d, want 2", code) + } + if !strings.Contains(errOut.String(), "usage: check-doc-commands.sh ...") { + t.Errorf("usage text = %q", errOut.String()) + } +} + +func TestRunMissingPageFails(t *testing.T) { + var out, errOut bytes.Buffer + if code := run([]string{"testdata/missing.md"}, &out, &errOut); code != 1 { + t.Errorf("run() = %d, want 1", code) + } + if !strings.Contains(errOut.String(), "missing.md") { + t.Errorf("stderr = %q, want the missing page named", errOut.String()) + } +} + +func TestRunPageWithoutBlocksPasses(t *testing.T) { + var out, errOut bytes.Buffer + if code := run([]string{"testdata/no-blocks.md"}, &out, &errOut); code != 0 { + t.Errorf("run() = %d, want 0; stdout=%q stderr=%q", code, out.String(), errOut.String()) + } +} diff --git a/free/ci/tools/doccmds/extract.go b/free/ci/tools/doccmds/extract.go new file mode 100644 index 000000000..a8c08432b --- /dev/null +++ b/free/ci/tools/doccmds/extract.go @@ -0,0 +1,83 @@ +package main + +import ( + "fmt" + "strings" +) + +// expectFailMarker, placed on the line directly above a bash fence, marks the block as expected to fail. +const expectFailMarker = "" + +// Block is one fenced bash block taken from a doc page. +type Block struct { + // Number is the 1-based position of the block among the page's bash blocks. + Number int + // Line is the 1-based line of the opening fence. + Line int + // Script is the block body without its fences. + Script string + // ExpectFail is true when the expect-fail marker sits directly above the opening fence. + ExpectFail bool +} + +// ExtractBlocks returns the fenced bash blocks of content in page order. Fenced blocks in other +// languages are skipped whole, so a bash-looking line inside them never runs. An unclosed bash fence +// is an error; an unclosed fence in another language runs to the end of the page, as Markdown does. +func ExtractBlocks(content string) ([]Block, error) { + lines := strings.Split(content, "\n") + var blocks []Block + for i := 0; i < len(lines); i++ { + ticks, lang, ok := parseOpenFence(lines[i]) + if !ok { + continue + } + end := closingFence(lines, i+1, ticks) + if end < 0 { + if lang == "bash" { + return nil, fmt.Errorf("line %d: unclosed bash fence", i+1) + } + break + } + if lang == "bash" { + blocks = append(blocks, Block{ + Number: len(blocks) + 1, + Line: i + 1, + Script: strings.Join(lines[i+1:end], "\n"), + ExpectFail: i > 0 && strings.TrimSpace(lines[i-1]) == expectFailMarker, + }) + } + i = end + } + return blocks, nil +} + +// parseOpenFence reports whether line opens a code fence. It returns the backtick count and the +// first word of the info string, which is the language. +func parseOpenFence(line string) (int, string, bool) { + t := strings.TrimSpace(line) + if !strings.HasPrefix(t, "```") { + return 0, "", false + } + ticks := len(t) - len(strings.TrimLeft(t, "`")) + info := strings.TrimSpace(t[ticks:]) + if strings.Contains(info, "`") { + return 0, "", false + } + lang := "" + if fields := strings.Fields(info); len(fields) > 0 { + lang = fields[0] + } + return ticks, lang, true +} + +// closingFence returns the index of the first line at or after from that closes a fence opened with +// ticks backticks, or -1 when none does. +func closingFence(lines []string, from, ticks int) int { + for j := from; j < len(lines); j++ { + t := strings.TrimSpace(lines[j]) + if len(t) >= ticks && strings.Trim(t, "`") == "" { + return j + } + } + return -1 +} diff --git a/free/ci/tools/doccmds/extract_test.go b/free/ci/tools/doccmds/extract_test.go new file mode 100644 index 000000000..d1e78ce82 --- /dev/null +++ b/free/ci/tools/doccmds/extract_test.go @@ -0,0 +1,87 @@ +package main + +import ( + "reflect" + "strings" + "testing" +) + +func TestExtractBlocksNumbersAndMarker(t *testing.T) { + content := strings.Join([]string{ + "# Page", + "```sh", + "exit 9", + "```", + "```bash", + "echo one", + "echo two", + "```", + "", + "```bash title=second", + "exit 2", + "```", + "", + "", + "```bash", + "exit 4", + "```", + }, "\n") + got, err := ExtractBlocks(content) + if err != nil { + t.Fatalf("ExtractBlocks: %v", err) + } + want := []Block{ + {Number: 1, Line: 5, Script: "echo one\necho two"}, + {Number: 2, Line: 10, Script: "exit 2", ExpectFail: true}, + {Number: 3, Line: 15, Script: "exit 4"}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("blocks = %+v\nwant %+v", got, want) + } +} + +func TestExtractBlocksSkipsNestedFence(t *testing.T) { + // A four-backtick fence holds a bash fence as plain text, so no block is extracted. + content := "````text\n```bash\nexit 4\n```\n````\n" + got, err := ExtractBlocks(content) + if err != nil || len(got) != 0 { + t.Errorf("ExtractBlocks = %+v, %v; want no blocks and no error", got, err) + } +} + +func TestExtractBlocksUnclosedBashFails(t *testing.T) { + _, err := ExtractBlocks("```bash\nexit 1\n") + if err == nil || err.Error() != "line 1: unclosed bash fence" { + t.Errorf("err = %v, want unclosed bash fence on line 1", err) + } +} + +func TestExtractBlocksUnclosedOtherFenceRunsToEnd(t *testing.T) { + got, err := ExtractBlocks("```bash\nexit 1\n```\n```text\nfoo\n") + if err != nil || len(got) != 1 { + t.Errorf("ExtractBlocks = %+v, %v; want one block and no error", got, err) + } +} + +func TestParseOpenFence(t *testing.T) { + cases := []struct { + line string + ticks int + lang string + ok bool + }{ + {"```bash", 3, "bash", true}, + {" ```bash title=x", 3, "bash", true}, + {"````sh", 4, "sh", true}, + {"```", 3, "", true}, + {"``` a`b", 0, "", false}, + {"text ```bash", 0, "", false}, + } + for _, c := range cases { + ticks, lang, ok := parseOpenFence(c.line) + if ticks != c.ticks || lang != c.lang || ok != c.ok { + t.Errorf("parseOpenFence(%q) = %d, %q, %v; want %d, %q, %v", + c.line, ticks, lang, ok, c.ticks, c.lang, c.ok) + } + } +} diff --git a/free/ci/tools/doccmds/main.go b/free/ci/tools/doccmds/main.go new file mode 100644 index 000000000..562c304a8 --- /dev/null +++ b/free/ci/tools/doccmds/main.go @@ -0,0 +1,44 @@ +// Command doccmds runs the bash blocks of doc pages in clean fixture projects and fails when a block +// exits in a way its marker does not allow. A block under must exit +// non-zero; every other block must exit 0. It is run by .github/scripts/check-doc-commands.sh. +// +// Usage: doccmds ... +// Exit codes: 0 all blocks behave, 1 a block or page failed, 2 usage. +package main + +import ( + "fmt" + "io" + "os" +) + +// usageText is printed when no page is given. +const usageText = "usage: check-doc-commands.sh ..." + +func main() { + os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) +} + +// run checks every page and writes one line per failure to stdout and page errors to stderr. It returns +// the process exit code. +func run(pages []string, stdout, stderr io.Writer) int { + if len(pages) == 0 { + fmt.Fprintln(stderr, usageText) + return 2 + } + opts := Options{Runner: shellRunner{timeout: blockTimeout}, Fixture: newFixture} + status := 0 + for _, page := range pages { + failures, err := CheckPage(page, opts) + if err != nil { + fmt.Fprintf(stderr, "%v\n", err) + status = 1 + continue + } + for _, f := range failures { + fmt.Fprintln(stdout, f.Error()) + status = 1 + } + } + return status +} diff --git a/free/ci/tools/doccmds/runner.go b/free/ci/tools/doccmds/runner.go new file mode 100644 index 000000000..29448cc0c --- /dev/null +++ b/free/ci/tools/doccmds/runner.go @@ -0,0 +1,71 @@ +package main + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "time" +) + +// blockTimeout bounds one doc block, so a hung command fails the check instead of stalling CI. +const blockTimeout = 10 * time.Minute + +// shellRunner runs a block with bash in the fixture directory. +type shellRunner struct { + timeout time.Duration +} + +// Run executes script with bash in dir and returns its exit code. Block output goes to stderr so the +// failure report on stdout stays readable. +func (r shellRunner) Run(dir, script string) (int, error) { + ctx, cancel := context.WithTimeout(context.Background(), r.timeout) + defer cancel() + cmd := exec.CommandContext(ctx, "bash", "-c", script) + cmd.Dir = dir + cmd.Env = fixtureEnv(dir) + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + err := cmd.Run() + if ctx.Err() == context.DeadlineExceeded { + return -1, fmt.Errorf("timed out after %s", r.timeout) + } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return exitErr.ExitCode(), nil + } + if err != nil { + return -1, err + } + return 0, nil +} + +// fixtureEnv is the whole environment a block sees. Nothing else from the caller leaks in, so a block +// cannot read a credential from the CI environment. +func fixtureEnv(dir string) []string { + return []string{ + "PATH=" + os.Getenv("PATH"), + "HOME=" + dir, + "TMPDIR=" + os.TempDir(), + "LANG=C", + "NSELF_V15=1", + } +} + +// newFixture makes an empty temp project with a git repo. It returns the directory and a function that +// removes it. +func newFixture() (string, func(), error) { + dir, err := os.MkdirTemp("", "doccmds-") + if err != nil { + return "", nil, err + } + cleanup := func() { _ = os.RemoveAll(dir) } + cmd := exec.Command("git", "init", "-q", dir) + cmd.Env = fixtureEnv(dir) + if out, err := cmd.CombinedOutput(); err != nil { + cleanup() + return "", nil, fmt.Errorf("git init: %w: %s", err, out) + } + return dir, cleanup, nil +} diff --git a/free/ci/tools/doccmds/testdata/expect-fail.md b/free/ci/tools/doccmds/testdata/expect-fail.md new file mode 100644 index 000000000..a025804d1 --- /dev/null +++ b/free/ci/tools/doccmds/testdata/expect-fail.md @@ -0,0 +1,20 @@ +# Expect-fail page + +Block 1 is marked and fails, so it passes. Block 2 is marked but exits 0, so it fails. +Block 3 has the marker separated by a blank line, so the marker does not apply and it fails. + + +```bash +exit 1 +``` + + +```bash +true +``` + + + +```bash +exit 5 +``` diff --git a/free/ci/tools/doccmds/testdata/fail.md b/free/ci/tools/doccmds/testdata/fail.md new file mode 100644 index 000000000..c0d8d6320 --- /dev/null +++ b/free/ci/tools/doccmds/testdata/fail.md @@ -0,0 +1,15 @@ +# Fail page + +Block 2 and block 3 exit non-zero without a marker. + +```bash +echo ok +``` + +```bash +exit 3 +``` + +```bash +exit 7 +``` diff --git a/free/ci/tools/doccmds/testdata/no-blocks.md b/free/ci/tools/doccmds/testdata/no-blocks.md new file mode 100644 index 000000000..585db67de --- /dev/null +++ b/free/ci/tools/doccmds/testdata/no-blocks.md @@ -0,0 +1,7 @@ +# No blocks page + +Prose only. The only fence is Go, so nothing runs. + +```go +package main +``` diff --git a/free/ci/tools/doccmds/testdata/pass.md b/free/ci/tools/doccmds/testdata/pass.md new file mode 100644 index 000000000..5c17ff016 --- /dev/null +++ b/free/ci/tools/doccmds/testdata/pass.md @@ -0,0 +1,15 @@ +# Pass page + +Two bash blocks that exit 0, and one shell block that must be skipped. + +```bash +echo ok +``` + +```sh +exit 9 +``` + +```bash +true +``` From 600f7793b2b3f20e10b6601c5aac8e12deb1ee11 Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Mon, 26 Oct 2026 12:44:45 -0400 Subject: [PATCH 2/4] feat(p7-node-11): lost-runner reliability suite (scenario 9 + reliabilit Ticket: p7-node-11 Swarm-Run: p7f-plugins-r1 --- free/ci/test/reliability/carrier_test.go | 216 +++++++++++++++++ .../reliability/coordinator_outage_test.go | 193 +++++++++++++++ free/ci/test/reliability/doc.go | 19 ++ free/ci/test/reliability/scenario9_test.go | 162 +++++++++++++ free/ci/test/reliability/sim_agent_test.go | 144 ++++++++++++ free/ci/test/reliability/sim_world_test.go | 219 ++++++++++++++++++ 6 files changed, 953 insertions(+) create mode 100644 free/ci/test/reliability/carrier_test.go create mode 100644 free/ci/test/reliability/coordinator_outage_test.go create mode 100644 free/ci/test/reliability/doc.go create mode 100644 free/ci/test/reliability/scenario9_test.go create mode 100644 free/ci/test/reliability/sim_agent_test.go create mode 100644 free/ci/test/reliability/sim_world_test.go diff --git a/free/ci/test/reliability/carrier_test.go b/free/ci/test/reliability/carrier_test.go new file mode 100644 index 000000000..b3e02d8f9 --- /dev/null +++ b/free/ci/test/reliability/carrier_test.go @@ -0,0 +1,216 @@ +package reliability + +// Purpose: carrier faults. Agent carrier: an interrupted connection (network drop and restore, kill -9 of +// the coordinator mid log stream, a partial log upload) resumes with no duplicate log line, +// result or artifact reference. SSH carrier: session loss marks the attempt lost and it is +// retried elsewhere; no resume is claimed. +// Inputs: the coordinator's lease log writer (internal/transport/agent) and the sim world. +// Outputs: assertions only. +// Constraints: no sleeps; artifact references are recorded only behind the lease fence, as the hub does. + +import ( + "bufio" + "crypto/sha256" + "encoding/json" + "fmt" + "os" + "testing" + + "github.com/nself-org/plugins/free/ci/internal/protocol" + agentcarrier "github.com/nself-org/plugins/free/ci/internal/transport/agent" + "github.com/nself-org/plugins/free/ci/internal/transport/ssh" +) + +// logSeqs reads the line_seq values of a lease log file in order. +func logSeqs(t *testing.T, path string) []uint64 { + t.Helper() + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + var out []uint64 + sc := bufio.NewScanner(f) + for sc.Scan() { + var l struct { + LineSeq uint64 `json:"line_seq"` + } + if err := json.Unmarshal(sc.Bytes(), &l); err != nil { + t.Fatalf("log line %q: %v", sc.Text(), err) + } + out = append(out, l.LineSeq) + } + if err := sc.Err(); err != nil { + t.Fatal(err) + } + return out +} + +func wantSeqs(t *testing.T, path string, n uint64) { + t.Helper() + got := logSeqs(t, path) + if uint64(len(got)) != n { + t.Fatalf("log has %d lines %v, want %d", len(got), got, n) + } + for i, s := range got { + if s != uint64(i)+1 { + t.Fatalf("log line %d has line_seq %d: %v", i, s, got) + } + } +} + +// send appends lines from..to of a lease; it returns how many were written (not duplicates). +func send(t *testing.T, lw *agentcarrier.LogWriter, leaseID string, from, to uint64) int { + t.Helper() + n := 0 + for s := from; s <= to; s++ { + ok, err := lw.Append(protocol.Log{LeaseID: leaseID, LineSeq: s, Stream: "stdout", Text: fmt.Sprintf("line %d", s)}) + if err != nil { + t.Fatal(err) + } + if ok { + n++ + } + } + return n +} + +func openLog(t *testing.T, path string) *agentcarrier.LogWriter { + t.Helper() + lw, err := agentcarrier.OpenLog(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = lw.Close() }) + return lw +} + +// artifacts is the coordinator's artifact index: a reference is kept only while the lease fence holds. +type artifacts map[string]int + +func (x artifacts) add(w *world, a *agent, path string) error { + if err := w.engine.Fence(w.ctx, a.leaseID, a.epoch); err != nil { + return err + } + sum := sha256.Sum256([]byte(path)) + x[fmt.Sprintf("%s/%s/%x", a.leaseID, path, sum[:4])]++ + return nil +} + +// wantOnce asserts one artifact reference, recorded once. +func (x artifacts) wantOnce(t *testing.T) { + t.Helper() + for _, n := range x { + if n != 1 || len(x) != 1 { + t.Fatalf("artifact index %v", x) + } + } + if len(x) != 1 { + t.Fatalf("artifact index %v", x) + } +} + +func TestFaultAgentCarrierResume(t *testing.T) { + w := newWorld(t) + a1 := w.job("jres", "test", true, 1) + a := w.startAgent("node-a", "lease-a", w.grant(a1, "lease-a", "node-a", 1)) + path := agentcarrier.LogFile(t.TempDir(), "lease-a") + lw := openLog(t, path) + idx := artifacts{} + if n := send(t, lw, "lease-a", 1, 4); n != 4 { + t.Fatalf("first burst wrote %d", n) + } + if err := idx.add(w, a, "dist/app.tar"); err != nil { + t.Fatal(err) + } + a.partitioned = true // Wi-Fi drop: the acks for lines 3-4 and the artifact never reach the agent + w.run(heartbeat, a) + a.partitioned = false + if err := a.reconnect(); err != nil { + t.Fatal(err) + } + if n := send(t, lw, "lease-a", 3, 6); n != 2 { // the agent resends from its last acknowledged line + t.Fatalf("resend wrote %d lines, want 2", n) + } + idx.wantOnce(t) + if ran, err := a.finish("jres"); !ran || err != nil { + t.Fatalf("finish: ran=%t err=%v", ran, err) + } + wantCode(t, "resent result", a.lateResult(), "E605") // the finalizing CAS refuses a second result + wantCode(t, "resent artifact reference", idx.add(w, a, "dist/app.tar"), "E662") + idx.wantOnce(t) + if err := lw.Sync(); err != nil { + t.Fatal(err) + } + wantSeqs(t, path, 6) + w.wantAttempts("jres", "finalizing/") + w.wantEffect("jres", "node-a") +} + +func TestFaultCoordinatorKilledMidLogStream(t *testing.T) { + path := agentcarrier.LogFile(t.TempDir(), "lease-k") + lw, err := agentcarrier.OpenLog(path) + if err != nil { + t.Fatal(err) + } + send(t, lw, "lease-k", 1, 3) + if err := lw.Sync(); err != nil { + t.Fatal(err) + } + _ = lw.Close() + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + if _, err := f.WriteString(`{"line_seq":4,"stream":"std`); err != nil { // kill -9 mid write + t.Fatal(err) + } + _ = f.Close() + lw = openLog(t, path) + if lw.Last() != 3 { + t.Fatalf("recovered tail %d, want 3", lw.Last()) + } + if n := send(t, lw, "lease-k", 2, 5); n != 2 { + t.Fatalf("resume wrote %d, want 2", n) + } + wantSeqs(t, path, 5) +} + +func TestFaultPartialLogUpload(t *testing.T) { + path := agentcarrier.LogFile(t.TempDir(), "lease-p") + lw := openLog(t, path) + send(t, lw, "lease-p", 1, 2) + // The upload of lines 3-5 was cut after line 3 reached the coordinator. + send(t, lw, "lease-p", 3, 3) + if n := send(t, lw, "lease-p", 3, 5); n != 2 { + t.Fatalf("re-upload wrote %d, want 2", n) + } + if err := lw.Sync(); err != nil { + t.Fatal(err) + } + wantSeqs(t, path, 5) +} + +func TestFaultSSHSessionLoss(t *testing.T) { + w := newWorld(t) + a1 := w.job("jssh", "test", true, 1) + a := w.startAgent("node-a", "lease-a", w.grant(a1, "lease-a", "node-a", 1)) + w.run(heartbeat, a) + l, err := w.store.LeaseGet(w.ctx, "lease-a") + if err != nil { + t.Fatal(err) + } + a.dead = true // the ssh session carried the job; it ends with the session + if err := w.store.LeaseFail(w.ctx, l, w.clock(), ssh.ReasonSessionLost, false); err != nil { + t.Fatal(err) + } + if got := w.retryReasons(a1); len(got) != 1 || got[0] != ssh.ReasonSessionLost { + t.Fatalf("retry reasons %v", got) + } + wantCode(t, "resume over a new session", w.engine.Reconnect(w.ctx, "lease-a", a.epoch, "running"), "E662") + wantCode(t, "late result", a.lateResult(), "E662") + rows := w.wantAttempts("jssh", "retryable/lost", "queued/") + retryElsewhere(t, w, "jssh", rows) + w.wantAttempts("jssh", "retryable/lost", "finalizing/") + w.wantEffect("jssh", "node-b") +} diff --git a/free/ci/test/reliability/coordinator_outage_test.go b/free/ci/test/reliability/coordinator_outage_test.go new file mode 100644 index 000000000..00481d921 --- /dev/null +++ b/free/ci/test/reliability/coordinator_outage_test.go @@ -0,0 +1,193 @@ +package reliability + +// Purpose: coordinator outages (EPIC D6, Acceptance 5), kill -9 of the coordinator, a VPS reboot of a +// node and drain hand-back. Down 20 s (< agent TTL) with 20 running agents: 0 requeues. Down +// 5 minutes: every agent self-fences, each attempt goes lost (agent.self_fenced) and is retried +// exactly once, each side-effect file is written once, and the coordinator expires no lease for +// its own downtime. Drain hand-back never overlaps two attempts of a job. +// Inputs: the sim world; a coordinator restart is a new lease.Engine over the same store. +// Outputs: assertions only. +// Constraints: no sleeps. + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/nself-org/plugins/free/ci/internal/coordinator/drain" + "github.com/nself-org/plugins/free/ci/internal/model" +) + +// fleet grants n jobs on n runners and lets one heartbeat land. +func fleet(t *testing.T, w *world, n int) []*agent { + t.Helper() + agents := make([]*agent, n) + for i := range agents { + id := fmt.Sprintf("j%02d", i) + runner := fmt.Sprintf("node-%02d", i) + lid := "lease-" + id + agents[i] = w.startAgent(runner, lid, w.grant(w.job(id, "test", true, 1), lid, runner, 1)) + } + w.run(heartbeat, agents...) + return agents +} + +// outage stops the coordinator for d, then starts a new incarnation with restart grace. +func outage(t *testing.T, w *world, d time.Duration, agents []*agent) { + t.Helper() + w.down = true + w.run(d, agents...) + w.engine = w.coordinator() + w.down = false + if _, err := w.engine.Start(w.ctx, "coordinator-2", 2); err != nil { + t.Fatal(err) + } +} + +func TestFaultCoordinatorDownShort(t *testing.T) { + w := newWorld(t) + agents := fleet(t, w, 20) + outage(t, w, 20*time.Second, agents) + for i, a := range agents { + if !a.fencedAt.IsZero() { + t.Fatalf("agent %d fenced during a 20 s outage", i) + } + if err := a.reconnect(); err != nil { + t.Fatalf("reconnect %d: %v", i, err) + } + } + if n, err := w.engine.Sweep(w.ctx); err != nil || n != 0 { + t.Fatalf("short outage requeued %d: %v", n, err) + } + for i, a := range agents { + id := fmt.Sprintf("j%02d", i) + if ran, err := a.finish(id); !ran || err != nil { + t.Fatalf("job %s after short outage: ran=%t err=%v", id, ran, err) + } + w.wantAttempts(id, "finalizing/") + w.wantEffect(id, a.runner) + } +} + +func TestFaultCoordinatorDownLong(t *testing.T) { + w := newWorld(t) + agents := fleet(t, w, 20) + down := w.clock() + outage(t, w, 5*time.Minute, agents) + if n, err := w.engine.Sweep(w.ctx); err != nil || n != 0 { + t.Fatalf("coordinator expired %d leases for its own downtime: %v", n, err) + } + for i, a := range agents { + id := fmt.Sprintf("j%02d", i) + if a.fencedAt.IsZero() || !a.fencedAt.After(down) || a.fencedAt.Sub(down) > coordTTL { + t.Fatalf("agent %d fenced at %v (outage began %v)", i, a.fencedAt, down) + } + if ran, _ := a.finish(id); ran { + t.Fatalf("fenced job %s wrote its side effect", id) + } + if err := a.reconnect(); err != nil { + t.Fatalf("fence report %d: %v", i, err) + } + wantCode(t, "duplicate fence report", a.reconnect(), "E662") + if got := w.retryReasons(id + ".a1"); len(got) != 1 || got[0] != "agent.self_fenced" { + t.Fatalf("job %s retry reasons %v", id, got) + } + } + for i, a := range agents { + id := fmt.Sprintf("j%02d", i) + rows := w.wantAttempts(id, "retryable/lost", "queued/") + next := fmt.Sprintf("node-%02d", (i+1)%len(agents)) + lid := "lease-r-" + id + b := w.startAgent(next, lid, w.grant(rows[1].ID, lid, next, 2)) + if ran, err := b.finish(id); !ran || err != nil { + t.Fatalf("retry %s: ran=%t err=%v", id, ran, err) + } + wantCode(t, "late result of the fenced attempt", a.lateResult(), "E662") + w.wantAttempts(id, "retryable/lost", "finalizing/") + w.wantEffect(id, next) + } +} + +func TestFaultCoordinatorKilledMidRun(t *testing.T) { + w := newWorld(t) + agents := fleet(t, w, 3) + outage(t, w, 0, agents) // kill -9 and immediate restart: the new incarnation re-adopts every lease + w.run(3*heartbeat, agents...) + if n, err := w.engine.Sweep(w.ctx); err != nil || n != 0 { + t.Fatalf("restart requeued %d: %v", n, err) + } + for i, a := range agents { + id := fmt.Sprintf("j%02d", i) + if ran, err := a.finish(id); !ran || err != nil { + t.Fatalf("job %s: ran=%t err=%v", id, ran, err) + } + w.wantEffect(id, a.runner) + } +} + +func TestFaultNodeReboot(t *testing.T) { + w := newWorld(t) + a1 := w.job("jboot", "test", true, 1) + a := w.startAgent("node-a", "lease-a", w.grant(a1, "lease-a", "node-a", 1)) + w.run(heartbeat, a) + // The container restarts: the job process is gone and the new agent process holds no live job, so it + // reports the lease it finds in its spool as self-fenced. + a.dead = true + w.run(heartbeat, a) + if err := w.engine.Reconnect(w.ctx, "lease-a", a.epoch, "self_fenced"); err != nil { + t.Fatal(err) + } + rows := w.wantAttempts("jboot", "retryable/lost", "queued/") + if prev := w.previousRunner(rows[1].ID); prev != "node-a" { + t.Fatalf("previous runner %q", prev) + } + b := w.startAgent("node-b", "lease-b", w.grant(rows[1].ID, "lease-b", "node-b", 1)) + if ran, err := b.finish("jboot"); !ran || err != nil { + t.Fatalf("retry: ran=%t err=%v", ran, err) + } + w.wantEffect("jboot", "node-b") +} + +func TestFaultDrainHandBack(t *testing.T) { + w := newWorld(t) + a1 := w.job("jdrain", "test", true, 1) + a := w.startAgent("node-a", "lease-a", w.grant(a1, "lease-a", "node-a", 1)) + w.run(heartbeat, a) + leases := &drain.EngineLeases{Engine: w.engine, JobInfo: func(context.Context, string) (string, bool, error) { return "test", true, nil }} + running, err := leases.Running(w.ctx) + if err != nil || len(running) != 1 || !running[0].HandBackable() { + t.Fatalf("running leases %+v %v", running, err) + } + a.partitioned = true // the draining node stops answering; the drain waits for expiry + w.run(coordTTL+heartbeat, a) + if n := w.active("jdrain"); n != 1 { + t.Fatalf("active attempts before hand-back = %d", n) + } + if err := leases.HandBack(w.ctx, running[0], true, "drain"); err != nil { + t.Fatal(err) + } + if n := w.active("jdrain"); n != 0 { + t.Fatalf("active attempts after hand-back = %d, want 0 before the retry is leased", n) + } + wantCode(t, "old lease after hand-back", w.engine.Fence(w.ctx, "lease-a", a.epoch), "E662") + rows := w.wantAttempts("jdrain", "retryable/lost", "queued/") + draining := simRunner("node-a") + draining.Capability.Availability.State.Value = ptr("draining") + res := place("jdrain", 2, "node-a", draining, simRunner("node-b")) + if len(res.Placements) != 1 || res.Placements[0].RunnerID != "node-b" || !contains(reasonsFor(res, "node-a"), string(model.NodeDraining)) { + t.Fatalf("placement after drain %+v", res) + } + b := w.startAgent("node-b", "lease-b", w.grant(rows[1].ID, "lease-b", "node-b", 1)) + if n := w.active("jdrain"); n != 1 { + t.Fatalf("active attempts after retry lease = %d", n) + } + a.partitioned = false + if ran, _ := a.finish("jdrain"); ran { + t.Fatal("handed-back job wrote its side effect") + } + if ran, err := b.finish("jdrain"); !ran || err != nil { + t.Fatalf("retry: ran=%t err=%v", ran, err) + } + w.wantEffect("jdrain", "node-b") +} diff --git a/free/ci/test/reliability/doc.go b/free/ci/test/reliability/doc.go new file mode 100644 index 000000000..0af28c524 --- /dev/null +++ b/free/ci/test/reliability/doc.go @@ -0,0 +1,19 @@ +// Package reliability holds the lost-runner fault suite (scenario 9 and the +// reliability items of P7-NODE-11). +// +// Two layers run the same assertions: +// +// - The sim layer (sim_*_test.go and the TestFault* tests) drives the real +// store, lease engine, agent fence, drain hand-back, placement and log +// writer with injected wall and monotonic clocks. Freezes, partitions, +// suspends, clock jumps, crashes and coordinator outages are clock and +// carrier events, so no test sleeps. It runs in every `go test`. +// - The Docker layer (docker_*_test.go) applies the same faults to real +// sshd containers through sdk/go/simharness: docker pause, a network +// partition, and a container restart. It runs only with INTEGRATION=1 on +// a Linux Docker Engine; without it simharness skips the test. +// +// Every test asserts store state, the evidence attempts and their +// failure_class, process timestamps, and that the fixture job's side-effect +// file is written at most once. +package reliability diff --git a/free/ci/test/reliability/scenario9_test.go b/free/ci/test/reliability/scenario9_test.go new file mode 100644 index 000000000..fcddd939c --- /dev/null +++ b/free/ci/test/reliability/scenario9_test.go @@ -0,0 +1,162 @@ +package reliability + +// Purpose: scenario 9. A node frozen (SIGSTOP / docker pause) or partitioned past the TTL self-fences +// before the coordinator's expiry; the attempt goes lost and is retried on another eligible +// node (retry.anti_affinity); the late result after SIGCONT or heal is rejected with E662; the +// side-effect file is written once; the evidence lists both attempts with failure_class. +// A deterministic failure is not retried; a lost release job goes needs-operator. +// Inputs: the sim world (real store, lease engine, agent fence, scheduler) with injected clocks. +// Outputs: assertions only. +// Constraints: no sleeps; the Docker variant of the same faults is docker_faults_test.go. + +import ( + "testing" + "time" + + "github.com/nself-org/plugins/free/ci/internal/model" + "github.com/nself-org/plugins/free/ci/internal/store" +) + +// retryElsewhere checks the retry placement and runs the retry to completion on node-b. +func retryElsewhere(t *testing.T, w *world, jobID string, rows []store.AttemptView) *agent { + t.Helper() + prev := w.previousRunner(rows[1].ID) + if prev != "node-a" { + t.Fatalf("retry anti-affinity runner = %q, want node-a", prev) + } + res := place(jobID, rows[1].N, prev, simRunner("node-a"), simRunner("node-b")) + if len(res.Placements) != 1 || res.Placements[0].RunnerID != "node-b" { + t.Fatalf("retry placement = %+v, want node-b", res.Placements) + } + if got := reasonsFor(res, "node-a"); !contains(got, string(model.RetryAntiAffinity)) { + t.Fatalf("node-a reasons = %v, want %s", got, model.RetryAntiAffinity) + } + b := w.startAgent("node-b", "lease-b", w.grant(rows[1].ID, "lease-b", "node-b", 1)) + w.run(heartbeat, b) + if ran, err := b.finish(jobID); !ran || err != nil { + t.Fatalf("retry on node-b: ran=%t err=%v", ran, err) + } + return b +} + +// fencedBeforeExpiry asserts the agent's fence deadline falls before the coordinator's lease expiry. +func fencedBeforeExpiry(t *testing.T, a *agent, expiresAt int64) { + t.Helper() + deadline := a.lastAck.Add(a.fence.TTL) + if !deadline.Before(time.Unix(0, expiresAt)) { + t.Fatalf("agent fence deadline %v not before coordinator expiry %v", deadline, time.Unix(0, expiresAt)) + } + if !a.fencedAt.IsZero() && !a.fencedAt.Before(time.Unix(0, expiresAt)) { + t.Fatalf("agent fenced at %v, after coordinator expiry %v", a.fencedAt, time.Unix(0, expiresAt)) + } +} + +func TestFaultScenario9Freeze(t *testing.T) { + w := newWorld(t) + a1 := w.job("j9", "test", true, 1) + a := w.startAgent("node-a", "lease-a", w.grant(a1, "lease-a", "node-a", 1)) + w.run(2*heartbeat, a) + l, err := w.store.LeaseGet(w.ctx, "lease-a") + if err != nil { + t.Fatal(err) + } + a.frozen = true // SIGSTOP + w.run(time.Minute, a) + fencedBeforeExpiry(t, a, l.ExpiresAt) + if n, err := w.engine.Sweep(w.ctx); err != nil || n != 1 { + t.Fatalf("sweep after freeze: %d %v", n, err) + } + a.frozen = false // SIGCONT: the wake check latches the fence before job work resumes + if ran, _ := a.finish("j9"); ran || a.fencedAt.IsZero() { + t.Fatalf("frozen job resumed: ran=%t fencedAt=%v", ran, a.fencedAt) + } + wantCode(t, "late result after SIGCONT", a.lateResult(), "E662") + rows := w.wantAttempts("j9", "retryable/lost", "queued/") + if got := w.retryReasons(a1); len(got) != 1 || got[0] != "lease.expired" { + t.Fatalf("retry reasons %v", got) + } + retryElsewhere(t, w, "j9", rows) + wantCode(t, "late result after the retry", a.lateResult(), "E662") + w.wantAttempts("j9", "retryable/lost", "finalizing/") + w.wantEffect("j9", "node-b") + if w.engine.StaleCount() != 2 { + t.Fatalf("stale rejections = %d, want 2", w.engine.StaleCount()) + } +} + +func TestFaultScenario9Partition(t *testing.T) { + w := newWorld(t) + a1 := w.job("j9p", "test", true, 1) + a := w.startAgent("node-a", "lease-a", w.grant(a1, "lease-a", "node-a", 1)) + w.run(2*heartbeat, a) + l, err := w.store.LeaseGet(w.ctx, "lease-a") + if err != nil { + t.Fatal(err) + } + a.partitioned = true // netem loss 100 % / network disconnect + w.run(3*heartbeat, a) + if a.fencedAt.IsZero() { + t.Fatal("partitioned agent did not self-fence") + } + fencedBeforeExpiry(t, a, l.ExpiresAt) + if ran, _ := a.finish("j9p"); ran { + t.Fatal("fenced job wrote its side effect") + } + a.partitioned = false // heal: the agent reports its fence + if err := a.reconnect(); err != nil { + t.Fatal(err) + } + if got := w.retryReasons(a1); len(got) != 1 || got[0] != "agent.self_fenced" { + t.Fatalf("retry reasons %v", got) + } + wantCode(t, "late result after heal", a.lateResult(), "E662") + wantCode(t, "second fence report", a.reconnect(), "E662") + w.advance(coordTTL) + if n, err := w.engine.Sweep(w.ctx); err != nil || n != 0 { + t.Fatalf("sweep after self-fence: %d %v", n, err) + } + rows := w.wantAttempts("j9p", "retryable/lost", "queued/") + retryElsewhere(t, w, "j9p", rows) + w.wantAttempts("j9p", "retryable/lost", "finalizing/") + w.wantEffect("j9p", "node-b") +} + +func TestFaultDeterministicNotRetried(t *testing.T) { + w := newWorld(t) + a1 := w.job("jdet", "test", true, 1) + a := w.startAgent("node-a", "lease-a", w.grant(a1, "lease-a", "node-a", 1)) + w.run(heartbeat, a) + w.effect("jdet", "node-a") + _, epoch := w.state(a1) + if err := w.store.Transition(w.ctx, a1, "running", "failed", store.TransitionDetail{Epoch: epoch, FailureClass: "code"}); err != nil { + t.Fatal(err) + } + w.advance(2 * coordTTL) + if n, err := w.engine.Sweep(w.ctx); err != nil || n != 0 { + t.Fatalf("deterministic failure swept: %d %v", n, err) + } + w.wantAttempts("jdet", "failed/code") + w.wantEffect("jdet", "node-a") + if got := w.retryReasons(a1); len(got) != 0 { + t.Fatalf("deterministic failure retried: %v", got) + } +} + +func TestFaultLostReleaseNeedsOperator(t *testing.T) { + w := newWorld(t) + a1 := w.job("jrel", "release", false, 1) + a := w.startAgent("node-a", "lease-a", w.grant(a1, "lease-a", "node-a", 1)) + w.run(heartbeat, a) + a.dead = true // kill -9 of the agent running the release + w.run(2*coordTTL, a) + if n, err := w.engine.Sweep(w.ctx); err != nil || n != 1 { + t.Fatalf("sweep: %d %v", n, err) + } + w.wantAttempts("jrel", "needs-operator/lost") + if got := w.retryReasons(a1); len(got) != 0 { + t.Fatalf("release retried: %v", got) + } + if lines := w.effectLines("jrel"); len(lines) != 0 { + t.Fatalf("release side effect %v", lines) + } +} diff --git a/free/ci/test/reliability/sim_agent_test.go b/free/ci/test/reliability/sim_agent_test.go new file mode 100644 index 000000000..729df54e0 --- /dev/null +++ b/free/ci/test/reliability/sim_agent_test.go @@ -0,0 +1,144 @@ +package reliability + +// Purpose: the node side of the sim: an agent with its own wall and monotonic clocks and the real +// lease.AgentFence, plus the placement view (sched.Place) a retry goes through. +// Inputs: the world, a runner name and the lease the agent holds. +// Outputs: heartbeats, job completion (which writes the side-effect file) and late carrier frames. +// Constraints: a frozen agent's clocks keep moving (time passes) but it runs no code; a suspended agent's +// monotonic clock stops while its wall clock jumps; no sleeps. + +import ( + "time" + + "github.com/nself-org/plugins/free/ci/internal/lease" + "github.com/nself-org/plugins/free/ci/internal/model" + "github.com/nself-org/plugins/free/ci/internal/sched" + "github.com/nself-org/plugins/free/ci/internal/trust" +) + +// agent is one node agent holding one lease. +type agent struct { + w *world + runner string + leaseID string + epoch int64 + wall time.Time + mono time.Duration + fence *lease.AgentFence + fencedAt time.Time // world time at which the fence latched (zero: never) + frozen bool // SIGSTOP / docker pause: no code runs + partitioned bool // the carrier cannot reach the coordinator + dead bool // kill -9: never runs again + acks int + lastAck time.Time // world time of the last acknowledged heartbeat +} + +func (w *world) startAgent(runner, leaseID string, epoch int64) *agent { + a := &agent{w: w, runner: runner, leaseID: leaseID, epoch: epoch, wall: w.clock()} + a.fence = lease.NewAgentFence(func() time.Time { return a.wall }, func() time.Duration { return a.mono }, coordTTL, heartbeat) + a.fence.OnFence = func() { a.fencedAt = w.clock() } + a.lastAck = w.clock() + return a +} + +// run moves time forward in heartbeat steps; every live, unfrozen agent heartbeats at each step. +func (w *world) run(d time.Duration, agents ...*agent) { + for left := d; left > 0; left -= heartbeat { + step := min(left, heartbeat) + w.advance(step) + for _, a := range agents { + a.wall = a.wall.Add(step) + a.mono += step + if !a.frozen && !a.dead { + a.beat() + } + } + } +} + +// beat is one heartbeat: the fence is checked first, then the coordinator is asked to renew. +func (a *agent) beat() { + if a.fence.Check() || a.partitioned || a.w.down { + return + } + if a.w.engine.Renew(a.w.ctx, a.leaseID, a.epoch, 0) == nil { + a.fence.Ack() + a.acks++ + a.lastAck = a.w.clock() + } +} + +// finish completes the fixture job: on a fenced agent the job is killed before its side effect; +// otherwise the side effect is written and the result is sent. +func (a *agent) finish(jobID string) (ran bool, err error) { + if a.dead || a.fence.Check() { + return false, nil + } + a.w.effect(jobID, a.runner) + return true, a.w.engine.Result(a.w.ctx, a.leaseID, a.epoch) +} + +// lateResult is a result frame buffered before the fault and delivered after it. +func (a *agent) lateResult() error { return a.w.engine.Result(a.w.ctx, a.leaseID, a.epoch) } + +// reconnect is the agent's report after a carrier or coordinator outage. +func (a *agent) reconnect() error { + status := "running" + if a.fence.Check() { + status = "self_fenced" + } + return a.w.engine.Reconnect(a.w.ctx, a.leaseID, a.epoch, status) +} + +func ptr[T any](v T) *T { return &v } + +// simRunner is an eligible linux/amd64 node speaking protocol 1. +func simRunner(id string) sched.Runner { + c := model.Capability{Schema: "ci.runner-capability/v1"} + c.Lifecycle.AuthorizedProjects = []string{"project"} + c.Lifecycle.Eligible = true + c.Protocol.Versions = []int{1} + c.Protocol.Min = 1 + c.Platform.OS.Value = ptr("linux") + c.Platform.Arch.Value = ptr("amd64") + c.Resources.CPU.Value = ptr(float64(8)) + c.Resources.MemMB.Value = ptr(int64(8192)) + c.Location.Kind.Value = ptr("lan") + return sched.Runner{ID: id, Location: "lan", Class: "owned", Provider: "operator", Avail: sched.Avail{Slots: 1, CPU: 8, MemMB: 8192}, Capability: c, + Decl: trust.Declaration{Accepts: []model.TrustClass{"owner", "internal", "collaborator", "untrusted"}, Isolation: "vm", Network: "privileged", UIDSeparation: true, Ownership: "operator", NoLANRoute: true}} +} + +// place runs the scheduler for one job attempt; previous is the runner retry anti-affinity excludes. +func place(jobKey string, attempt int, previous string, runners ...sched.Runner) sched.Result { + w := sched.World{Runners: runners, Policy: sched.Policy{Preset: "balanced"}, Costs: map[string]sched.Cost{}, Previous: map[string]string{}} + if previous != "" { + w.Previous[jobKey] = previous + } + j := sched.Job{Key: jobKey, ID: jobKey, Project: "project", Attempt: attempt, Pipeline: sched.Pipeline{ID: "pipe", Support: []string{"linux/amd64"}}, + Requirements: model.PlacementRequirements{Platform: "linux/amd64", CPU: 1, MemMB: 128}, Trust: "owner", Network: "lan", TimeoutMs: 60000} + return sched.Place(w, []sched.Job{j}) +} + +// reasonsFor returns the placement reason codes the scheduler gave one runner. +func reasonsFor(r sched.Result, runner string) []string { + var out []string + for _, x := range r.Explanations { + for _, alt := range x.Alternatives { + if alt.RunnerID == runner { + for _, q := range alt.Reasons { + out = append(out, q.Code) + } + } + } + } + return out +} + +func contains(values []string, want string) bool { + for _, v := range values { + if v == want { + return true + } + } + return false +} diff --git a/free/ci/test/reliability/sim_world_test.go b/free/ci/test/reliability/sim_world_test.go new file mode 100644 index 000000000..0b5fc48ce --- /dev/null +++ b/free/ci/test/reliability/sim_world_test.go @@ -0,0 +1,219 @@ +package reliability + +// Purpose: the coordinator side of the sim: one real store and lease engine behind an injected clock, +// fixture jobs, grants on named runners and the fixture job's side-effect file. +// Inputs: a *testing.T; every time moves through world.advance. +// Outputs: helpers that fail the test on any unexpected store error. +// Constraints: no sleeps; times start at time.Unix(1, 0); the side-effect file lives in the test's TempDir. + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nself-org/plugins/free/ci/internal/lease" + "github.com/nself-org/plugins/free/ci/internal/store" +) + +const ( + coordTTL = 45 * time.Second + heartbeat = 10 * time.Second +) + +// world is one coordinator store with its lease engine and an injected clock. +type world struct { + t *testing.T + ctx context.Context + mu sync.Mutex + now time.Time + down bool // the coordinator process is not running + store *store.Store + engine *lease.Engine + effects string + audits []store.AuditRow +} + +func newWorld(t *testing.T) *world { + t.Helper() + w := &world{t: t, ctx: context.Background(), now: time.Unix(1, 0), effects: t.TempDir()} + s, err := store.Open(filepath.Join(t.TempDir(), "state.db"), store.Options{Clock: w.clock}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + w.store = s + s.SetAuditSealer(func(_ []byte, row store.AuditRow) (store.AuditChain, error) { + w.mu.Lock() + w.audits = append(w.audits, row) + w.mu.Unlock() + return store.AuditChain{}, nil + }) + w.engine = w.coordinator() + return w +} + +// retryReasons lists the reasons of the retries queued for an attempt (audit action retry-queued). +func (w *world) retryReasons(attemptID string) []string { + w.mu.Lock() + defer w.mu.Unlock() + var out []string + for _, row := range w.audits { + if row.Action == "retry-queued" && row.Target == attemptID { + out = append(out, row.Detail) + } + } + return out +} + +// coordinator builds a fresh lease engine over the same store: a restarted coordinator process. +func (w *world) coordinator() *lease.Engine { + return lease.New(w.store, nil, lease.Config{Clock: w.clock, Heartbeat: heartbeat, TTL: coordTTL}) +} + +func (w *world) clock() time.Time { + w.mu.Lock() + defer w.mu.Unlock() + return w.now +} + +func (w *world) advance(d time.Duration) { + w.mu.Lock() + w.now = w.now.Add(d) + w.mu.Unlock() +} + +// job creates a pipeline, a job and its first attempt ".a1". +func (w *world) job(id, kind string, idempotent bool, infraMax int) string { + w.t.Helper() + p := store.PipelineRow{ID: "p-" + id, Project: "project", Revision: "r", Trigger: "local", SourceTrust: "owner", PrivacyZone: "local-only", PolicyDigest: "p", InputDigest: "i", SelectionMode: "full", Status: "queued"} + if err := w.store.CreatePipeline(w.ctx, p); err != nil { + w.t.Fatal(err) + } + if err := w.store.CreateJob(w.ctx, store.JobRow{ID: id, PipelineID: p.ID, Name: id, Kind: kind, Idempotent: idempotent, InfraMax: infraMax, InputDigest: "i"}); err != nil { + w.t.Fatal(err) + } + if err := w.store.CreateAttempt(w.ctx, store.AttemptRow{ID: id + ".a1", JobID: id, N: 1}, ""); err != nil { + w.t.Fatal(err) + } + return id + ".a1" +} + +// grant leases a queued attempt to runner (one host per runner) and starts it; it returns the lease epoch. +func (w *world) grant(attemptID, leaseID, runner string, incarnation int64) int64 { + w.t.Helper() + d := store.Demand{AttemptID: attemptID, LeaseID: leaseID, RunnerID: runner, CPU: 1, MemMB: 1, CapacityCPU: 100, CapacityMemMB: 100, TTL: coordTTL} + epoch, err := w.engine.Grant(w.ctx, "host-"+runner, d, incarnation) + if err != nil { + w.t.Fatalf("grant %s on %s: %v", attemptID, runner, err) + } + _, attemptEpoch := w.state(attemptID) + if err := w.store.Transition(w.ctx, attemptID, "leased", "running", store.TransitionDetail{Epoch: attemptEpoch}); err != nil { + w.t.Fatal(err) + } + return epoch +} + +func (w *world) state(attemptID string) (string, int64) { + w.t.Helper() + st, epoch, err := w.store.AttemptStateEpoch(w.ctx, attemptID) + if err != nil { + w.t.Fatal(err) + } + return string(st), epoch +} + +func (w *world) attempts(jobID string) []store.AttemptView { + w.t.Helper() + rows, err := w.store.AttemptsByJob(w.ctx, jobID) + if err != nil { + w.t.Fatal(err) + } + return rows +} + +// active counts attempts of a job that hold or may hold a lease. +func (w *world) active(jobID string) int { + n := 0 + for _, a := range w.attempts(jobID) { + if a.State == "leased" || a.State == "running" || a.State == "finalizing" { + n++ + } + } + return n +} + +// wantAttempts checks the evidence: the state and failure_class of every attempt in order. +func (w *world) wantAttempts(jobID string, want ...string) []store.AttemptView { + w.t.Helper() + rows := w.attempts(jobID) + got := make([]string, len(rows)) + for i, a := range rows { + got[i] = a.State + "/" + a.FailureClass + } + if strings.Join(got, ",") != strings.Join(want, ",") { + w.t.Fatalf("job %s attempts = %v, want %v", jobID, got, want) + } + return rows +} + +func (w *world) previousRunner(attemptID string) string { + w.t.Helper() + r, err := w.store.LeasePreviousRunner(w.ctx, attemptID) + if err != nil { + w.t.Fatal(err) + } + return r +} + +// effect is the fixture job's side effect: one line per execution, stamped with the world clock. +func (w *world) effect(jobID, runner string) { + w.t.Helper() + f, err := os.OpenFile(filepath.Join(w.effects, jobID), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + w.t.Fatal(err) + } + defer func() { _ = f.Close() }() + if _, err := fmt.Fprintf(f, "%s %d\n", runner, w.clock().Unix()); err != nil { + w.t.Fatal(err) + } +} + +// effects returns the lines of a job's side-effect file. +func (w *world) effectLines(jobID string) []string { + w.t.Helper() + b, err := os.ReadFile(filepath.Join(w.effects, jobID)) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + w.t.Fatal(err) + } + return strings.Fields(strings.ReplaceAll(strings.TrimSpace(string(b)), " ", "@")) +} + +// wantEffect asserts the side-effect file holds exactly one line, written by runner. +func (w *world) wantEffect(jobID, runner string) { + w.t.Helper() + lines := w.effectLines(jobID) + if len(lines) != 1 || !strings.HasPrefix(lines[0], runner+"@") { + w.t.Fatalf("side effect of %s = %v, want one write by %s", jobID, lines, runner) + } +} + +func hasCode(err error, code string) bool { + var coded *store.Error + return errors.As(err, &coded) && coded.Code == code +} + +func wantCode(t *testing.T, what string, err error, code string) { + t.Helper() + if !hasCode(err, code) { + t.Fatalf("%s: got %v, want %s", what, err, code) + } +} From 6c5cb587a789400b52a70102ee487642501a966a Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Mon, 26 Oct 2026 12:53:40 -0400 Subject: [PATCH 3/4] feat(p7-node-17): scenario legs 2 (personal LAN) and 3 (VPS) through pla Ticket: p7-node-17 Swarm-Run: p7f-plugins-r1 --- free/ci/test/sim/perf_test.go | 150 ++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 free/ci/test/sim/perf_test.go diff --git a/free/ci/test/sim/perf_test.go b/free/ci/test/sim/perf_test.go new file mode 100644 index 000000000..ec1d4f55a --- /dev/null +++ b/free/ci/test/sim/perf_test.go @@ -0,0 +1,150 @@ +//go:build integration + +package sim + +// Purpose: the perfbench/v1 document of the node scenario legs (P7-NODE-17): every measured leg adds +// named samples, and TestMain writes one document after the run when -perf-json is set. +// Inputs: -perf-json ; samples from TestScenario2, TestScenario3 and TestWriteLoad. +// Outputs: one JSON file with the command, revision and environment of the run and one entry per +// metric (count, min, p50, p95, p99, max, mean). +// Constraints: nothing is written when no leg measured anything; the numbers are only ever measured, +// never defaulted, so a leg that did not run leaves its metric out and `perfbench check` +// reports it missing. + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "os/exec" + "runtime" + "sort" + "strings" + "sync" + "testing" + "time" +) + +var ( + perfJSON = flag.String("perf-json", "", "write the perfbench/v1 measurements of the node scenario legs to this file") + writeLoadFor = flag.Duration("write-load-for", 10*time.Minute, "how long TestWriteLoad holds the load (the acceptance leg is 10m)") +) + +// metric is one perfbench/v1 entry. +type metric struct { + Name string `json:"name"` + Unit string `json:"unit"` + Leg string `json:"leg"` + Count int `json:"count"` + Min float64 `json:"min"` + P50 float64 `json:"p50"` + P95 float64 `json:"p95"` + P99 float64 `json:"p99"` + Max float64 `json:"max"` + Mean float64 `json:"mean"` + Context string `json:"context,omitempty"` +} + +var perf struct { + mu sync.Mutex + metrics []metric +} + +// record adds the samples of one metric; an empty sample set records nothing. +func record(leg, name, unit, context string, samples []float64) { + if len(samples) == 0 { + return + } + s := append([]float64(nil), samples...) + sort.Float64s(s) + sum := 0.0 + for _, v := range s { + sum += v + } + m := metric{Name: name, Unit: unit, Leg: leg, Count: len(s), Min: s[0], P50: quantile(s, 0.50), P95: quantile(s, 0.95), + P99: quantile(s, 0.99), Max: s[len(s)-1], Mean: sum / float64(len(s)), Context: context} + perf.mu.Lock() + perf.metrics = append(perf.metrics, m) + perf.mu.Unlock() +} + +// quantile is the nearest-rank quantile of sorted samples. +func quantile(sorted []float64, q float64) float64 { + if len(sorted) == 0 { + return 0 + } + i := int(q*float64(len(sorted))+0.999999) - 1 + if i < 0 { + i = 0 + } + if i >= len(sorted) { + i = len(sorted) - 1 + } + return sorted[i] +} + +func ms(d time.Duration) float64 { return float64(d.Microseconds()) / 1000 } + +// environment names where the numbers were measured. +func environment() map[string]any { + env := map[string]any{"goos": runtime.GOOS, "goarch": runtime.GOARCH, "cpus": runtime.NumCPU(), "go": runtime.Version()} + if b, err := os.ReadFile("/proc/sys/kernel/osrelease"); err == nil { + env["kernel"] = strings.TrimSpace(string(b)) + } + for k, v := range map[string]string{"runner_os": "RUNNER_OS", "runner_image": "ImageOS", "runner_image_version": "ImageVersion", + "github_run_id": "GITHUB_RUN_ID", "github_run_attempt": "GITHUB_RUN_ATTEMPT", "github_repository": "GITHUB_REPOSITORY"} { + if s := os.Getenv(v); s != "" { + env[k] = s + } + } + env["hosted"] = os.Getenv("GITHUB_ACTIONS") == "true" + return env +} + +// revision is the commit of the checkout the legs ran from. +func revision() string { + out, err := exec.Command("git", "rev-parse", "HEAD").Output() + if err != nil { + return "unknown" + } + return strings.TrimSpace(string(out)) +} + +func writePerf(path string) error { + perf.mu.Lock() + defer perf.mu.Unlock() + if len(perf.metrics) == 0 { + return nil + } + sort.SliceStable(perf.metrics, func(i, j int) bool { return perf.metrics[i].Name < perf.metrics[j].Name }) + doc := map[string]any{"schema": "perfbench/v1", "suite": "plugins/free/ci/test/sim", "command": strings.Join(os.Args, " "), + "revision": revision(), "measured_at": time.Now().UTC().Format(time.RFC3339), "environment": environment(), "metrics": perf.metrics} + b, err := json.MarshalIndent(doc, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, append(b, '\n'), 0o600) +} + +func TestMain(m *testing.M) { + flag.Parse() + code := m.Run() + if *perfJSON != "" { + if err := writePerf(*perfJSON); err != nil { + fmt.Fprintf(os.Stderr, "perf-json: %v\n", err) + if code == 0 { + code = 1 + } + } + } + os.Exit(code) +} + +// integration fails the leg unless INTEGRATION=1, the opt-in every Docker and load leg of the plugins +// repo uses (simharness skips on its own; the load leg has no Docker, so it asks here). +func integration(t *testing.T) { + t.Helper() + if os.Getenv("INTEGRATION") != "1" { + t.Skip("set INTEGRATION=1 to run the node scenario legs") + } +} From df3ad66b2d43188aa0e6a9c753445be817aa898c Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Mon, 26 Oct 2026 13:39:11 -0400 Subject: [PATCH 4/4] feat(p7-ci-65): doc-command check: plugins:.github/scripts/check-doc-com Ticket: p7-ci-65 Swarm-Run: p7f-plugins-r1-r2 --- .github/scripts/check-doc-commands.sh | 30 ++----------- free/ci/tools/doccmds/check.go | 2 +- free/ci/tools/doccmds/main.go | 21 ++++++++- free/ci/tools/doccmds/main_test.go | 63 +++++++++++++++++++++++++++ free/ci/tools/doccmds/runner.go | 2 + 5 files changed, 89 insertions(+), 29 deletions(-) create mode 100644 free/ci/tools/doccmds/main_test.go diff --git a/.github/scripts/check-doc-commands.sh b/.github/scripts/check-doc-commands.sh index 2b7dc5d05..7b89103cc 100755 --- a/.github/scripts/check-doc-commands.sh +++ b/.github/scripts/check-doc-commands.sh @@ -1,28 +1,4 @@ -#!/usr/bin/env bash -# check-doc-commands.sh — run the bash blocks of doc pages in clean fixture projects and check their exit codes. -# +#!/bin/sh # Usage: bash .github/scripts/check-doc-commands.sh ... -# A bash block must exit 0. A block directly under must exit non-zero. -# Each block runs in a fresh temp project with a git repo and NSELF_V15=1. Nothing in the working tree changes. -# All logic lives in free/ci/tools/doccmds. This script only finds free/ci and execs it. -set -euo pipefail - -if [ "$#" -eq 0 ]; then - echo "usage: check-doc-commands.sh ..." >&2 - exit 2 -fi - -script_dir="$(cd "$(dirname "$0")" && pwd)" -ci_dir="$(cd "$script_dir/../../free/ci" && pwd)" - -# go -C changes directory, so pages given relative to the caller's directory are made absolute first. -pages=() -for page in "$@"; do - case "$page" in - /*) pages+=("$page") ;; - *) pages+=("$PWD/$page") ;; - esac -done - -export CGO_ENABLED=0 -exec go -C "$ci_dir" run -mod=vendor ./tools/doccmds "${pages[@]}" +exec env CGO_ENABLED=0 GOFLAGS=-mod=vendor DOCCMDS_CALLER_DIR="$PWD" \ + go -C "$(dirname "$0")/../../free/ci" run ./tools/doccmds "$@" diff --git a/free/ci/tools/doccmds/check.go b/free/ci/tools/doccmds/check.go index 3465d2e9e..e62b1f588 100644 --- a/free/ci/tools/doccmds/check.go +++ b/free/ci/tools/doccmds/check.go @@ -67,8 +67,8 @@ func checkBlock(page string, b Block, opts Options) (Failure, bool) { if err != nil { return fail(-1, fmt.Sprintf("fixture: %v", err)) } + defer cleanup() code, err := opts.Runner.Run(dir, b.Script) - cleanup() if err != nil { return fail(-1, fmt.Sprintf("could not run: %v", err)) } diff --git a/free/ci/tools/doccmds/main.go b/free/ci/tools/doccmds/main.go index 562c304a8..7157c4a77 100644 --- a/free/ci/tools/doccmds/main.go +++ b/free/ci/tools/doccmds/main.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "os" + "path/filepath" ) // usageText is printed when no page is given. @@ -22,11 +23,29 @@ func main() { // run checks every page and writes one line per failure to stdout and page errors to stderr. It returns // the process exit code. func run(pages []string, stdout, stderr io.Writer) int { + base := os.Getenv("DOCCMDS_CALLER_DIR") + resolved := make([]string, len(pages)) + for i, page := range pages { + resolved[i] = pagePath(base, page) + } + opts := Options{Runner: shellRunner{timeout: blockTimeout}, Fixture: newFixture} + return runWithOptions(resolved, stdout, stderr, opts) +} + +// pagePath preserves paths relative to the wrapper's caller across go -C. +func pagePath(base, page string) string { + if base == "" || filepath.IsAbs(page) { + return page + } + return filepath.Join(base, page) +} + +// runWithOptions checks pages using injected execution and fixture creation. +func runWithOptions(pages []string, stdout, stderr io.Writer, opts Options) int { if len(pages) == 0 { fmt.Fprintln(stderr, usageText) return 2 } - opts := Options{Runner: shellRunner{timeout: blockTimeout}, Fixture: newFixture} status := 0 for _, page := range pages { failures, err := CheckPage(page, opts) diff --git a/free/ci/tools/doccmds/main_test.go b/free/ci/tools/doccmds/main_test.go new file mode 100644 index 000000000..5c4a83c57 --- /dev/null +++ b/free/ci/tools/doccmds/main_test.go @@ -0,0 +1,63 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "testing" +) + +func TestRunFixturePages(t *testing.T) { + cases := []struct { + page string + code int + out string + }{ + {"pass.md", 0, ""}, + {"fail.md", 1, "testdata/fail.md: block 2: exit 3\ntestdata/fail.md: block 3: exit 7\n"}, + {"expect-fail.md", 1, "testdata/expect-fail.md: block 2: exit 0 (expected non-zero: marked expect-fail)\ntestdata/expect-fail.md: block 3: exit 5\n"}, + {"no-blocks.md", 0, ""}, + } + for _, tc := range cases { + t.Run(tc.page, func(t *testing.T) { + r := &fakeRunner{codes: map[string]int{ + "exit 1": 1, "exit 3": 3, "exit 5": 5, "exit 7": 7, + }} + fx := &fakeFixture{} + var out, errOut bytes.Buffer + code := runWithOptions([]string{"testdata/" + tc.page}, &out, &errOut, + Options{Runner: r, Fixture: fx.fixture}) + if code != tc.code || out.String() != tc.out || errOut.Len() != 0 { + t.Fatalf("code=%d stdout=%q stderr=%q; want code=%d stdout=%q stderr empty", + code, out.String(), errOut.String(), tc.code, tc.out) + } + if fx.made != len(r.calls) || fx.cleaned != fx.made { + t.Errorf("calls=%d fixtures=%d cleanups=%d", len(r.calls), fx.made, fx.cleaned) + } + }) + } +} + +func TestPagePath(t *testing.T) { + base := filepath.Join(string(filepath.Separator), "caller") + abs := filepath.Join(base, "page.md") + for _, tc := range []struct{ base, page, want string }{ + {base, "page.md", abs}, + {base, abs, abs}, + {"", "page.md", "page.md"}, + } { + if got := pagePath(tc.base, tc.page); got != tc.want { + t.Errorf("pagePath(%q, %q) = %q; want %q", tc.base, tc.page, got, tc.want) + } + } +} + +func TestWrapperExecutable(t *testing.T) { + info, err := os.Stat("../../../../.github/scripts/check-doc-commands.sh") + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0755 { + t.Errorf("wrapper mode = %04o; want 0755", got) + } +} diff --git a/free/ci/tools/doccmds/runner.go b/free/ci/tools/doccmds/runner.go index 29448cc0c..4895743f6 100644 --- a/free/ci/tools/doccmds/runner.go +++ b/free/ci/tools/doccmds/runner.go @@ -50,6 +50,8 @@ func fixtureEnv(dir string) []string { "TMPDIR=" + os.TempDir(), "LANG=C", "NSELF_V15=1", + "CGO_ENABLED=0", + "GOFLAGS=-mod=vendor", } }