diff --git a/.github/workflows/ci-isolation-linux.yml b/.github/workflows/ci-isolation-linux.yml index a289ba98e..96243944a 100644 --- a/.github/workflows/ci-isolation-linux.yml +++ b/.github/workflows/ci-isolation-linux.yml @@ -144,7 +144,7 @@ jobs: echo "tags: $TAGS" echo "packages: ${selected[*]}" # -p 1: the packages share one Docker daemon (and its memory); a container of one package was SIGKILLed mid-run while the others ran (run 39385968795) - go test -mod=vendor -count=1 -p 1 -v -tags "$TAGS" "${selected[@]}" 2>&1 | tee "$RUNNER_TEMP/integration.txt" + go test -mod=vendor -count=1 -p 1 -timeout 25m -v -tags "$TAGS" "${selected[@]}" 2>&1 | tee "$RUNNER_TEMP/integration.txt" if grep -q -- '--- SKIP' "$RUNNER_TEMP/integration.txt"; then echo "a test skipped: a missing prerequisite must fail, never skip" >&2 exit 1 diff --git a/free/ci/cmd/runs_wait.go b/free/ci/cmd/runs_wait.go index 63cd1156e..cd7cd3f26 100644 --- a/free/ci/cmd/runs_wait.go +++ b/free/ci/cmd/runs_wait.go @@ -31,7 +31,6 @@ import ( "github.com/nself-org/plugins/free/ci/internal/detach" cirun "github.com/nself-org/plugins/free/ci/internal/run" - "github.com/nself-org/plugins/free/ci/internal/store" ) const ( @@ -168,137 +167,3 @@ func printWait(asJSON bool, v detach.View, d cirun.RunView, terminal bool, waite } return code } - -// runSource reads one run's state: from a RemoteResolver when the run lives on -// another machine, else from the local store. -type runSource struct { - resolver cirun.RemoteResolver - ref string - remote bool - st *store.Store - row store.PipelineRow - found bool - read bool // a local read has finished - firstRead context.Context // when set, bounds the first local read instead of the caller's context - last time.Time - // Detail is the RunView of the latest View call. - Detail cirun.RunView -} - -// recoverEvery bounds how often a poll runs dead-coordinator recovery. -const recoverEvery = 2 * time.Second - -func openRunSource(ctx context.Context, ref, coordinator string) (*runSource, error) { - return openRunSourceGrace(ctx, ctx, ref, coordinator) -} - -// openRunSourceGrace is openRunSource with a separate context for opening the local store, which -// bounds remote lookups by ctx and the store open by storeCtx. -func openRunSourceGrace(ctx, storeCtx context.Context, ref, coordinator string) (*runSource, error) { - if coordinator != "" { - if err := os.Setenv(detach.EnvCoordinator, coordinator); err != nil { - return nil, err - } - } - s := &runSource{ref: ref} - for _, r := range cirun.Registered.Remote { - v, ok, err := r.Resolve(ctx, ref) - if err != nil && cirun.RemoteFatal(r) { // an active coordinator target never falls back to the local store (P7-NODE-19) - return nil, err - } - if err == nil && ok { - s.remote, s.Detail, s.resolver = true, v, r - return s, nil - } - } - // recovery after dead coordinators runs in View, under the caller's deadline - st, err := openStoreCtx(storeCtx, runsStorePath()) - if err != nil { - return nil, err - } - s.st = st - return s, nil -} - -// Close releases the store. -func (s *runSource) Close() { - if s.st != nil { - _ = s.st.Close() - } -} - -// View returns the run's current state. -func (s *runSource) View(ctx context.Context) (detach.View, error) { - if s.remote { - for _, r := range cirun.Registered.Remote { - v, ok, err := r.Resolve(ctx, s.ref) - if err != nil { - return detach.View{}, err - } - if ok { - s.Detail = v - return toView(v), nil - } - } - return detach.View{}, fmt.Errorf("run %q is no longer known to its coordinator", s.ref) - } - if !s.read && s.firstRead != nil { - ctx = s.firstRead - } - if time.Since(s.last) >= recoverEvery { - s.last = time.Now() - _ = cirun.Recover(ctx, s.st, "", os.Getenv("NSELF_CI_HOME")) - } - if !s.found { - row, err := cirun.ResolveRun(ctx, s.st, s.ref) - if err != nil { - return detach.View{}, err - } - s.row, s.found = row, true - } else if row, ok, err := s.st.PipelineByID(ctx, s.row.ID); err != nil { - return detach.View{}, err - } else if ok { - s.row = row - } - v, err := cirun.ViewRun(ctx, s.st, s.row) - if err != nil { - return detach.View{}, err - } - s.Detail = v - s.read = true - return toView(v), nil -} - -func toView(v cirun.RunView) detach.View { - out := detach.View{RunID: v.ID, State: v.Status} - for _, j := range v.Jobs { - out.Jobs = append(out.Jobs, detach.JobState{Name: j.Name, State: j.State, FailureClass: j.FailureClass}) - } - return out -} - -// openStoreCtx opens the run store but gives up when ctx ends (a held SQLite -// write lock can make the open wait for seconds); the open finishes in the -// background and closes its store. -func openStoreCtx(ctx context.Context, path string) (*store.Store, error) { - type result struct { - st *store.Store - err error - } - ch := make(chan result, 1) - go func() { - st, err := cirun.OpenStore(path) - ch <- result{st, err} - }() - select { - case r := <-ch: - return r.st, r.err - case <-ctx.Done(): - go func() { - if r := <-ch; r.st != nil { - _ = r.st.Close() - } - }() - return nil, ctx.Err() - } -} diff --git a/free/ci/cmd/runs_wait_poll.go b/free/ci/cmd/runs_wait_poll.go new file mode 100644 index 000000000..8653bb698 --- /dev/null +++ b/free/ci/cmd/runs_wait_poll.go @@ -0,0 +1,153 @@ +package main + +// Purpose: the read side of `nself ci runs wait` (P7-CI-63, Epic D30): runSource reads one run's +// state per poll, from a RemoteResolver when the run lives on another machine, else from the local +// store (split from runs_wait.go to keep each file within the 300-line budget). +// Inputs: a run reference, the registered RemoteResolvers, NSELF_CI_HOME. +// Outputs: detach.View per poll, plus the RunView detail of the latest poll. +// Constraints: every call is bounded by the caller's context; a held SQLite lock never outlives it. + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/nself-org/plugins/free/ci/internal/detach" + cirun "github.com/nself-org/plugins/free/ci/internal/run" + "github.com/nself-org/plugins/free/ci/internal/store" +) + +// runSource reads one run's state: from a RemoteResolver when the run lives on +// another machine, else from the local store. +type runSource struct { + resolver cirun.RemoteResolver + ref string + remote bool + st *store.Store + row store.PipelineRow + found bool + read bool // a local read has finished + firstRead context.Context // when set, bounds the first local read instead of the caller's context + last time.Time + // Detail is the RunView of the latest View call. + Detail cirun.RunView +} + +// recoverEvery bounds how often a poll runs dead-coordinator recovery. +const recoverEvery = 2 * time.Second + +func openRunSource(ctx context.Context, ref, coordinator string) (*runSource, error) { + return openRunSourceGrace(ctx, ctx, ref, coordinator) +} + +// openRunSourceGrace is openRunSource with a separate context for opening the local store, which +// bounds remote lookups by ctx and the store open by storeCtx. +func openRunSourceGrace(ctx, storeCtx context.Context, ref, coordinator string) (*runSource, error) { + if coordinator != "" { + if err := os.Setenv(detach.EnvCoordinator, coordinator); err != nil { + return nil, err + } + } + s := &runSource{ref: ref} + for _, r := range cirun.Registered.Remote { + v, ok, err := r.Resolve(ctx, ref) + if err != nil && cirun.RemoteFatal(r) { // an active coordinator target never falls back to the local store (P7-NODE-19) + return nil, err + } + if err == nil && ok { + s.remote, s.Detail, s.resolver = true, v, r + return s, nil + } + } + // recovery after dead coordinators runs in View, under the caller's deadline + st, err := openStoreCtx(storeCtx, runsStorePath()) + if err != nil { + return nil, err + } + s.st = st + return s, nil +} + +// Close releases the store. +func (s *runSource) Close() { + if s.st != nil { + _ = s.st.Close() + } +} + +// View returns the run's current state. +func (s *runSource) View(ctx context.Context) (detach.View, error) { + if s.remote { + for _, r := range cirun.Registered.Remote { + v, ok, err := r.Resolve(ctx, s.ref) + if err != nil { + return detach.View{}, err + } + if ok { + s.Detail = v + return toView(v), nil + } + } + return detach.View{}, fmt.Errorf("run %q is no longer known to its coordinator", s.ref) + } + if !s.read && s.firstRead != nil { + ctx = s.firstRead + } + if time.Since(s.last) >= recoverEvery { + s.last = time.Now() + _ = cirun.Recover(ctx, s.st, "", os.Getenv("NSELF_CI_HOME")) + } + if !s.found { + row, err := cirun.ResolveRun(ctx, s.st, s.ref) + if err != nil { + return detach.View{}, err + } + s.row, s.found = row, true + } else if row, ok, err := s.st.PipelineByID(ctx, s.row.ID); err != nil { + return detach.View{}, err + } else if ok { + s.row = row + } + v, err := cirun.ViewRun(ctx, s.st, s.row) + if err != nil { + return detach.View{}, err + } + s.Detail = v + s.read = true + return toView(v), nil +} + +func toView(v cirun.RunView) detach.View { + out := detach.View{RunID: v.ID, State: v.Status} + for _, j := range v.Jobs { + out.Jobs = append(out.Jobs, detach.JobState{Name: j.Name, State: j.State, FailureClass: j.FailureClass}) + } + return out +} + +// openStoreCtx opens the run store but gives up when ctx ends (a held SQLite +// write lock can make the open wait for seconds); the open finishes in the +// background and closes its store. +func openStoreCtx(ctx context.Context, path string) (*store.Store, error) { + type result struct { + st *store.Store + err error + } + ch := make(chan result, 1) + go func() { + st, err := cirun.OpenStore(path) + ch <- result{st, err} + }() + select { + case r := <-ch: + return r.st, r.err + case <-ctx.Done(): + go func() { + if r := <-ch; r.st != nil { + _ = r.st.Close() + } + }() + return nil, ctx.Err() + } +} diff --git a/free/ci/internal/evidence/inputs_test.go b/free/ci/internal/evidence/inputs_test.go index e4d22bea4..6400df2e2 100644 --- a/free/ci/internal/evidence/inputs_test.go +++ b/free/ci/internal/evidence/inputs_test.go @@ -232,3 +232,35 @@ func TestWorktreeAddRunsNoHooksOfTheRevision(t *testing.T) { t.Fatal("the revision's post-checkout hook ran") } } + +// The detector fixtures' lockfiles enter the input digests. The proof lives here, not in internal/gates: +// this package reaches internal/run, which imports internal/gates, so a gates test cannot import it. +func TestInputsOfFixtureLockfiles(t *testing.T) { + detectData := filepath.Join("..", "detect", "testdata") + for _, c := range []struct { + dir string + want []string + }{ + {filepath.Join("python", "uv"), []string{"uv.lock"}}, + {filepath.Join("python", "poetry"), []string{"poetry.lock"}}, + {filepath.Join("python", "pip"), []string{"requirements.txt", "requirements-dev.txt"}}, + {filepath.Join("swift", "swiftpm"), []string{"Package.resolved"}}, + } { + d, err := InputsOf(os.DirFS(filepath.Join(detectData, c.dir))) + if err != nil { + t.Fatalf("%s: %v", c.dir, err) + } + have := map[string]bool{} + for _, l := range d.Lockfiles { + have[l.Path] = true + } + if len(have) != len(c.want) { + t.Errorf("%s: lockfiles %+v, want exactly %v", c.dir, d.Lockfiles, c.want) + } + for _, w := range c.want { + if !have[w] { + t.Errorf("%s: %s missing from input digests %+v", c.dir, w, d.Lockfiles) + } + } + } +} diff --git a/free/ci/internal/gates/python_integration_test.go b/free/ci/internal/gates/python_integration_test.go index b8080d607..dcb4c15cb 100644 --- a/free/ci/internal/gates/python_integration_test.go +++ b/free/ci/internal/gates/python_integration_test.go @@ -5,8 +5,9 @@ package gates // Purpose: TestPythonFixtures (P7-CI-64) runs the Python checks of the uv and // pip fixtures for real on the verify host (python3 and uv on PATH, network // for PyPI): a passing project passes, a failing test fails, ruff findings -// stay advisory, a missing tool or a missing pytest is E614 and never a pass, -// and the lockfiles enter the evidence input digests. +// stay advisory, a missing tool or a missing pytest is E614 and never a pass. +// The lockfile-to-input-digest proof lives in internal/evidence (TestInputsOfFixtureLockfiles): +// internal/evidence reaches internal/run, which imports this package, so a test here cannot import it. // Constraints: build tag integration, INTEGRATION=1; a missing prerequisite is // t.Fatal, never t.Skip, so a green run proves the legs ran. @@ -19,7 +20,6 @@ import ( "testing" "github.com/nself-org/plugins/free/ci/internal/detect" - "github.com/nself-org/plugins/free/ci/internal/evidence" ciexec "github.com/nself-org/plugins/free/ci/internal/exec" "github.com/nself-org/plugins/free/ci/internal/model" ) @@ -126,33 +126,4 @@ func TestPythonFixtures(t *testing.T) { t.Fatalf("python-test without uv: %+v", c) } }) - - t.Run("lockfiles enter the evidence input digests", func(t *testing.T) { - for fixture, want := range map[string][]string{ - "uv": {"uv.lock"}, - "poetry": {"poetry.lock"}, - "pip": {"requirements.txt", "requirements-dev.txt"}, - } { - d, err := evidence.InputsOf(os.DirFS(filepath.Join("..", "detect", "testdata", "python", fixture))) - if err != nil { - t.Fatal(err) - } - have := map[string]bool{} - for _, l := range d.Lockfiles { - have[l.Path] = true - } - for _, w := range want { - if !have[w] { - t.Errorf("%s: %s missing from input digests %+v", fixture, w, d.Lockfiles) - } - } - } - d, err := evidence.InputsOf(os.DirFS(filepath.Join("..", "detect", "testdata", "swift", "swiftpm"))) - if err != nil { - t.Fatal(err) - } - if len(d.Lockfiles) != 1 || d.Lockfiles[0].Path != "Package.resolved" { - t.Errorf("swiftpm: Package.resolved missing from input digests %+v", d.Lockfiles) - } - }) } diff --git a/free/ci/internal/isolation/gvisor/adhoc_integration_test.go b/free/ci/internal/isolation/gvisor/adhoc_integration_test.go index 79a80cde6..c0d7b0738 100644 --- a/free/ci/internal/isolation/gvisor/adhoc_integration_test.go +++ b/free/ci/internal/isolation/gvisor/adhoc_integration_test.go @@ -9,22 +9,40 @@ package gvisor import ( "bytes" + "context" "encoding/json" + "fmt" "os" osexec "os/exec" "path/filepath" "strings" + "syscall" "testing" + "time" ) +// runBudget bounds one command of these tests. The sandboxed ad-hoc run takes about 35 s on a hosted runner, +// so five minutes is generous; a command past it is a hang, and the test says so instead of waiting for the +// 10-minute go test alarm (which names no cause and kills the whole package). +const runBudget = 5 * time.Minute + func runCmd(t *testing.T, dir string, env []string, name string, args ...string) (string, string, int) { t.Helper() - cmd := osexec.Command(name, args...) + ctx, cancel := context.WithTimeout(context.Background(), runBudget) + defer cancel() + cmd := osexec.CommandContext(ctx, name, args...) cmd.Dir, cmd.Env = dir, append(os.Environ(), env...) + // SIGQUIT makes a Go binary print every goroutine's stack, which is the evidence a hang needs. + cmd.Cancel = func() error { return cmd.Process.Signal(syscall.SIGQUIT) } + cmd.WaitDelay = 15 * time.Second var stdout, stderr bytes.Buffer cmd.Stdout, cmd.Stderr = &stdout, &stderr err := cmd.Run() code := 0 + if ctx.Err() != nil { + t.Fatalf("%s %v did not finish within %s (hang, not a skip): sent SIGQUIT\nstdout: %s\nstderr (goroutine dump): %s\nprocesses and containers at the time:\n%s", + name, args, runBudget, stdout.String(), tail(stderr.String(), 12000), hangContext()) + } if ee, ok := err.(*osexec.ExitError); ok { code = ee.ExitCode() } else if err != nil { @@ -33,6 +51,23 @@ func runCmd(t *testing.T, dir string, env []string, name string, args ...string) return stdout.String(), stderr.String(), code } +// hangContext lists the processes and containers a hung run left, for the failure message. +func hangContext() string { + var b strings.Builder + for _, c := range [][]string{{"ps", "-eo", "pid,ppid,stat,etime,args"}, {"docker", "ps", "-a", "--format", "{{.ID}} {{.Status}} {{.Names}} {{.Image}}"}} { + out, err := osexec.Command(c[0], c[1:]...).CombinedOutput() + fmt.Fprintf(&b, "$ %s (err %v)\n%s\n", strings.Join(c, " "), err, tail(string(out), 4000)) + } + return b.String() +} + +func tail(s string, n int) string { + if len(s) > n { + return "..." + s[len(s)-n:] + } + return s +} + func adhocRig(t *testing.T) (bin, repo, home string) { t.Helper() tmp := t.TempDir()