diff --git a/free/ci/internal/gates/result_classifier.go b/free/ci/internal/gates/result_classifier.go new file mode 100644 index 000000000..7bbb322c5 --- /dev/null +++ b/free/ci/internal/gates/result_classifier.go @@ -0,0 +1,28 @@ +package gates + +import "strings" + +// MissingResult identifies a missing catalogue tool from a failed command's +// output, rather than mistaking an installed launcher for an installed tool. +func MissingResult(check string, exit int, output string) (tool, remediation string) { + if exit == 0 { + return "", "" + } + switch check { + case "python-test": + if pytestMissing(output) { + return "pytest", "Add pytest to the project's dependencies" + } + case "python-types": + for _, marker := range []string{"No module named mypy", "Failed to spawn: `mypy`", "mypy: command not found", "Command not found: mypy"} { + if strings.Contains(output, marker) { + return "mypy", "Add mypy to the project's dependencies" + } + } + case "xcode-test": + if strings.Contains(output, "requires Xcode") && strings.Contains(strings.ToLower(output), "command line tools") { + return "xcodebuild", "Run on a Mac with Xcode installed and selected with xcode-select" + } + } + return "", "" +} diff --git a/free/ci/internal/gates/result_classifier_test.go b/free/ci/internal/gates/result_classifier_test.go new file mode 100644 index 000000000..a599a4749 --- /dev/null +++ b/free/ci/internal/gates/result_classifier_test.go @@ -0,0 +1,24 @@ +package gates + +import "testing" + +func TestMissingResult(t *testing.T) { + for _, tc := range []struct { + check, output, tool string + exit int + }{ + {"python-test", "No module named pytest", "pytest", 1}, + {"python-types", "Failed to spawn: `mypy`", "mypy", 1}, + {"xcode-test", "xcodebuild requires Xcode; Command Line Tools installed", "xcodebuild", 1}, + {"python-test", "No module named pytest", "", 0}, + {"python-test", "test failed", "", 1}, + {"custom", "No module named pytest", "", 1}, + } { + t.Run(tc.check+"/"+tc.output, func(t *testing.T) { + tool, fix := MissingResult(tc.check, tc.exit, tc.output) + if tool != tc.tool || (tool != "" && fix == "") { + t.Fatalf("tool = %q, remediation = %q", tool, fix) + } + }) + } +} diff --git a/free/ci/internal/gates/run.go b/free/ci/internal/gates/run.go index 7053eea59..9811ae5fc 100644 --- a/free/ci/internal/gates/run.go +++ b/free/ci/internal/gates/run.go @@ -100,9 +100,9 @@ func (r Runner) Run(ctx context.Context, root string, p PlannedCheck, cfg model. c.Excerpt = res.err.Error() return c } - if d.ID == "python-test" && res.exit != 0 && pytestMissing(res.output) { - c.Tool = "pytest" - return unavailable(c, "Add pytest to the project's dependencies") + if tool, fix := MissingResult(d.ID, res.exit, res.output); tool != "" { + c.Tool = tool + return unavailable(c, fix) } if advisoryOffline(d.ID, res.output, res.exit) { if !p.Required { diff --git a/free/ci/internal/run/catalogue_result.go b/free/ci/internal/run/catalogue_result.go new file mode 100644 index 000000000..da9bd3a52 --- /dev/null +++ b/free/ci/internal/run/catalogue_result.go @@ -0,0 +1,23 @@ +package run + +import ( + ciexec "github.com/nself-org/plugins/free/ci/internal/exec" + "github.com/nself-org/plugins/free/ci/internal/gates" +) + +// classifyCatalogue applies the same missing-tool verdict as the direct +// catalogue runner. It cannot turn a cancellation or a successful job into a failure. +func classifyCatalogue(check string, res *ciexec.AttemptResult, o *JobOutcome) { + if res.Attempt.State != "failed" { + return + } + tool, fix := gates.MissingResult(check, res.ExitCode, o.Tail) + if tool == "" { + return + } + res.Attempt.FailureClass = "infra" + res.Err = &Error{Code: "E614", Message: tool + " unavailable; " + fix} + o.Tool, o.Code, o.FailureClass = tool, "E614", "infra" + o.Remediation, o.retryable = fix, false + o.Excerpt = res.Err.Error() +} diff --git a/free/ci/internal/run/catalogue_result_test.go b/free/ci/internal/run/catalogue_result_test.go new file mode 100644 index 000000000..ee7a44485 --- /dev/null +++ b/free/ci/internal/run/catalogue_result_test.go @@ -0,0 +1,37 @@ +package run + +import ( + "github.com/nself-org/plugins/free/ci/internal/model" + "testing" + + ciexec "github.com/nself-org/plugins/free/ci/internal/exec" +) + +func TestCatalogueMissingTools(t *testing.T) { + for _, tc := range []struct { + check, output, state string + exit int + missing bool + }{ + {"python-test", "No module named pytest", "failed", 1, true}, + {"python-types", "No module named mypy", "failed", 1, true}, + {"xcode-test", "xcode-select: error: tool 'xcodebuild' requires Xcode, but active developer directory is a command line tools instance", "failed", 1, true}, + {"python-test", "assertion failed", "failed", 1, false}, + {"custom", "No module named pytest", "failed", 1, false}, + {"python-test", "No module named pytest", "passed", 0, false}, + {"python-test", "No module named pytest", "cancelled", 1, false}, + } { + t.Run(tc.check+"/"+tc.state+"/"+tc.output, func(t *testing.T) { + var res ciexec.AttemptResult + res.Attempt.State, res.Attempt.FailureClass, res.ExitCode = model.JobState(tc.state), "code", tc.exit + o := JobOutcome{Tail: tc.output, retryable: true} + classifyCatalogue(tc.check, &res, &o) + if (o.Code == "E614") != tc.missing { + t.Fatalf("code = %q", o.Code) + } + if tc.missing && (res.Attempt.FailureClass != "infra" || o.Remediation == "" || o.retryable) { + t.Fatal("missing tool verdict lost") + } + }) + } +} diff --git a/free/ci/internal/run/jobrun.go b/free/ci/internal/run/jobrun.go index a83d2d318..724e504b3 100644 --- a/free/ci/internal/run/jobrun.go +++ b/free/ci/internal/run/jobrun.go @@ -193,6 +193,7 @@ func (rs *runState) runLeased(ctx context.Context, pj plan.PlannedJob, job model } rs.judge(ctx, info, full, &res, &o) rs.diagnose(&o, &res) + classifyCatalogue(pj.Check, &res, &o) oc := Outcome{State: res.Attempt.State, FailureClass: res.Attempt.FailureClass, ExitCode: res.ExitCode} defer func() { if rs.hooks.Cache != nil { diff --git a/free/ci/internal/run/safety_test.go b/free/ci/internal/run/safety_test.go index 584f544e4..6ae08f484 100644 --- a/free/ci/internal/run/safety_test.go +++ b/free/ci/internal/run/safety_test.go @@ -45,13 +45,25 @@ func TestStatusNotPostedForPartialCancelledOrOutsideConfig(t *testing.T) { }) } // Ctrl-C: the run context ends while a job runs; nothing is posted - r := newRig(t, map[string]model.Job{"a": sh("sleep 60")}, gitFake{}) + marker := filepath.Join(t.TempDir(), "started") + r := newRig(t, map[string]model.Job{"a": sh("touch " + strconv.Quote(marker) + "; sleep 60")}, gitFake{}) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - r.C.Hooks.AttemptStarts = []AttemptStartHook{startFn(func(AttemptInfo) error { - cancel() - return nil - })} + go func() { + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if _, err := os.Stat(marker); err == nil { + cancel() + return + } + } + } + }() res, err := r.C.Run(ctx) if err != nil || res.Exit == 0 || len(r.Rep.all()) != 0 || !strings.Contains(res.Summary.Status, report.ReasonCancelled) { t.Fatalf("exit %d err %v status %q posted %+v", res.Exit, err, res.Summary.Status, r.Rep.all())