diff --git a/README.md b/README.md index 5ede7cf..09335b1 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ Run `tinyagent --help` (or plain `tinyagent` at a terminal) for the overview and - Project rules from agent failures (`tinyagent ratchet add`): each rule records its incident and an expiry or review condition, is added to the system prompt, and can be enforced as a permission deny rule; see [Project rules](docs/ratchet.md). - A task planner that can require a plan before any change. - Run budgets: turn cap, cost limit and token limit (`--token-limit`), with an 80% warning to the model; delegates, workers and `--attempts` share the budget, and a run that hits it can be resumed. -- Verify before stop (`--verify-on-stop`): a run that changed files isn't accepted until a test or build command passes; otherwise it is marked unverified. +- Verify before stop (`--verify-on-stop`): a run that changed files isn't accepted until a test or build command passes; otherwise it is marked unverified. Optional computational checks (`--verify-check`, e.g. lint, type check, tests) run in order, cheapest first, before the run may finish; see [Computational checks](docs/verify.md#computational-checks). - Help everywhere: `tinyagent` or `tinyagent ` alone at a terminal prints help; `--help --verbose` lists every flag by section. - Undo for file edits, workspace rewind, resume and follow-up runs. - Sandboxed commands with a network allowlist, permission rules, hooks, untrusted-content lockdown and file-access rules. diff --git a/cmd/tinyagent/verify.go b/cmd/tinyagent/verify.go index 0a14221..bf209d2 100644 --- a/cmd/tinyagent/verify.go +++ b/cmd/tinyagent/verify.go @@ -10,6 +10,7 @@ import ( type verifyFlags struct { enabled bool commands stringList + checks stringList maxReprompts int } @@ -17,11 +18,12 @@ type verifyFlags struct { var verifyGroup = struct { name string flags [][3]string -}{"Verify before stop", [][3]string{{"", "max-verify-reprompts", "n"}, {"", "verify-command", "cmd"}, {"", "verify-on-stop", ""}}} +}{"Verify before stop", [][3]string{{"", "max-verify-reprompts", "n"}, {"", "verify-check", "cmd"}, {"", "verify-command", "cmd"}, {"", "verify-on-stop", ""}}} func registerVerifyFlags(fs *flag.FlagSet, v *verifyFlags) { fs.BoolVar(&v.enabled, "verify-on-stop", false, "don't accept a final answer while files changed since the last successful test/build run: send the model back to verify, up to --max-verify-reprompts times, then mark the run unverified (default off; config verify_on_stop.enabled)") fs.Var(&v.commands, "verify-command", "a bash command that counts as verification for --verify-on-stop when it exits 0, e.g. \"go test ./...\"; repeatable, replaces verify_on_stop.commands") + fs.Var(&v.checks, "verify-check", "a computational check for --verify-on-stop: a fast, deterministic bash command (lint, type check, tests) that passes on exit 0; when the model finishes with unverified changes tinyagent runs the checks itself in the given order, cheapest first, and sends the model back with the first failure; repeatable, replaces verify_on_stop.checks") fs.IntVar(&v.maxReprompts, "max-verify-reprompts", config.DefaultVerifyMaxReprompts, "with --verify-on-stop, how many times to send the model back to verify before stopping unverified (1 to 10; config verify_on_stop.max_reprompts)") } @@ -34,6 +36,9 @@ func applyVerifyFlags(fs *flag.FlagSet, cfg *config.AgentConfig, v verifyFlags) if flagWasSet(fs, "verify-command") { cfg.VerifyOnStop.Commands = append([]string(nil), v.commands...) } + if flagWasSet(fs, "verify-check") { + cfg.VerifyOnStop.Checks = append([]string(nil), v.checks...) + } if flagWasSet(fs, "max-verify-reprompts") { cfg.VerifyOnStop.MaxReprompts = v.maxReprompts } diff --git a/cmd/tinyagent/verify_checks_test.go b/cmd/tinyagent/verify_checks_test.go new file mode 100644 index 0000000..cf99897 --- /dev/null +++ b/cmd/tinyagent/verify_checks_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "strings" + "testing" + + "github.com/vicgupta/tinyagent/internal/config" +) + +func TestApplyVerifyCheckFlag(t *testing.T) { + var o options + fs := newFlagSet(&o) + if err := fs.Parse([]string{"--verify-check", "gofmt -l .", "--verify-check", "go test ./..."}); err != nil { + t.Fatal(err) + } + cfg := config.Default() + cfg.VerifyOnStop.Checks = []string{"from config"} + if err := applyVerifyFlags(fs, &cfg, o.verify); err != nil { + t.Fatal(err) + } + if got := strings.Join(cfg.VerifyOnStop.Checks, "|"); got != "gofmt -l .|go test ./..." { + t.Errorf("checks = %q", got) + } + + var o2 options + fs = newFlagSet(&o2) + cfg = config.Default() + cfg.VerifyOnStop.Checks = []string{"from config"} + if err := applyVerifyFlags(fs, &cfg, o2.verify); err != nil || len(cfg.VerifyOnStop.Checks) != 1 { + t.Errorf("unset flag changed the checks: %v, %v", cfg.VerifyOnStop.Checks, err) + } + + var o3 options + fs = newFlagSet(&o3) + if err := fs.Parse([]string{"--verify-check", " "}); err == nil { + t.Error("blank --verify-check accepted") + } +} diff --git a/docs/config.md b/docs/config.md index e1f6b52..7295d9a 100644 --- a/docs/config.md +++ b/docs/config.md @@ -169,7 +169,8 @@ hooks: # omitted by default; see Tool hooks below verify_on_stop: # off by default; see Verify before stop below enabled: false # --verify-on-stop commands: [go test ./...] # --verify-command (repeatable) replaces this list - patterns: [] # Go regexps; built-in test/build patterns when both lists are empty + patterns: [] # Go regexps; built-in test/build patterns when commands, checks and patterns are all empty + checks: [] # --verify-check (repeatable) replaces this list; computational checks run in order at stop, cheapest first max_reprompts: 2 # --max-verify-reprompts; default 2, range 1..10 attempts: # omitted by default; only used with --attempts, see Best-of-N attempts below checks: # replaced as a whole by --attempt-check @@ -185,7 +186,7 @@ profiles: # omitted by default; named overlays selected with read_only: true ### Verify before stop -`verify_on_stop` (or `--verify-on-stop`, `--verify-command`, `--max-verify-reprompts`) keeps a run that changed files from finishing until a test or build command has passed, re-prompting the model up to `max_reprompts` times and otherwise marking the run unverified. See [Verify before stop](verify.md#verify-before-stop) for the full rules. +`verify_on_stop` (or `--verify-on-stop`, `--verify-command`, `--verify-check`, `--max-verify-reprompts`) keeps a run that changed files from finishing until a test or build command has passed, re-prompting the model up to `max_reprompts` times and otherwise marking the run unverified. `verify_on_stop.checks` lists computational checks (lint, type check, tests) that tinyagent runs itself, in order, when the model finishes with unverified changes; see [Computational checks](verify.md#computational-checks). See [Verify before stop](verify.md#verify-before-stop) for the full rules. ### Profiles diff --git a/docs/json-output.md b/docs/json-output.md index c50a4fa..e6ea10f 100644 --- a/docs/json-output.md +++ b/docs/json-output.md @@ -29,6 +29,7 @@ Every event has a `type`. Event names and fields in schema version 1 are: | `ratchet.loaded` | `path` (the rules file), `loaded` (active rule count), `ids` (active rule ids), `expired` (ids of expired rules that were skipped), `enforced` (active rules added as permission deny rules); once after `run.started` for a fresh run when `.tinyagent/ratchet.toml` exists and `--no-ratchet` isn't set. See [Project rules](ratchet.md). Never includes rule text | | `budget.warning` | `step`, `usage` (what was used, as text), `tokens`, `token_limit`, `cost_usd`, `cost_limit`; once per run when a token or cost budget reaches 80% (see [Token and cost budgets](runs.md#token-and-cost-budgets)) | | `verify.reprompt` | `reprompt`, `max_reprompts`; the model gave a final answer with unverified changes and was sent back to verify (see [Verify before stop](verify.md#verify-before-stop)) | +| `verify.check` | `index` (1-based position in `verify_on_stop.checks`), `command`, `passed`; one per [computational check](verify.md#computational-checks) tinyagent runs when the model finishes with unverified changes, emitted before any `verify.reprompt` | | `verify.unverified` | `reason`, `reprompts`; the run finishes with unverified changes | | `memory.saved` | `op` (`add`, `replace` or `forget`), `id`, `status` (`active`, `pending` or `forgotten`), `redacted`, `untrusted_context`; one per successful `remember` write, emitted before its `tool.result`. Never includes entry text | | `lsp.server` | `name`, `state` (`started`, `unavailable`, `crashed` or `stopped`), `reason` (for `unavailable` and `crashed`); one per language-server state change (see [Language servers (LSP)](long-runs.md#language-servers-lsp)) | diff --git a/docs/subcommands.md b/docs/subcommands.md index 4fa0b05..e768e0d 100644 --- a/docs/subcommands.md +++ b/docs/subcommands.md @@ -85,6 +85,7 @@ Six subcommands are shortcuts for existing flags. They rewrite the command line | `--time-limit` | 0 (unlimited) | Max wall-clock seconds; overrides `wall_time_seconds` | | `--verify-on-stop` | config `verify_on_stop.enabled`, else off | Don't accept a final answer while files changed since the last successful test/build command: send the model back to verify, up to `--max-verify-reprompts` times, then mark the run unverified. See [Verify before stop](verify.md#verify-before-stop) | | `--verify-command ` | config `verify_on_stop.commands`, else common test/build commands | A `bash` command that counts as verification for `--verify-on-stop`; repeatable, replaces the config list | +| `--verify-check ` | config `verify_on_stop.checks`, else none | A computational check for `--verify-on-stop`: a fast, deterministic `bash` command (lint, type check, tests) that passes on exit 0. When the model finishes with unverified changes, tinyagent runs the checks itself in the given order (cheapest first), stops at the first failure and sends the model back with its output; all passing lets the run finish. Also counts as a verification command. Repeatable, replaces the config list. See [Computational checks](verify.md#computational-checks) | | `--max-verify-reprompts ` | config `verify_on_stop.max_reprompts`, else 2 | With `--verify-on-stop`, how many times to send the model back before stopping unverified (1 to 10) | | `--max-format-errors` | config `max_format_errors`, else 3 | Stop with `RepeatedFormatError` after this many consecutive steps whose tool calls are all malformed; must be >= 1. See [Malformed tool calls](troubleshooting.md#malformed-tool-calls) | | `--max-output-tokens` | config `max_output_tokens`, else 0 | Per-request output-token budget sent to the provider; 0 keeps the provider default (8192 `max_tokens` on Anthropic). Not supported by `codex`. Unrelated to `max_output_chars`, which caps tool output. See [Output truncation](troubleshooting.md#output-truncation) | diff --git a/docs/verify.md b/docs/verify.md index 31ccc38..70ca3bc 100644 --- a/docs/verify.md +++ b/docs/verify.md @@ -11,6 +11,7 @@ tinyagent --verify-on-stop --verify-command "go test ./..." --max-verify-repromp |---|---|---|---| | `--verify-on-stop` | `verify_on_stop.enabled` | off | Turn the guard on (`--verify-on-stop=false` turns it off) | | `--verify-command ` | `verify_on_stop.commands` | common test/build commands | A `bash` command that counts as verification; repeatable, replaces the config list | +| `--verify-check ` | `verify_on_stop.checks` | none | A computational check tinyagent runs itself before the run may finish, in order; repeatable, replaces the config list. See [Computational checks](#computational-checks) | | | `verify_on_stop.patterns` | none | Regular expressions; a matching `bash` command counts as verification | | `--max-verify-reprompts ` | `verify_on_stop.max_reprompts` | 2 | How many times the model is sent back before the run is marked unverified (1 to 10) | @@ -22,10 +23,37 @@ With `verify_on_stop.enabled` (or `--verify-on-stop`), a run that changed files When the model gives a final answer (with or without the completion marker) while changes are pending, it gets a fixed user message asking it to verify, naming the configured commands, and the run continues as normal steps, so step, cost and time limits still apply. Tool calls in a submitting message are not run and get an error result saying so. After `max_reprompts` re-prompts, or straight away when `bash` cannot run (denied by the permission policy, a hook or lockdown, or unavailable as in read-only mode), the run finishes with its normal status and is marked unverified: the summary line ends with `, unverified changes`, the `--json` result has `"unverified": true`, and the trajectory has `unverified: true`. `verify_reprompts` counts the re-prompts in both. The guard runs before stop hooks, does not apply to subagent runs or `--output-schema` runs, and keeps its state across `--resume`. +### Computational checks + +Sensors come in two tiers. Computational checks are fast and deterministic (linters, type checkers, structural checks, tests: pass means exit 0); inferential review asks a model to judge the change and is slower. `verify_on_stop.checks` (or `--verify-check`, repeatable) configures the computational tier. List them cheapest first: + +```yaml +verify_on_stop: + enabled: true + checks: + - gofmt -l . | (! grep .) + - go vet ./... + - go test ./... +``` + +```bash +tinyagent --verify-on-stop --verify-check "go vet ./..." --verify-check "go test ./..." -t "..." +``` + +When the model gives a final answer while changes are pending, tinyagent runs the checks itself, one after another in the listed order, as `bash` tool calls, so the permission policy, hooks, lockdown and sandbox apply to them exactly as to the model's own commands. It stops at the first failing check: + +- **All pass:** the pending changes are cleared and the run finishes verified, with no re-prompt. +- **One fails:** the model is sent back as described above, and the re-prompt names the failing check (`Computational check 2 of 3 failed`) and includes its output; later checks don't run. This counts against `max_reprompts`, and at the cap the run finishes unverified. +- **One is refused** (denied by the policy, a hook or lockdown): the run finishes unverified straight away. + +Each check also counts as a verification command when the model runs it, so a model that ran the checks itself after its edits isn't sent back. Configuring checks (like `commands`) replaces the built-in test and build patterns. The checks run only when changes are pending and the guard is on, so with `checks` empty, or `verify_on_stop` off, behaviour is unchanged. Checks never run for subagent or `--output-schema` runs. Each check that runs emits a `verify.check` event in `--json`. + +The inferential tier, running the existing [review](runs.md#reviewing-changes) once at the end after every check passes, is not wired into verify-on-stop yet; for now, run `tinyagent --review` after the run. + ## Where it shows up - **Summary line:** ends with `, unverified changes`. -- **`--json`:** `verify.reprompt` (`reprompt`, `max_reprompts`) each time the model is sent back, `verify.unverified` (`reason`, `reprompts`) when the run ends unverified, and the `result` event's `unverified` and `verify_reprompts` fields; see [JSON output](json-output.md#machine-readable-output). +- **`--json`:** `verify.check` (`index`, `command`, `passed`) for each computational check, `verify.reprompt` (`reprompt`, `max_reprompts`) each time the model is sent back, `verify.unverified` (`reason`, `reprompts`) when the run ends unverified, and the `result` event's `unverified` and `verify_reprompts` fields; see [JSON output](json-output.md#machine-readable-output). - **Trajectory (`-o`):** `unverified: true` and `verify_reprompts`. The config block is shown in the [example config](config.md#config). diff --git a/internal/agent/loop.go b/internal/agent/loop.go index fa89479..f2d0099 100644 --- a/internal/agent/loop.go +++ b/internal/agent/loop.go @@ -600,7 +600,7 @@ func (a *Agent) Run(ctx context.Context, task string) *trajectory.Trajectory { } if strings.Contains(msg.Content, SubmitMarker) || len(msg.ToolCalls) == 0 { - if done, again := a.verifyStop(t, v, msg); again || done != nil { + if done, again := a.verifyStop(ctx, t, v, msg); again || done != nil { if done != nil { return done } diff --git a/internal/agent/verify.go b/internal/agent/verify.go index 034f12f..a28221d 100644 --- a/internal/agent/verify.go +++ b/internal/agent/verify.go @@ -1,6 +1,7 @@ package agent import ( + "context" "encoding/json" "fmt" "regexp" @@ -66,7 +67,7 @@ func (a *Agent) noteVerify(t *trajectory.Trajectory, call model.ToolCall, res to // model was sent back to verify, so the loop must continue; a non-nil done // is the final trajectory after a failed save. Otherwise the run may finish, // with t.Unverified set when changes are still unverified. -func (a *Agent) verifyStop(t *trajectory.Trajectory, v *view, msg model.Message) (done *trajectory.Trajectory, again bool) { +func (a *Agent) verifyStop(ctx context.Context, t *trajectory.Trajectory, v *view, msg model.Message) (done *trajectory.Trajectory, again bool) { if !a.verifyOn() || a.subagentID != "" { return nil, false } @@ -78,7 +79,12 @@ func (a *Agent) verifyStop(t *trajectory.Trajectory, v *view, msg model.Message) case t.VerifyBlocked || a.Tools.ReadOnly || !a.Tools.Permits("bash"): a.markUnverified(t, "verification commands cannot run (bash is refused or unavailable)") return nil, false - case t.VerifyReprompts >= a.maxVerifyReprompts(): + } + failed, ok := a.runVerifyChecks(ctx, t) + if !ok { + return nil, false + } + if t.VerifyReprompts >= a.maxVerifyReprompts() { a.markUnverified(t, fmt.Sprintf("still unverified after %d verify re-prompts", t.VerifyReprompts)) return nil, false } @@ -93,7 +99,7 @@ func (a *Agent) verifyStop(t *trajectory.Trajectory, v *view, msg model.Message) if a.Verbose && a.Err != nil { fmt.Fprintf(a.Err, "verify: unverified changes; asking the model to verify (%d/%d)\n", t.VerifyReprompts, a.maxVerifyReprompts()) } - t.Messages = append(t.Messages, model.Message{Role: "user", Content: fmt.Sprintf(verifyNotice, verifyHint(a.Config.VerifyOnStop))}) + t.Messages = append(t.Messages, model.Message{Role: "user", Content: fmt.Sprintf(verifyNotice, verifyHint(a.Config.VerifyOnStop)) + failed}) v.add(t, t.Messages[len(t.Messages)-1]) if err := a.step(t); err != nil { return a.saveFailed(t, err), false @@ -112,11 +118,12 @@ func (a *Agent) markUnverified(t *trajectory.Trajectory, reason string) { // verifyHint names the configured verify commands for the notice. func verifyHint(c config.VerifyOnStop) string { - if len(c.Commands) == 0 { + all := append(append([]string(nil), c.Checks...), c.Commands...) + if len(all) == 0 { return "" } - q := make([]string, len(c.Commands)) - for i, cmd := range c.Commands { + q := make([]string, len(all)) + for i, cmd := range all { q[i] = "`" + strings.TrimSpace(cmd) + "`" } return " (configured: " + strings.Join(q, ", ") + ")" @@ -143,14 +150,14 @@ func isVerifyCommand(c config.VerifyOnStop, cmd string) bool { if cmd == "" { return false } - for _, want := range c.Commands { + for _, want := range append(append([]string(nil), c.Checks...), c.Commands...) { want = strings.TrimSpace(want) if want != "" && (cmd == want || strings.HasPrefix(cmd, want+" ")) { return true } } patterns := c.Patterns - if len(c.Commands) == 0 && len(patterns) == 0 { + if len(c.Commands) == 0 && len(c.Checks) == 0 && len(patterns) == 0 { patterns = config.DefaultVerifyPatterns } for _, p := range patterns { diff --git a/internal/agent/verify_checks.go b/internal/agent/verify_checks.go new file mode 100644 index 0000000..313bf36 --- /dev/null +++ b/internal/agent/verify_checks.go @@ -0,0 +1,53 @@ +package agent + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/vicgupta/tinyagent/internal/model" + "github.com/vicgupta/tinyagent/internal/trajectory" +) + +// checkFailedNotice is appended to the verify notice when a computational +// check failed; %d is its position, %s its command and its output. +const checkFailedNotice = "\n\nComputational check %d of %d failed: `%s`\n%s" + +// runVerifyChecks runs the computational checks (verify_on_stop.checks) +// in their configured order, cheapest first, stopping at the first +// failure. Each runs as a bash tool call, so the permission policy, hooks, +// lockdown and sandbox apply. With every check passing, the pending +// changes are cleared and ok is false: the run may finish verified. A +// failure returns its description for the re-prompt and ok true. A refused +// check marks the run unverified and returns ok false. With no checks +// configured it returns ok true so the plain re-prompt applies. +func (a *Agent) runVerifyChecks(ctx context.Context, t *trajectory.Trajectory) (failed string, ok bool) { + checks := a.Config.VerifyOnStop.Checks + if len(checks) == 0 { + return "", true + } + for i, cmd := range checks { + cmd = strings.TrimSpace(cmd) + args, _ := json.Marshal(map[string]string{"command": cmd}) + call := model.ToolCall{ID: fmt.Sprintf("verify-check-%d-%d", t.VerifyReprompts, i+1), Type: "function"} + call.Function.Name, call.Function.Arguments = "bash", string(args) + res := a.runTool(ctx, t, call) + refused := a.toolRefused + passed := !refused && bashSucceeded(res) + a.event("verify.check", map[string]any{"index": i + 1, "command": cmd, "passed": passed}) + if a.Verbose && a.Err != nil { + fmt.Fprintf(a.Err, "verify: check %d/%d `%s`: %s\n", i+1, len(checks), cmd, map[bool]string{true: "passed", false: "failed"}[passed]) + } + if refused { + t.VerifyBlocked = true + a.markUnverified(t, fmt.Sprintf("computational check `%s` cannot run (bash is refused)", cmd)) + return "", false + } + if !passed { + return fmt.Sprintf(checkFailedNotice, i+1, len(checks), cmd, res.Text), true + } + } + t.VerifyPending = false + return "", false +} diff --git a/internal/agent/verify_checks_test.go b/internal/agent/verify_checks_test.go new file mode 100644 index 0000000..32c6ffc --- /dev/null +++ b/internal/agent/verify_checks_test.go @@ -0,0 +1,146 @@ +package agent + +import ( + "context" + "strings" + "testing" + + "github.com/vicgupta/tinyagent/internal/config" + "github.com/vicgupta/tinyagent/internal/model" +) + +// checkAgent returns a verify agent whose computational checks are checks, +// and a pointer to the verify events it emits with their fields. +func checkAgent(t *testing.T, m model.Model, checks ...string) (*Agent, *[]string, *[]map[string]any) { + t.Helper() + a := newAgent(t, m) + a.Config.VerifyOnStop = config.VerifyOnStop{Enabled: true, Checks: checks, MaxReprompts: 2} + var kinds []string + var fields []map[string]any + a.OnEvent = func(kind string, f map[string]any) { + if strings.HasPrefix(kind, "verify.") { + kinds = append(kinds, kind) + fields = append(fields, f) + } + } + return a, &kinds, &fields +} + +func TestVerifyChecksAllPassFinishVerified(t *testing.T) { + m := &scripted{replies: []model.Message{ + edit(), + {Role: "assistant", Content: "Done."}, + }} + a, kinds, fields := checkAgent(t, m, "true lint", "true test") + tr := a.Run(context.Background(), "task") + if tr.ExitStatus != StatusCompleted || tr.Submission != "Done." || tr.Unverified || tr.VerifyPending || tr.VerifyReprompts != 0 { + t.Fatalf("trajectory = %+v", tr) + } + if strings.Join(*kinds, ",") != "verify.check,verify.check" { + t.Fatalf("events = %v", *kinds) + } + if (*fields)[0]["command"] != "true lint" || (*fields)[1]["command"] != "true test" || (*fields)[1]["passed"] != true { + t.Errorf("checks ran out of order or failed: %v", *fields) + } + if verifyNotices(tr.Messages) != 0 { + t.Error("passing checks still re-prompted the model") + } +} + +func TestVerifyChecksStopAtFirstFailure(t *testing.T) { + m := &scripted{replies: []model.Message{ + edit(), + {Role: "assistant", Content: "Done."}, + {Role: "assistant", Content: "Fixed."}, + {Role: "assistant", Content: "Still done."}, + }} + a, kinds, fields := checkAgent(t, m, "true lint", "echo lint-broke; false", "true test") + tr := a.Run(context.Background(), "task") + if tr.ExitStatus != StatusCompleted || !tr.Unverified || tr.VerifyReprompts != 2 { + t.Fatalf("trajectory = %+v", tr) + } + // First stop: check 1 passes, check 2 fails, check 3 never runs. + if (*fields)[0]["passed"] != true || (*fields)[1]["passed"] != false || (*kinds)[2] != "verify.reprompt" { + t.Errorf("events = %v %v", *kinds, *fields) + } + notice := m.seen[2][len(m.seen[2])-1].Content + if !strings.Contains(notice, "check 2 of 3 failed") || !strings.Contains(notice, "lint-broke") { + t.Errorf("notice does not show the failing check: %q", notice) + } + for _, f := range *fields { + if f["command"] == "true test" { + t.Error("a check after the failing one ran") + } + } +} + +func TestVerifyChecksFixedAfterReprompt(t *testing.T) { + m := &scripted{replies: []model.Message{ + edit(), + {Role: "assistant", Content: "Done."}, + {Role: "assistant", ToolCalls: []model.ToolCall{call("w2", "write_file", `{"path":"fixed","content":"ok"}`)}}, + {Role: "assistant", Content: "Fixed."}, + }} + a, _, _ := checkAgent(t, m, "test -f fixed") + tr := a.Run(context.Background(), "task") + if tr.ExitStatus != StatusCompleted || tr.Submission != "Fixed." || tr.Unverified || tr.VerifyReprompts != 1 { + t.Fatalf("trajectory = %+v", tr) + } +} + +func TestVerifyChecksModelRunCounts(t *testing.T) { + m := &scripted{replies: []model.Message{ + edit(), + bashMsg("v", "true lint"), + {Role: "assistant", Content: "Done."}, + }} + a, kinds, _ := checkAgent(t, m, "true lint") + tr := a.Run(context.Background(), "task") + if tr.Unverified || tr.VerifyPending || len(*kinds) != 0 { + t.Fatalf("a passing check run by the model should verify: %+v, events %v", tr, *kinds) + } +} + +func TestVerifyChecksRefused(t *testing.T) { + m := &scripted{replies: []model.Message{ + edit(), + {Role: "assistant", Content: "Done."}, + }} + a, kinds, _ := checkAgent(t, m, "true lint") + withPolicy(t, a, config.Permissions{Rules: []config.PermissionRule{ + {Tool: "bash", Command: "true lint", Decision: "deny"}, + }}) + tr := a.Run(context.Background(), "task") + if tr.ExitStatus != StatusCompleted || !tr.Unverified || tr.VerifyReprompts != 0 { + t.Fatalf("trajectory = %+v", tr) + } + if strings.Join(*kinds, ",") != "verify.check,verify.unverified" { + t.Errorf("events = %v", *kinds) + } +} + +func TestVerifyChecksOffByDefault(t *testing.T) { + m := &scripted{replies: []model.Message{ + edit(), + {Role: "assistant", Content: "Done."}, + }} + a, kinds, _ := checkAgent(t, m, "false") + a.Config.VerifyOnStop.Enabled = false + tr := a.Run(context.Background(), "task") + if tr.ExitStatus != StatusCompleted || tr.Unverified || len(*kinds) != 0 { + t.Fatalf("checks ran with the guard off: %+v, %v", tr, *kinds) + } +} + +func TestIsVerifyCommandChecks(t *testing.T) { + c := config.VerifyOnStop{Checks: []string{"golangci-lint run"}} + if !isVerifyCommand(c, "golangci-lint run ./...") { + t.Error("a check should count as verification") + } + if isVerifyCommand(c, "go test ./...") { + t.Error("checks should replace the default patterns") + } + if h := verifyHint(c); !strings.Contains(h, "`golangci-lint run`") { + t.Errorf("hint = %q", h) + } +} diff --git a/internal/config/verify.go b/internal/config/verify.go index d15e4b9..479abda 100644 --- a/internal/config/verify.go +++ b/internal/config/verify.go @@ -27,6 +27,13 @@ type VerifyOnStop struct { // command matches any of them also verifies. When neither Commands nor // Patterns is set, DefaultVerifyPatterns apply. Patterns []string `yaml:"patterns,omitempty"` + // Checks are computational sensors: fast, deterministic bash commands + // (lint, type check, tests) that pass on exit 0. When the model gives a + // final answer with changes pending, the harness runs them itself, in + // the listed order (put the cheapest first), stopping at the first + // failure. Each also counts as a verification command when the model + // runs it. --verify-check replaces this list. Empty by default. + Checks []string `yaml:"checks,omitempty"` // MaxReprompts caps how many times one run is sent back to verify // (1 to MaxVerifyReprompts; default DefaultVerifyMaxReprompts). After // the cap the run stops and is marked unverified. @@ -53,6 +60,11 @@ func (v VerifyOnStop) Validate() error { return fmt.Errorf("verify_on_stop.commands: empty command") } } + for _, c := range v.Checks { + if strings.TrimSpace(c) == "" { + return fmt.Errorf("verify_on_stop.checks: empty command") + } + } for _, p := range v.Patterns { if _, err := regexp.Compile(p); err != nil { return fmt.Errorf("verify_on_stop.patterns: %q: %v", p, err) diff --git a/internal/config/verify_checks_test.go b/internal/config/verify_checks_test.go new file mode 100644 index 0000000..e5a9f98 --- /dev/null +++ b/internal/config/verify_checks_test.go @@ -0,0 +1,26 @@ +package config + +import ( + "errors" + "strings" + "testing" +) + +func TestVerifyOnStopChecksLoad(t *testing.T) { + v := Default().VerifyOnStop + if len(v.Checks) != 0 { + t.Errorf("checks on by default: %v", v.Checks) + } + cfg, _, err := loadYAML(t, "verify_on_stop:\n enabled: true\n checks: [\"gofmt -l .\", \"go vet ./...\"]\n") + if err != nil { + t.Fatal(err) + } + if got := strings.Join(cfg.VerifyOnStop.Checks, "|"); got != "gofmt -l .|go vet ./..." { + t.Errorf("checks = %q", got) + } + _, _, err = loadYAML(t, "verify_on_stop:\n checks: [\"\"]\n") + var ie *InvalidError + if err == nil || !errors.As(err, &ie) || !strings.Contains(err.Error(), "verify_on_stop.checks") { + t.Errorf("empty check: err = %v", err) + } +}