Part of #250: computational checks for verify-on-stop

README.md

@@ -117,7 +117,7 @@ Run `tinyagent --help` (or plain `tinyagent` at a terminal) for the overview and
 - Project rules from agent failures (`tinyagent ratchet add`): each rule records its incident and an expiry or review condition, is added to the system prompt, and can be enforced as a permission deny rule; see [Project rules](docs/ratchet.md).
 - A task planner that can require a plan before any change.
 - Run budgets: turn cap, cost limit and token limit (`--token-limit`), with an 80% warning to the model; delegates, workers and `--attempts` share the budget, and a run that hits it can be resumed.
-- Verify before stop (`--verify-on-stop`): a run that changed files isn't accepted until a test or build command passes; otherwise it is marked unverified.
+- Verify before stop (`--verify-on-stop`): a run that changed files isn't accepted until a test or build command passes; otherwise it is marked unverified. Optional computational checks (`--verify-check`, e.g. lint, type check, tests) run in order, cheapest first, before the run may finish; see [Computational checks](docs/verify.md#computational-checks).
 - Help everywhere: `tinyagent` or `tinyagent <subcommand>` alone at a terminal prints help; `--help --verbose` lists every flag by section.
 - Undo for file edits, workspace rewind, resume and follow-up runs.
 - Sandboxed commands with a network allowlist, permission rules, hooks, untrusted-content lockdown and file-access rules.

cmd/tinyagent/verify.go

@@ -10,18 +10,20 @@ import (
 type verifyFlags struct {
 	enabled      bool
 	commands     stringList
+	checks       stringList
 	maxReprompts int
 }
 
 // verifyGroup lists the verify-before-stop flags in --help --verbose.
 var verifyGroup = struct {
 	name  string
 	flags [][3]string
-}{"Verify before stop", [][3]string{{"", "max-verify-reprompts", "n"}, {"", "verify-command", "cmd"}, {"", "verify-on-stop", ""}}}
+}{"Verify before stop", [][3]string{{"", "max-verify-reprompts", "n"}, {"", "verify-check", "cmd"}, {"", "verify-command", "cmd"}, {"", "verify-on-stop", ""}}}
 
 func registerVerifyFlags(fs *flag.FlagSet, v *verifyFlags) {
 	fs.BoolVar(&v.enabled, "verify-on-stop", false, "don't accept a final answer while files changed since the last successful test/build run: send the model back to verify, up to --max-verify-reprompts times, then mark the run unverified (default off; config verify_on_stop.enabled)")
 	fs.Var(&v.commands, "verify-command", "a bash command that counts as verification for --verify-on-stop when it exits 0, e.g. \"go test ./...\"; repeatable, replaces verify_on_stop.commands")
+	fs.Var(&v.checks, "verify-check", "a computational check for --verify-on-stop: a fast, deterministic bash command (lint, type check, tests) that passes on exit 0; when the model finishes with unverified changes tinyagent runs the checks itself in the given order, cheapest first, and sends the model back with the first failure; repeatable, replaces verify_on_stop.checks")
 	fs.IntVar(&v.maxReprompts, "max-verify-reprompts", config.DefaultVerifyMaxReprompts, "with --verify-on-stop, how many times to send the model back to verify before stopping unverified (1 to 10; config verify_on_stop.max_reprompts)")
 }
 
@@ -34,6 +36,9 @@ func applyVerifyFlags(fs *flag.FlagSet, cfg *config.AgentConfig, v verifyFlags)
 	if flagWasSet(fs, "verify-command") {
 		cfg.VerifyOnStop.Commands = append([]string(nil), v.commands...)
 	}
+	if flagWasSet(fs, "verify-check") {
+		cfg.VerifyOnStop.Checks = append([]string(nil), v.checks...)
+	}
 	if flagWasSet(fs, "max-verify-reprompts") {
 		cfg.VerifyOnStop.MaxReprompts = v.maxReprompts
 	}

cmd/tinyagent/verify_checks_test.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/vicgupta/tinyagent/internal/config"
+)
+
+func TestApplyVerifyCheckFlag(t *testing.T) {
+	var o options
+	fs := newFlagSet(&o)
+	if err := fs.Parse([]string{"--verify-check", "gofmt -l .", "--verify-check", "go test ./..."}); err != nil {
+		t.Fatal(err)
+	}
+	cfg := config.Default()
+	cfg.VerifyOnStop.Checks = []string{"from config"}
+	if err := applyVerifyFlags(fs, &cfg, o.verify); err != nil {
+		t.Fatal(err)
+	}
+	if got := strings.Join(cfg.VerifyOnStop.Checks, "|"); got != "gofmt -l .|go test ./..." {
+		t.Errorf("checks = %q", got)
+	}
+
+	var o2 options
+	fs = newFlagSet(&o2)
+	cfg = config.Default()
+	cfg.VerifyOnStop.Checks = []string{"from config"}
+	if err := applyVerifyFlags(fs, &cfg, o2.verify); err != nil || len(cfg.VerifyOnStop.Checks) != 1 {
+		t.Errorf("unset flag changed the checks: %v, %v", cfg.VerifyOnStop.Checks, err)
+	}
+
+	var o3 options
+	fs = newFlagSet(&o3)
+	if err := fs.Parse([]string{"--verify-check", " "}); err == nil {
+		t.Error("blank --verify-check accepted")
+	}
+}

docs/config.md

@@ -169,7 +169,8 @@ hooks:                       # omitted by default; see Tool hooks below
 verify_on_stop:              # off by default; see Verify before stop below
   enabled: false             # --verify-on-stop
   commands: [go test ./...]  # --verify-command (repeatable) replaces this list
-  patterns: []               # Go regexps; built-in test/build patterns when both lists are empty
+  patterns: []               # Go regexps; built-in test/build patterns when commands, checks and patterns are all empty
+  checks: []                 # --verify-check (repeatable) replaces this list; computational checks run in order at stop, cheapest first
   max_reprompts: 2           # --max-verify-reprompts; default 2, range 1..10
 attempts:                    # omitted by default; only used with --attempts, see Best-of-N attempts below
   checks:                    # replaced as a whole by --attempt-check
@@ -185,7 +186,7 @@ profiles:                    # omitted by default; named overlays selected with
     read_only: true
 ### Verify before stop
 
-`verify_on_stop` (or `--verify-on-stop`, `--verify-command`, `--max-verify-reprompts`) keeps a run that changed files from finishing until a test or build command has passed, re-prompting the model up to `max_reprompts` times and otherwise marking the run unverified. See [Verify before stop](verify.md#verify-before-stop) for the full rules.
+`verify_on_stop` (or `--verify-on-stop`, `--verify-command`, `--verify-check`, `--max-verify-reprompts`) keeps a run that changed files from finishing until a test or build command has passed, re-prompting the model up to `max_reprompts` times and otherwise marking the run unverified. `verify_on_stop.checks` lists computational checks (lint, type check, tests) that tinyagent runs itself, in order, when the model finishes with unverified changes; see [Computational checks](verify.md#computational-checks). See [Verify before stop](verify.md#verify-before-stop) for the full rules.
 
 ### Profiles
 

docs/json-output.md

@@ -29,6 +29,7 @@ Every event has a `type`. Event names and fields in schema version 1 are:
 | `ratchet.loaded` | `path` (the rules file), `loaded` (active rule count), `ids` (active rule ids), `expired` (ids of expired rules that were skipped), `enforced` (active rules added as permission deny rules); once after `run.started` for a fresh run when `.tinyagent/ratchet.toml` exists and `--no-ratchet` isn't set. See [Project rules](ratchet.md). Never includes rule text |
 | `budget.warning` | `step`, `usage` (what was used, as text), `tokens`, `token_limit`, `cost_usd`, `cost_limit`; once per run when a token or cost budget reaches 80% (see [Token and cost budgets](runs.md#token-and-cost-budgets)) |
 | `verify.reprompt` | `reprompt`, `max_reprompts`; the model gave a final answer with unverified changes and was sent back to verify (see [Verify before stop](verify.md#verify-before-stop)) |
+| `verify.check` | `index` (1-based position in `verify_on_stop.checks`), `command`, `passed`; one per [computational check](verify.md#computational-checks) tinyagent runs when the model finishes with unverified changes, emitted before any `verify.reprompt` |
 | `verify.unverified` | `reason`, `reprompts`; the run finishes with unverified changes |
 | `memory.saved` | `op` (`add`, `replace` or `forget`), `id`, `status` (`active`, `pending` or `forgotten`), `redacted`, `untrusted_context`; one per successful `remember` write, emitted before its `tool.result`. Never includes entry text |
 | `lsp.server` | `name`, `state` (`started`, `unavailable`, `crashed` or `stopped`), `reason` (for `unavailable` and `crashed`); one per language-server state change (see [Language servers (LSP)](long-runs.md#language-servers-lsp)) |

docs/subcommands.md

@@ -85,6 +85,7 @@ Six subcommands are shortcuts for existing flags. They rewrite the command line
 | `--time-limit` | 0 (unlimited) | Max wall-clock seconds; overrides `wall_time_seconds` |
 | `--verify-on-stop` | config `verify_on_stop.enabled`, else off | Don't accept a final answer while files changed since the last successful test/build command: send the model back to verify, up to `--max-verify-reprompts` times, then mark the run unverified. See [Verify before stop](verify.md#verify-before-stop) |
 | `--verify-command <cmd>` | config `verify_on_stop.commands`, else common test/build commands | A `bash` command that counts as verification for `--verify-on-stop`; repeatable, replaces the config list |
+| `--verify-check <cmd>` | config `verify_on_stop.checks`, else none | A computational check for `--verify-on-stop`: a fast, deterministic `bash` command (lint, type check, tests) that passes on exit 0. When the model finishes with unverified changes, tinyagent runs the checks itself in the given order (cheapest first), stops at the first failure and sends the model back with its output; all passing lets the run finish. Also counts as a verification command. Repeatable, replaces the config list. See [Computational checks](verify.md#computational-checks) |
 | `--max-verify-reprompts <n>` | config `verify_on_stop.max_reprompts`, else 2 | With `--verify-on-stop`, how many times to send the model back before stopping unverified (1 to 10) |
 | `--max-format-errors` | config `max_format_errors`, else 3 | Stop with `RepeatedFormatError` after this many consecutive steps whose tool calls are all malformed; must be >= 1. See [Malformed tool calls](troubleshooting.md#malformed-tool-calls) |
 | `--max-output-tokens` | config `max_output_tokens`, else 0 | Per-request output-token budget sent to the provider; 0 keeps the provider default (8192 `max_tokens` on Anthropic). Not supported by `codex`. Unrelated to `max_output_chars`, which caps tool output. See [Output truncation](troubleshooting.md#output-truncation) |

docs/verify.md

@@ -11,6 +11,7 @@ tinyagent --verify-on-stop --verify-command "go test ./..." --max-verify-repromp
 |---|---|---|---|
 | `--verify-on-stop` | `verify_on_stop.enabled` | off | Turn the guard on (`--verify-on-stop=false` turns it off) |
 | `--verify-command <cmd>` | `verify_on_stop.commands` | common test/build commands | A `bash` command that counts as verification; repeatable, replaces the config list |
+| `--verify-check <cmd>` | `verify_on_stop.checks` | none | A computational check tinyagent runs itself before the run may finish, in order; repeatable, replaces the config list. See [Computational checks](#computational-checks) |
 | | `verify_on_stop.patterns` | none | Regular expressions; a matching `bash` command counts as verification |
 | `--max-verify-reprompts <n>` | `verify_on_stop.max_reprompts` | 2 | How many times the model is sent back before the run is marked unverified (1 to 10) |
 
@@ -22,10 +23,37 @@ With `verify_on_stop.enabled` (or `--verify-on-stop`), a run that changed files
 
 When the model gives a final answer (with or without the completion marker) while changes are pending, it gets a fixed user message asking it to verify, naming the configured commands, and the run continues as normal steps, so step, cost and time limits still apply. Tool calls in a submitting message are not run and get an error result saying so. After `max_reprompts` re-prompts, or straight away when `bash` cannot run (denied by the permission policy, a hook or lockdown, or unavailable as in read-only mode), the run finishes with its normal status and is marked unverified: the summary line ends with `, unverified changes`, the `--json` result has `"unverified": true`, and the trajectory has `unverified: true`. `verify_reprompts` counts the re-prompts in both. The guard runs before stop hooks, does not apply to subagent runs or `--output-schema` runs, and keeps its state across `--resume`.
 
+### Computational checks
+
+Sensors come in two tiers. Computational checks are fast and deterministic (linters, type checkers, structural checks, tests: pass means exit 0); inferential review asks a model to judge the change and is slower. `verify_on_stop.checks` (or `--verify-check`, repeatable) configures the computational tier. List them cheapest first:
+
+```yaml
+verify_on_stop:
+  enabled: true
+  checks:
+    - gofmt -l . | (! grep .)
+    - go vet ./...
+    - go test ./...
+```
+
+```bash
+tinyagent --verify-on-stop --verify-check "go vet ./..." --verify-check "go test ./..." -t "..."
+```
+
+When the model gives a final answer while changes are pending, tinyagent runs the checks itself, one after another in the listed order, as `bash` tool calls, so the permission policy, hooks, lockdown and sandbox apply to them exactly as to the model's own commands. It stops at the first failing check:
+
+- **All pass:** the pending changes are cleared and the run finishes verified, with no re-prompt.
+- **One fails:** the model is sent back as described above, and the re-prompt names the failing check (`Computational check 2 of 3 failed`) and includes its output; later checks don't run. This counts against `max_reprompts`, and at the cap the run finishes unverified.
+- **One is refused** (denied by the policy, a hook or lockdown): the run finishes unverified straight away.
+
+Each check also counts as a verification command when the model runs it, so a model that ran the checks itself after its edits isn't sent back. Configuring checks (like `commands`) replaces the built-in test and build patterns. The checks run only when changes are pending and the guard is on, so with `checks` empty, or `verify_on_stop` off, behaviour is unchanged. Checks never run for subagent or `--output-schema` runs. Each check that runs emits a `verify.check` event in `--json`.
+
+The inferential tier, running the existing [review](runs.md#reviewing-changes) once at the end after every check passes, is not wired into verify-on-stop yet; for now, run `tinyagent --review` after the run.
+
 ## Where it shows up
 
 - **Summary line:** ends with `, unverified changes`.
-- **`--json`:** `verify.reprompt` (`reprompt`, `max_reprompts`) each time the model is sent back, `verify.unverified` (`reason`, `reprompts`) when the run ends unverified, and the `result` event's `unverified` and `verify_reprompts` fields; see [JSON output](json-output.md#machine-readable-output).
+- **`--json`:** `verify.check` (`index`, `command`, `passed`) for each computational check, `verify.reprompt` (`reprompt`, `max_reprompts`) each time the model is sent back, `verify.unverified` (`reason`, `reprompts`) when the run ends unverified, and the `result` event's `unverified` and `verify_reprompts` fields; see [JSON output](json-output.md#machine-readable-output).
 - **Trajectory (`-o`):** `unverified: true` and `verify_reprompts`.
 
 The config block is shown in the [example config](config.md#config).

internal/agent/loop.go

@@ -600,7 +600,7 @@ func (a *Agent) Run(ctx context.Context, task string) *trajectory.Trajectory {
 		}
 
 		if strings.Contains(msg.Content, SubmitMarker) || len(msg.ToolCalls) == 0 {
-			if done, again := a.verifyStop(t, v, msg); again || done != nil {
+			if done, again := a.verifyStop(ctx, t, v, msg); again || done != nil {
 				if done != nil {
 					return done
 				}

internal/agent/verify.go

@@ -1,6 +1,7 @@
 package agent
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"regexp"
@@ -66,7 +67,7 @@ func (a *Agent) noteVerify(t *trajectory.Trajectory, call model.ToolCall, res to
 // model was sent back to verify, so the loop must continue; a non-nil done
 // is the final trajectory after a failed save. Otherwise the run may finish,
 // with t.Unverified set when changes are still unverified.
-func (a *Agent) verifyStop(t *trajectory.Trajectory, v *view, msg model.Message) (done *trajectory.Trajectory, again bool) {
+func (a *Agent) verifyStop(ctx context.Context, t *trajectory.Trajectory, v *view, msg model.Message) (done *trajectory.Trajectory, again bool) {
 	if !a.verifyOn() || a.subagentID != "" {
 		return nil, false
 	}
@@ -78,7 +79,12 @@ func (a *Agent) verifyStop(t *trajectory.Trajectory, v *view, msg model.Message)
 	case t.VerifyBlocked || a.Tools.ReadOnly || !a.Tools.Permits("bash"):
 		a.markUnverified(t, "verification commands cannot run (bash is refused or unavailable)")
 		return nil, false
-	case t.VerifyReprompts >= a.maxVerifyReprompts():
+	}
+	failed, ok := a.runVerifyChecks(ctx, t)
+	if !ok {
+		return nil, false
+	}
+	if t.VerifyReprompts >= a.maxVerifyReprompts() {
 		a.markUnverified(t, fmt.Sprintf("still unverified after %d verify re-prompts", t.VerifyReprompts))
 		return nil, false
 	}
@@ -93,7 +99,7 @@ func (a *Agent) verifyStop(t *trajectory.Trajectory, v *view, msg model.Message)
 	if a.Verbose && a.Err != nil {
 		fmt.Fprintf(a.Err, "verify: unverified changes; asking the model to verify (%d/%d)\n", t.VerifyReprompts, a.maxVerifyReprompts())
 	}
-	t.Messages = append(t.Messages, model.Message{Role: "user", Content: fmt.Sprintf(verifyNotice, verifyHint(a.Config.VerifyOnStop))})
+	t.Messages = append(t.Messages, model.Message{Role: "user", Content: fmt.Sprintf(verifyNotice, verifyHint(a.Config.VerifyOnStop)) + failed})
 	v.add(t, t.Messages[len(t.Messages)-1])
 	if err := a.step(t); err != nil {
 		return a.saveFailed(t, err), false
@@ -112,11 +118,12 @@ func (a *Agent) markUnverified(t *trajectory.Trajectory, reason string) {
 
 // verifyHint names the configured verify commands for the notice.
 func verifyHint(c config.VerifyOnStop) string {
-	if len(c.Commands) == 0 {
+	all := append(append([]string(nil), c.Checks...), c.Commands...)
+	if len(all) == 0 {
 		return ""
 	}
-	q := make([]string, len(c.Commands))
-	for i, cmd := range c.Commands {
+	q := make([]string, len(all))
+	for i, cmd := range all {
 		q[i] = "`" + strings.TrimSpace(cmd) + "`"
 	}
 	return " (configured: " + strings.Join(q, ", ") + ")"
@@ -143,14 +150,14 @@ func isVerifyCommand(c config.VerifyOnStop, cmd string) bool {
 	if cmd == "" {
 		return false
 	}
-	for _, want := range c.Commands {
+	for _, want := range append(append([]string(nil), c.Checks...), c.Commands...) {
 		want = strings.TrimSpace(want)
 		if want != "" && (cmd == want || strings.HasPrefix(cmd, want+" ")) {
 			return true
 		}
 	}
 	patterns := c.Patterns
-	if len(c.Commands) == 0 && len(patterns) == 0 {
+	if len(c.Commands) == 0 && len(c.Checks) == 0 && len(patterns) == 0 {
 		patterns = config.DefaultVerifyPatterns
 	}
 	for _, p := range patterns {