Fix #250: inferential review tier for verify-on-stop

README.md

@@ -117,7 +117,7 @@ Run `tinyagent --help` (or plain `tinyagent` at a terminal) for the overview and
 - Project rules from agent failures (`tinyagent ratchet add`): each rule records its incident and an expiry or review condition, is added to the system prompt, and can be enforced as a permission deny rule; see [Project rules](docs/ratchet.md).
 - A task planner that can require a plan before any change.
 - Run budgets: turn cap, cost limit and token limit (`--token-limit`), with an 80% warning to the model; delegates, workers and `--attempts` share the budget, and a run that hits it can be resumed.
-- Verify before stop (`--verify-on-stop`): a run that changed files isn't accepted until a test or build command passes; otherwise it is marked unverified. Optional computational checks (`--verify-check`, e.g. lint, type check, tests) run in order, cheapest first, before the run may finish; see [Computational checks](docs/verify.md#computational-checks).
+- Verify before stop (`--verify-on-stop`): a run that changed files isn't accepted until a test or build command passes; otherwise it is marked unverified. Optional computational checks (`--verify-check`, e.g. lint, type check, tests) run in order, cheapest first, before the run may finish; see [Computational checks](docs/verify.md#computational-checks). An optional inferential review (`--verify-review`) runs last, once every check passes, and reports the model's findings; see [Inferential review](docs/verify.md#inferential-review).
 - Help everywhere: `tinyagent` or `tinyagent <subcommand>` alone at a terminal prints help; `--help --verbose` lists every flag by section.
 - Undo for file edits, workspace rewind, resume and follow-up runs.
 - Sandboxed commands with a network allowlist, permission rules, hooks, untrusted-content lockdown and file-access rules.

cmd/tinyagent/verify.go

@@ -11,19 +11,21 @@ type verifyFlags struct {
 	enabled      bool
 	commands     stringList
 	checks       stringList
+	review       bool
 	maxReprompts int
 }
 
 // verifyGroup lists the verify-before-stop flags in --help --verbose.
 var verifyGroup = struct {
 	name  string
 	flags [][3]string
-}{"Verify before stop", [][3]string{{"", "max-verify-reprompts", "n"}, {"", "verify-check", "cmd"}, {"", "verify-command", "cmd"}, {"", "verify-on-stop", ""}}}
+}{"Verify before stop", [][3]string{{"", "max-verify-reprompts", "n"}, {"", "verify-check", "cmd"}, {"", "verify-command", "cmd"}, {"", "verify-on-stop", ""}, {"", "verify-review", ""}}}
 
 func registerVerifyFlags(fs *flag.FlagSet, v *verifyFlags) {
 	fs.BoolVar(&v.enabled, "verify-on-stop", false, "don't accept a final answer while files changed since the last successful test/build run: send the model back to verify, up to --max-verify-reprompts times, then mark the run unverified (default off; config verify_on_stop.enabled)")
 	fs.Var(&v.commands, "verify-command", "a bash command that counts as verification for --verify-on-stop when it exits 0, e.g. \"go test ./...\"; repeatable, replaces verify_on_stop.commands")
 	fs.Var(&v.checks, "verify-check", "a computational check for --verify-on-stop: a fast, deterministic bash command (lint, type check, tests) that passes on exit 0; when the model finishes with unverified changes tinyagent runs the checks itself in the given order, cheapest first, and sends the model back with the first failure; repeatable, replaces verify_on_stop.checks")
+	fs.BoolVar(&v.review, "verify-review", false, "with --verify-on-stop and at least one --verify-check, after every check passes ask the model once, without tools, to review the run's changes and report its findings (report only: never re-prompts or changes the status; default off; config verify_on_stop.review)")
 	fs.IntVar(&v.maxReprompts, "max-verify-reprompts", config.DefaultVerifyMaxReprompts, "with --verify-on-stop, how many times to send the model back to verify before stopping unverified (1 to 10; config verify_on_stop.max_reprompts)")
 }
 
@@ -39,6 +41,9 @@ func applyVerifyFlags(fs *flag.FlagSet, cfg *config.AgentConfig, v verifyFlags)
 	if flagWasSet(fs, "verify-check") {
 		cfg.VerifyOnStop.Checks = append([]string(nil), v.checks...)
 	}
+	if flagWasSet(fs, "verify-review") {
+		cfg.VerifyOnStop.Review = v.review
+	}
 	if flagWasSet(fs, "max-verify-reprompts") {
 		cfg.VerifyOnStop.MaxReprompts = v.maxReprompts
 	}

cmd/tinyagent/verify_checks_test.go

@@ -36,3 +36,25 @@ func TestApplyVerifyCheckFlag(t *testing.T) {
 		t.Error("blank --verify-check accepted")
 	}
 }
+
+func TestApplyVerifyReviewFlag(t *testing.T) {
+	var o options
+	fs := newFlagSet(&o)
+	if err := fs.Parse([]string{"--verify-check", "go vet ./...", "--verify-review"}); err != nil {
+		t.Fatal(err)
+	}
+	cfg := config.Default()
+	if err := applyVerifyFlags(fs, &cfg, o.verify); err != nil || !cfg.VerifyOnStop.Review {
+		t.Fatalf("review = %v, err = %v", cfg.VerifyOnStop.Review, err)
+	}
+
+	var o2 options
+	fs = newFlagSet(&o2)
+	if err := fs.Parse([]string{"--verify-review"}); err != nil {
+		t.Fatal(err)
+	}
+	cfg = config.Default()
+	if err := applyVerifyFlags(fs, &cfg, o2.verify); err == nil {
+		t.Error("--verify-review without checks accepted")
+	}
+}

docs/config.md

@@ -171,6 +171,7 @@ verify_on_stop:              # off by default; see Verify before stop below
   commands: [go test ./...]  # --verify-command (repeatable) replaces this list
   patterns: []               # Go regexps; built-in test/build patterns when commands, checks and patterns are all empty
   checks: []                 # --verify-check (repeatable) replaces this list; computational checks run in order at stop, cheapest first
+  review: false              # --verify-review; after every check passes, one model review of the changes (report only; needs checks)
   max_reprompts: 2           # --max-verify-reprompts; default 2, range 1..10
 attempts:                    # omitted by default; only used with --attempts, see Best-of-N attempts below
   checks:                    # replaced as a whole by --attempt-check
@@ -186,7 +187,7 @@ profiles:                    # omitted by default; named overlays selected with
     read_only: true
 ### Verify before stop
 
-`verify_on_stop` (or `--verify-on-stop`, `--verify-command`, `--verify-check`, `--max-verify-reprompts`) keeps a run that changed files from finishing until a test or build command has passed, re-prompting the model up to `max_reprompts` times and otherwise marking the run unverified. `verify_on_stop.checks` lists computational checks (lint, type check, tests) that tinyagent runs itself, in order, when the model finishes with unverified changes; see [Computational checks](verify.md#computational-checks). See [Verify before stop](verify.md#verify-before-stop) for the full rules.
+`verify_on_stop` (or `--verify-on-stop`, `--verify-command`, `--verify-check`, `--max-verify-reprompts`) keeps a run that changed files from finishing until a test or build command has passed, re-prompting the model up to `max_reprompts` times and otherwise marking the run unverified. `verify_on_stop.checks` lists computational checks (lint, type check, tests) that tinyagent runs itself, in order, when the model finishes with unverified changes; see [Computational checks](verify.md#computational-checks). `verify_on_stop.review` (or `--verify-review`) then asks the model once to review the changes, report only; see [Inferential review](verify.md#inferential-review). See [Verify before stop](verify.md#verify-before-stop) for the full rules.
 
 ### Profiles
 

docs/json-output.md

@@ -30,6 +30,7 @@ Every event has a `type`. Event names and fields in schema version 1 are:
 | `budget.warning` | `step`, `usage` (what was used, as text), `tokens`, `token_limit`, `cost_usd`, `cost_limit`; once per run when a token or cost budget reaches 80% (see [Token and cost budgets](runs.md#token-and-cost-budgets)) |
 | `verify.reprompt` | `reprompt`, `max_reprompts`; the model gave a final answer with unverified changes and was sent back to verify (see [Verify before stop](verify.md#verify-before-stop)) |
 | `verify.check` | `index` (1-based position in `verify_on_stop.checks`), `command`, `passed`; one per [computational check](verify.md#computational-checks) tinyagent runs when the model finishes with unverified changes, emitted before any `verify.reprompt` |
+| `verify.review` | `findings` (the review text), or `error` when the request failed; the [inferential review](verify.md#inferential-review) that runs once after every computational check passed, report only |
 | `verify.unverified` | `reason`, `reprompts`; the run finishes with unverified changes |
 | `memory.saved` | `op` (`add`, `replace` or `forget`), `id`, `status` (`active`, `pending` or `forgotten`), `redacted`, `untrusted_context`; one per successful `remember` write, emitted before its `tool.result`. Never includes entry text |
 | `lsp.server` | `name`, `state` (`started`, `unavailable`, `crashed` or `stopped`), `reason` (for `unavailable` and `crashed`); one per language-server state change (see [Language servers (LSP)](long-runs.md#language-servers-lsp)) |

docs/subcommands.md

@@ -86,6 +86,7 @@ Six subcommands are shortcuts for existing flags. They rewrite the command line
 | `--verify-on-stop` | config `verify_on_stop.enabled`, else off | Don't accept a final answer while files changed since the last successful test/build command: send the model back to verify, up to `--max-verify-reprompts` times, then mark the run unverified. See [Verify before stop](verify.md#verify-before-stop) |
 | `--verify-command <cmd>` | config `verify_on_stop.commands`, else common test/build commands | A `bash` command that counts as verification for `--verify-on-stop`; repeatable, replaces the config list |
 | `--verify-check <cmd>` | config `verify_on_stop.checks`, else none | A computational check for `--verify-on-stop`: a fast, deterministic `bash` command (lint, type check, tests) that passes on exit 0. When the model finishes with unverified changes, tinyagent runs the checks itself in the given order (cheapest first), stops at the first failure and sends the model back with its output; all passing lets the run finish. Also counts as a verification command. Repeatable, replaces the config list. See [Computational checks](verify.md#computational-checks) |
+| `--verify-review` | config `verify_on_stop.review`, else off | The inferential tier for `--verify-on-stop`: after every `--verify-check` passes at stop, ask the model once, without tools, to review the file changes the run made. Findings are report only (`verify.review` event, stderr, trajectory `verify_review`); they never re-prompt the model or change the run's status. Needs at least one check. See [Inferential review](verify.md#inferential-review) |
 | `--max-verify-reprompts <n>` | config `verify_on_stop.max_reprompts`, else 2 | With `--verify-on-stop`, how many times to send the model back before stopping unverified (1 to 10) |
 | `--max-format-errors` | config `max_format_errors`, else 3 | Stop with `RepeatedFormatError` after this many consecutive steps whose tool calls are all malformed; must be >= 1. See [Malformed tool calls](troubleshooting.md#malformed-tool-calls) |
 | `--max-output-tokens` | config `max_output_tokens`, else 0 | Per-request output-token budget sent to the provider; 0 keeps the provider default (8192 `max_tokens` on Anthropic). Not supported by `codex`. Unrelated to `max_output_chars`, which caps tool output. See [Output truncation](troubleshooting.md#output-truncation) |

docs/verify.md

@@ -12,6 +12,7 @@ tinyagent --verify-on-stop --verify-command "go test ./..." --max-verify-repromp
 | `--verify-on-stop` | `verify_on_stop.enabled` | off | Turn the guard on (`--verify-on-stop=false` turns it off) |
 | `--verify-command <cmd>` | `verify_on_stop.commands` | common test/build commands | A `bash` command that counts as verification; repeatable, replaces the config list |
 | `--verify-check <cmd>` | `verify_on_stop.checks` | none | A computational check tinyagent runs itself before the run may finish, in order; repeatable, replaces the config list. See [Computational checks](#computational-checks) |
+| `--verify-review` | `verify_on_stop.review` | off | After every check passes, ask the model once to review the run's changes; report only. Needs at least one check. See [Inferential review](#inferential-review) |
 | | `verify_on_stop.patterns` | none | Regular expressions; a matching `bash` command counts as verification |
 | `--max-verify-reprompts <n>` | `verify_on_stop.max_reprompts` | 2 | How many times the model is sent back before the run is marked unverified (1 to 10) |
 
@@ -48,12 +49,30 @@ When the model gives a final answer while changes are pending, tinyagent runs th
 
 Each check also counts as a verification command when the model runs it, so a model that ran the checks itself after its edits isn't sent back. Configuring checks (like `commands`) replaces the built-in test and build patterns. The checks run only when changes are pending and the guard is on, so with `checks` empty, or `verify_on_stop` off, behaviour is unchanged. Checks never run for subagent or `--output-schema` runs. Each check that runs emits a `verify.check` event in `--json`.
 
-The inferential tier, running the existing [review](runs.md#reviewing-changes) once at the end after every check passes, is not wired into verify-on-stop yet; for now, run `tinyagent --review` after the run.
+### Inferential review
+
+`verify_on_stop.review` (or `--verify-review`) adds the inferential tier, run last: once every computational check has passed at stop, tinyagent sends the model one extra request, without tools, asking it to review the file changes the run made, in the spirit of [`--review`](runs.md#reviewing-changes). The order is always cheap checks first, review last, and the review runs at most once per stop that passes the checks.
+
+```yaml
+verify_on_stop:
+  enabled: true
+  checks: [go vet ./..., go test ./...]
+  review: true
+```
+
+```bash
+tinyagent --verify-on-stop --verify-check "go test ./..." --verify-review -t "..."
+```
+
+- **What the reviewer sees:** the task and the run's `write_file`, `edit_file` and `apply_patch` calls with their arguments and the start of each result, in order (capped at 100,000 characters). It does not read the repository or a Git diff, so changes made through `bash` are not included.
+- **Report only:** findings are written to stderr, emitted as a `verify.review` event and kept in the trajectory as `verify_review`. They never send the model back, never mark the run unverified and never change its status; act on them with `--resume` or a follow-up run. A failed review request is reported (`verify.review` with `error`) and the run finishes normally.
+- **Cost:** the request counts toward the run's tokens and cost.
+- **When it runs:** only with the guard on, at least one check configured (a config with `review` and no `checks` is rejected), changes pending at stop and every check passing. It never runs for subagent or `--output-schema` runs. Off by default.
 
 ## Where it shows up
 
 - **Summary line:** ends with `, unverified changes`.
-- **`--json`:** `verify.check` (`index`, `command`, `passed`) for each computational check, `verify.reprompt` (`reprompt`, `max_reprompts`) each time the model is sent back, `verify.unverified` (`reason`, `reprompts`) when the run ends unverified, and the `result` event's `unverified` and `verify_reprompts` fields; see [JSON output](json-output.md#machine-readable-output).
-- **Trajectory (`-o`):** `unverified: true` and `verify_reprompts`.
+- **`--json`:** `verify.check` (`index`, `command`, `passed`) for each computational check, `verify.review` (`findings`, or `error`) when the [inferential review](#inferential-review) ran, `verify.reprompt` (`reprompt`, `max_reprompts`) each time the model is sent back, `verify.unverified` (`reason`, `reprompts`) when the run ends unverified, and the `result` event's `unverified` and `verify_reprompts` fields; see [JSON output](json-output.md#machine-readable-output).
+- **Trajectory (`-o`):** `unverified: true`, `verify_reprompts`, and `verify_review` (the review findings, when it ran).
 
 The config block is shown in the [example config](config.md#config).

internal/agent/verify.go

@@ -82,6 +82,9 @@ func (a *Agent) verifyStop(ctx context.Context, t *trajectory.Trajectory, v *vie
 	}
 	failed, ok := a.runVerifyChecks(ctx, t)
 	if !ok {
+		if !t.Unverified && !t.VerifyPending {
+			a.runVerifyReview(ctx, t)
+		}
 		return nil, false
 	}
 	if t.VerifyReprompts >= a.maxVerifyReprompts() {