feat(verify): configurable per-command verify timeout ([verify] command_timeout_min)

CHANGELOG.md

@@ -40,6 +40,8 @@ breaking changes may land in a minor release.
   writes count as session activity for the idle notice (#858).
 - Name the tool call an idle Claude session is waiting on: `session-idle` carries
   `in_tool`, and the TUI shows `· idle <age> · in <tool>` (#858).
+- Add `[verify] command_timeout_min` (default 30): per-command verify timeout,
+  previously fixed at 30 minutes (#857).
 
 ### Changed
 
@@ -54,6 +56,8 @@ breaking changes may land in a minor release.
   run-state reference pages live in docs/reference/ (the settings tables generated from
   the schema); sweep and escalation recovery have their own guides; a docs hygiene test
   guards links, anchors, tracker refs, block length and the catalog shape.
+- Name the per-command time limit in a timed-out verify command's failure reason;
+  `--reverify` also names the key to raise.
 
 ### Fixed
 

docs/FEATURES.md

@@ -53,7 +53,7 @@ bmad-loop drives a BMAD sprint unattended: a deterministic Python loop launches
   - With probes configured, a failing, timed-out or unstartable probe stops verification before any `[verify]` command runs.
   - `bmad-loop validate --probes` runs every probe, not stopping at the first failure, and fails validate if any probe fails.
   - A `repo_root` key in `_bmad/bmm/config.yaml` moves verify commands and git checks to a monorepo's code root.
-- **Config:** `[verify] commands`, `[verify] env_fault_rc`, `[environment] probes`, `[environment] probe_timeout_s`; `bmad-loop validate --probes`
+- **Config:** `[verify] commands`, `[verify] command_timeout_min`, `[verify] env_fault_rc`, `[environment] probes`, `[environment] probe_timeout_s`; `bmad-loop validate --probes`
 - **Tested by:** `tests/test_verify.py`, `tests/test_engine.py`, `tests/test_cli.py`, `tests/test_bmadconfig.py`
 - **Docs:** [Configuration: [verify]](reference/configuration.md#verify), [Configuration: [environment]](reference/configuration.md#environment), [Environment faults](escalations-guide.md#environment-faults), [Re-verify kept work](escalations-guide.md#resolve-reverify)
 

docs/reference/configuration.md

@@ -101,6 +101,7 @@ Post-implementation verification commands.
 | Key | Type | Default | Allowed | Description |
 | --- | --- | --- | --- | --- |
 | `commands` | array of strings | `[]` |  |  |
+| `command_timeout_min` | integer | `30` | ≥ 1 | per-command wall-clock bound; a command running past it is killed and counts as an ordinary (fixable) verify failure · a graceful stop waits for the running command |
 | `stream_capture_kb` | integer | `256` | ≥ 0 | per-stream cap on the verifier stdout/stderr retained under the run's verify/ directory (the tail is kept; the journal records the full size and a truncation flag) · 0 = capture nothing |
 | `env_fault_rc` | integer | `0` | 0–255 | exit status a verify command uses to declare an environment fault (pause, attempt not charged) instead of a code failure · 0 = disabled; 75 (EX_TEMPFAIL) is a good choice |
 

src/bmad_loop/cli.py

@@ -5744,6 +5744,14 @@ def _reverify(project: Path, cwd: Path) -> str | None:
                     f"(rc={result.returncode}, [verify] env_fault_rc)"
                 )
             return f"{result.command!r} could not run: {fault}"
+        if verify._timed_out_leg(result):
+            # Operator-facing, unlike the engine's neutral reason: naming the key
+            # here is the point — only the operator decides the gate is slow.
+            return (
+                f"{result.command!r} timed out after {pol.verify.command_timeout_min} min — "
+                "raise [verify] command_timeout_min in policy.toml if this gate is "
+                "legitimately this slow"
+            )
         if result.returncode != 0:
             return f"{result.command!r} failed (rc {result.returncode}):\n{result.output_tail}"
     print("  verify commands passed")

src/bmad_loop/data/settings/core.toml

@@ -208,6 +208,13 @@ description = "post-implementation verification commands"
 key = "commands"
 kind = "lines"
 [[section.field]]
+key = "command_timeout_min"
+kind = "int"
+minimum = 1
+default_ref = "VerifyPolicy.command_timeout_min"
+label = "command timeout (min)"
+description = "per-command wall-clock bound; a command running past it is killed and counts as an ordinary (fixable) verify failure · a graceful stop waits for the running command"
+[[section.field]]
 key = "stream_capture_kb"
 kind = "int"
 minimum = 0

src/bmad_loop/engine.py

@@ -6411,7 +6411,10 @@ def _verify_commands_with_results(
         sequence = self._journal_verify_command_results(task, verification_stage, results)
         self._stop_if_verify_interrupted(results)
         outcome = verify.verify_command_results_outcome(
-            list(results), self.workspace.root, env_fault_rc=self.policy.verify.env_fault_rc
+            list(results),
+            self.workspace.root,
+            env_fault_rc=self.policy.verify.env_fault_rc,
+            command_timeout_min=self.policy.verify.command_timeout_min,
         )
         return outcome, VerifyCommandRecords(
             results=results, stage=verification_stage, sequence=sequence

src/bmad_loop/policy.py

@@ -180,6 +180,12 @@ class LimitsPolicy:
 @dataclass(frozen=True)
 class VerifyPolicy:
     commands: tuple[str, ...] = ()
+    # command_timeout_min is the wall-clock bound on EACH [verify] command, in
+    # minutes, across the dev, review and --reverify passes. A command running
+    # past it is killed and fails as an ordinary, fixable verify failure — the
+    # same leg as a nonzero exit — so a slow-but-healthy gate (a mutation suite)
+    # needs a bigger bound, not a different failure class.
+    command_timeout_min: int = 30
     # stream_capture_kb bounds, per stream, the verifier stdout/stderr retained
     # under the run's `verify/` directory for plugins and post-mortems (#641).
     # A tail is kept, matching every other bound on this output — the merged
@@ -191,8 +197,8 @@ class VerifyPolicy:
     # 256 KiB is chosen against what the store is FOR: a repair session or a
     # plugin reading a failing suite's tail. A verbose pytest/ruff failure runs
     # tens of KB, so the cap is generous enough that the realistic case is never
-    # cut, while a chatty command under COMMAND_TIMEOUT_S (30 minutes) can no
-    # longer emit hundreds of MB per attempt. Worst case is bounded and small:
+    # cut, while a chatty command under [verify] command_timeout_min (default
+    # 30 minutes) can no longer emit hundreds of MB per attempt. Worst case is bounded and small:
     # commands x 2 streams x attempts x 256 KiB. It sits far under the file-store
     # precedent it is modelled on (scm.failed_diff_max_mb = 5) and far above the
     # inline-journal caps, which is the right side of both.
@@ -1062,11 +1068,18 @@ def loads(text: str, plugin_schemas: dict[str, Any] | None = None) -> Policy:
 
     verify = VerifyPolicy(
         commands=_typed_str_tuple(verify_d, "verify", "commands") or (),
+        command_timeout_min=_typed_int(
+            verify_d, "verify", "command_timeout_min", VerifyPolicy.command_timeout_min
+        ),
         stream_capture_kb=_typed_int(
             verify_d, "verify", "stream_capture_kb", VerifyPolicy.stream_capture_kb
         ),
         env_fault_rc=_typed_int(verify_d, "verify", "env_fault_rc", VerifyPolicy.env_fault_rc),
     )
+    if verify.command_timeout_min < 1:
+        raise PolicyError(
+            f"verify.command_timeout_min must be >= 1: got {verify.command_timeout_min}"
+        )
     if verify.stream_capture_kb < 0:
         raise PolicyError(f"verify.stream_capture_kb must be >= 0: got {verify.stream_capture_kb}")
     if not 0 <= verify.env_fault_rc <= 255:
@@ -1456,6 +1469,7 @@ def _fold_deprecated_engine(
 [verify]
 # Deterministic gates run by the orchestrator after a clean review, before commit.
 commands = []                # e.g. ["pytest -q", "ruff check ."]
+command_timeout_min = 30     # per-command wall-clock bound (minutes); a command running past it is killed and fails as an ordinary verify failure
 stream_capture_kb = 256      # per-stream cap (KiB) on the verifier stdout/stderr retained under the run's verify/ directory; the TAIL is kept and the journal records the full byte count plus a truncation flag. 0 = capture nothing (records still land, with null pointers)
 env_fault_rc = 0             # exit status a [verify] command uses to declare an environment fault (pause, attempt not charged) instead of a code failure. 0 = disabled; 75 (EX_TEMPFAIL) is a good choice
 

src/bmad_loop/runsetup.py

@@ -159,8 +159,9 @@ def config_digest(
       surface. Hashed only when non-empty, for the ``session_id_flag`` reason
       below: an unset list runs nothing, and leaving the key out keeps the payload
       byte-identical to a digest stamped before the field existed.
-      ``verify.env_fault_rc`` and ``environment.probe_timeout_s`` are not: they
-      classify or bound a command's run, and neither can name a program.
+      ``verify.env_fault_rc``, ``verify.command_timeout_min`` and
+      ``environment.probe_timeout_s`` are not: they classify or bound a
+      command's run, and none can name a program.
     * ``sorted(plugins.enabled)`` — set semantics, so order is not meaningful.
     * per :data:`ROLES`, every field that decides **which program runs and with
       what flags and environment**. That rule, not a hand-picked list, is what

src/bmad_loop/verify.py

@@ -69,7 +69,6 @@
 from .sprintstatus import STATUS_ORDER, story_status
 
 GIT_TIMEOUT_S = 120
-COMMAND_TIMEOUT_S = 30 * 60
 
 # The oldest git bmad-loop SUPPORTS — the version this project tests against, writes
 # code against, and will help you with. INCLUSIVE: 2.34 itself clears it. (The
@@ -9566,16 +9565,21 @@ def run_verify_commands(policy: Policy, cwd: Path) -> list[CommandResult]:
     Every child runs through :func:`childrun.run_child`, which kills the whole
     process tree on a timeout or a hard stop (DW-353). A hard stop is read off
     the ambient probe the outermost ``Engine.run()`` installs; outside an engine
-    run (``cli._reverify``) none is installed and no command is ever interrupted."""
+    run (``cli._reverify``) none is installed and no command is ever interrupted.
+
+    Each command is bounded by ``[verify] command_timeout_min`` on its own, not
+    the pass as a whole: a command running past it is killed and lands on the
+    timeout leg, and the next command still gets its full bound."""
     results = []
     interrupted = False
+    # One per-command bound for the whole pass, passed positionally: test fakes
+    # of _run_shell_command take (command, cwd, timeout).
+    timeout_s = policy.verify.command_timeout_min * 60
     for command in policy.verify.commands:
         if interrupted:
             results.append(_interrupted_result(command, "", "", tail=_NOT_STARTED_TAIL))
             continue
-        # COMMAND_TIMEOUT_S is read here, at call time, so a patched module
-        # value reaches the child.
-        result = _run_shell_command(command, cwd, COMMAND_TIMEOUT_S)
+        result = _run_shell_command(command, cwd, timeout_s)
         interrupted = result.interrupted
         results.append(result)
     return results
@@ -9715,7 +9719,11 @@ def _env_fault_outcome(result: CommandResult, reason: str, cause: str) -> Verify
 
 
 def verify_command_results_outcome(
-    results: list[CommandResult], cwd: Path, *, env_fault_rc: int = 0
+    results: list[CommandResult],
+    cwd: Path,
+    *,
+    env_fault_rc: int = 0,
+    command_timeout_min: int | None = None,
 ) -> VerifyOutcome:
     """Classify already-observed verifier results without discarding them.
 
@@ -9730,6 +9738,15 @@ def verify_command_results_outcome(
     broken environment. Note the first loop inspects rc=0 results too — on
     Windows an unrunnable command is a silent pass, not a failure (#302).
 
+    A command killed at its time limit (:func:`_timed_out_leg`) is an ordinary
+    fixable failure like any other nonzero exit. When ``command_timeout_min`` is
+    given, the reason's second line names that limit instead of the bare
+    ``"timed out"`` tail, so a reader learns a time limit was hit and which. The
+    wording stays neutral on purpose: this reason is written verbatim into the
+    repair session's feedback, and it must not invite the agent to raise the
+    limit in policy.toml — that is the operator's call (``--reverify`` names the
+    key). The first line is unchanged on every leg.
+
     An interrupted result (DW-353) is not classifiable at all and raises
     :class:`VerifyInterrupted` before anything else is read: the engine raises
     ``RunStopped`` on one before reaching here, so this is a backstop."""
@@ -9745,9 +9762,13 @@ def verify_command_results_outcome(
             return _env_fault_outcome(result, reason, cause)
     for result in results:
         if result.returncode != 0:
+            detail = result.output_tail
+            if command_timeout_min is not None and _timed_out_leg(result):
+                detail = (
+                    f"timed out after {command_timeout_min} min (the per-command verify time limit)"
+                )
             return VerifyOutcome.retry(
-                f"verify command failed (rc={result.returncode}): {result.command}\n"
-                f"{result.output_tail}",
+                f"verify command failed (rc={result.returncode}): {result.command}\n{detail}",
                 fixable=True,
             )
     return VerifyOutcome.passed()
@@ -9789,9 +9810,10 @@ def preflight_required(policy: Policy) -> bool:
     return bool(policy.verify.commands) and bool(policy.environment.probes)
 
 
-def _probe_timed_out(result: CommandResult) -> bool:
-    # `_run_shell_command`'s timeout leg: rc -1 with the literal tail. A
-    # signal-killed child (rc -1 is SIGHUP) carries its own output instead.
+def _timed_out_leg(result: CommandResult) -> bool:
+    # `_run_shell_command`'s timeout leg — for a probe and a [verify] command
+    # alike: rc -1 with the literal tail. A signal-killed child (rc -1 is
+    # SIGHUP) carries its own output instead.
     return result.returncode == -1 and result.output_tail == "timed out" and not result.spawn_error
 
 
@@ -9802,7 +9824,7 @@ def probe_failure_reason(result: CommandResult, timeout_s: int, *, cwd: Path | N
     #302) name why; without it that leg reads ``"rc=0"``."""
     if result.spawn_error is not None:
         return f"could not be started: {result.spawn_error}"
-    if _probe_timed_out(result):
+    if _timed_out_leg(result):
         return f"timed out after {timeout_s}s"
     if result.returncode == 0 and cwd is not None:
         unrunnable = env_fault_reason(result, cwd)
@@ -9815,7 +9837,7 @@ def _probe_failed(result: CommandResult, cwd: Path) -> bool:
     return (
         result.spawn_error is not None
         or result.returncode != 0
-        or _probe_timed_out(result)
+        or _timed_out_leg(result)
         # rc 0 is not proof on win32: cmd exits 0 for a file it cannot execute
         or env_fault_reason(result, cwd) is not None
     )
@@ -9876,7 +9898,7 @@ def check_environment_probes(policy: Policy, cwd: Path) -> tuple[ProbeCheck, ...
     nothing, and is not one of the compositions tests/test_portability_guard.py
     pins to the preflight runners. What it shares with the run is the verdict —
     the same :func:`_run_shell_command` runner, ``probe_timeout_s`` bound,
-    :func:`_probe_timed_out`/:func:`_probe_failed` predicates and
+    :func:`_timed_out_leg`/:func:`_probe_failed` predicates and
     :func:`probe_failure_reason` phrasing — so validate and the run cannot
     disagree on a given probe RESULT. They can still see different environments:
     validate probes the run's code root (``repo_root``), which is where the run
@@ -9891,7 +9913,7 @@ def check_environment_probes(policy: Policy, cwd: Path) -> tuple[ProbeCheck, ...
         if result.interrupted:
             checks.append(ProbeCheck(result, "interrupted", "interrupted by a hard stop request"))
             break
-        if _probe_timed_out(result):
+        if _timed_out_leg(result):
             checks.append(
                 ProbeCheck(result, "timeout", probe_failure_reason(result, timeout_s, cwd=cwd))
             )
@@ -9965,7 +9987,12 @@ def verify_commands_outcome(
     results = run_verify_commands(policy, cwd)
     if on_results is not None:
         on_results(tuple(results))
-    return verify_command_results_outcome(results, cwd, env_fault_rc=policy.verify.env_fault_rc)
+    return verify_command_results_outcome(
+        results,
+        cwd,
+        env_fault_rc=policy.verify.env_fault_rc,
+        command_timeout_min=policy.verify.command_timeout_min,
+    )
 
 
 def _verify_review_commands(

tests/test_cli.py

@@ -14481,6 +14481,30 @@ def test_reverify_answers_with_the_first_ordinary_failure_not_a_later_one(tmp_pa
     assert "verify commands passed" not in capsys.readouterr().out
 
 
+def test_reverify_timeout_names_the_limit_and_the_key(tmp_path, monkeypatch):
+    """A `--reverify` command killed at its time limit is reported to the
+    OPERATOR with the limit and the key that sets it — unlike the engine's
+    neutral reason, which reaches an agent. The configured minutes reach the
+    runner as seconds. Faked runner; nothing sleeps toward a deadline.
+    Ablation: revert the timeout arm in `_reverify` and the message falls back to
+    `failed (rc -1)` with neither the limit nor the key."""
+    timeouts: list[float] = []
+
+    def timed_out(command, cwd, timeout):
+        timeouts.append(timeout)
+        return verify.CommandResult(command, -1, "timed out")
+
+    monkeypatch.setattr(verify, "_run_shell_command", timed_out)
+    _write_policy(tmp_path, '[verify]\ncommands = ["slow-gate"]\ncommand_timeout_min = 7\n')
+
+    reason = cli._reverify(tmp_path, tmp_path)
+
+    assert timeouts == [420]
+    assert reason is not None
+    assert repr("slow-gate") in reason
+    assert "7 min" in reason and "[verify] command_timeout_min" in reason
+
+
 def test_reverify_walks_every_command_when_they_all_pass(tmp_path, capsys):
     """The control for the two rows above: with nothing to return on, the walk
     reaches the end and reports the pass.