From 14d1434025b44a5a6cc387b3dac0023655b1aadf Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 13:55:25 -0700 Subject: [PATCH 1/2] Send tool-call arguments back as valid JSON: strict providers refused every request after one broken call --- src/transcript.ts | 35 ++++++++++++++++++++++++++++++++++- test/wire-args.test.ts | 30 ++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 test/wire-args.test.ts diff --git a/src/transcript.ts b/src/transcript.ts index d67119b..f5e6823 100644 --- a/src/transcript.ts +++ b/src/transcript.ts @@ -19,6 +19,7 @@ * No filesystem access here — archiving lives in archive.ts so this whole * module stays pure and testable. */ +import { parseLenient } from "./lenient-json.js"; import { estTokens, type Bom, type Msg } from "./types.js"; export const STALE_FAILURE_PREFIX = "[molt: superseded]"; @@ -154,7 +155,7 @@ export class Transcript { // in order; a system message anywhere later goes as a user message. const out: Omit[] = []; for (const { molt: _molt, ...m } of this.all()) { - if (m.role !== "system") out.push(m); + if (m.role !== "system") out.push(Array.isArray(m.tool_calls) ? { ...m, tool_calls: m.tool_calls.map(wireCall) } : m); else if (out.length === 0) out.push({ ...m }); else if (out.length === 1 && out[0]!.role === "system") out[0] = { ...out[0]!, content: `${out[0]!.content ?? ""}\n\n${m.content ?? ""}` }; else out.push({ role: "user", content: m.content ?? "" }); @@ -617,3 +618,35 @@ export function toolDetail(name: string, args: Record): string return raw.replace(/\s+/g, " ").trim(); } + +/** + * A tool call as it goes back to the provider: arguments always valid JSON. + * + * Maat reads a model's slightly broken arguments leniently, which is right for + * running the call, but it sent the broken text back in the history. Strict + * providers then refuse every later request: DeepSeek V4 Pro on StreamLake + * answered "Assistant tool call function.arguments must be valid JSON" and + * ended 6 of 12 runs (2026-10-07). The transcript keeps what the model wrote; + * only the wire copy is repaired, or replaced by {} when nothing can be read. + */ +export function wireArgs(text: string): string { + try { + JSON.parse(text); + return text; + } catch { + try { + const v = parseLenient(text); + if (v && typeof v === "object") return JSON.stringify(v); + } catch { + /* fall through */ + } + return "{}"; + } +} + +function wireCall(c: T): T { + const a = c.function?.arguments; + if (typeof a !== "string") return c; + const fixed = wireArgs(a); + return fixed === a ? c : { ...c, function: { ...c.function!, arguments: fixed } }; +} diff --git a/test/wire-args.test.ts b/test/wire-args.test.ts new file mode 100644 index 0000000..234e04a --- /dev/null +++ b/test/wire-args.test.ts @@ -0,0 +1,30 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { Transcript, wireArgs } from "../src/transcript.js"; + +describe("tool-call arguments on the wire", () => { + it("keeps valid JSON as written", () => { + assert.equal(wireArgs('{"path":"a.txt"}'), '{"path":"a.txt"}'); + }); + + it("repairs what the lenient reader can read, and sends {} for the rest", () => { + // StreamLake refused every later request once a broken call was in the history. + assert.deepEqual(JSON.parse(wireArgs('{"content":"a\nb","re":"\\d"}')), { content: "a\nb", re: "\\d" }); + assert.equal(wireArgs("not json at all"), "{}"); + }); + + it("repairs only the copy sent to the provider", () => { + const t = new Transcript("sys"); + t.push({ role: "user", content: "go" }); + t.push({ + role: "assistant", + content: null, + tool_calls: [{ id: "c1", type: "function", function: { name: "read_file", arguments: '{"path":"a.txt","re":"\\d+"}' } }], + }); + const wire = t.wire(); + const call = wire.find((m) => m.tool_calls)!.tool_calls![0]!; + assert.doesNotThrow(() => JSON.parse(call.function.arguments)); + const kept = t.all().find((m) => m.tool_calls)!.tool_calls![0]!; + assert.equal(kept.function.arguments, '{"path":"a.txt","re":"\\d+"}'); + }); +}); From 3079b1ab74385ba8190b7839beb8dad19b44088d Mon Sep 17 00:00:00 2001 From: Tyler Skelton Date: Wed, 7 Oct 2026 14:21:19 -0700 Subject: [PATCH 2/2] Captures keep the model's own tool-call arguments; the wire always sends an object - capture() recorded transcript.wire(), which now holds repaired arguments, so a model's broken JSON showed up as {} in the training record. wire() takes repairArgs: false and the capture uses it; the repair stays on the request path only. - Valid JSON that is not an object went out unchanged ([] / 42 / "x"), which the engine refuses and items-iterating chat templates fail on. Blank and null now go as {}, anything else that is not an object as {"_unparsed": ""}, which is also what an unreadable payload now gets instead of {}: the next turn sees its own mistake, not a call that seems to have sent nothing. --- src/engine.ts | 5 +++- src/transcript.ts | 32 +++++++++++++++----- test/wire-args.test.ts | 67 ++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 93 insertions(+), 11 deletions(-) diff --git a/src/engine.ts b/src/engine.ts index af55c8d..5b89a7d 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -4082,7 +4082,10 @@ export class Engine { provider: this.provider, task, claim, - transcript: this.transcript.wire(), + // What the model wrote, broken arguments included: the repair in + // wire() is for the provider, and a model's broken JSON is exactly + // what a training record must keep. + transcript: this.transcript.wire({ repairArgs: false }), ledger: this.sessionLedger(), turnLedger: this.turnLedger(), did: [...this.did], diff --git a/src/transcript.ts b/src/transcript.ts index f5e6823..2b8d995 100644 --- a/src/transcript.ts +++ b/src/transcript.ts @@ -146,7 +146,11 @@ export class Transcript { * Messages formatted for the wire: molt's own metadata removed, since * providers reject unknown fields with varying degrees of politeness. */ - wire(): Omit[] { + wire(opts: { repairArgs?: boolean } = {}): Omit[] { + // Repaired tool-call arguments are for the request only (wireArgs). A + // record of what the model did (a capture) passes repairArgs: false and + // keeps the arguments exactly as the model wrote them. + const repair = opts.repairArgs !== false; // One system message, and it comes first. The pinned task and a shed's // digest were system messages of their own, which OpenAI-style APIs take // and strict chat templates refuse: Qwen's raises "System message must be @@ -155,7 +159,7 @@ export class Transcript { // in order; a system message anywhere later goes as a user message. const out: Omit[] = []; for (const { molt: _molt, ...m } of this.all()) { - if (m.role !== "system") out.push(Array.isArray(m.tool_calls) ? { ...m, tool_calls: m.tool_calls.map(wireCall) } : m); + if (m.role !== "system") out.push(repair && Array.isArray(m.tool_calls) ? { ...m, tool_calls: m.tool_calls.map(wireCall) } : m); else if (out.length === 0) out.push({ ...m }); else if (out.length === 1 && out[0]!.role === "system") out[0] = { ...out[0]!, content: `${out[0]!.content ?? ""}\n\n${m.content ?? ""}` }; else out.push({ role: "user", content: m.content ?? "" }); @@ -627,21 +631,33 @@ export function toolDetail(name: string, args: Record): string * providers then refuse every later request: DeepSeek V4 Pro on StreamLake * answered "Assistant tool call function.arguments must be valid JSON" and * ended 6 of 12 runs (2026-10-07). The transcript keeps what the model wrote; - * only the wire copy is repaired, or replaced by {} when nothing can be read. + * only the wire copy is repaired. + * + * Always a JSON object, the one shape the engine runs and the one chat + * templates that iterate `arguments | items` accept: blank or `null` is `{}`; + * anything else that is not an object (an array, a number, a string) or that + * cannot be read at all goes as `{"_unparsed": ""}`, so + * the next turn sees its own mistake rather than a call that seems to have + * sent nothing. */ export function wireArgs(text: string): string { + const isObject = (v: unknown): v is Record => !!v && typeof v === "object" && !Array.isArray(v); + if (!text.trim()) return "{}"; + let v: unknown; try { - JSON.parse(text); - return text; + v = JSON.parse(text); } catch { try { - const v = parseLenient(text); - if (v && typeof v === "object") return JSON.stringify(v); + const lenient = parseLenient(text); + if (isObject(lenient)) return JSON.stringify(lenient); } catch { /* fall through */ } - return "{}"; + return JSON.stringify({ _unparsed: text }); } + if (isObject(v)) return text; + if (v === null) return "{}"; + return JSON.stringify({ _unparsed: text }); } function wireCall(c: T): T { diff --git a/test/wire-args.test.ts b/test/wire-args.test.ts index 234e04a..01c3cf6 100644 --- a/test/wire-args.test.ts +++ b/test/wire-args.test.ts @@ -1,16 +1,34 @@ import { describe, it } from "node:test"; import assert from "node:assert/strict"; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { Transcript, wireArgs } from "../src/transcript.js"; +import { Engine } from "../src/engine.js"; +import { parseBar } from "../src/bar.js"; +import { Receipts } from "../src/receipts.js"; +import { allowAll, drain } from "./helpers.js"; describe("tool-call arguments on the wire", () => { it("keeps valid JSON as written", () => { assert.equal(wireArgs('{"path":"a.txt"}'), '{"path":"a.txt"}'); }); - it("repairs what the lenient reader can read, and sends {} for the rest", () => { + it("repairs what the lenient reader can read, and wraps the rest so the mistake stays visible", () => { // StreamLake refused every later request once a broken call was in the history. assert.deepEqual(JSON.parse(wireArgs('{"content":"a\nb","re":"\\d"}')), { content: "a\nb", re: "\\d" }); - assert.equal(wireArgs("not json at all"), "{}"); + assert.deepEqual(JSON.parse(wireArgs("not json at all")), { _unparsed: "not json at all" }); + }); + + it("sends an object whatever valid JSON the model wrote", () => { + // The engine refuses anything but an object, and chat templates that + // iterate `arguments | items` fail on the rest. + assert.equal(wireArgs("null"), "{}"); + assert.equal(wireArgs(""), "{}"); + assert.equal(wireArgs(" "), "{}"); + for (const raw of ["[]", "[1,2]", "42", '"x"', "true"]) { + assert.deepEqual(JSON.parse(wireArgs(raw)), { _unparsed: raw }, raw); + } }); it("repairs only the copy sent to the provider", () => { @@ -26,5 +44,50 @@ describe("tool-call arguments on the wire", () => { assert.doesNotThrow(() => JSON.parse(call.function.arguments)); const kept = t.all().find((m) => m.tool_calls)!.tool_calls![0]!; assert.equal(kept.function.arguments, '{"path":"a.txt","re":"\\d+"}'); + const raw = t.wire({ repairArgs: false }).find((m) => m.tool_calls)!.tool_calls![0]!; + assert.equal(raw.function.arguments, '{"path":"a.txt","re":"\\d+"}'); + }); + + it("a capture keeps the arguments the model wrote; the next request carries the repair", async () => { + const dir = mkdtempSync(join(tmpdir(), "maat-wire-")); + try { + const cap = join(dir, "captures"); + // A raw newline inside a string: not JSON, but the lenient reader runs it. + const broken = '{"path": "r.txt", "content": "real\nline\n"}'; + const bodies: string[] = []; + let n = 0; + const fetchFn = (async (_u: string, init?: RequestInit) => { + bodies.push(String(init?.body ?? "")); + n += 1; + const message = + n === 1 + ? { role: "assistant", content: null, tool_calls: [{ id: "c1", type: "function", function: { name: "write_file", arguments: broken } }] } + : { role: "assistant", content: "Wrote r.txt." }; + return { + ok: true, + status: 200, + json: async () => ({ choices: [{ message }], usage: { prompt_tokens: 100, completion_tokens: 20 } }), + text: async () => "", + } as unknown as Response; + }) as unknown as typeof fetch; + const engine = new Engine({ + baseUrl: "http://mock/v1", model: "m", cwd: dir, fetchFn, stream: false, + bar: parseBar("version: 1\nchecks:\n - name: landed\n builtin: files-changed\n"), + receipts: new Receipts(dir), captureDir: cap, + }); + await drain(engine.run("add r.txt", allowAll)); + assert.equal(readFileSync(join(dir, "r.txt"), "utf8"), "real\nline\n"); + // The request after the call sent valid JSON. + const sent = JSON.parse(bodies[1]!) as { messages: { tool_calls?: { function: { arguments: string } }[] }[] }; + const sentArgs = sent.messages.find((m) => m.tool_calls)!.tool_calls![0]!.function.arguments; + assert.deepEqual(JSON.parse(sentArgs), { path: "r.txt", content: "real\nline\n" }); + // The capture kept the broken text. + const files = readdirSync(cap); + assert.equal(files.length, 1); + const c = JSON.parse(readFileSync(join(cap, files[0]!), "utf8")) as { transcript: { tool_calls?: { function: { arguments: string } }[] }[] }; + assert.equal(c.transcript.find((m) => m.tool_calls)!.tool_calls![0]!.function.arguments, broken); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); });