diff --git a/CHANGELOG.md b/CHANGELOG.md index bcfc5c62..b1973cc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## 0.8.3 - Unreleased +- Fixed provider JSON extraction to recover structured results after unmatched braces in leading prose, thanks @goutamadwant and @notlakshitha. - Updated Node setup to 7.1.0, CodeQL to 4.38.3, and both the TruffleHog action and scanner to 3.99.2, thanks @dependabot[bot]. - Updated Oxfmt to 0.72.0 and Oxlint to 1.87.0 for repository formatting and linting, thanks @dependabot[bot]. diff --git a/docs/providers.md b/docs/providers.md index 0ae0f310..3781a667 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -45,6 +45,7 @@ Codex invocation: - fix: workspace-write sandbox - output: strict JSON schema via `--output-schema` - final message capture: `--output-last-message` +- JSON extraction accepts fenced results and structured objects after leading prose, including prose with an unmatched brace; recovered results still undergo schema validation. - timeout: 300 seconds by default, matching the Cursor provider; override with `CLAWPATCH_CODEX_TIMEOUT_MS` or `CLAWPATCH_PROVIDER_TIMEOUT_MS` diff --git a/src/provider-json.ts b/src/provider-json.ts index 41ee1bf7..dbfcb209 100644 --- a/src/provider-json.ts +++ b/src/provider-json.ts @@ -11,46 +11,83 @@ export function extractJson(text: string): unknown | null { return JSON.parse(candidate); } catch {} } - let firstBrace = text.indexOf("{"); - while (firstBrace !== -1) { - let depth = 0; - let inString = false; - let escape = false; - for (let i = firstBrace; i < text.length; i += 1) { - const ch = text[i]; - if (escape) { - escape = false; - continue; + const candidates: Array<{ start: number; end: number; parent: number | null }> = []; + const stack: number[] = []; + let inString = false; + let escape = false; + for (let i = 0; i < text.length; i += 1) { + const ch = text[i]; + if (stack.length === 0) { + if (ch === "{") { + stack.push(i); + } else if (ch === '"') { + const closingQuote = findProseQuoteEnd(text, i); + if (closingQuote !== -1) i = closingQuote; } - if (ch === "\\") { - escape = true; - continue; - } - if (ch === '"') { - inString = !inString; - continue; - } - if (!inString) { - if (ch === "{") depth += 1; - else if (ch === "}") { - depth -= 1; - if (depth === 0) { - const candidate = text.slice(firstBrace, i + 1); - try { - return JSON.parse(candidate); - } catch { - firstBrace = text.indexOf("{", i + 1); - break; - } - } - } + continue; + } + if (escape) { + escape = false; + continue; + } + if (ch === "\\") { + escape = true; + continue; + } + if (ch === '"') { + inString = !inString; + continue; + } + if (inString) continue; + if (ch === "{") { + stack.push(i); + } else if (ch === "}" && stack.length > 0) { + const start = stack.pop(); + if (start !== undefined) { + candidates.push({ start, end: i + 1, parent: stack.at(-1) ?? null }); } } - if (depth !== 0) { - firstBrace = -1; + } + + for (const candidate of candidates) { + if (candidate.parent !== null) continue; + try { + return JSON.parse(text.slice(candidate.start, candidate.end)); + } catch {} + } + + const unmatched = new Set(stack); + let recovered: unknown | null = null; + for (const candidate of candidates) { + if (candidate.parent === null || !unmatched.has(candidate.parent)) continue; + try { + recovered = JSON.parse(text.slice(candidate.start, candidate.end)); + } catch {} + } + return recovered; +} + +function findProseQuoteEnd(text: string, start: number): number { + let escape = false; + for (let i = start + 1; i < text.length; i += 1) { + const ch = text[i]; + if (escape) { + escape = false; + continue; + } + if (ch === "\\") { + escape = true; + continue; } + if (ch !== '"') continue; + const next = text[i + 1]; + if (next !== undefined && /[\p{L}\p{N}_]/u.test(next)) return -1; + let previous = i - 1; + while (previous > start && /\s/u.test(text[previous] ?? "")) previous -= 1; + if (text[previous] === "{" && text.slice(start + 1, i).trim() !== "{") return -1; + return i; } - return null; + return -1; } export function parseCodexJson(raw: string): unknown { diff --git a/src/provider.test.ts b/src/provider.test.ts index 25eedda1..4b8c1863 100644 --- a/src/provider.test.ts +++ b/src/provider.test.ts @@ -163,6 +163,43 @@ describe("extractJson", () => { }); }); + it("ignores malformed prose between balanced candidates", () => { + const input = 'thinking { not-json } note "unfinished final {"ok":true}'; + + expect(extractJson(input)).toEqual({ ok: true }); + }); + + it("ignores unmatched braces inside quoted prose", () => { + const input = 'Plan: mention "{" or "use { here" then final {"ok":true}'; + + expect(extractJson(input)).toEqual({ ok: true }); + }); + + it("recovers JSON with an escaped first key after an unmatched prose quote", () => { + const input = 'note "unfinished final {"\\u006futcome":"fixed","reasoning":"ok"}'; + + expect(extractJson(input)).toEqual({ outcome: "fixed", reasoning: "ok" }); + }); + + it.each([ + ['Plan: wrap keys in { braces.\n{"ok":true}', { ok: true }], + ['Looking at fn() { ... the result:\n{"findings":[]}', { findings: [] }], + ])("skips an unmatched brace before valid JSON", (input, expected) => { + expect(extractJson(input)).toEqual(expected); + }); + + it("uses the final JSON object after an unmatched malformed preamble", () => { + const input = + 'draft { outer: {"outcome":"fixed","reasoning":"draft","commands":[]}\n' + + 'final {"outcome":"fixed","reasoning":"final","commands":[]}'; + + expect(extractJson(input)).toEqual({ + outcome: "fixed", + reasoning: "final", + commands: [], + }); + }); + it("does not parse nested JSON from malformed preambles", () => { const input = 'draft { outer: {"outcome":"draft","reasoning":"x","commands":[]} } final ' + @@ -178,6 +215,7 @@ describe("extractJson", () => { it("returns null for text with no valid JSON", () => { expect(extractJson("no json here at all")).toBeNull(); expect(extractJson("just some words { unbalanced")).toBeNull(); + expect(extractJson("{".repeat(40_000))).toBeNull(); }); });