diff --git a/packages/ai/src/protocols/anthropic-messages.ts b/packages/ai/src/protocols/anthropic-messages.ts index daa4b0476fe4..36eb59151acc 100644 --- a/packages/ai/src/protocols/anthropic-messages.ts +++ b/packages/ai/src/protocols/anthropic-messages.ts @@ -814,7 +814,10 @@ const lowerMedia = Effect.fnUntraced(function* (part: MediaPart, request: LLMReq // Tool results may carry structured text, images, and documents. Keep media as provider-native // content instead of JSON-stringifying base64 into a prompt string. const lowerToolResultContentItem = Effect.fnUntraced(function* (item: Tool.Content, request: LLMRequest) { - if (item.type === "text") return { type: "text" as const, text: item.text } satisfies AnthropicTextBlock + if (item.type === "text") { + if (item.text.trim() === "") return + return { type: "text" as const, text: item.text } satisfies AnthropicTextBlock + } return yield* lowerMedia(ProviderShared.toolFileMedia(item), request) }) @@ -825,7 +828,8 @@ const lowerToolResultContent = Effect.fnUntraced(function* (part: ToolResultPart if (part.result.type === "error") return lowerToolErrorContent(part, rich ?? []) // Text and json results stay as a string for backward compatibility with existing cassettes. if (part.result.type !== "content") return ProviderShared.toolResultText(part) - return yield* Effect.forEach(part.result.value, (item) => lowerToolResultContentItem(item, request)) + const blocks = yield* Effect.forEach(part.result.value, (item) => lowerToolResultContentItem(item, request)) + return blocks.flatMap((block) => (block === undefined ? [] : [block])) }) const lowerToolErrorContent = (part: ToolResultPart, content: ReadonlyArray) => { diff --git a/packages/ai/src/protocols/bedrock-converse.ts b/packages/ai/src/protocols/bedrock-converse.ts index 1bc7a9a4c008..8cdc5d897b06 100644 --- a/packages/ai/src/protocols/bedrock-converse.ts +++ b/packages/ai/src/protocols/bedrock-converse.ts @@ -317,23 +317,37 @@ const lowerToolCall = (part: ToolCallPart, normalizeID: (id: string) => string): }, }) +const omittedToolErrorNote = (mime: string) => { + if (mime.startsWith("image/")) return "[image omitted]" + if (mime === "application/pdf") return "[document omitted]" + return "[file omitted]" +} + const lowerToolResultContent = Effect.fnUntraced(function* ( part: ToolResultPart, documentNames: Set, + omitErrorMedia: boolean, ) { if (part.result.type === "text") return [{ text: ProviderShared.toolResultText(part) }] if (part.result.type === "error") { const rich = ProviderShared.toolErrorContent(part) const text = { text: ProviderShared.toolResultText(part) } if (rich === undefined) return [text] - const blocks: Array> = [text] + const blocks: Array> = [] for (const item of rich) { if (item.type === "text") { + if (item.text.trim() === "") continue blocks.push({ text: item.text }) continue } + // Claude and Nova reject image and document blocks inside an error tool result. + if (omitErrorMedia) { + blocks.push({ text: omittedToolErrorNote(item.mime) }) + continue + } blocks.push(...(yield* BedrockMedia.lower(ProviderShared.toolFileMedia(item), documentNames))) } + if (blocks.length === 0 || part.result.empty !== true) blocks.unshift(text) return blocks } if (part.result.type === "json") return [{ json: part.result.value }] @@ -341,6 +355,7 @@ const lowerToolResultContent = Effect.fnUntraced(function* ( const content: Array> = [] for (const item of part.result.value) { if (item.type === "text") { + if (item.text.trim() === "") continue content.push({ text: item.text }) continue } @@ -354,11 +369,12 @@ const lowerToolResult = Effect.fnUntraced(function* ( part: ToolResultPart, documentNames: Set, normalizeID: (id: string) => string, + omitErrorMedia: boolean, ) { return { toolResult: { toolUseId: normalizeID(part.id), - content: yield* lowerToolResultContent(part, documentNames), + content: yield* lowerToolResultContent(part, documentNames, omitErrorMedia), status: part.result.type === "error" ? "error" : "success", }, } satisfies BedrockToolResultBlock @@ -462,7 +478,7 @@ const lowerMessages = Effect.fnUntraced(function* ( for (const part of message.content) { if (!ProviderShared.supportsContent(part, ["tool-result"])) return yield* ProviderShared.unsupportedContent("Bedrock Converse", "tool", ["tool-result"]) - const result = yield* lowerToolResult(part, documentNames, normalizeID) + const result = yield* lowerToolResult(part, documentNames, normalizeID, keepToolImagesInline(request.model.id)) const images: BedrockMedia.ImageBlock[] = hoistImages ? result.toolResult.content.filter((item) => "image" in item) : [] diff --git a/packages/ai/src/schema/messages.ts b/packages/ai/src/schema/messages.ts index a61e71b6af4c..6fb5cc60e555 100644 --- a/packages/ai/src/schema/messages.ts +++ b/packages/ai/src/schema/messages.ts @@ -70,6 +70,9 @@ const toolResultValueSchema = Schema.Union([ type: Schema.Literal("error"), value: Schema.Unknown, content: Schema.optional(Schema.Array(Tool.Content)), + // Set when history replaced a blank error value. Providers can tell that apart from a real + // error whose text is the same placeholder. + empty: Schema.optional(Schema.Boolean), }), Schema.Struct({ type: Schema.Literal("content"), diff --git a/packages/ai/src/tool-history.ts b/packages/ai/src/tool-history.ts index ddf7d1ce95f9..d255c8ec56cd 100644 --- a/packages/ai/src/tool-history.ts +++ b/packages/ai/src/tool-history.ts @@ -2,6 +2,7 @@ import { Message, ToolResultPart, type ToolCallPart } from "./schema/messages.js const EMPTY_TOOL_OUTPUT = "(no tool output)" const MISSING_TOOL_RESULT = "Tool result missing" +const blankText = (value: unknown) => typeof value === "string" && value.trim() === "" export function normalizeToolHistory(messages: ReadonlyArray) { const normalized: Message[] = [] @@ -86,13 +87,17 @@ function normalizeToolResult(part: ToolResultPart, call: ToolCallPart | undefine call === undefined || (part.name === call.name && part.namespace === call.namespace) ? part : { ...part, name: call.name, namespace: call.namespace } - if (named.result.type === "text" && named.result.value === "") + if (named.result.type === "text" && blankText(named.result.value)) return { ...named, result: { type: "text", value: EMPTY_TOOL_OUTPUT } } - if (named.result.type === "error" && named.result.value === "") - return { ...named, result: { ...named.result, value: EMPTY_TOOL_OUTPUT } } + if (named.result.type === "error" && blankText(named.result.value)) + return { ...named, result: { ...named.result, value: EMPTY_TOOL_OUTPUT, empty: true } } if (named.result.type !== "content") return named + // Exact "" is not a separator. Drop it, then replace the array only when every remaining + // item is blank and there is no media. Whitespace such as "\n" stays: Mistral joins text + // with "", so ["foo", "\n", "bar"] must remain "foo\nbar". const value = named.result.value.filter((item) => item.type !== "text" || item.text !== "") - if (value.length === 0) return { ...named, result: { type: "text", value: EMPTY_TOOL_OUTPUT } } + if (value.every((item) => item.type === "text" && blankText(item.text))) + return { ...named, result: { type: "text", value: EMPTY_TOOL_OUTPUT } } if (value.length === named.result.value.length) return named return { ...named, result: { type: "content", value } } } diff --git a/packages/ai/test/provider/anthropic-messages.test.ts b/packages/ai/test/provider/anthropic-messages.test.ts index 768d5db5bb5a..fbc0e93759dc 100644 --- a/packages/ai/test/provider/anthropic-messages.test.ts +++ b/packages/ai/test/provider/anthropic-messages.test.ts @@ -823,6 +823,47 @@ describe("Anthropic Messages route", () => { }), ) + it.effect("drops whitespace-only tool-result text and keeps non-blank text", () => + Effect.gen(function* () { + const spaced = " Keep this spacing. " + const prepared = yield* compileRequest( + LLM.request({ + model, + cache: "none", + messages: [ + Message.assistant([ToolCallPart.make({ id: "call_1", name: "read", input: {} })]), + Message.tool({ + id: "call_1", + name: "read", + resultType: "content", + result: [ + { type: "text", text: " \n\t" }, + { type: "text", text: spaced }, + { type: "file", uri: "data:image/png;base64,AAECAw==", mime: "image/png" }, + ], + }), + ], + }), + ) + const blank = yield* compileRequest( + LLM.request({ + model, + cache: "none", + messages: [ + Message.assistant([ToolCallPart.make({ id: "call_1", name: "read", input: {} })]), + Message.tool({ id: "call_1", name: "read", resultType: "text", result: " \n\t" }), + ], + }), + ) + + expect(expectToolResult(prepared.body).content).toEqual([ + { type: "text", text: spaced }, + { type: "image", source: { type: "base64", media_type: "image/png", data: "AAECAw==" } }, + ]) + expect(expectToolResult(blank.body).content).toBe("(no tool output)") + }), + ) + it.effect("lowers single-image tool-result content as a structured image block", () => Effect.gen(function* () { const prepared = yield* compileRequest( @@ -1036,7 +1077,7 @@ describe("Anthropic Messages route", () => { const prepared = yield* compileErrorToolResult(" \n", [{ type: "text", text: "\t " }]) const result = expectToolResult(prepared.body) expect(result.is_error).toBe(true) - expect(result.content).toEqual([{ type: "text", text: "[tool error]" }]) + expect(result.content).toEqual([{ type: "text", text: "(no tool output)" }]) expectNoEmptyText(result.content) }), ) diff --git a/packages/ai/test/provider/bedrock-converse.test.ts b/packages/ai/test/provider/bedrock-converse.test.ts index a2de0b26b6eb..a09dd118e2f3 100644 --- a/packages/ai/test/provider/bedrock-converse.test.ts +++ b/packages/ai/test/provider/bedrock-converse.test.ts @@ -648,6 +648,53 @@ describe("Bedrock Converse route", () => { }), ) + it.effect("drops blank tool-result text and keeps the image", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + id: "req_tool_blank_text", + model, + messages: [ + Message.user("Capture the screen."), + Message.assistant([ToolCallPart.make({ id: "tool_1", name: "screenshot", input: {} })]), + Message.tool({ + id: "tool_1", + name: "screenshot", + result: { + type: "content", + value: [ + { type: "text", text: " " }, + { type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png" }, + ], + }, + }), + ], + cache: "none", + }), + ) + + expect(prepared.body.messages).toEqual([ + { role: "user", content: [{ text: "Capture the screen." }] }, + { + role: "assistant", + content: [{ toolUse: { toolUseId: "tool_1", name: "screenshot", input: {} } }], + }, + { + role: "user", + content: [ + { + toolResult: { + toolUseId: "tool_1", + content: [{ image: { format: "png", source: { bytes: "AAAA" } } }], + status: "success", + }, + }, + ], + }, + ]) + }), + ) + it.effect("hoists tool-result images beside the result for Bedrock GPT models", () => Effect.gen(function* () { const prepared = yield* compileRequest( @@ -819,7 +866,7 @@ describe("Bedrock Converse route", () => { }), ) - it.effect("lowers rich tool failures with status error and an image", () => + it.effect("lowers rich Claude tool failures as text notes", () => Effect.gen(function* () { const prepared = yield* compileRequest( LLM.request({ @@ -853,7 +900,7 @@ describe("Bedrock Converse route", () => { text: JSON.stringify({ error: { type: "tool.execution", message: "snapshot failed" }, content: [] }), }, { text: "could not capture" }, - { image: { format: "png", source: { bytes: "AAAA" } } }, + { text: "[image omitted]" }, ], }, }, @@ -865,6 +912,132 @@ describe("Bedrock Converse route", () => { }), ) + it.effect("drops blank rich text from a Claude or Nova error and sends an omission note", () => + Effect.gen(function* () { + for (const id of [model.id, "us.amazon.nova-pro-v1:0"]) { + const prepared = yield* compileRequest( + LLM.request({ + model: AmazonBedrock.configure({ baseURL: "https://bedrock-runtime.test", apiKey: "test-bearer" }).model( + id, + ), + cache: "none", + messages: [ + Message.assistant([ToolCallPart.make({ id: "tool_1", name: "snapshot", input: {} })]), + Message.tool({ + id: "tool_1", + name: "snapshot", + result: { + type: "error", + value: "\n", + content: [ + { type: "text", text: "\n" }, + { type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png" }, + { type: "file", uri: "data:application/pdf;base64,QkI=", mime: "application/pdf" }, + { type: "file", uri: "data:text/plain;base64,Zg==", mime: "text/plain" }, + ], + }, + }), + ], + }), + ) + + expect(prepared.body.messages).toEqual([ + { + role: "assistant", + content: [{ toolUse: { toolUseId: "tool_1", name: "snapshot", input: {} } }], + }, + { + role: "user", + content: [ + { + toolResult: { + toolUseId: "tool_1", + status: "error", + content: [{ text: "[image omitted]" }, { text: "[document omitted]" }, { text: "[file omitted]" }], + }, + }, + ], + }, + ]) + } + }), + ) + + it.effect("keeps an error whose text is the empty-output placeholder when media is present", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + model, + cache: "none", + messages: [ + Message.assistant([ToolCallPart.make({ id: "tool_1", name: "snapshot", input: {} })]), + Message.tool({ + id: "tool_1", + name: "snapshot", + result: { + type: "error", + value: "(no tool output)", + content: [{ type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png" }], + }, + }), + ], + }), + ) + + expect(prepared.body.messages[1]?.content).toEqual([ + { + toolResult: { + toolUseId: "tool_1", + status: "error", + content: [{ text: "(no tool output)" }, { text: "[image omitted]" }], + }, + }, + ]) + }), + ) + + it.effect("hoists an image from a blank error for models that do not keep tool images inline", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + model: AmazonBedrock.configure({ baseURL: "https://bedrock-runtime.test", apiKey: "test-bearer" }).model( + "global.openai.gpt-6-sol", + ), + cache: "none", + messages: [ + Message.assistant([ToolCallPart.make({ id: "tool_1", name: "snapshot", input: {} })]), + Message.tool({ + id: "tool_1", + name: "snapshot", + result: { + type: "error", + value: "\n", + content: [ + { type: "text", text: "\n" }, + { type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png" }, + ], + }, + }), + ], + }), + ) + + expect(prepared.body.messages[1]).toEqual({ + role: "user", + content: [ + { + toolResult: { + toolUseId: "tool_1", + status: "error", + content: [{ text: "See attached image." }], + }, + }, + { image: { format: "png", source: { bytes: "AAAA" } } }, + ], + }) + }), + ) + it.effect("decodes text-delta + messageStop + metadata usage from binary event stream", () => Effect.gen(function* () { const body = eventStreamBody( diff --git a/packages/ai/test/provider/mistral-chat.test.ts b/packages/ai/test/provider/mistral-chat.test.ts index 742ea1e6da43..b72a47048e5c 100644 --- a/packages/ai/test/provider/mistral-chat.test.ts +++ b/packages/ai/test/provider/mistral-chat.test.ts @@ -353,6 +353,31 @@ describe("Mistral Chat", () => { }), ) + it.effect("keeps a whitespace separator when joining text-only tool results", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + model, + messages: [ + Message.tool({ + id: "Ab12Cd34E", + name: "lookup", + resultType: "content", + result: [ + { type: "text", text: "foo" }, + { type: "text", text: "\n" }, + { type: "text", text: "bar" }, + ], + }), + ], + }), + ) + expect(prepared.body.messages).toEqual([ + { role: "tool", tool_call_id: "Ab12Cd34E", name: "lookup", content: "foo\nbar" }, + ]) + }), + ) + it.effect("streams ordered thinking and text and replays native thinking metadata", () => Effect.gen(function* () { const response = yield* LLMClient.generate(request).pipe( diff --git a/packages/ai/test/tool-history.test.ts b/packages/ai/test/tool-history.test.ts index 80eac7a36943..add75ba94110 100644 --- a/packages/ai/test/tool-history.test.ts +++ b/packages/ai/test/tool-history.test.ts @@ -58,7 +58,7 @@ describe("tool history normalization", () => { expect(normalizeToolHistory([Message.assistant(hosted)])).toEqual([Message.assistant(hosted)]) }) - test("normalizes empty results without changing whitespace or media", () => { + test("normalizes empty and whitespace-only results without changing media", () => { const media = { type: "file" as const, uri: "data:image/png;base64,AQID", mime: "image/png" } const normalized = normalizeToolHistory([ Message.assistant([ @@ -78,9 +78,60 @@ describe("tool history normalization", () => { expect(normalized.slice(1).map((message) => message.content[0])).toEqual([ { type: "tool-result", id: "text", name: "text", result: { type: "text", value: "(no tool output)" } }, { type: "tool-result", id: "content", name: "content", result: { type: "text", value: "(no tool output)" } }, - { type: "tool-result", id: "error", name: "error", result: { type: "error", value: "(no tool output)" } }, + { + type: "tool-result", + id: "error", + name: "error", + result: { type: "error", value: "(no tool output)", empty: true }, + }, { type: "tool-result", id: "mixed", name: "mixed", result: { type: "content", value: [media] } }, - { type: "tool-result", id: "whitespace", name: "whitespace", result: { type: "text", value: " " } }, + { + type: "tool-result", + id: "whitespace", + name: "whitespace", + result: { type: "text", value: "(no tool output)" }, + }, + ]) + }) + + test("treats whitespace-only text and content as empty and leaves mixed content unchanged", () => { + const media = { type: "file" as const, uri: "data:image/png;base64,AQID", mime: "image/png" } + const spaced = " Keep this spacing. " + const separated = [ + { type: "text" as const, text: "foo" }, + { type: "text" as const, text: "\n" }, + { type: "text" as const, text: "bar" }, + ] + const mixed = [{ type: "text" as const, text: " \n\t" }, { type: "text" as const, text: spaced }, media] + const normalized = normalizeToolHistory([ + Message.assistant([ + toolCall("text"), + toolCall("error"), + toolCall("content"), + toolCall("spaced"), + toolCall("separated"), + toolCall("mixed"), + ]), + toolResult("text", " \n\t", "text", "text"), + toolResult("error", " \n\t", "error", "error"), + toolResult("content", [{ type: "text", text: " \n\t" }], "content", "content"), + toolResult("spaced", spaced, "spaced", "text"), + toolResult("separated", separated, "separated", "content"), + toolResult("mixed", mixed, "mixed", "content"), + ]) + + expect(normalized.slice(1).map((message) => message.content[0])).toEqual([ + { type: "tool-result", id: "text", name: "text", result: { type: "text", value: "(no tool output)" } }, + { + type: "tool-result", + id: "error", + name: "error", + result: { type: "error", value: "(no tool output)", empty: true }, + }, + { type: "tool-result", id: "content", name: "content", result: { type: "text", value: "(no tool output)" } }, + { type: "tool-result", id: "spaced", name: "spaced", result: { type: "text", value: spaced } }, + { type: "tool-result", id: "separated", name: "separated", result: { type: "content", value: separated } }, + { type: "tool-result", id: "mixed", name: "mixed", result: { type: "content", value: mixed } }, ]) })