Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions packages/ai/src/protocols/anthropic-messages.ts
Original file line number Diff line number Diff line change
Expand Up @@ -814,7 +814,10 @@ const lowerMedia = Effect.fnUntraced(function* (part: MediaPart, request: LLMReq
// Tool results may carry structured text, images, and documents. Keep media as provider-native
// content instead of JSON-stringifying base64 into a prompt string.
const lowerToolResultContentItem = Effect.fnUntraced(function* (item: Tool.Content, request: LLMRequest) {
if (item.type === "text") return { type: "text" as const, text: item.text } satisfies AnthropicTextBlock
if (item.type === "text") {
if (item.text.trim() === "") return
return { type: "text" as const, text: item.text } satisfies AnthropicTextBlock
}
return yield* lowerMedia(ProviderShared.toolFileMedia(item), request)
})

Expand All @@ -825,7 +828,8 @@ const lowerToolResultContent = Effect.fnUntraced(function* (part: ToolResultPart
if (part.result.type === "error") return lowerToolErrorContent(part, rich ?? [])
// Text and json results stay as a string for backward compatibility with existing cassettes.
if (part.result.type !== "content") return ProviderShared.toolResultText(part)
return yield* Effect.forEach(part.result.value, (item) => lowerToolResultContentItem(item, request))
const blocks = yield* Effect.forEach(part.result.value, (item) => lowerToolResultContentItem(item, request))
return blocks.flatMap((block) => (block === undefined ? [] : [block]))
})

const lowerToolErrorContent = (part: ToolResultPart, content: ReadonlyArray<Tool.Content>) => {
Expand Down
22 changes: 19 additions & 3 deletions packages/ai/src/protocols/bedrock-converse.ts
Original file line number Diff line number Diff line change
Expand Up @@ -317,30 +317,45 @@ const lowerToolCall = (part: ToolCallPart, normalizeID: (id: string) => string):
},
})

const omittedToolErrorNote = (mime: string) => {
if (mime.startsWith("image/")) return "[image omitted]"
if (mime === "application/pdf") return "[document omitted]"
return "[file omitted]"
}

const lowerToolResultContent = Effect.fnUntraced(function* (
part: ToolResultPart,
documentNames: Set<string>,
omitErrorMedia: boolean,
) {
if (part.result.type === "text") return [{ text: ProviderShared.toolResultText(part) }]
if (part.result.type === "error") {
const rich = ProviderShared.toolErrorContent(part)
const text = { text: ProviderShared.toolResultText(part) }
if (rich === undefined) return [text]
const blocks: Array<Schema.Schema.Type<typeof BedrockToolResultContentItem>> = [text]
const blocks: Array<Schema.Schema.Type<typeof BedrockToolResultContentItem>> = []
for (const item of rich) {
if (item.type === "text") {
if (item.text.trim() === "") continue
blocks.push({ text: item.text })
continue
}
// Claude and Nova reject image and document blocks inside an error tool result.
if (omitErrorMedia) {
blocks.push({ text: omittedToolErrorNote(item.mime) })
continue
}
blocks.push(...(yield* BedrockMedia.lower(ProviderShared.toolFileMedia(item), documentNames)))
}
if (blocks.length === 0 || part.result.empty !== true) blocks.unshift(text)
return blocks
}
if (part.result.type === "json") return [{ json: part.result.value }]

const content: Array<Schema.Schema.Type<typeof BedrockToolResultContentItem>> = []
for (const item of part.result.value) {
if (item.type === "text") {
if (item.text.trim() === "") continue
content.push({ text: item.text })
continue
}
Expand All @@ -354,11 +369,12 @@ const lowerToolResult = Effect.fnUntraced(function* (
part: ToolResultPart,
documentNames: Set<string>,
normalizeID: (id: string) => string,
omitErrorMedia: boolean,
) {
return {
toolResult: {
toolUseId: normalizeID(part.id),
content: yield* lowerToolResultContent(part, documentNames),
content: yield* lowerToolResultContent(part, documentNames, omitErrorMedia),
status: part.result.type === "error" ? "error" : "success",
},
} satisfies BedrockToolResultBlock
Expand Down Expand Up @@ -462,7 +478,7 @@ const lowerMessages = Effect.fnUntraced(function* (
for (const part of message.content) {
if (!ProviderShared.supportsContent(part, ["tool-result"]))
return yield* ProviderShared.unsupportedContent("Bedrock Converse", "tool", ["tool-result"])
const result = yield* lowerToolResult(part, documentNames, normalizeID)
const result = yield* lowerToolResult(part, documentNames, normalizeID, keepToolImagesInline(request.model.id))
const images: BedrockMedia.ImageBlock[] = hoistImages
? result.toolResult.content.filter((item) => "image" in item)
: []
Expand Down
3 changes: 3 additions & 0 deletions packages/ai/src/schema/messages.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,9 @@ const toolResultValueSchema = Schema.Union([
type: Schema.Literal("error"),
value: Schema.Unknown,
content: Schema.optional(Schema.Array(Tool.Content)),
// Set when history replaced a blank error value. Providers can tell that apart from a real
// error whose text is the same placeholder.
empty: Schema.optional(Schema.Boolean),
}),
Schema.Struct({
type: Schema.Literal("content"),
Expand Down
13 changes: 9 additions & 4 deletions packages/ai/src/tool-history.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { Message, ToolResultPart, type ToolCallPart } from "./schema/messages.js

const EMPTY_TOOL_OUTPUT = "(no tool output)"
const MISSING_TOOL_RESULT = "Tool result missing"
const blankText = (value: unknown) => typeof value === "string" && value.trim() === ""

export function normalizeToolHistory(messages: ReadonlyArray<Message>) {
const normalized: Message[] = []
Expand Down Expand Up @@ -86,13 +87,17 @@ function normalizeToolResult(part: ToolResultPart, call: ToolCallPart | undefine
call === undefined || (part.name === call.name && part.namespace === call.namespace)
? part
: { ...part, name: call.name, namespace: call.namespace }
if (named.result.type === "text" && named.result.value === "")
if (named.result.type === "text" && blankText(named.result.value))
return { ...named, result: { type: "text", value: EMPTY_TOOL_OUTPUT } }
if (named.result.type === "error" && named.result.value === "")
return { ...named, result: { ...named.result, value: EMPTY_TOOL_OUTPUT } }
if (named.result.type === "error" && blankText(named.result.value))
return { ...named, result: { ...named.result, value: EMPTY_TOOL_OUTPUT, empty: true } }
if (named.result.type !== "content") return named
// Exact "" is not a separator. Drop it, then replace the array only when every remaining
// item is blank and there is no media. Whitespace such as "\n" stays: Mistral joins text
// with "", so ["foo", "\n", "bar"] must remain "foo\nbar".
const value = named.result.value.filter((item) => item.type !== "text" || item.text !== "")
if (value.length === 0) return { ...named, result: { type: "text", value: EMPTY_TOOL_OUTPUT } }
if (value.every((item) => item.type === "text" && blankText(item.text)))
return { ...named, result: { type: "text", value: EMPTY_TOOL_OUTPUT } }
if (value.length === named.result.value.length) return named
return { ...named, result: { type: "content", value } }
}
43 changes: 42 additions & 1 deletion packages/ai/test/provider/anthropic-messages.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -823,6 +823,47 @@ describe("Anthropic Messages route", () => {
}),
)

it.effect("drops whitespace-only tool-result text and keeps non-blank text", () =>
Effect.gen(function* () {
const spaced = " Keep this spacing. "
const prepared = yield* compileRequest(
LLM.request({
model,
cache: "none",
messages: [
Message.assistant([ToolCallPart.make({ id: "call_1", name: "read", input: {} })]),
Message.tool({
id: "call_1",
name: "read",
resultType: "content",
result: [
{ type: "text", text: " \n\t" },
{ type: "text", text: spaced },
{ type: "file", uri: "data:image/png;base64,AAECAw==", mime: "image/png" },
],
}),
],
}),
)
const blank = yield* compileRequest(
LLM.request({
model,
cache: "none",
messages: [
Message.assistant([ToolCallPart.make({ id: "call_1", name: "read", input: {} })]),
Message.tool({ id: "call_1", name: "read", resultType: "text", result: " \n\t" }),
],
}),
)

expect(expectToolResult(prepared.body).content).toEqual([
{ type: "text", text: spaced },
{ type: "image", source: { type: "base64", media_type: "image/png", data: "AAECAw==" } },
])
expect(expectToolResult(blank.body).content).toBe("(no tool output)")
}),
)

it.effect("lowers single-image tool-result content as a structured image block", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
Expand Down Expand Up @@ -1036,7 +1077,7 @@ describe("Anthropic Messages route", () => {
const prepared = yield* compileErrorToolResult(" \n", [{ type: "text", text: "\t " }])
const result = expectToolResult(prepared.body)
expect(result.is_error).toBe(true)
expect(result.content).toEqual([{ type: "text", text: "[tool error]" }])
expect(result.content).toEqual([{ type: "text", text: "(no tool output)" }])
expectNoEmptyText(result.content)
}),
)
Expand Down
177 changes: 175 additions & 2 deletions packages/ai/test/provider/bedrock-converse.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -648,6 +648,53 @@ describe("Bedrock Converse route", () => {
}),
)

it.effect("drops blank tool-result text and keeps the image", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
LLM.request({
id: "req_tool_blank_text",
model,
messages: [
Message.user("Capture the screen."),
Message.assistant([ToolCallPart.make({ id: "tool_1", name: "screenshot", input: {} })]),
Message.tool({
id: "tool_1",
name: "screenshot",
result: {
type: "content",
value: [
{ type: "text", text: " " },
{ type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png" },
],
},
}),
],
cache: "none",
}),
)

expect(prepared.body.messages).toEqual([
{ role: "user", content: [{ text: "Capture the screen." }] },
{
role: "assistant",
content: [{ toolUse: { toolUseId: "tool_1", name: "screenshot", input: {} } }],
},
{
role: "user",
content: [
{
toolResult: {
toolUseId: "tool_1",
content: [{ image: { format: "png", source: { bytes: "AAAA" } } }],
status: "success",
},
},
],
},
])
}),
)

it.effect("hoists tool-result images beside the result for Bedrock GPT models", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
Expand Down Expand Up @@ -819,7 +866,7 @@ describe("Bedrock Converse route", () => {
}),
)

it.effect("lowers rich tool failures with status error and an image", () =>
it.effect("lowers rich Claude tool failures as text notes", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
LLM.request({
Expand Down Expand Up @@ -853,7 +900,7 @@ describe("Bedrock Converse route", () => {
text: JSON.stringify({ error: { type: "tool.execution", message: "snapshot failed" }, content: [] }),
},
{ text: "could not capture" },
{ image: { format: "png", source: { bytes: "AAAA" } } },
{ text: "[image omitted]" },
],
},
},
Expand All @@ -865,6 +912,132 @@ describe("Bedrock Converse route", () => {
}),
)

it.effect("drops blank rich text from a Claude or Nova error and sends an omission note", () =>
Effect.gen(function* () {
for (const id of [model.id, "us.amazon.nova-pro-v1:0"]) {
const prepared = yield* compileRequest(
LLM.request({
model: AmazonBedrock.configure({ baseURL: "https://bedrock-runtime.test", apiKey: "test-bearer" }).model(
id,
),
cache: "none",
messages: [
Message.assistant([ToolCallPart.make({ id: "tool_1", name: "snapshot", input: {} })]),
Message.tool({
id: "tool_1",
name: "snapshot",
result: {
type: "error",
value: "\n",
content: [
{ type: "text", text: "\n" },
{ type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png" },
{ type: "file", uri: "data:application/pdf;base64,QkI=", mime: "application/pdf" },
{ type: "file", uri: "data:text/plain;base64,Zg==", mime: "text/plain" },
],
},
}),
],
}),
)

expect(prepared.body.messages).toEqual([
{
role: "assistant",
content: [{ toolUse: { toolUseId: "tool_1", name: "snapshot", input: {} } }],
},
{
role: "user",
content: [
{
toolResult: {
toolUseId: "tool_1",
status: "error",
content: [{ text: "[image omitted]" }, { text: "[document omitted]" }, { text: "[file omitted]" }],
},
},
],
},
])
}
}),
)

it.effect("keeps an error whose text is the empty-output placeholder when media is present", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
LLM.request({
model,
cache: "none",
messages: [
Message.assistant([ToolCallPart.make({ id: "tool_1", name: "snapshot", input: {} })]),
Message.tool({
id: "tool_1",
name: "snapshot",
result: {
type: "error",
value: "(no tool output)",
content: [{ type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png" }],
},
}),
],
}),
)

expect(prepared.body.messages[1]?.content).toEqual([
{
toolResult: {
toolUseId: "tool_1",
status: "error",
content: [{ text: "(no tool output)" }, { text: "[image omitted]" }],
},
},
])
}),
)

it.effect("hoists an image from a blank error for models that do not keep tool images inline", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
LLM.request({
model: AmazonBedrock.configure({ baseURL: "https://bedrock-runtime.test", apiKey: "test-bearer" }).model(
"global.openai.gpt-6-sol",
),
cache: "none",
messages: [
Message.assistant([ToolCallPart.make({ id: "tool_1", name: "snapshot", input: {} })]),
Message.tool({
id: "tool_1",
name: "snapshot",
result: {
type: "error",
value: "\n",
content: [
{ type: "text", text: "\n" },
{ type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png" },
],
},
}),
],
}),
)

expect(prepared.body.messages[1]).toEqual({
role: "user",
content: [
{
toolResult: {
toolUseId: "tool_1",
status: "error",
content: [{ text: "See attached image." }],
},
},
{ image: { format: "png", source: { bytes: "AAAA" } } },
],
})
}),
)

it.effect("decodes text-delta + messageStop + metadata usage from binary event stream", () =>
Effect.gen(function* () {
const body = eventStreamBody(
Expand Down
Loading
Loading