From 70dc294b5354383298cfc2dff1a39bfc8c4c87f9 Mon Sep 17 00:00:00 2001 From: Steven Roussey Date: Thu, 1 Oct 2026 20:45:49 -0700 Subject: [PATCH] fix(ai): name the field a nullable section's submission got wrong An answer section typed object-or-null that failed its schema was reported as "does not match any schema of [...]", with the whole value and both branches echoed back. The model never learned which field was wrong. In a production run of 108k filings, 15 of 26 answer-budget failures were one model resubmitting the same extra property until its time ran out. When exactly one branch of a failing anyOf/oneOf takes the value's JSON type, AgentTask now reports that branch's own errors, re-rooted at the value's path: "Additional property `trust_total` in `#/offering_terms/trust_total` is not allowed". Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QnfZoVpU6VAoGoTkN3jtmv --- packages/ai/src/task/AgentTask.ts | 9 ++- packages/ai/src/task/ToolCallingUtils.ts | 71 +++++++++++++++++++ .../src/test/ai/describeSchemaErrors.test.ts | 47 ++++++++++++ 3 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 packages/test/src/test/ai/describeSchemaErrors.test.ts diff --git a/packages/ai/src/task/AgentTask.ts b/packages/ai/src/task/AgentTask.ts index 208ebec8a..212a4af52 100644 --- a/packages/ai/src/task/AgentTask.ts +++ b/packages/ai/src/task/AgentTask.ts @@ -57,7 +57,12 @@ import { collectToolUseIds, uniquifyToolCallIds } from "./ToolCallIds"; import type { ToolCallingTaskInput, ToolCallingTaskOutput } from "./ToolCallingTask"; import { ToolCallingInputSchema, ToolCallingTask } from "./ToolCallingTask"; import type { ToolCall, ToolDefinition } from "./ToolCallingUtils"; -import { compileToolValidators, sanitizeToolArgs, ToolCallError } from "./ToolCallingUtils"; +import { + compileToolValidators, + describeSchemaErrors, + sanitizeToolArgs, + ToolCallError, +} from "./ToolCallingUtils"; import { RetryableJobError } from "@workglow/job-queue"; /** Rounds before the loop gives up on the model reaching an answer. */ @@ -906,7 +911,7 @@ export class AgentTask extends Task error.message).join("; ") || "invalid arguments"; + const detail = describeSchemaErrors(check.errors); // The answer itself failed its schema: say so as a correction to make, // which is what gets a model to resubmit rather than give up. const text = diff --git a/packages/ai/src/task/ToolCallingUtils.ts b/packages/ai/src/task/ToolCallingUtils.ts index 54fb36a5d..d31c492bc 100644 --- a/packages/ai/src/task/ToolCallingUtils.ts +++ b/packages/ai/src/task/ToolCallingUtils.ts @@ -215,6 +215,77 @@ export function compileToolValidators( return validators; } +/** One schema error as a validator reports it: the parts this module reads. */ +interface SchemaErrorLike { + readonly code?: string; + readonly message: string; + readonly data?: unknown; +} + +/** The JSON type a value has, in the spelling a schema's `type` uses. */ +function jsonTypeOf(value: unknown): string { + if (value === null) return "null"; + if (Array.isArray(value)) return "array"; + if (typeof value === "number") return Number.isInteger(value) ? "integer" : "number"; + return typeof value; +} + +function branchAccepts(branch: unknown, type: string): boolean { + if (branch === null || typeof branch !== "object") return false; + const declared = (branch as { type?: unknown }).type; + if (declared === undefined) return false; + const types = Array.isArray(declared) ? declared : [declared]; + return types.includes(type) || (type === "integer" && types.includes("number")); +} + +/** + * The validation errors as one line a model can act on. + * + * An `anyOf` that fails reports only that no branch matched, with the whole + * value and every branch echoed back. For the commonest shape — an object or + * null — that hides the one thing to fix: the object branch's own complaint, + * such as a property its schema does not allow. Where exactly one branch takes + * the value's JSON type, its errors are reported instead, at the path of the + * value that failed. + */ +export function describeSchemaErrors(errors: ReadonlyArray<{ readonly message: string }>): string { + const lines = errors.flatMap((error) => explainSchemaError(error as SchemaErrorLike, 0)); + return lines.join("; ") || "invalid arguments"; +} + +function explainSchemaError(error: SchemaErrorLike, depth: number): string[] { + const data = error.data as + | { readonly pointer?: unknown; readonly value?: unknown; readonly anyOf?: unknown } + | undefined; + if ( + depth > 4 || + (error.code !== "any-of-error" && error.code !== "one-of-error") || + data === undefined || + !Array.isArray(data.anyOf ?? (data as { oneOf?: unknown }).oneOf) + ) { + return [error.message]; + } + const branches = (data.anyOf ?? (data as { oneOf?: unknown }).oneOf) as unknown[]; + const candidates = branches.filter((branch) => branchAccepts(branch, jsonTypeOf(data.value))); + if (candidates.length !== 1) return [error.message]; + let inner: SchemaErrorLike[]; + try { + const result = compileSchema(candidates[0] as JsonSchema).validate(data.value); + if (result.valid) return [error.message]; + inner = result.errors as SchemaErrorLike[]; + } catch { + return [error.message]; + } + // The branch was validated on its own, so its paths start at `#`: re-root them + // at the value's place in the whole answer. + const at = typeof data.pointer === "string" ? data.pointer : "#"; + return inner.flatMap((e) => + explainSchemaError(e, depth + 1).map((line) => + at === "#" ? line : line.replace(/`#(?=[/`])/g, `\`${at}`) + ) + ); +} + /** * Filter tool calls whose `input` doesn't match the tool's compiled * `inputSchema`. Tools without a compiled validator (compile failed, diff --git a/packages/test/src/test/ai/describeSchemaErrors.test.ts b/packages/test/src/test/ai/describeSchemaErrors.test.ts new file mode 100644 index 000000000..9df9ebbf9 --- /dev/null +++ b/packages/test/src/test/ai/describeSchemaErrors.test.ts @@ -0,0 +1,47 @@ +/** + * @license + * Copyright 2026 Steven Roussey + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describeSchemaErrors } from "@workglow/ai"; +import { compileSchema } from "@workglow/util/schema"; +import { describe, expect, it } from "vitest"; + +const answer = compileSchema({ + type: "object", + properties: { + offering_terms: { + anyOf: [ + { + type: "object", + properties: { price: { type: "number" } }, + required: ["price"], + additionalProperties: false, + }, + { type: "null" }, + ], + }, + }, + required: ["offering_terms"], +}); + +describe("describeSchemaErrors", () => { + it("reports a nullable object's own errors, at their path in the answer", () => { + const { errors } = answer.validate({ offering_terms: { price: 10, trust_total: 5 } }); + const detail = describeSchemaErrors(errors); + expect(detail).toContain("trust_total"); + expect(detail).toContain("#/offering_terms"); + expect(detail).not.toContain("does not match any schema"); + }); + + it("keeps the any-of message when no single branch takes the value's type", () => { + const { errors } = answer.validate({ offering_terms: "ten" }); + expect(describeSchemaErrors(errors)).toContain("does not match any schema"); + }); + + it("passes other errors through", () => { + const { errors } = answer.validate({}); + expect(describeSchemaErrors(errors)).toContain("offering_terms"); + }); +});