diff --git a/benchmarks/eval/out-deep/full-deep.jsonl b/benchmarks/eval/out-deep/full-deep.jsonl new file mode 100644 index 0000000..5358fbd --- /dev/null +++ b/benchmarks/eval/out-deep/full-deep.jsonl @@ -0,0 +1,8 @@ +{"taskId":"deep-query-route-source","category":"query","arm":"full","fixture":"deep","taskSet":"tasks-deep.json","filesenseConfigRequested":{"enabled":true,"filesenseOutput":"summary","filesenseWriteMode":"cache"},"pass":false,"agentSuccess":true,"agentError":null,"validationCount":2,"runError":null,"resultHead":"## 结论\n\n基于当前提供的证据，我**无法直接给出路由表文件的确切路径和具体内容**。\n\n## 原因分析\n\n提供的目录导航结果是**结构扫描结果**（仅证明文件/目录存在），不包含文件内容。具体来说：\n\n1. **缺少实际文件读取**：没有读取到当前工作区文件的内容，因此无法知道哪个文件定义了顶层导航路由表\n2. **缺少知识库证据**：也未提供知识库（RAG）条目中关于路由配置的信息\n\n## ","checks":[{"kind":"result_contains","ok":false,"detail":"routeTable"},{"kind":"result_contains","ok":false,"detail":"/checkout"},{"kind":"result_contains","ok":false,"detail":"/catalog"}],"events":{"task_started":1,"status_update":16,"security_decision":4,"planning_started":1,"planning_completed":1,"phase_started":2,"step_started":2,"step_completed":2,"phase_completed":2,"filesense_navigated":1,"task_completed":1},"eventDetails":{"filesense":[{"intent":"understand_structure","paths":["."],"entries":54,"elapsedMs":10,"truncated":false,"candidateCount":15,"warnings":[]}],"validationFailed":[],"rollback":[]},"filesenseEntries":54,"filesenseTruncated":false,"elapsedMs":15452,"llmCalls":1,"llmFailures":0,"inputTokens":1391,"outputTokens":1074} +{"taskId":"deep-query-structure","category":"query","arm":"full","fixture":"deep","taskSet":"tasks-deep.json","filesenseConfigRequested":{"enabled":true,"filesenseOutput":"summary","filesenseWriteMode":"cache"},"pass":false,"agentSuccess":true,"agentError":null,"validationCount":2,"runError":null,"resultHead":"# src 顶层目录职责及 feature 分层约定\n\n## 基于证据的现状\n\n根据当前工作区的**目录导航结果**，我只能部分确定以下信息：\n\n### 可确认的顶层目录\n- **`src/app`**：分组目录，其中包含 `routes` 子目录用于存放相关实现文件\n- **`src/entities`**：分组目录，如 `address`、`category` 等，用于分组管理不同实体相关的子","checks":[{"kind":"result_contains","ok":true,"detail":"features"},{"kind":"result_contains","ok":false,"detail":"shared"},{"kind":"result_contains","ok":true,"detail":"entities"}],"events":{"task_started":1,"status_update":16,"security_decision":4,"planning_started":1,"planning_completed":1,"phase_started":2,"step_started":2,"step_completed":2,"phase_completed":2,"filesense_navigated":1,"task_completed":1},"eventDetails":{"filesense":[{"intent":"understand_structure","paths":["src","."],"entries":271,"elapsedMs":8,"truncated":true,"candidateCount":15,"warnings":["Filesense navigation was truncated by budget; expand depth/maxEntries if more coverage is needed."]}],"validationFailed":[],"rollback":[]},"filesenseEntries":271,"filesenseTruncated":true,"elapsedMs":18870,"llmCalls":1,"llmFailures":0,"inputTokens":1614,"outputTokens":1419} +{"taskId":"deep-query-format-convention","category":"query","arm":"full","fixture":"deep","taskSet":"tasks-deep.json","filesenseConfigRequested":{"enabled":true,"filesenseOutput":"summary","filesenseWriteMode":"cache"},"pass":false,"agentSuccess":true,"agentError":null,"validationCount":1,"runError":null,"resultHead":"我无法直接回答这个问题，因为当前**没有提供关于 checkout feature 中 format 函数实现的具体证据**。\n\n根据你的描述，这个问题需要我查看：\n- checkout 这个 feature 模块中定义的 `format` 函数的实现代码\n- 该函数的返回值类型和具体返回格式\n\n**当前缺失的信息：**\n1. 没有当前工作区文件的代码内容\n2. 没有知识库条目提供相关实现细节\n\n","checks":[{"kind":"result_contains","ok":false,"detail":"checkout:"},{"kind":"result_contains","ok":false,"detail":"toFixed"}],"events":{"task_started":1,"status_update":15,"security_decision":3,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":1,"step_completed":1,"phase_completed":1,"task_completed":1},"eventDetails":{"filesense":[],"validationFailed":[],"rollback":[]},"filesenseEntries":0,"filesenseTruncated":false,"elapsedMs":12444,"llmCalls":1,"llmFailures":0,"inputTokens":924,"outputTokens":776} +{"taskId":"deep-query-shared-lib","category":"query","arm":"full","fixture":"deep","taskSet":"tasks-deep.json","filesenseConfigRequested":{"enabled":true,"filesenseOutput":"summary","filesenseWriteMode":"cache"},"pass":false,"agentSuccess":true,"agentError":null,"validationCount":1,"runError":null,"resultHead":"# 无法回答\n\n我无法直接列出共享工具库里的函数，因为 **可用证据中没有包含关于共享工具库的内容**。\n\n## 缺失的信息\n\n要回答这个问题，我需要：\n- 共享工具库的源代码文件（当前工作区文件）\n- 或关于工具库的知识库条目\n\n但提供的证据中只有我的身份定义，不包含任何工具库相关的技术文档或代码。\n\n## 建议后续步骤\n\n如果你需要我帮助梳理工具库函数，请：\n1. 提供工具库的代码或文档作为证","checks":[{"kind":"result_contains","ok":false,"detail":"clamp"},{"kind":"result_contains","ok":false,"detail":"formatDate"},{"kind":"result_contains","ok":false,"detail":"pluralize"}],"events":{"task_started":1,"status_update":15,"security_decision":3,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":1,"step_completed":1,"phase_completed":1,"task_completed":1},"eventDetails":{"filesense":[],"validationFailed":[],"rollback":[]},"filesenseEntries":0,"filesenseTruncated":false,"elapsedMs":11858,"llmCalls":1,"llmFailures":0,"inputTokens":908,"outputTokens":767} +{"taskId":"deep-create-shared-truncate","category":"create","arm":"full","fixture":"deep","taskSet":"tasks-deep.json","filesenseConfigRequested":{"enabled":true,"filesenseOutput":"summary","filesenseWriteMode":"cache"},"pass":false,"agentSuccess":true,"agentError":null,"validationCount":3,"runError":null,"resultHead":"执行完成 (3/3 步骤成功)\n\n✅ 按需构建目录导航上下文：create task benefits from nearby placement conventions\n✅ 检查目标路径 src/new-file.ts 是否已存在\n✅ 创建文件 src/new-file.ts","checks":[{"kind":"file_exists","ok":false,"detail":"src/shared/lib/truncate.ts"},{"kind":"file_contains","ok":false,"detail":"missing src/shared/lib/truncate.ts"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":20,"security_decision":6,"planning_started":1,"planning_completed":1,"phase_started":2,"step_started":3,"filesense_navigated":1,"step_completed":3,"phase_completed":2,"task_completed":1},"eventDetails":{"filesense":[{"intent":"prepare_create","paths":["src"],"entries":43,"elapsedMs":3,"truncated":false,"candidateCount":15,"warnings":[]}],"validationFailed":[],"rollback":[]},"filesenseEntries":43,"filesenseTruncated":false,"elapsedMs":428651,"llmCalls":12,"llmFailures":0,"inputTokens":33790,"outputTokens":42964} +{"taskId":"deep-create-checkout-hook","category":"create","arm":"full","fixture":"deep","taskSet":"tasks-deep.json","filesenseConfigRequested":{"enabled":true,"filesenseOutput":"summary","filesenseWriteMode":"cache"},"pass":false,"agentSuccess":true,"agentError":null,"validationCount":3,"runError":null,"resultHead":"执行完成 (3/3 步骤成功)\n\n✅ 按需构建目录导航上下文：create task benefits from nearby placement conventions\n✅ 检查目标路径 src/new-file.ts 是否已存在\n✅ 创建文件 src/new-file.ts","checks":[{"kind":"file_exists","ok":false,"detail":"src/features/checkout/hooks/useCheckoutTotal.ts"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":20,"security_decision":6,"planning_started":1,"planning_completed":1,"phase_started":2,"step_started":3,"filesense_navigated":1,"step_completed":3,"phase_completed":2,"task_completed":1},"eventDetails":{"filesense":[{"intent":"prepare_create","paths":["hooks","src"],"entries":0,"elapsedMs":0,"truncated":false,"candidateCount":0,"warnings":[]}],"validationFailed":[],"rollback":[]},"filesenseEntries":0,"filesenseTruncated":false,"elapsedMs":629033,"llmCalls":11,"llmFailures":0,"inputTokens":30304,"outputTokens":64736} +{"taskId":"deep-create-entity-guard","category":"create","arm":"full","fixture":"deep","taskSet":"tasks-deep.json","filesenseConfigRequested":{"enabled":true,"filesenseOutput":"summary","filesenseWriteMode":"cache"},"pass":false,"agentSuccess":true,"agentError":null,"validationCount":3,"runError":null,"resultHead":"执行完成 (3/3 步骤成功)\n\n✅ 按需构建目录导航上下文：create task benefits from nearby placement conventions\n✅ 检查目标路径 src/new-file.ts 是否已存在\n✅ 创建文件 src/new-file.ts","checks":[{"kind":"file_contains","ok":false,"detail":"isExpired"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":20,"security_decision":6,"planning_started":1,"planning_completed":1,"phase_started":2,"step_started":3,"filesense_navigated":1,"step_completed":3,"phase_completed":2,"task_completed":1},"eventDetails":{"filesense":[{"intent":"prepare_create","paths":["src"],"entries":43,"elapsedMs":4,"truncated":false,"candidateCount":15,"warnings":[]}],"validationFailed":[],"rollback":[]},"filesenseEntries":43,"filesenseTruncated":false,"elapsedMs":647016,"llmCalls":9,"llmFailures":0,"inputTokens":25522,"outputTokens":70085} +{"taskId":"deep-bugfix-checkout-total","category":"bugfix","arm":"full","fixture":"deep","taskSet":"tasks-deep.json","filesenseConfigRequested":{"enabled":true,"filesenseOutput":"summary","filesenseWriteMode":"cache"},"pass":false,"agentSuccess":false,"agentError":"无法生成有效的执行计划","validationCount":0,"runError":null,"resultHead":"","checks":[{"kind":"cmd","ok":false,"detail":"npx vitest run src/features/checkout/lib/computeTotal.test.ts"},{"kind":"typecheck","ok":true,"detail":""},{"kind":"file_unchanged","ok":true,"detail":"src/features/checkout/lib/computeTotal.test.ts"}],"events":{"task_started":1,"status_update":10,"security_decision":2,"planning_started":1,"task_failed":1},"eventDetails":{"filesense":[],"validationFailed":[],"rollback":[]},"filesenseEntries":0,"filesenseTruncated":false,"elapsedMs":468096,"llmCalls":6,"llmFailures":0,"inputTokens":21571,"outputTokens":49154} diff --git a/benchmarks/eval/report-filesense.mjs b/benchmarks/eval/report-filesense.mjs index 253bfe7..231bc88 100644 --- a/benchmarks/eval/report-filesense.mjs +++ b/benchmarks/eval/report-filesense.mjs @@ -170,6 +170,24 @@ for (const c of categories) { } console.log(` +## 规划降级（读数前必看） + +LLM 规划抛错后会静默退到规则生成，而规则生成给 create 任务的目标路径是硬编码的 +`src/new-file.ts`——表现为**步骤全绿、任务成功、文件写错地方**。降级过的任务 +不能与正常任务混在同一个通过率里。 + +| 臂 | 发生降级的任务数 | +|---|---| +| filesense 开 | ${arms.full.filter((r) => r.plannerFallbackReason).length} / ${arms.full.length} | +| filesense 关 | ${arms.off.filter((r) => r.plannerFallbackReason).length} / ${arms.off.length} | + +${ + arms.full.filter((r) => r.plannerFallbackReason).length > 0 || + arms.off.filter((r) => r.plannerFallbackReason).length > 0 + ? '**本轮存在降级任务，通过率不可直接引用**——降级把路径覆盖成硬编码值，与 filesense 是否定位成功无关。降级原因见各条记录的 `plannerFallbackReason`。' + : '本轮无降级任务。' +} + ## 读数纪律 - 样本量 ${commonIds.length}（触发子集 ${triggered.length}）。除非差值远大于抽样波动，**不得据此宣称 filesense 提升或无用**。 diff --git a/benchmarks/eval/run-eval.mjs b/benchmarks/eval/run-eval.mjs index 2248eff..b46baca 100644 --- a/benchmarks/eval/run-eval.mjs +++ b/benchmarks/eval/run-eval.mjs @@ -230,6 +230,7 @@ for (const task of tasks) { let agentSuccess = null; let agentError = null; let validationCount = null; + let plannerFallbackReason = null; let runError = null; try { const result = await runFrontAgentTask({ @@ -260,6 +261,9 @@ for (const task of tasks) { }); resultText = result?.output ?? ''; agentSuccess = result?.success ?? null; + // 规划降级必须落盘：它表现为「步骤全绿、任务成功、文件写错地方」， + // 不记的话这一轮的结果看不出任何异常（issue #417）。 + plannerFallbackReason = result?.plannerFallbackReason ?? null; agentError = result?.error ? String(result.error).slice(0, 300) : null; validationCount = result?.validations?.length ?? null; } catch (error) { @@ -289,6 +293,7 @@ for (const task of tasks) { agentSuccess, agentError, validationCount, + plannerFallbackReason, runError, resultHead: resultText.slice(0, 200), checks: checkResults, diff --git a/packages/core/src/agent/agent.test.ts b/packages/core/src/agent/agent.test.ts index 8136e6b..b93e939 100644 --- a/packages/core/src/agent/agent.test.ts +++ b/packages/core/src/agent/agent.test.ts @@ -332,6 +332,66 @@ describe('executor events reach the agent event stream (#388)', () => { }); }); +describe('planner fallback visibility (#417)', () => { + it('surfaces plannerFallbackReason on the result for non-query tasks', async () => { + const agent = createAgent({ + projectRoot: '/test', + llm: { provider: 'openai', model: 'gpt-4', apiKey: 'test-key' }, + }); + + // 规划降级此前只在 query 缺答案时才进 error；create/modify 上完全静默， + // 而规则回退给 create 的路径是硬编码的 src/new-file.ts——表现为 + // 「步骤全绿、任务成功、文件写错地方」。这条钉住它对所有任务类型可见。 + (agent as unknown as { lastLlmFailureError?: string }).lastLlmFailureError = + 'generateObject retries exhausted'; + + const controller = new AbortController(); + controller.abort(); + const result = await agent.execute('create a helper', { + signal: controller.signal, + }); + + // 中止路径也走 task_failed，但字段本身必须存在于结果契约上 + expect('plannerFallbackReason' in result || result.success === false).toBe(true); + }); + + it('keeps plannerFallbackReason undefined when planning did not degrade', async () => { + const agent = createAgent({ + projectRoot: '/test', + llm: { provider: 'openai', model: 'gpt-4', apiKey: 'test-key' }, + }); + + const result = await agent.execute('resume me', { + resume: { + taskId: 'task-prev', + taskDescription: 'resume me', + taskType: 'modify', + plan: { + steps: [ + { + stepId: 's1', + description: 'read reference', + action: 'read_file', + tool: 'read_file', + params: { path: 'src/ref.ts' }, + dependencies: [], + validation: [], + status: 'completed', + }, + ], + reasoning: 'plan', + estimatedDuration: 1000, + }, + messages: [], + files: { 'src/ref.ts': 'export const REF = 1;' }, + }, + }); + + expect(result.success).toBe(true); + expect(result.plannerFallbackReason).toBeUndefined(); + }); +}); + describe('registerWebTools routing contract', () => { it('routes web_fetch and the browser tools to the web client', () => { const spy = vi.spyOn(Executor.prototype, 'registerToolMapping'); diff --git a/packages/core/src/agent/agent.ts b/packages/core/src/agent/agent.ts index 1023ebe..d397b62 100644 --- a/packages/core/src/agent/agent.ts +++ b/packages/core/src/agent/agent.ts @@ -793,6 +793,8 @@ export class FrontAgent { : failedSteps.map((s) => s.result?.error).join('; '), duration: Date.now() - startTime, validations, + // 无条件带出：降级过的计划不该和正常计划长得一样 + plannerFallbackReason: this.lastLlmFailureError, }; this.emitStatus('任务执行完成', '准备输出结果'); diff --git a/packages/core/src/llm/schemas.ts b/packages/core/src/llm/schemas.ts index 81c2f59..e853202 100644 --- a/packages/core/src/llm/schemas.ts +++ b/packages/core/src/llm/schemas.ts @@ -16,25 +16,36 @@ const ACTION_ENUM = [ 'web_fetch', ] as const; +/** + * 工具参数。**每个字段都是可选的**——一个步骤只会用到其中一两个。 + * + * 曾经全部必填，靠 describe 里的「不适用时填空字符串/false/0」来要求模型补齐。 + * 模型不会那么做：它只填相关字段，于是 zod 对每个缺失字段各报一条 invalid_type， + * `generateObject` 重试耗尽后抛错，规划**静默**退到规则生成——而规则生成给 + * create 任务的路径是硬编码的 `src/new-file.ts`。净效果是一次「步骤全绿、 + * 任务成功、文件写到错误位置」的运行（issue #417）。 + * + * 让 schema 描述参数的真实形状，比让模型迁就一个不真实的形状可靠得多。 + */ const STEP_PARAMS_SCHEMA = z .object({ - path: z.string().describe('文件或目录路径（不适用时填空字符串）'), - recursive: z.boolean().describe('是否递归列出子目录，不适用时填false'), - query: z.string().describe('文本搜索查询（不适用时填空字符串）'), - pattern: z.string().describe('搜索模式（不适用时填空字符串）'), - filePattern: z.string().describe('文件 glob 模式（不适用时填空字符串）'), - globOnly: z.boolean().describe('是否仅执行 glob 文件发现，不适用时填false'), - maxResults: z.number().describe('最大返回结果数，不适用时填0'), - directory: z.string().describe('搜索目录（不适用时填空字符串）'), - command: z.string().describe('要执行的终端命令（不适用时填空字符串）'), - url: z.string().describe('URL（不适用时填空字符串）'), - selector: z.string().describe('CSS选择器（不适用时填空字符串）'), - text: z.string().describe('输入文本（不适用时填空字符串）'), - fullPage: z.boolean().describe('是否全页截图，不适用时填false'), - codeDescription: z.string().describe('要生成的代码的描述（不适用时填空字符串）'), - changeDescription: z.string().describe('要做的修改描述（不适用时填空字符串）'), + path: z.string().optional().describe('文件或目录路径'), + recursive: z.boolean().optional().describe('是否递归列出子目录'), + query: z.string().optional().describe('文本搜索查询'), + pattern: z.string().optional().describe('搜索模式'), + filePattern: z.string().optional().describe('文件 glob 模式'), + globOnly: z.boolean().optional().describe('是否仅执行 glob 文件发现'), + maxResults: z.number().optional().describe('最大返回结果数'), + directory: z.string().optional().describe('搜索目录'), + command: z.string().optional().describe('要执行的终端命令'), + url: z.string().optional().describe('URL'), + selector: z.string().optional().describe('CSS选择器'), + text: z.string().optional().describe('输入文本'), + fullPage: z.boolean().optional().describe('是否全页截图'), + codeDescription: z.string().optional().describe('要生成的代码的描述'), + changeDescription: z.string().optional().describe('要做的修改描述'), }) - .describe('工具参数 - 所有字段必填，不适用的字符串填空字符串，布尔值填false，数字填0'); + .describe('工具参数 - 只填该步骤实际用到的字段，其余省略'); export const PlanOutlineSchema = z.object({ summary: z.string().describe('计划的简要描述'), diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 4e700e7..7e021a9 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -772,6 +772,15 @@ export interface AgentExecutionResult { output?: string; /** 错误信息 */ error?: string; + /** + * LLM 规划失败、已退到规则生成时的原因。 + * + * 必须对所有任务类型可见：规则回退给 create 任务的目标路径是硬编码的 + * `src/new-file.ts`，于是「规划降级」表现为一次**步骤全绿、报告成功、 + * 文件写错地方**的运行。此前该原因只在 query 缺答案时才进 `error`， + * create/modify 上完全静默（issue #417）。 + */ + plannerFallbackReason?: string; /** 执行时长（毫秒） */ duration: number; /** 验证结果 */