diff --git a/plugins/tracing/src/parse.ts b/plugins/tracing/src/parse.ts index 5c70645..86c6438 100644 --- a/plugins/tracing/src/parse.ts +++ b/plugins/tracing/src/parse.ts @@ -94,6 +94,16 @@ function extractToolError(payload: EventMsgPayload): string | undefined { type MutableTurn = Turn & { lastAgentMessage?: string; userInputFallback?: string }; +// A turn started by /goal receives its objective inside an internal context +// wrapper; surface the objective itself rather than the raw wrapper. +const GOAL_OBJECTIVE_RE = + /]*\bsource="goal"[^>]*>[\s\S]*?([\s\S]*?)<\/objective>/; + +function extractGoalObjective(text: string): string | null { + const m = text.match(GOAL_OBJECTIVE_RE); + return m ? m[1].trim() : null; +} + const TURN_OPENING_EVENTS = new Set(["user_message", "item_completed", "agent_message"]); function newTurn(startTime: number): MutableTurn { @@ -259,7 +269,7 @@ export function parseSession(lines: RolloutLine[]): { if (isInjectedContext) { collect(injectedContext, text); } else if (!turn!.userInputFallback) { - turn!.userInputFallback = text; + turn!.userInputFallback = extractGoalObjective(text) ?? text; } } } else if (p.type === "function_call") { @@ -373,10 +383,10 @@ export function parseSession(lines: RolloutLine[]): { else if (!turn) continue; if (et === "user_message" && typeof p.message === "string") { - if (!turn!.userInput) turn!.userInput = p.message; + if (!turn!.userInput && !turn!.userInputFallback) turn!.userInput = p.message; } else if (et === "item_completed" && p.item?.type === "UserMessage") { const text = extractMessageText(p.item.content); - if (text && !turn!.userInput) turn!.userInput = text; + if (text && !turn!.userInput && !turn!.userInputFallback) turn!.userInput = text; } else if (et === "item_completed" && p.item?.type === "SubAgentActivity") { if (p.item.kind === "started" && typeof p.item.agent_thread_id === "string") { recordSubagentThread(p.item.agent_thread_id, p.item.id); diff --git a/plugins/tracing/test/parse-goal-attribution.test.ts b/plugins/tracing/test/parse-goal-attribution.test.ts new file mode 100644 index 0000000..4352b74 --- /dev/null +++ b/plugins/tracing/test/parse-goal-attribution.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; + +import { parseSession } from "../src/parse.js"; + +// Synthetic fixture from issue #110: a /goal-started turn where a later +// in-flight user message arrives before task_complete. +const GOAL_TURN_WITH_PROGRESS_MESSAGE = [ + { timestamp: "2000-01-01T00:00:00Z", type: "session_meta", payload: { id: "example-thread" } }, + { + timestamp: "2000-01-01T00:00:01Z", + type: "event_msg", + payload: { type: "task_started", turn_id: "example-turn" }, + }, + { + timestamp: "2000-01-01T00:00:02Z", + type: "response_item", + payload: { + type: "message", + role: "user", + content: [ + { + type: "input_text", + text: '\nComplete the example task\n', + }, + ], + }, + }, + { + timestamp: "2000-01-01T00:00:03Z", + type: "event_msg", + payload: { + type: "item_completed", + item: { type: "UserMessage", content: [{ type: "text", text: "What is the progress?" }] }, + }, + }, + { + timestamp: "2000-01-01T00:00:04Z", + type: "event_msg", + payload: { type: "task_complete", turn_id: "example-turn" }, + }, + // eslint-disable-next-line @typescript-eslint/no-explicit-any +] as any[]; + +describe("goal turn input attribution (#110)", () => { + it("keeps the initiating goal objective as the turn input instead of a later in-flight user message", () => { + const { turns } = parseSession(GOAL_TURN_WITH_PROGRESS_MESSAGE); + expect(turns).toHaveLength(1); + expect(turns[0].userInput).toBe("Complete the example task"); + }); + + it("keeps the goal objective when no later user message arrives", () => { + const lines = GOAL_TURN_WITH_PROGRESS_MESSAGE.filter( + (line) => !(line.type === "event_msg" && line.payload?.type === "item_completed"), + ); + const { turns } = parseSession(lines); + expect(turns).toHaveLength(1); + expect(turns[0].userInput).toBe("Complete the example task"); + }); + + it("does not expose the raw internal context wrapper as the turn input", () => { + const { turns } = parseSession(GOAL_TURN_WITH_PROGRESS_MESSAGE); + expect(turns[0].userInput).not.toContain("codex_internal_context"); + }); +});