Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions plugins/tracing/src/parse.ts
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,16 @@ function extractToolError(payload: EventMsgPayload): string | undefined {

type MutableTurn = Turn & { lastAgentMessage?: string; userInputFallback?: string };

// A turn started by /goal receives its objective inside an internal context
// wrapper; surface the objective itself rather than the raw wrapper.
const GOAL_OBJECTIVE_RE =
/<codex_internal_context\b[^>]*\bsource="goal"[^>]*>[\s\S]*?<objective>([\s\S]*?)<\/objective>/;

function extractGoalObjective(text: string): string | null {
const m = text.match(GOAL_OBJECTIVE_RE);
return m ? m[1].trim() : null;
}

const TURN_OPENING_EVENTS = new Set(["user_message", "item_completed", "agent_message"]);

function newTurn(startTime: number): MutableTurn {
Expand Down Expand Up @@ -259,7 +269,7 @@ export function parseSession(lines: RolloutLine[]): {
if (isInjectedContext) {
collect(injectedContext, text);
} else if (!turn!.userInputFallback) {
turn!.userInputFallback = text;
turn!.userInputFallback = extractGoalObjective(text) ?? text;
}
}
} else if (p.type === "function_call") {
Expand Down Expand Up @@ -373,10 +383,10 @@ export function parseSession(lines: RolloutLine[]): {
else if (!turn) continue;

if (et === "user_message" && typeof p.message === "string") {
if (!turn!.userInput) turn!.userInput = p.message;
if (!turn!.userInput && !turn!.userInputFallback) turn!.userInput = p.message;
} else if (et === "item_completed" && p.item?.type === "UserMessage") {
const text = extractMessageText(p.item.content);
if (text && !turn!.userInput) turn!.userInput = text;
if (text && !turn!.userInput && !turn!.userInputFallback) turn!.userInput = text;
} else if (et === "item_completed" && p.item?.type === "SubAgentActivity") {
if (p.item.kind === "started" && typeof p.item.agent_thread_id === "string") {
recordSubagentThread(p.item.agent_thread_id, p.item.id);
Expand Down
64 changes: 64 additions & 0 deletions plugins/tracing/test/parse-goal-attribution.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import { describe, expect, it } from "vitest";

import { parseSession } from "../src/parse.js";

// Synthetic fixture from issue #110: a /goal-started turn where a later
// in-flight user message arrives before task_complete.
const GOAL_TURN_WITH_PROGRESS_MESSAGE = [
{ timestamp: "2000-01-01T00:00:00Z", type: "session_meta", payload: { id: "example-thread" } },
{
timestamp: "2000-01-01T00:00:01Z",
type: "event_msg",
payload: { type: "task_started", turn_id: "example-turn" },
},
{
timestamp: "2000-01-01T00:00:02Z",
type: "response_item",
payload: {
type: "message",
role: "user",
content: [
{
type: "input_text",
text: '<codex_internal_context source="goal">\n<objective>Complete the example task</objective>\n</codex_internal_context>',
},
],
},
},
{
timestamp: "2000-01-01T00:00:03Z",
type: "event_msg",
payload: {
type: "item_completed",
item: { type: "UserMessage", content: [{ type: "text", text: "What is the progress?" }] },
},
},
{
timestamp: "2000-01-01T00:00:04Z",
type: "event_msg",
payload: { type: "task_complete", turn_id: "example-turn" },
},
// eslint-disable-next-line @typescript-eslint/no-explicit-any
] as any[];

describe("goal turn input attribution (#110)", () => {
it("keeps the initiating goal objective as the turn input instead of a later in-flight user message", () => {
const { turns } = parseSession(GOAL_TURN_WITH_PROGRESS_MESSAGE);
expect(turns).toHaveLength(1);
expect(turns[0].userInput).toBe("Complete the example task");
});

it("keeps the goal objective when no later user message arrives", () => {
const lines = GOAL_TURN_WITH_PROGRESS_MESSAGE.filter(
(line) => !(line.type === "event_msg" && line.payload?.type === "item_completed"),
);
const { turns } = parseSession(lines);
expect(turns).toHaveLength(1);
expect(turns[0].userInput).toBe("Complete the example task");
});

it("does not expose the raw internal context wrapper as the turn input", () => {
const { turns } = parseSession(GOAL_TURN_WITH_PROGRESS_MESSAGE);
expect(turns[0].userInput).not.toContain("codex_internal_context");
});
});