Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions e2e/acp.e2e.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import { expect, test } from "@playwright/test";

/**
* Issue #65 (ADR 0013 §5): run detail shows what an ACP agent reported —
* context against the window and cost per step, the run's summed cost, and
* the descriptive tool title in the transcript. Seeded as `run-static-acp`
* by `e2e/server.ts`; the live context fold is covered by `run-events.test.ts`.
*/
test("run detail shows context, cost and tool titles for an ACP run", async ({ page }) => {

Check warning on line 9 in e2e/acp.e2e.ts

View workflow job for this annotation

GitHub Actions / check

effecttsgo(async-function)

e2e/acp.e2e.ts:9:71: This code declares an async function, consider representing this async control flow with Effect values and `Effect.gen`.
await page.goto("/runs/run-static-acp");

// Run cost: both steps, the cancelled one included ($0.039 + $0.028).
await expect(page.getByTestId("run-cost")).toHaveText("$0.067");

const implement = page
.locator('[data-testid="step-row"][data-kind="agent"]')
.filter({ hasText: "implement" });
await expect(implement.getByTestId("step-meta")).toHaveText(/15\.3k\/200k ctx · \$0\.039/);
const review = page
.locator('[data-testid="step-row"][data-kind="agent"]')
.filter({ hasText: "review" });
await expect(review).toHaveAttribute("data-status", "cancelled");
await expect(review.getByTestId("step-meta")).toHaveText(/9\.1k\/200k ctx · \$0\.028/);

await implement.click();
await expect(page.getByTestId("step-context")).toContainText("15.3k / 200k · 8%");
await expect(page.getByTestId("step-context-meter")).toHaveAttribute("data-share", "0.077");
await expect(page.getByTestId("step-cost")).toHaveText("$0.039");

await page.getByTestId("open-transcript").click();
const tool = page.getByTestId("transcript-tool");
await expect(tool).toHaveCount(1);
await expect(tool).toContainText("Edit math.ts");
await expect(tool).toContainText("edit · complete");
});

test("an old opencode log falls back to the derived context, without cost", async ({ page }) => {

Check warning on line 37 in e2e/acp.e2e.ts

View workflow job for this annotation

GitHub Actions / check

effecttsgo(async-function)

e2e/acp.e2e.ts:37:77: This code declares an async function, consider representing this async control flow with Effect values and `Effect.gen`.
await page.goto("/runs/run-static-corpus");
await expect(page.getByTestId("run-meta")).toContainText("cost");
await expect(page.getByTestId("run-cost")).toHaveCount(0);
const implement = page
.locator('[data-testid="step-row"][data-kind="agent"]')
.filter({ hasText: "implement" })
.first();
await expect(implement.getByTestId("step-meta")).toHaveText(/\d+(\.\d)?k ctx$/);
await implement.click();
await expect(page.getByTestId("step-context")).toContainText("(final turn)");
await expect(page.getByTestId("step-context-meter")).toHaveCount(0);
});
100 changes: 100 additions & 0 deletions e2e/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -245,9 +245,109 @@ async function seedCorpusRuns(root: string, db: ReturnType<typeof openStore>): P
payload: { _tag: "RunFinished", durationMs: 5 },
});

seedAcpRun(db, workDir, ts);

return remoteDir;
}

/**
* Issue #65: an agent step as the ACP adapter records it, shaped after the
* recorded finding-13 Claude run — `acp.usage` chunks while it runs, a tool
* call whose descriptive title arrives in an `acp.tool-call` chunk, and
* `AgentStepFinished.context` / `.cost`. A second step is cancelled
* mid-turn and keeps what it reported.
*/
function seedAcpRun(db: ReturnType<typeof openStore>, dir: string, ts: number): void {
const runId = "run-static-acp";
let seq = 0;
const append = (payload: Parameters<typeof appendEvent>[1]["payload"]) =>
appendEvent(db, { runId, seq: seq++, ts: ts + seq, payload });
const chunk = (stepId: string, value: Record<string, unknown>) =>
append({ _tag: "AgentChunk", stepId, chunkType: String(value.type), chunk: value as never });
const usage = (stepId: string, used: number, cost?: number) =>
chunk(stepId, {
type: "CUSTOM",
name: "acp.usage",
value: {
context: { used, size: 200_000 },
...(cost !== undefined && { cost: { amount: cost, currency: "USD" } }),
},
});

append({ _tag: "RunStarted", workflowId: "acp-hello", dir, input: {} });
append({
_tag: "AgentStepStarted",
stepId: "step-0",
name: "implement",
agent: "claude",
model: "haiku",
prompt: "add multiply to math.ts",
structured: false,
});
chunk("step-0", { type: "RUN_STARTED", runId: "r0", threadId: "t" });
usage("step-0", 14_874);
chunk("step-0", { type: "TOOL_CALL_START", toolCallId: "toolu_1", toolCallName: "edit" });
chunk("step-0", {
type: "TOOL_CALL_ARGS",
toolCallId: "toolu_1",
delta: '{"title":"Edit"}',
args: '{"title":"Edit"}',
});
chunk("step-0", {
type: "TOOL_CALL_END",
toolCallId: "toolu_1",
toolCallName: "edit",
input: { title: "Edit" },
});
chunk("step-0", {
type: "CUSTOM",
name: "acp.tool-call",
value: { toolCallId: "toolu_1", title: "Edit math.ts", input: { file_path: "math.ts" } },
});
chunk("step-0", {
type: "TOOL_CALL_RESULT",
toolCallId: "toolu_1",
messageId: "m1",
content: "The file math.ts has been updated.",
});
usage("step-0", 15_305, 0.0391796);
chunk("step-0", { type: "RUN_FINISHED", runId: "r0", threadId: "t", finishReason: "stop" });
append({
_tag: "AgentStepFinished",
stepId: "step-0",
name: "implement",
outcome: "completed",
chunkCount: 8,
durationMs: 7_800,
finalText: "Added multiply.",
usage: { inputTokens: 3, outputTokens: 120, cachedInputTokens: 36_386, reasoningTokens: 0 },
context: { used: 15_305, size: 200_000 },
cost: { amount: 0.0391796, currency: "USD" },
});
append({
_tag: "AgentStepStarted",
stepId: "step-1",
name: "review",
agent: "claude",
model: "haiku",
prompt: "review the change",
structured: false,
});
usage("step-1", 9_120);
append({
_tag: "AgentStepFinished",
stepId: "step-1",
name: "review",
outcome: "cancelled",
chunkCount: 1,
durationMs: 1_200,
finalText: "",
context: { used: 9_120, size: 200_000 },
cost: { amount: 0.0275702, currency: "USD" },
});
append({ _tag: "RunCancelled", durationMs: 9_100 });
}

async function main(): Promise<void> {
const port = Number(process.env.FACTORY_E2E_PORT ?? "5199");
const dbPath = process.env.FACTORY_E2E_DB ?? join(import.meta.dir, "../.factory/e2e/factory.db");
Expand Down
53 changes: 53 additions & 0 deletions src/events.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -419,3 +419,56 @@ describe("AgentStepStarted.agent (ADR 0013 §2)", () => {
expect(event.payload._tag === "AgentStepStarted" && "agent" in event.payload).toBe(false);
});
});

describe("AgentStepFinished.context / .cost (ADR 0013 §5)", () => {
const finished = (extra: Record<string, unknown>) => ({
runId: "r",
seq: 1,
ts: 0,
payload: {
_tag: "AgentStepFinished",
stepId: "step-0",
name: "implement",
outcome: "completed",
chunkCount: 3,
durationMs: 10,
finalText: "done",
...extra,
},
});
const encode = SchemaParser.encodeUnknownSync(RunEvent);

test("round-trips context and cost beside usage", () => {
const raw = finished({
usage: { inputTokens: 1, outputTokens: 2, cachedInputTokens: 3, reasoningTokens: 0 },
context: { used: 15_305, size: 200_000 },
cost: { amount: 0.0391796, currency: "USD" },
});
const event = decodeRunEvent(raw);
expect(event.payload).toMatchObject({
context: { used: 15_305, size: 200_000 },
cost: { amount: 0.0391796, currency: "USD" },
});
expect(encode(event) as unknown).toEqual(raw);
});

test("a log written before the fields existed has neither, and still decodes", () => {
const event = decodeRunEvent(
finished({
usage: {
inputTokens: 260,
outputTokens: 288,
cachedInputTokens: 14208,
reasoningTokens: 0,
},
}),
);
expect("context" in event.payload).toBe(false);
expect("cost" in event.payload).toBe(false);
});

test("rejects a context that is not a token count", () => {
expect(() => decodeRunEvent(finished({ context: { used: 1.5, size: 10 } }))).toThrow();
expect(() => decodeRunEvent(finished({ cost: { amount: 0.1 } }))).toThrow();
});
});
62 changes: 47 additions & 15 deletions src/events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -213,27 +213,27 @@ export const RunEventPayload = Schema.TaggedUnion({
/** From the `opencode.session-id` CUSTOM chunk. Fresh per step (D10). */
sessionId: Schema.optional(Schema.String),
/**
* Token counts from the step's `RUN_FINISHED.usage` chunk.
* Token counts from the step's `RUN_FINISHED.usage` chunk, **as the agent
* reports them** (ADR 0013 §5). What they cover depends on the step's
* `agent` (`AgentStepStarted.agent`):
*
* - **opencode**: the *last* assistant message only. One step runs
* opencode's whole agent loop, often dozens of model calls, and only the
* final one is reported. Logs from before ACP (no `agent`) mean the same.
* - **claude**: the *whole turn*, summed over its model calls, so the
* cached-read figure can be a multiple of the context size.
*
* Do not bill off these; `cost` exists for that. For the context in use,
* read `context`; `agentStepContextTokens` derives it from these
* components only for logs written before `context` existed.
*
* Absent when the step produced no `RUN_FINISHED` (cancelled mid-stream, or
* a failure that surfaced as `RUN_ERROR` — the adapter emits one or the
* other, never both).
*
* ONLY THE FINAL ASSISTANT MESSAGE IS REPRESENTED HERE. One `ctx.agent()`
* step is one opencode `session.prompt()`, which internally runs opencode's
* whole agent loop — commonly 50-150 assistant messages. Each has its own
* token counts, but `@tanstack/ai-opencode` builds `RUN_FINISHED.usage`
* from the terminal `done` message alone and drops the rest (its translator
* ignores `message.updated`, which is where the per-message counts arrive).
* So these are the *last turn's* numbers, not the step's cumulative spend —
* do not bill off them. Recovering the true cumulative figure needs
* opencode's own message store, keyed by `sessionId`.
*
* Stored as the four raw components, deliberately without a total: the
* adapter's own `usage.totalTokens` is `input + output` and silently omits
* cache reads, which for a coding session are ~99% of the context. Consumers
* derive their own sum (see `agentStepContextTokens`); the components are
* individually correct, so that derivation survives an upstream fix.
* adapters' own `usage.totalTokens` is `input + output` and silently omits
* cache reads, which for a coding session are ~99% of the context.
*/
usage: Schema.optional(
Schema.Struct({
Expand All @@ -244,6 +244,24 @@ export const RunEventPayload = Schema.TaggedUnion({
reasoningTokens: Integer,
}),
),
/**
* The context in use and the window size, as the agent measured them
* (ADR 0013 §5), from the step's last `usage` signal (ACP
* `usage_update`). Present for cancelled and failed steps too, as far as
* the agent got. Absent in logs written before it existed and for steps
* whose agent never reported usage; there, `agentStepContextTokens(usage)`
* is the fallback.
*/
context: Schema.optional(Schema.Struct({ used: Integer, size: Integer })),
/**
* What the agent says the step cost (ADR 0013 §5): the latest cost its
* `usage` signals carried. A step is one ACP session, and the agents
* report the session's cost so far, so the latest figure is the step's.
* Recorded as reported: opencode says `0 USD` for a free model. Claude
* reports cost only at the end of a turn, so a step cancelled mid-turn
* usually has none.
*/
cost: Schema.optional(Schema.Struct({ amount: Schema.Finite, currency: Schema.String })),
/** From a `RUN_ERROR` chunk, or the abort reason. */
error: Schema.optional(Schema.String),
},
Expand Down Expand Up @@ -366,6 +384,16 @@ export type AgentStepUsage = NonNullable<
Extract<RunEventPayload, { _tag: "AgentStepFinished" }>["usage"]
>;

/** `AgentStepFinished.context`: tokens in context and the window size. */
export type AgentStepContext = NonNullable<
Extract<RunEventPayload, { _tag: "AgentStepFinished" }>["context"]
>;

/** `AgentStepFinished.cost`: an amount in a currency, as the agent reported it. */
export type AgentStepCost = NonNullable<
Extract<RunEventPayload, { _tag: "AgentStepFinished" }>["cost"]
>;

/**
* Size of the context the step's final assistant turn ran against: prompt plus
* completion plus the cached prefix. This is the number opencode itself
Expand All @@ -376,6 +404,10 @@ export type AgentStepUsage = NonNullable<
* Cache reads are added rather than assumed subsumed because opencode reports
* `input` exclusive of them (the Anthropic convention). It is emphatically not
* the step's cumulative token spend — see `AgentStepFinished.usage`.
*
* Superseded by `AgentStepFinished.context.used` where present (ADR 0013 §5);
* kept as the fallback for logs written before that field, which all ran on
* opencode, where this derivation is exact.
*/
export function agentStepContextTokens(usage: AgentStepUsage): number {
return usage.inputTokens + usage.outputTokens + usage.cachedInputTokens;
Expand Down
21 changes: 21 additions & 0 deletions src/runtime/acp-adapter.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,27 @@ describe("acpAdapter", () => {
});
});

test("carries each tool call title the translator drops as an acp.tool-call chunk", async () => {
const c = await collect("tools");
const chunks = c.yields.map((y) => y.chunk as { type: string; name?: string; value?: unknown });
const tools = chunks.filter((ch) => ch.type === "CUSTOM" && ch.name === "acp.tool-call");
expect(tools.map((ch) => ch.value)).toEqual([
{ toolCallId: "t1", title: "Edit" },
{ toolCallId: "t1", title: "Edit math.ts", input: { file_path: "math.ts" } },
]);
// The translator keeps the first, generic title only.
const end = chunks.find((ch) => ch.type === "TOOL_CALL_END") as { input?: unknown };
expect(end.input).toEqual({ title: "Edit" });
// Each info chunk follows the call's TOOL_CALL_START, and carries no signal.
expect(chunks.indexOf(tools[0]!)).toBeGreaterThan(
chunks.findIndex((ch) => ch.type === "TOOL_CALL_START"),
);
expect(c.yields.filter((y) => tools.includes(y.chunk as never)).map((y) => y.signal)).toEqual([
undefined,
undefined,
]);
});

test("parses structured output from the last message and yields it before RUN_FINISHED", async () => {
const c = await collect("json", { outputSchema: { type: "object" } });
expect(signals(c, "structuredOutput")).toEqual([
Expand Down
Loading
Loading