diff --git a/src/web-ui/src/flow_chat/components/modern/BackgroundRunItem.tsx b/src/web-ui/src/flow_chat/components/modern/BackgroundRunItem.tsx index 950595561c..f677e1f8cb 100644 --- a/src/web-ui/src/flow_chat/components/modern/BackgroundRunItem.tsx +++ b/src/web-ui/src/flow_chat/components/modern/BackgroundRunItem.tsx @@ -4,6 +4,7 @@ import type { DialogTurn } from '../../types/flow-chat'; import { UserMessageItem } from './UserMessageItem'; import { ModelRoundItem } from './ModelRoundItem'; import { backgroundDeliveryFromMetadata } from '../../utils/backgroundDelivery'; +import { getTurnOutputDurationMs } from '../../utils/turnUsageMetrics'; import './BackgroundRunItem.scss'; /** A delivery receipt is presentation only; expanding it reads the original execution. */ @@ -15,6 +16,7 @@ export function BackgroundRunItem({ turn, absoluteTurnIndex, forceOpen = false, const [open, setOpen] = useState(forceOpen); useEffect(() => { if (forceOpen) setOpen(true); }, [forceOpen]); const completed = ['completed', 'finishing'].includes(turn.status); + const turnOutputDurationMs = getTurnOutputDurationMs(turn.modelRounds); return (
{turn.modelRounds.map((round, index) => )} + turnStartedAt={turn.startTime} turnEndedAt={turn.endTime} + turnOutputDurationMs={turnOutputDurationMs} turnTokenUsage={turn.tokenUsage} />)} ) : null}
diff --git a/src/web-ui/src/flow_chat/components/modern/ModelRoundItem.tsx b/src/web-ui/src/flow_chat/components/modern/ModelRoundItem.tsx index 1133c34dc5..77044a589c 100644 --- a/src/web-ui/src/flow_chat/components/modern/ModelRoundItem.tsx +++ b/src/web-ui/src/flow_chat/components/modern/ModelRoundItem.tsx @@ -371,6 +371,7 @@ export const ModelRoundItem = React.memo( turnStartedAt, turnEndedAt, turnDurationMs, + turnOutputDurationMs, turnTokenUsage, canvasArtifactItems = [], expandedThinkingItemIds = [], @@ -551,7 +552,9 @@ export const ModelRoundItem = React.memo( ); const completedAt = turnEndedAt ?? round.endTime; - const usageDurationMs = turnDurationMs ?? + // Throughput must divide by streaming time, not turn wall time: the latter + // also counts tool execution and waiting and would report a deflated rate. + const usageDurationMs = turnOutputDurationMs ?? turnDurationMs ?? (typeof turnStartedAt === 'number' && typeof completedAt === 'number' ? Math.max(0, completedAt - turnStartedAt) : undefined); const effectiveDurationMs = turnDurationMs ?? diff --git a/src/web-ui/src/flow_chat/components/modern/VirtualItemRenderer.tsx b/src/web-ui/src/flow_chat/components/modern/VirtualItemRenderer.tsx index 384e970afd..e370e13a5e 100644 --- a/src/web-ui/src/flow_chat/components/modern/VirtualItemRenderer.tsx +++ b/src/web-ui/src/flow_chat/components/modern/VirtualItemRenderer.tsx @@ -100,6 +100,7 @@ export const VirtualItemRenderer = React.memo( turnStartedAt={item.turnStartedAt} turnEndedAt={item.turnEndedAt} turnDurationMs={item.turnDurationMs} + turnOutputDurationMs={item.turnOutputDurationMs} turnTokenUsage={item.turnTokenUsage} canvasArtifactItems={item.canvasArtifactItems} expandedThinkingItemIds={item.layoutHints?.expandedThinkingItemIds ?? []} diff --git a/src/web-ui/src/flow_chat/components/modern/modelRoundItemMemo.ts b/src/web-ui/src/flow_chat/components/modern/modelRoundItemMemo.ts index 4060db0eb8..5acb023f56 100644 --- a/src/web-ui/src/flow_chat/components/modern/modelRoundItemMemo.ts +++ b/src/web-ui/src/flow_chat/components/modern/modelRoundItemMemo.ts @@ -13,6 +13,7 @@ export interface ModelRoundItemProps { turnStartedAt?: number; turnEndedAt?: number; turnDurationMs?: number; + turnOutputDurationMs?: number; turnTokenUsage?: TokenUsage; canvasArtifactItems?: FlowToolItem[]; expandedThinkingItemIds?: string[]; @@ -59,6 +60,7 @@ export function areModelRoundItemPropsEqual(prev: ModelRoundItemProps, next: Mod prev.turnStartedAt === next.turnStartedAt && prev.turnEndedAt === next.turnEndedAt && prev.turnDurationMs === next.turnDurationMs && + prev.turnOutputDurationMs === next.turnOutputDurationMs && prev.turnTokenUsage === next.turnTokenUsage && prev.canvasArtifactItems === next.canvasArtifactItems ); diff --git a/src/web-ui/src/flow_chat/store/modernFlowChatStore.test.ts b/src/web-ui/src/flow_chat/store/modernFlowChatStore.test.ts index 0fc29fa7b2..905eef7759 100644 --- a/src/web-ui/src/flow_chat/store/modernFlowChatStore.test.ts +++ b/src/web-ui/src/flow_chat/store/modernFlowChatStore.test.ts @@ -647,6 +647,8 @@ describe('sessionToVirtualItems explore grouping', () => { modelRounds: [makeRound({ id: 'round-with-answer', items: [makeTextItem('text-final', 'Here is the answer.')], + streamDurationMs: 1200, + firstChunkMs: 200, })], status: 'completed', startTime: 900, @@ -667,6 +669,7 @@ describe('sessionToVirtualItems explore grouping', () => { turnStartedAt: 900, turnEndedAt: 2400, turnDurationMs: 1500, + turnOutputDurationMs: 1000, turnTokenUsage: { inputTokens: 1200, outputTokens: 300, diff --git a/src/web-ui/src/flow_chat/store/modernFlowChatStore.ts b/src/web-ui/src/flow_chat/store/modernFlowChatStore.ts index 34a64a29bf..e4e28bd403 100644 --- a/src/web-ui/src/flow_chat/store/modernFlowChatStore.ts +++ b/src/web-ui/src/flow_chat/store/modernFlowChatStore.ts @@ -13,6 +13,7 @@ import { flowChatStore } from './FlowChatStore'; import { getTurnCompletionNotice } from '../utils/turnCompletionNotice'; import { createAbsoluteSessionTurnIndexResolver } from '../utils/flowChatTurnOrdinal'; import { collectCanvasArtifactToolItems } from '../utils/canvasArtifactPresentation'; +import { getTurnOutputDurationMs } from '../utils/turnUsageMetrics'; import { projectAdjacentFlowGroups } from '../grouping/groupProjection'; import { hasModelRoundNarrative } from '../grouping/roundGroups'; import { isSuppressedBackgroundTurn } from '../utils/backgroundDelivery'; @@ -261,6 +262,7 @@ export function sessionToVirtualItems(session: Session | null): VirtualItem[] { }); const isTurnComplete = turn.status === 'completed' || turn.status === 'cancelled' || turn.status === 'error'; + const turnOutputDurationMs = getTurnOutputDurationMs(turn.modelRounds); const canvasArtifactItems = collectCanvasArtifactToolItems(turn.modelRounds); const canvasAttachmentHostRoundId = [...renderEntries] .reverse() @@ -293,6 +295,7 @@ export function sessionToVirtualItems(session: Session | null): VirtualItem[] { turnDurationMs: typeof turn.endTime === 'number' ? Math.max(0, turn.endTime - turn.startTime) : undefined, + turnOutputDurationMs, turnTokenUsage: turn.tokenUsage, canvasArtifactItems: isTurnComplete && round.id === canvasAttachmentHostRoundId ? canvasArtifactItems diff --git a/src/web-ui/src/flow_chat/types/flow-chat-projection.ts b/src/web-ui/src/flow_chat/types/flow-chat-projection.ts index 5358f8cd72..a7cb6b0adc 100644 --- a/src/web-ui/src/flow_chat/types/flow-chat-projection.ts +++ b/src/web-ui/src/flow_chat/types/flow-chat-projection.ts @@ -64,6 +64,8 @@ type VirtualContent = turnStartedAt?: number; turnEndedAt?: number; turnDurationMs?: number; + /** Streaming window only; the throughput denominator, not turn wall time. */ + turnOutputDurationMs?: number; turnTokenUsage?: TokenUsage; canvasArtifactItems?: FlowToolItem[]; } diff --git a/src/web-ui/src/flow_chat/utils/turnUsageMetrics.test.ts b/src/web-ui/src/flow_chat/utils/turnUsageMetrics.test.ts index 402cf8185d..74e8779151 100644 --- a/src/web-ui/src/flow_chat/utils/turnUsageMetrics.test.ts +++ b/src/web-ui/src/flow_chat/utils/turnUsageMetrics.test.ts @@ -1,12 +1,19 @@ import { describe, expect, it } from 'vitest'; -import type { TokenUsage } from '../types/flow-chat'; -import { getTurnUsageMetrics } from './turnUsageMetrics'; +import type { ModelRound, TokenUsage } from '../types/flow-chat'; +import { getTurnOutputDurationMs, getTurnUsageMetrics } from './turnUsageMetrics'; const usage: TokenUsage = { inputTokens: 10_000, cachedTokens: 7_200, outputTokens: 2_800, totalTokens: 12_800, timestamp: 1, }; +function round(timing: Pick): ModelRound { + return { + id: 'round', index: 0, items: [], isStreaming: false, isComplete: true, + status: 'completed', startTime: 0, ...timing, + }; +} + describe('getTurnUsageMetrics', () => { it('uses input tokens as the cache denominator and output tokens as the rate numerator', () => { expect(getTurnUsageMetrics(usage, 40_000)).toEqual({ @@ -40,4 +47,45 @@ describe('getTurnUsageMetrics', () => { expect(getTurnUsageMetrics({ ...usage, cachedTokens: NaN, totalTokens: Infinity, outputTokens: -1 }, 1_000)) .toEqual({ totalTokens: null, cacheHitRate: null, outputRate: null, speedLevel: null }); }); + + it('divides by streaming time so tool execution cannot deflate the rate', () => { + const rounds = [round({ streamDurationMs: 12_000, firstChunkMs: 2_000 })]; + // The turn took 90s wall time; the model only streamed for 10s. + expect(getTurnUsageMetrics({ ...usage, outputTokens: 2_000 }, getTurnOutputDurationMs(rounds))) + .toMatchObject({ outputRate: 200, speedLevel: 4 }); + }); +}); + +describe('getTurnOutputDurationMs', () => { + it('subtracts time to the first upstream item from the stream window', () => { + expect(getTurnOutputDurationMs([round({ streamDurationMs: 12_000, firstChunkMs: 2_000 })])).toBe(10_000); + }); + + it('falls back to the visible-output mark when the first chunk is unrecorded', () => { + expect(getTurnOutputDurationMs([round({ streamDurationMs: 9_000, firstVisibleOutputMs: 1_500 })])).toBe(7_500); + }); + + it('uses the whole window when no first-output mark exists', () => { + expect(getTurnOutputDurationMs([round({ streamDurationMs: 9_000 })])).toBe(9_000); + }); + + it('sums every round of a multi-round turn', () => { + expect(getTurnOutputDurationMs([ + round({ streamDurationMs: 5_000, firstChunkMs: 1_000 }), + round({ streamDurationMs: 7_000, firstChunkMs: 2_000 }), + ])).toBe(9_000); + }); + + it('never reports a negative or invented window', () => { + expect(getTurnOutputDurationMs([round({ streamDurationMs: 1_000, firstChunkMs: 4_000 })])).toBeUndefined(); + expect(getTurnOutputDurationMs([round({ streamDurationMs: 2_000, firstChunkMs: 2_000 })])).toBeUndefined(); + }); + + it('skips rounds without stream timing and reports nothing when none have it', () => { + expect(getTurnOutputDurationMs([round({}), round({ streamDurationMs: 6_000, firstChunkMs: 1_000 })])) + .toBe(5_000); + expect(getTurnOutputDurationMs([round({}), round({})])).toBeUndefined(); + expect(getTurnOutputDurationMs([])).toBeUndefined(); + expect(getTurnOutputDurationMs(undefined)).toBeUndefined(); + }); }); diff --git a/src/web-ui/src/flow_chat/utils/turnUsageMetrics.ts b/src/web-ui/src/flow_chat/utils/turnUsageMetrics.ts index 64dbd4a3a7..4d46ed93f5 100644 --- a/src/web-ui/src/flow_chat/utils/turnUsageMetrics.ts +++ b/src/web-ui/src/flow_chat/utils/turnUsageMetrics.ts @@ -1,18 +1,54 @@ -import type { TokenUsage } from '../types/flow-chat'; +import type { ModelRound, TokenUsage } from '../types/flow-chat'; function isCount(value: unknown): value is number { return typeof value === 'number' && Number.isFinite(value) && value >= 0; } -export function getTurnUsageMetrics(usage?: TokenUsage, durationMs?: number) { +function isPositiveMs(value: unknown): value is number { + return typeof value === 'number' && Number.isFinite(value) && value > 0; +} + +/** + * Round-level milliseconds spent actually receiving model output. + * + * `streamDurationMs` already excludes request handshake and provider queueing, + * but it still starts before the first token arrives. Subtract the time to the + * first upstream item so prompt prefill and reasoning warm-up are not charged + * to generation. + */ +function roundOutputDurationMs(round: ModelRound): number | undefined { + if (!isPositiveMs(round.streamDurationMs)) return undefined; + const firstOutputMs = isPositiveMs(round.firstChunkMs) ? round.firstChunkMs + : isPositiveMs(round.firstVisibleOutputMs) ? round.firstVisibleOutputMs : 0; + return Math.max(0, round.streamDurationMs - Math.min(firstOutputMs, round.streamDurationMs)); +} + +/** + * Milliseconds a turn spent streaming model output, summed over its rounds. + * + * Turn wall time also covers tool execution, permission waits and retries, so + * dividing output tokens by it understates throughput by a wide margin. Returns + * undefined when no round reports stream timing (older sessions), letting the + * caller fall back instead of inventing a rate. + */ +export function getTurnOutputDurationMs(rounds?: readonly ModelRound[]): number | undefined { + if (!rounds?.length) return undefined; + let totalMs = 0; + for (const round of rounds) { + totalMs += roundOutputDurationMs(round) ?? 0; + } + return totalMs > 0 ? totalMs : undefined; +} + +export function getTurnUsageMetrics(usage?: TokenUsage, outputDurationMs?: number) { const totalTokens = isCount(usage?.totalTokens) ? usage.totalTokens : null; const cacheHitRate = isCount(usage?.cachedTokens) && isCount(usage?.inputTokens) && usage.inputTokens > 0 && usage.cachedTokens <= usage.inputTokens ? usage.cachedTokens / usage.inputTokens : null; - // Turn wall time is available live and after hydration, including tool/wait time. - // It must not be presented as the provider's pure streaming throughput. - const rate = isCount(usage?.outputTokens) && isCount(durationMs) && durationMs > 0 - ? usage.outputTokens / (durationMs / 1000) : null; + // `outputDurationMs` must be the streaming window from `getTurnOutputDurationMs`, + // never turn wall time: tool execution and waiting would deflate the rate. + const rate = isCount(usage?.outputTokens) && isCount(outputDurationMs) && outputDurationMs > 0 + ? usage.outputTokens / (outputDurationMs / 1000) : null; const outputRate = rate !== null && Number.isFinite(rate) ? rate : null; const speedLevel: 1 | 2 | 3 | 4 | null = outputRate === null ? null : outputRate < 15 ? 1 : outputRate < 40 ? 2 : outputRate < 80 ? 3 : 4; diff --git a/src/web-ui/src/locales/en-US/flow-chat.json b/src/web-ui/src/locales/en-US/flow-chat.json index d6db059fa2..3ad4dce79c 100644 --- a/src/web-ui/src/locales/en-US/flow-chat.json +++ b/src/web-ui/src/locales/en-US/flow-chat.json @@ -1732,7 +1732,7 @@ "averageRate": "Average output", "speedLevel": "Speed level", "rateValue": "{{value}} t/s", - "rateBasis": "Output tokens divided by total turn time, including tools and waiting; speed levels change at 15, 40 and 80 tokens/s", + "rateBasis": "Output tokens divided by the time from the first token to end of stream, excluding tool execution, queueing and waiting; speed levels change at 15, 40 and 80 tokens/s", "slow": "Slow", "normal": "Normal", "fast": "Fast", diff --git a/src/web-ui/src/locales/zh-CN/flow-chat.json b/src/web-ui/src/locales/zh-CN/flow-chat.json index ea5e6a4471..4cf1ab09bc 100644 --- a/src/web-ui/src/locales/zh-CN/flow-chat.json +++ b/src/web-ui/src/locales/zh-CN/flow-chat.json @@ -1732,7 +1732,7 @@ "averageRate": "平均输出", "speedLevel": "速度档位", "rateValue": "{{value}} t/s", - "rateBasis": "输出 Token 除以本轮总耗时，包含工具执行与等待；档位分界为 15、40、80 Token/秒", + "rateBasis": "输出 Token 除以从首个 Token 到流结束的时长，不含工具执行、排队与等待；档位分界为 15、40、80 Token/秒", "slow": "慢", "normal": "一般", "fast": "快", diff --git a/src/web-ui/src/locales/zh-TW/flow-chat.json b/src/web-ui/src/locales/zh-TW/flow-chat.json index 6eb4f23aa4..2fe94fd61e 100644 --- a/src/web-ui/src/locales/zh-TW/flow-chat.json +++ b/src/web-ui/src/locales/zh-TW/flow-chat.json @@ -1732,7 +1732,7 @@ "averageRate": "平均輸出", "speedLevel": "速度檔位", "rateValue": "{{value}} t/s", - "rateBasis": "輸出 Token 除以本輪總耗時，包含工具執行與等待；檔位分界為 15、40、80 Token/秒", + "rateBasis": "輸出 Token 除以從首個 Token 到流結束的時長，不含工具執行、排隊與等待；檔位分界為 15、40、80 Token/秒", "slow": "慢", "normal": "一般", "fast": "快",