From 4356ab6e682a66f7c8b3ef3210f91f3b2fe69686 Mon Sep 17 00:00:00 2001 From: Bob Lee Date: Sun, 18 Oct 2026 17:55:04 +0800 Subject: [PATCH 1/2] fix(flow-chat): report token throughput from streaming time The per-turn t/s figure divided output tokens by turn wall time, which also covers tool execution, permission waits and retries. On tool-heavy turns that understated the model's actual generation speed by an order of magnitude. Derive the rate from the streaming window instead. Per model round, subtract the time to the first upstream item from stream_duration_ms so handshake, queueing and prefill are not charged to generation, then sum those windows across the turn's rounds to match the turn-level output token count. Rounds without stream timing (older persisted sessions) keep the previous wall-time fallback instead of reporting an invented rate. The turn elapsed time shown as completion metadata still uses wall time, since that is a duration, not throughput. Co-authored-by: OpenBitFun <318544290+bitfun-ai@users.noreply.github.com> --- .../components/modern/BackgroundRunItem.tsx | 5 +- .../components/modern/ModelRoundItem.tsx | 5 +- .../components/modern/VirtualItemRenderer.tsx | 1 + .../components/modern/modelRoundItemMemo.ts | 2 + .../store/modernFlowChatStore.test.ts | 3 ++ .../flow_chat/store/modernFlowChatStore.ts | 3 ++ .../flow_chat/types/flow-chat-projection.ts | 2 + .../flow_chat/utils/turnUsageMetrics.test.ts | 52 ++++++++++++++++++- .../src/flow_chat/utils/turnUsageMetrics.ts | 48 ++++++++++++++--- src/web-ui/src/locales/en-US/flow-chat.json | 2 +- src/web-ui/src/locales/zh-CN/flow-chat.json | 2 +- src/web-ui/src/locales/zh-TW/flow-chat.json | 2 +- 12 files changed, 114 insertions(+), 13 deletions(-) diff --git a/src/web-ui/src/flow_chat/components/modern/BackgroundRunItem.tsx b/src/web-ui/src/flow_chat/components/modern/BackgroundRunItem.tsx index 950595561c..f677e1f8cb 100644 --- a/src/web-ui/src/flow_chat/components/modern/BackgroundRunItem.tsx +++ b/src/web-ui/src/flow_chat/components/modern/BackgroundRunItem.tsx @@ -4,6 +4,7 @@ import type { DialogTurn } from '../../types/flow-chat'; import { UserMessageItem } from './UserMessageItem'; import { ModelRoundItem } from './ModelRoundItem'; import { backgroundDeliveryFromMetadata } from '../../utils/backgroundDelivery'; +import { getTurnOutputDurationMs } from '../../utils/turnUsageMetrics'; import './BackgroundRunItem.scss'; /** A delivery receipt is presentation only; expanding it reads the original execution. */ @@ -15,6 +16,7 @@ export function BackgroundRunItem({ turn, absoluteTurnIndex, forceOpen = false, const [open, setOpen] = useState(forceOpen); useEffect(() => { if (forceOpen) setOpen(true); }, [forceOpen]); const completed = ['completed', 'finishing'].includes(turn.status); + const turnOutputDurationMs = getTurnOutputDurationMs(turn.modelRounds); return (
{turn.modelRounds.map((round, index) => )} + turnStartedAt={turn.startTime} turnEndedAt={turn.endTime} + turnOutputDurationMs={turnOutputDurationMs} turnTokenUsage={turn.tokenUsage} />)} ) : null}
diff --git a/src/web-ui/src/flow_chat/components/modern/ModelRoundItem.tsx b/src/web-ui/src/flow_chat/components/modern/ModelRoundItem.tsx index 1133c34dc5..77044a589c 100644 --- a/src/web-ui/src/flow_chat/components/modern/ModelRoundItem.tsx +++ b/src/web-ui/src/flow_chat/components/modern/ModelRoundItem.tsx @@ -371,6 +371,7 @@ export const ModelRoundItem = React.memo( turnStartedAt, turnEndedAt, turnDurationMs, + turnOutputDurationMs, turnTokenUsage, canvasArtifactItems = [], expandedThinkingItemIds = [], @@ -551,7 +552,9 @@ export const ModelRoundItem = React.memo( ); const completedAt = turnEndedAt ?? round.endTime; - const usageDurationMs = turnDurationMs ?? + // Throughput must divide by streaming time, not turn wall time: the latter + // also counts tool execution and waiting and would report a deflated rate. + const usageDurationMs = turnOutputDurationMs ?? turnDurationMs ?? (typeof turnStartedAt === 'number' && typeof completedAt === 'number' ? Math.max(0, completedAt - turnStartedAt) : undefined); const effectiveDurationMs = turnDurationMs ?? diff --git a/src/web-ui/src/flow_chat/components/modern/VirtualItemRenderer.tsx b/src/web-ui/src/flow_chat/components/modern/VirtualItemRenderer.tsx index 384e970afd..e370e13a5e 100644 --- a/src/web-ui/src/flow_chat/components/modern/VirtualItemRenderer.tsx +++ b/src/web-ui/src/flow_chat/components/modern/VirtualItemRenderer.tsx @@ -100,6 +100,7 @@ export const VirtualItemRenderer = React.memo( turnStartedAt={item.turnStartedAt} turnEndedAt={item.turnEndedAt} turnDurationMs={item.turnDurationMs} + turnOutputDurationMs={item.turnOutputDurationMs} turnTokenUsage={item.turnTokenUsage} canvasArtifactItems={item.canvasArtifactItems} expandedThinkingItemIds={item.layoutHints?.expandedThinkingItemIds ?? []} diff --git a/src/web-ui/src/flow_chat/components/modern/modelRoundItemMemo.ts b/src/web-ui/src/flow_chat/components/modern/modelRoundItemMemo.ts index 4060db0eb8..5acb023f56 100644 --- a/src/web-ui/src/flow_chat/components/modern/modelRoundItemMemo.ts +++ b/src/web-ui/src/flow_chat/components/modern/modelRoundItemMemo.ts @@ -13,6 +13,7 @@ export interface ModelRoundItemProps { turnStartedAt?: number; turnEndedAt?: number; turnDurationMs?: number; + turnOutputDurationMs?: number; turnTokenUsage?: TokenUsage; canvasArtifactItems?: FlowToolItem[]; expandedThinkingItemIds?: string[]; @@ -59,6 +60,7 @@ export function areModelRoundItemPropsEqual(prev: ModelRoundItemProps, next: Mod prev.turnStartedAt === next.turnStartedAt && prev.turnEndedAt === next.turnEndedAt && prev.turnDurationMs === next.turnDurationMs && + prev.turnOutputDurationMs === next.turnOutputDurationMs && prev.turnTokenUsage === next.turnTokenUsage && prev.canvasArtifactItems === next.canvasArtifactItems ); diff --git a/src/web-ui/src/flow_chat/store/modernFlowChatStore.test.ts b/src/web-ui/src/flow_chat/store/modernFlowChatStore.test.ts index 0fc29fa7b2..905eef7759 100644 --- a/src/web-ui/src/flow_chat/store/modernFlowChatStore.test.ts +++ b/src/web-ui/src/flow_chat/store/modernFlowChatStore.test.ts @@ -647,6 +647,8 @@ describe('sessionToVirtualItems explore grouping', () => { modelRounds: [makeRound({ id: 'round-with-answer', items: [makeTextItem('text-final', 'Here is the answer.')], + streamDurationMs: 1200, + firstChunkMs: 200, })], status: 'completed', startTime: 900, @@ -667,6 +669,7 @@ describe('sessionToVirtualItems explore grouping', () => { turnStartedAt: 900, turnEndedAt: 2400, turnDurationMs: 1500, + turnOutputDurationMs: 1000, turnTokenUsage: { inputTokens: 1200, outputTokens: 300, diff --git a/src/web-ui/src/flow_chat/store/modernFlowChatStore.ts b/src/web-ui/src/flow_chat/store/modernFlowChatStore.ts index 34a64a29bf..e4e28bd403 100644 --- a/src/web-ui/src/flow_chat/store/modernFlowChatStore.ts +++ b/src/web-ui/src/flow_chat/store/modernFlowChatStore.ts @@ -13,6 +13,7 @@ import { flowChatStore } from './FlowChatStore'; import { getTurnCompletionNotice } from '../utils/turnCompletionNotice'; import { createAbsoluteSessionTurnIndexResolver } from '../utils/flowChatTurnOrdinal'; import { collectCanvasArtifactToolItems } from '../utils/canvasArtifactPresentation'; +import { getTurnOutputDurationMs } from '../utils/turnUsageMetrics'; import { projectAdjacentFlowGroups } from '../grouping/groupProjection'; import { hasModelRoundNarrative } from '../grouping/roundGroups'; import { isSuppressedBackgroundTurn } from '../utils/backgroundDelivery'; @@ -261,6 +262,7 @@ export function sessionToVirtualItems(session: Session | null): VirtualItem[] { }); const isTurnComplete = turn.status === 'completed' || turn.status === 'cancelled' || turn.status === 'error'; + const turnOutputDurationMs = getTurnOutputDurationMs(turn.modelRounds); const canvasArtifactItems = collectCanvasArtifactToolItems(turn.modelRounds); const canvasAttachmentHostRoundId = [...renderEntries] .reverse() @@ -293,6 +295,7 @@ export function sessionToVirtualItems(session: Session | null): VirtualItem[] { turnDurationMs: typeof turn.endTime === 'number' ? Math.max(0, turn.endTime - turn.startTime) : undefined, + turnOutputDurationMs, turnTokenUsage: turn.tokenUsage, canvasArtifactItems: isTurnComplete && round.id === canvasAttachmentHostRoundId ? canvasArtifactItems diff --git a/src/web-ui/src/flow_chat/types/flow-chat-projection.ts b/src/web-ui/src/flow_chat/types/flow-chat-projection.ts index 5358f8cd72..a7cb6b0adc 100644 --- a/src/web-ui/src/flow_chat/types/flow-chat-projection.ts +++ b/src/web-ui/src/flow_chat/types/flow-chat-projection.ts @@ -64,6 +64,8 @@ type VirtualContent = turnStartedAt?: number; turnEndedAt?: number; turnDurationMs?: number; + /** Streaming window only; the throughput denominator, not turn wall time. */ + turnOutputDurationMs?: number; turnTokenUsage?: TokenUsage; canvasArtifactItems?: FlowToolItem[]; } diff --git a/src/web-ui/src/flow_chat/utils/turnUsageMetrics.test.ts b/src/web-ui/src/flow_chat/utils/turnUsageMetrics.test.ts index 402cf8185d..74e8779151 100644 --- a/src/web-ui/src/flow_chat/utils/turnUsageMetrics.test.ts +++ b/src/web-ui/src/flow_chat/utils/turnUsageMetrics.test.ts @@ -1,12 +1,19 @@ import { describe, expect, it } from 'vitest'; -import type { TokenUsage } from '../types/flow-chat'; -import { getTurnUsageMetrics } from './turnUsageMetrics'; +import type { ModelRound, TokenUsage } from '../types/flow-chat'; +import { getTurnOutputDurationMs, getTurnUsageMetrics } from './turnUsageMetrics'; const usage: TokenUsage = { inputTokens: 10_000, cachedTokens: 7_200, outputTokens: 2_800, totalTokens: 12_800, timestamp: 1, }; +function round(timing: Pick): ModelRound { + return { + id: 'round', index: 0, items: [], isStreaming: false, isComplete: true, + status: 'completed', startTime: 0, ...timing, + }; +} + describe('getTurnUsageMetrics', () => { it('uses input tokens as the cache denominator and output tokens as the rate numerator', () => { expect(getTurnUsageMetrics(usage, 40_000)).toEqual({ @@ -40,4 +47,45 @@ describe('getTurnUsageMetrics', () => { expect(getTurnUsageMetrics({ ...usage, cachedTokens: NaN, totalTokens: Infinity, outputTokens: -1 }, 1_000)) .toEqual({ totalTokens: null, cacheHitRate: null, outputRate: null, speedLevel: null }); }); + + it('divides by streaming time so tool execution cannot deflate the rate', () => { + const rounds = [round({ streamDurationMs: 12_000, firstChunkMs: 2_000 })]; + // The turn took 90s wall time; the model only streamed for 10s. + expect(getTurnUsageMetrics({ ...usage, outputTokens: 2_000 }, getTurnOutputDurationMs(rounds))) + .toMatchObject({ outputRate: 200, speedLevel: 4 }); + }); +}); + +describe('getTurnOutputDurationMs', () => { + it('subtracts time to the first upstream item from the stream window', () => { + expect(getTurnOutputDurationMs([round({ streamDurationMs: 12_000, firstChunkMs: 2_000 })])).toBe(10_000); + }); + + it('falls back to the visible-output mark when the first chunk is unrecorded', () => { + expect(getTurnOutputDurationMs([round({ streamDurationMs: 9_000, firstVisibleOutputMs: 1_500 })])).toBe(7_500); + }); + + it('uses the whole window when no first-output mark exists', () => { + expect(getTurnOutputDurationMs([round({ streamDurationMs: 9_000 })])).toBe(9_000); + }); + + it('sums every round of a multi-round turn', () => { + expect(getTurnOutputDurationMs([ + round({ streamDurationMs: 5_000, firstChunkMs: 1_000 }), + round({ streamDurationMs: 7_000, firstChunkMs: 2_000 }), + ])).toBe(9_000); + }); + + it('never reports a negative or invented window', () => { + expect(getTurnOutputDurationMs([round({ streamDurationMs: 1_000, firstChunkMs: 4_000 })])).toBeUndefined(); + expect(getTurnOutputDurationMs([round({ streamDurationMs: 2_000, firstChunkMs: 2_000 })])).toBeUndefined(); + }); + + it('skips rounds without stream timing and reports nothing when none have it', () => { + expect(getTurnOutputDurationMs([round({}), round({ streamDurationMs: 6_000, firstChunkMs: 1_000 })])) + .toBe(5_000); + expect(getTurnOutputDurationMs([round({}), round({})])).toBeUndefined(); + expect(getTurnOutputDurationMs([])).toBeUndefined(); + expect(getTurnOutputDurationMs(undefined)).toBeUndefined(); + }); }); diff --git a/src/web-ui/src/flow_chat/utils/turnUsageMetrics.ts b/src/web-ui/src/flow_chat/utils/turnUsageMetrics.ts index 64dbd4a3a7..4d46ed93f5 100644 --- a/src/web-ui/src/flow_chat/utils/turnUsageMetrics.ts +++ b/src/web-ui/src/flow_chat/utils/turnUsageMetrics.ts @@ -1,18 +1,54 @@ -import type { TokenUsage } from '../types/flow-chat'; +import type { ModelRound, TokenUsage } from '../types/flow-chat'; function isCount(value: unknown): value is number { return typeof value === 'number' && Number.isFinite(value) && value >= 0; } -export function getTurnUsageMetrics(usage?: TokenUsage, durationMs?: number) { +function isPositiveMs(value: unknown): value is number { + return typeof value === 'number' && Number.isFinite(value) && value > 0; +} + +/** + * Round-level milliseconds spent actually receiving model output. + * + * `streamDurationMs` already excludes request handshake and provider queueing, + * but it still starts before the first token arrives. Subtract the time to the + * first upstream item so prompt prefill and reasoning warm-up are not charged + * to generation. + */ +function roundOutputDurationMs(round: ModelRound): number | undefined { + if (!isPositiveMs(round.streamDurationMs)) return undefined; + const firstOutputMs = isPositiveMs(round.firstChunkMs) ? round.firstChunkMs + : isPositiveMs(round.firstVisibleOutputMs) ? round.firstVisibleOutputMs : 0; + return Math.max(0, round.streamDurationMs - Math.min(firstOutputMs, round.streamDurationMs)); +} + +/** + * Milliseconds a turn spent streaming model output, summed over its rounds. + * + * Turn wall time also covers tool execution, permission waits and retries, so + * dividing output tokens by it understates throughput by a wide margin. Returns + * undefined when no round reports stream timing (older sessions), letting the + * caller fall back instead of inventing a rate. + */ +export function getTurnOutputDurationMs(rounds?: readonly ModelRound[]): number | undefined { + if (!rounds?.length) return undefined; + let totalMs = 0; + for (const round of rounds) { + totalMs += roundOutputDurationMs(round) ?? 0; + } + return totalMs > 0 ? totalMs : undefined; +} + +export function getTurnUsageMetrics(usage?: TokenUsage, outputDurationMs?: number) { const totalTokens = isCount(usage?.totalTokens) ? usage.totalTokens : null; const cacheHitRate = isCount(usage?.cachedTokens) && isCount(usage?.inputTokens) && usage.inputTokens > 0 && usage.cachedTokens <= usage.inputTokens ? usage.cachedTokens / usage.inputTokens : null; - // Turn wall time is available live and after hydration, including tool/wait time. - // It must not be presented as the provider's pure streaming throughput. - const rate = isCount(usage?.outputTokens) && isCount(durationMs) && durationMs > 0 - ? usage.outputTokens / (durationMs / 1000) : null; + // `outputDurationMs` must be the streaming window from `getTurnOutputDurationMs`, + // never turn wall time: tool execution and waiting would deflate the rate. + const rate = isCount(usage?.outputTokens) && isCount(outputDurationMs) && outputDurationMs > 0 + ? usage.outputTokens / (outputDurationMs / 1000) : null; const outputRate = rate !== null && Number.isFinite(rate) ? rate : null; const speedLevel: 1 | 2 | 3 | 4 | null = outputRate === null ? null : outputRate < 15 ? 1 : outputRate < 40 ? 2 : outputRate < 80 ? 3 : 4; diff --git a/src/web-ui/src/locales/en-US/flow-chat.json b/src/web-ui/src/locales/en-US/flow-chat.json index d6db059fa2..69a18528ca 100644 --- a/src/web-ui/src/locales/en-US/flow-chat.json +++ b/src/web-ui/src/locales/en-US/flow-chat.json @@ -1732,7 +1732,7 @@ "averageRate": "Average output", "speedLevel": "Speed level", "rateValue": "{{value}} t/s", - "rateBasis": "Output tokens divided by total turn time, including tools and waiting; speed levels change at 15, 40 and 80 tokens/s", + "rateBasis": "Output tokens divided by the model's streaming time, excluding tool execution and waiting; speed levels change at 15, 40 and 80 tokens/s", "slow": "Slow", "normal": "Normal", "fast": "Fast", diff --git a/src/web-ui/src/locales/zh-CN/flow-chat.json b/src/web-ui/src/locales/zh-CN/flow-chat.json index ea5e6a4471..f33ea2bbe8 100644 --- a/src/web-ui/src/locales/zh-CN/flow-chat.json +++ b/src/web-ui/src/locales/zh-CN/flow-chat.json @@ -1732,7 +1732,7 @@ "averageRate": "平均输出", "speedLevel": "速度档位", "rateValue": "{{value}} t/s", - "rateBasis": "输出 Token 除以本轮总耗时，包含工具执行与等待；档位分界为 15、40、80 Token/秒", + "rateBasis": "输出 Token 除以模型实际输出时长（不含工具执行与等待），档位分界为 15、40、80 Token/秒", "slow": "慢", "normal": "一般", "fast": "快", diff --git a/src/web-ui/src/locales/zh-TW/flow-chat.json b/src/web-ui/src/locales/zh-TW/flow-chat.json index 6eb4f23aa4..9dcb3e2a30 100644 --- a/src/web-ui/src/locales/zh-TW/flow-chat.json +++ b/src/web-ui/src/locales/zh-TW/flow-chat.json @@ -1732,7 +1732,7 @@ "averageRate": "平均輸出", "speedLevel": "速度檔位", "rateValue": "{{value}} t/s", - "rateBasis": "輸出 Token 除以本輪總耗時，包含工具執行與等待；檔位分界為 15、40、80 Token/秒", + "rateBasis": "輸出 Token 除以模型實際輸出時長（不含工具執行與等待），檔位分界為 15、40、80 Token/秒", "slow": "慢", "normal": "一般", "fast": "快", From 1e5e6ec05781e13088f5e0c6f2c619de3d832c1c Mon Sep 17 00:00:00 2001 From: Bob Lee Date: Sun, 18 Oct 2026 17:56:37 +0800 Subject: [PATCH 2/2] fix(flow-chat): state the exact rate basis in the metrics tooltip The note described only what the rate excludes. The denominator also starts at the first token rather than when the stream began, so say that too and keep the tooltip reproducible from its own text. Co-authored-by: OpenBitFun <318544290+bitfun-ai@users.noreply.github.com> --- src/web-ui/src/locales/en-US/flow-chat.json | 2 +- src/web-ui/src/locales/zh-CN/flow-chat.json | 2 +- src/web-ui/src/locales/zh-TW/flow-chat.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/web-ui/src/locales/en-US/flow-chat.json b/src/web-ui/src/locales/en-US/flow-chat.json index 69a18528ca..3ad4dce79c 100644 --- a/src/web-ui/src/locales/en-US/flow-chat.json +++ b/src/web-ui/src/locales/en-US/flow-chat.json @@ -1732,7 +1732,7 @@ "averageRate": "Average output", "speedLevel": "Speed level", "rateValue": "{{value}} t/s", - "rateBasis": "Output tokens divided by the model's streaming time, excluding tool execution and waiting; speed levels change at 15, 40 and 80 tokens/s", + "rateBasis": "Output tokens divided by the time from the first token to end of stream, excluding tool execution, queueing and waiting; speed levels change at 15, 40 and 80 tokens/s", "slow": "Slow", "normal": "Normal", "fast": "Fast", diff --git a/src/web-ui/src/locales/zh-CN/flow-chat.json b/src/web-ui/src/locales/zh-CN/flow-chat.json index f33ea2bbe8..4cf1ab09bc 100644 --- a/src/web-ui/src/locales/zh-CN/flow-chat.json +++ b/src/web-ui/src/locales/zh-CN/flow-chat.json @@ -1732,7 +1732,7 @@ "averageRate": "平均输出", "speedLevel": "速度档位", "rateValue": "{{value}} t/s", - "rateBasis": "输出 Token 除以模型实际输出时长（不含工具执行与等待），档位分界为 15、40、80 Token/秒", + "rateBasis": "输出 Token 除以从首个 Token 到流结束的时长，不含工具执行、排队与等待；档位分界为 15、40、80 Token/秒", "slow": "慢", "normal": "一般", "fast": "快", diff --git a/src/web-ui/src/locales/zh-TW/flow-chat.json b/src/web-ui/src/locales/zh-TW/flow-chat.json index 9dcb3e2a30..2fe94fd61e 100644 --- a/src/web-ui/src/locales/zh-TW/flow-chat.json +++ b/src/web-ui/src/locales/zh-TW/flow-chat.json @@ -1732,7 +1732,7 @@ "averageRate": "平均輸出", "speedLevel": "速度檔位", "rateValue": "{{value}} t/s", - "rateBasis": "輸出 Token 除以模型實際輸出時長（不含工具執行與等待），檔位分界為 15、40、80 Token/秒", + "rateBasis": "輸出 Token 除以從首個 Token 到流結束的時長，不含工具執行、排隊與等待；檔位分界為 15、40、80 Token/秒", "slow": "慢", "normal": "一般", "fast": "快",