Show per-job inference stats (tok/s, tokens, time to first token) in the Jobs list

desktop/src/electron/service-bridge/modular-state.ts

@@ -14,7 +14,7 @@ import type { LogEntry, LogPage } from '@/shared/types/log'
 import type { NodeItemMetrics } from '@/shared/types/metrics'
 import type { NodeItem } from '@/shared/types/nodes'
 import type { ServiceError, ServiceErrorAction, ServiceErrorSeverity } from '@/shared/types/errors'
-import type { Workload, WorkloadState } from '@/shared/types/workloads'
+import type { Workload, WorkloadState, WorkloadStats } from '@/shared/types/workloads'
 import type { ClusterNode, Invite } from '@/shared/types/cluster'
 import type { WsInvokeResponse } from '@/shared/types/ws-channels'
 import {
@@ -368,9 +368,32 @@ function parseWorkload(value: JsonValue | undefined): Workload | null {
     }
     const scheduledOn = nullableStringValue(obj.scheduledOn)
     if (scheduledOn) workload.scheduledOn = scheduledOn
+    const stats = parseWorkloadStats(obj.stats)
+    if (stats) workload.stats = stats
     return workload
 }
 
+/**
+ * Parse a workload's optional `stats` object. A field is copied only when
+ * present with the right type, so a measurement the proxy omitted stays absent
+ * rather than becoming a zero the UI would render.
+ */
+function parseWorkloadStats(value: JsonValue | undefined): WorkloadStats | null {
+    const obj = objectValue(value)
+    if (!obj) return null
+    const stats: WorkloadStats = {}
+    const promptTokens = nullableNumberValue(obj.promptTokens)
+    if (promptTokens !== null) stats.promptTokens = promptTokens
+    const completionTokens = nullableNumberValue(obj.completionTokens)
+    if (completionTokens !== null) stats.completionTokens = completionTokens
+    const tokensPerSecond = nullableNumberValue(obj.tokensPerSecond)
+    if (tokensPerSecond !== null) stats.tokensPerSecond = tokensPerSecond
+    const ttftMs = nullableNumberValue(obj.ttftMs)
+    if (ttftMs !== null) stats.ttftMs = ttftMs
+    if (booleanValue(obj.estimated)) stats.estimated = true
+    return Object.keys(stats).length > 0 ? stats : null
+}
+
 /**
  * Parse a broker `workloads:get-initial` baseline (`{ workloads: workloadInfo[] }`,
  * ordered by createdAt). Each element is a bare `workloadInfo` — the same shape

desktop/src/shared/types/workloads.ts

@@ -30,4 +30,35 @@ export interface Workload {
     completedAt: number | null
     error: string | null
     requesterId: string | null
+    /**
+     * Inference statistics for a finished job. Present only on the terminal
+     * (`completed` / `failed`) transition, and only when the origin proxy could
+     * measure something; see {@link WorkloadStats}.
+     */
+    stats?: WorkloadStats
+}
+
+/**
+ * Inference statistics the origin proxy measures from the response body as it
+ * streams to the client (services/shared/inferstats). Every field is optional:
+ * absent means "not measured", never zero.
+ */
+export interface WorkloadStats {
+    /** Prompt size in tokens, as reported by the engine. */
+    promptTokens?: number
+    /**
+     * Tokens generated. Reported by the engine when its response carried a usage
+     * object; otherwise counted from stream chunks, in which case `estimated` is
+     * set.
+     */
+    completionTokens?: number
+    /** Decode throughput: `completionTokens` over generation time, one decimal. */
+    tokensPerSecond?: number
+    /** Time to first token in ms: request start to the first response body byte. */
+    ttftMs?: number
+    /**
+     * `completionTokens` (and so `tokensPerSecond`) were counted from stream
+     * chunks rather than reported by the engine, so treat them as approximate.
+     */
+    estimated?: boolean
 }

desktop/src/ui/components/Workloads/WorkloadItemCard.tsx

@@ -8,6 +8,7 @@ import { workloadExecutionNodeId } from '@/shared/utils/workloads'
 import { useNodesStore } from '@/ui/stores/nodes.store'
 import { formatModelDisplayName } from '@/ui/utils/format-model-display-name'
 import { getWorkloadColorBar } from '@/ui/utils/colors'
+import { formatWorkloadStats } from '@/ui/utils/format-workload-stats'
 import EngineIcon from '@/ui/components/EngineIcon'
 
 const formatDate = (timestamp: number) => {
@@ -56,6 +57,7 @@ function WorkloadItemCard({ workload }: { workload: Workload }) {
     })
     const ranOnLabel = workload.state === 'running' ? 'Running on' : 'Ran on'
     const barColor = useMemo(() => getWorkloadColorBar(workload.state), [workload.state])
+    const statsParts = useMemo(() => formatWorkloadStats(workload.stats), [workload.stats])
 
     const subtext = useMemo(() => {
         const state = workload.state
@@ -156,6 +158,15 @@ function WorkloadItemCard({ workload }: { workload: Workload }) {
                 <Flex align="center" gap="2">
                     {subtext}
                 </Flex>
+                {statsParts.length > 0 && (
+                    <Flex align="center" wrap="wrap" gap="1">
+                        {statsParts.map((part, index) => (
+                            <Text key={part} kind="body/regular/sm" className="text-subtle-color">
+                                {index > 0 ? `\u00b7 ${part}` : part}
+                            </Text>
+                        ))}
+                    </Flex>
+                )}
                 {workload.error && workload.state === 'failed' && (
                     <Text
                         kind="body/regular/sm"

desktop/src/ui/utils/format-workload-stats.ts

@@ -0,0 +1,39 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import type { WorkloadStats } from '@/shared/types/workloads'
+
+/**
+ * Summarize a job's inference statistics for the job card as short phrases the
+ * card joins with separators, e.g. `["42.3 tok/s", "512 tokens", "0.8 s to first
+ * token"]`. Counts the proxy only estimated (from stream chunks) are prefixed
+ * with `~`. Empty when nothing worth showing was measured.
+ */
+export function formatWorkloadStats(stats: WorkloadStats | undefined): string[] {
+    if (!stats) return []
+    const approx = stats.estimated ? '~' : ''
+    const parts: string[] = []
+    if (stats.tokensPerSecond) {
+        parts.push(`${approx}${formatRate(stats.tokensPerSecond)} tok/s`)
+    }
+    if (stats.completionTokens) {
+        parts.push(`${approx}${stats.completionTokens.toLocaleString()} tokens`)
+    } else if (stats.promptTokens) {
+        // An embeddings request has a prompt but generates nothing.
+        parts.push(`${stats.promptTokens.toLocaleString()} prompt tokens`)
+    }
+    if (stats.ttftMs) {
+        parts.push(`${formatDuration(stats.ttftMs)} to first token`)
+    }
+    return parts
+}
+
+function formatRate(tokensPerSecond: number): string {
+    return tokensPerSecond >= 100
+        ? Math.round(tokensPerSecond).toString()
+        : tokensPerSecond.toFixed(1)
+}
+
+function formatDuration(ms: number): string {
+    return ms < 1000 ? `${Math.round(ms)} ms` : `${(ms / 1000).toFixed(1)} s`
+}

desktop/tests/modular/format-workload-stats.test.ts

@@ -0,0 +1,41 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it } from 'vitest'
+import { formatWorkloadStats } from '@/ui/utils/format-workload-stats'
+
+describe('formatWorkloadStats', () => {
+    it('shows engine-reported throughput, token count and time to first token', () => {
+        expect(
+            formatWorkloadStats({
+                promptTokens: 12,
+                completionTokens: 1234,
+                tokensPerSecond: 42.34,
+                ttftMs: 812
+            })
+        ).toEqual(['42.3 tok/s', '1,234 tokens', '812 ms to first token'])
+    })
+
+    it('marks estimated counts and rounds fast rates to whole tokens', () => {
+        expect(
+            formatWorkloadStats({
+                completionTokens: 40,
+                tokensPerSecond: 133.7,
+                ttftMs: 1480,
+                estimated: true
+            })
+        ).toEqual(['~134 tok/s', '~40 tokens', '1.5 s to first token'])
+    })
+
+    it('falls back to prompt tokens for a job that generated nothing', () => {
+        expect(formatWorkloadStats({ promptTokens: 8, ttftMs: 30 })).toEqual([
+            '8 prompt tokens',
+            '30 ms to first token'
+        ])
+    })
+
+    it('is empty when nothing was measured', () => {
+        expect(formatWorkloadStats(undefined)).toEqual([])
+        expect(formatWorkloadStats({})).toEqual([])
+    })
+})

services/lmstudio-proxy/README.md

@@ -179,6 +179,12 @@ One lifecycle transition per forwarded inference request, carrying a single `wor
 {"jsonrpc":"2.0","method":"workload:started","params":{"workloadInfo":{"id":"17","model":"lmstudio-community/Qwen3-8B-GGUF","engine":"lmstudio","runId":"3ce8a1740b62df95","state":"running","originatedFrom":"","scheduledOn":"22222222-2222-2222-2222-222222222222","createdAt":1716998400000,"startedAt":1716998400000,"completedAt":null,"error":null,"requesterId":null}}}
 ```
 
+The terminal `workload:completed` / `workload:errored` carries the same object plus an additive `stats` block measured from the response body as it streamed — token counts from the engine's usage report (OpenAI `usage`, Ollama `eval_count`) or, failing that, counted from stream chunks (`estimated`), decode throughput and time to first token. The body is never buffered: only its last few kilobytes are inspected (`nvpair-shared/inferstats`).
+
+```json
+{"jsonrpc":"2.0","method":"workload:completed","params":{"workloadInfo":{"id":"17","model":"lmstudio-community/Qwen3-8B-GGUF","engine":"lmstudio","runId":"3ce8a1740b62df95","state":"completed","originatedFrom":"","scheduledOn":"22222222-2222-2222-2222-222222222222","createdAt":1716998400000,"startedAt":1716998400000,"completedAt":1716998412000,"error":null,"requesterId":null,"stats":{"promptTokens":12,"completionTokens":340,"tokensPerSecond":42.3,"ttftMs":812}}}}
+```
+
 #### `node/activity`
 
 Raised while a node's engine is streaming a response back through the proxy: every successful write of upstream body bytes reports the node that produced them. The broker relays it to `nvpair-node-scanner`, which treats it as proof of life and cancels that node's eviction — a node saturated by inference cannot answer a liveness probe, but it is demonstrably alive precisely because it is streaming. Coalesced to one report per node per 2s (`nvpair-shared/nodeactivity`), since a generation writes hundreds of chunks and the scanner treats one report as good for a minute. `msSince` is the age of the observation; the broker adds its own relay delay before passing it on.

services/lmstudio-proxy/proxy.go

@@ -29,6 +29,7 @@ import (
 	"nvpair-shared/clustertrust"
 	"nvpair-shared/cors"
 	"nvpair-shared/errors"
+	"nvpair-shared/inferstats"
 	"nvpair-shared/netmon"
 	"nvpair-shared/netpick"
 	"nvpair-shared/nodeactivity"
@@ -182,6 +183,11 @@ type Workload struct {
 	CompletedAt    *int64  `json:"completedAt"`
 	Error          *string `json:"error"`
 	RequesterID    *string `json:"requesterId"`
+	// Stats carries the inference statistics measured for the terminal
+	// (completed/errored) transition: token counts, decode throughput and time
+	// to first token. Additive: omitted while the workload is running and when
+	// nothing could be measured (see nvpair-shared/inferstats).
+	Stats *inferstats.Stats `json:"stats,omitempty"`
 }
 
 // workloadParams is the params envelope for a workload:* notification
@@ -238,6 +244,12 @@ type statusCapture struct {
 	// discovery cannot obtain for itself while the node is too busy to answer a
 	// probe. Called on the reverse proxy's copy goroutine, so it must be cheap.
 	upstreamAlive func()
+
+	// tap, when set, observes every body byte written to the client so the
+	// workload's terminal event can carry token counts and throughput. Like
+	// upstreamAlive it is set at the commit point, and only for a successful
+	// inference response, so the proxy's own error bodies are never measured.
+	tap *inferstats.Tap
 }
 
 // Unwrap exposes the underlying ResponseWriter so http.ResponseController can
@@ -277,6 +289,9 @@ func (sc *statusCapture) Write(b []byte) (int, error) {
 	if err == nil && sc.upstreamAlive != nil {
 		sc.upstreamAlive()
 	}
+	if err == nil && sc.tap != nil {
+		sc.tap.Observe(b)
+	}
 	return n, err
 }
 
@@ -1026,6 +1041,7 @@ func (p *Proxy) handleHTTP(w http.ResponseWriter, r *http.Request) {
 		finalStatus  int
 		started      bool
 		wl           *Workload
+		tap          *inferstats.Tap
 	)
 
 	// Emit workload:started up front, the moment we begin forwarding, naming
@@ -1050,6 +1066,10 @@ func (p *Proxy) handleHTTP(w http.ResponseWriter, r *http.Request) {
 			StartedAt:   &createdMs,
 		}
 		p.emitWorkload(workloadStartedMethod, *wl)
+		// The tap measures the response body for this workload's stats. It is
+		// allocated here, before the disconnect watcher goroutine starts, and
+		// armed only once a candidate commits (ModifyResponse below).
+		tap = inferstats.NewTap(start)
 	}
 
 	// The terminal workload transition (completed/errored) can be reached from
@@ -1076,6 +1096,11 @@ func (p *Proxy) handleHTTP(w http.ResponseWriter, r *http.Request) {
 			if errMsg != "" {
 				wl.Error = &errMsg
 			}
+			// Stats ride the terminal event only; a failed stream still reports
+			// what was generated before it broke.
+			if tap != nil {
+				wl.Stats = tap.Finish()
+			}
 			snapshot := *wl
 			wlMu.Unlock()
 			method := workloadCompletedMethod
@@ -1165,6 +1190,13 @@ func (p *Proxy) handleHTTP(w http.ResponseWriter, r *http.Request) {
 				// came from the node. Same goroutine as the body copy, so no
 				// synchronization is needed.
 				sc.upstreamAlive = func() { p.reportActivity(cand.id) }
+				// Measure the body only for a successful inference response: an
+				// error body carries no tokens, and a control request has no
+				// workload to attach stats to.
+				if tap != nil && resp.StatusCode < http.StatusBadRequest {
+					tap.Arm(resp.Header)
+					sc.tap = tap
+				}
 				// The engine may enforce its own origin policy. Honor it:
 				// overwriting a declared Access-Control-Allow-Origin would
 				// silently widen the user's policy, and a wildcard is invalid