From 8d43f70e88487ed6a3138afafd2ace0c2e15ec94 Mon Sep 17 00:00:00 2001 From: Fergus Finn Date: Fri, 13 Nov 2026 17:53:28 +0000 Subject: [PATCH 1/2] feat(compare): score a trace replay against production's record of the same requests batchbench-compare-trace-replay joins a manifest's metadata.requests and the agent's request events to the Dynamo trace rows they came from, and reports both sides over a scoring window: failures, first-token, time per output token and end-to-end percentiles, and the share within a latency bound. A production context file splits production by worker set and gives each set's throughput per GPU beside the replay's. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VDPqeFzGgF3vkuiyimwMxX --- README.md | 8 +- docs/export-plans.md | 44 ++++++ pyproject.toml | 1 + src/batchbench/trace_replay.py | 272 +++++++++++++++++++++++++++++++++ tests/test_trace_replay.py | 136 +++++++++++++++++ 5 files changed, 460 insertions(+), 1 deletion(-) create mode 100644 src/batchbench/trace_replay.py create mode 100644 tests/test_trace_replay.py diff --git a/README.md b/README.md index 825a09e..5961c0e 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,14 @@ wrapper, published as prebuilt wheels: uv pip install batchbench ``` -Three entrypoints: +Four entrypoints: | Command | What it does | |---|---| | `batchbench` | Independent requests at a fixed concurrency with controlled token shapes | | `batchbench-agent` | Stateful multi-turn sessions, sampled or replayed from a manifest | | `batchbench-export-plans` | Builds a replay manifest from prompt-chain records in ClickHouse or from Dynamo request traces | +| `batchbench-compare-trace-replay` | Compares a replay of Dynamo request traces with production's record of the same requests | ## Independent requests @@ -122,6 +123,11 @@ It can also build one from a directory of Dynamo frontend request traces expects, and how sessions are reconstructed, are described in [docs/export-plans.md](docs/export-plans.md). +`batchbench-compare-trace-replay` scores a replay of such a manifest against the +traces it came from: the same requests' first-token time, time per output token and +end-to-end time on both sides, and their share within a latency bound +([docs/export-plans.md](docs/export-plans.md#comparing-a-replay-with-production)). + ## Report Each run prints total prompt and completion tokens from the responses' `usage` and diff --git a/docs/export-plans.md b/docs/export-plans.md index 8b161e2..0ed3437 100644 --- a/docs/export-plans.md +++ b/docs/export-plans.md @@ -228,3 +228,47 @@ Besides the usual counts, the summary line reports, for traces: `files`, - **No `stream` or `max_tokens`.** Traces do not record them, so they are omitted. - **Timing starts at Dynamo.** `request_received_ms` is when the frontend received the request, after any gateway queueing. + +## Comparing a replay with production + +Each trajectory's `metadata.requests` names its requests' source rows, and +`batchbench-agent --request-events-jsonl` records each replayed request's client-side +timings by trajectory and invocation, so a replay can be scored request by request +against the traces it was built from: + +```bash +batchbench-compare-trace-replay \ + --manifest plans.jsonl --request-events events.jsonl \ + --dynamo-traces traces/2026/11/12 \ + --score-start 2026-11-12T18:00:00Z --score-end 2026-11-12T19:00:00Z \ + --slo-ttft-s 15 --slo-tpot-ms 33 --replay-gpus 16 \ + --production-context context.json --model zai-org/GLM-5.3-Flash \ + --json-out report.json +``` + +Only requests whose source row arrived inside the scoring window are scored, so a +manifest exported from an earlier start lets the replay's caches and queues settle +before scoring begins. For the requests in the window it reports, for the replay and +for production's rows of the same requests: how many there were and how many failed, +first-token time, time per output token (production's `avg_itl_ms`) and end-to-end +time percentiles, and the share that succeeded with a first token within +`--slo-ttft-s` and a time per output token within `--slo-tpot-ms`. A failed replayed +request counts against that share. With `--replay-gpus`, the replay's successful and +within-bound requests and output tokens per second per GPU over the window are added. + +A production context file names the worker set behind each decode worker id and each +set's GPUs over the window: + +```json +{"worker_sets": {"5214388210741902154": "isambard", "...": "b300"}, + "set_gpus": {"isambard": 175.7, "b300": 9.3}} +``` + +With it, production's rows are also split by the set that served them (rows whose +worker it doesn't name are `unmapped`). With `--model` as well, every request of that +model that arrived in the window is counted by set, replayed or not, and each set with +GPUs gets its requests, output tokens and within-bound requests per second per GPU. + +Replay times are the client's and production's are the frontend's, so the replay's +include the client's round trip to its server. + diff --git a/pyproject.toml b/pyproject.toml index 8d8ac56..3b0047e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ batchbench = "batchbench.__main__:main" batchbench-agent = "batchbench.agent:main" batchbench-harness = "batchbench.harness:main" batchbench-export-plans = "batchbench.export_plans:main" +batchbench-compare-trace-replay = "batchbench.trace_replay:main" 'batchbench.harness' = "batchbench.harness:main" [tool.maturin] diff --git a/src/batchbench/trace_replay.py b/src/batchbench/trace_replay.py new file mode 100644 index 0000000..54d78f7 --- /dev/null +++ b/src/batchbench/trace_replay.py @@ -0,0 +1,272 @@ +"""Compare a replay of Dynamo request traces with what production recorded for the same requests. + +A manifest exported with ``batchbench-export-plans --dynamo-traces`` names each request's source trace row in its +trajectory ``metadata.requests``; ``batchbench-agent --request-events-jsonl`` records the replay's client-side timings +by trajectory and 1-based invocation. Joining the two on the request's source id puts the replay's first-token time, +time per output token and end-to-end time beside production's for the same request. Only requests whose source +arrived inside the scoring window count, so a replay that started with a link window and ran past the window's end +is scored on its steady part. + +A production context file, when given, names the production worker set that served each request (by its decode +worker id) and each set's GPUs, so production's figures are split by set and each set's throughput per GPU over the +window is reported beside the replay's:: + + {"worker_sets": {"": "", ...}, "set_gpus": {"": , ...}} +""" + +from __future__ import annotations + +import argparse +import json +import math +import statistics +import sys +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any, Iterable, Mapping, Optional, Sequence + +from .dynamo_traces import TRACE_EVENT, _trace_lines, TraceStats, trace_files +from .export_plans import ExportError + +UNMAPPED = "unmapped" + + +@dataclass +class Timing: + ttft_s: Optional[float] + tpot_s: Optional[float] + e2e_s: Optional[float] + output_tokens: Optional[int] + ok: bool + worker_set: str = "" + + +def parse_time(value: str) -> float: + """Milliseconds since the epoch of an ISO-8601 time (UTC when it names no zone).""" + + ts = datetime.fromisoformat(value.replace("Z", "+00:00")) + if ts.tzinfo is None: + ts = ts.replace(tzinfo=timezone.utc) + return ts.timestamp() * 1000.0 + + +def load_sources(lines: Iterable[str]) -> dict[tuple[str, int], tuple[str, int]]: + """Each manifest request's source row, ``(trajectory_id, invocation) -> (request_id, received_ms)``.""" + + sources = {} + for line in lines: + if not line.strip(): + continue + trajectory = json.loads(line) + for invocation, request in enumerate((trajectory.get("metadata") or {}).get("requests") or [], start=1): + if request.get("request_id"): + sources[(trajectory["trajectory_id"], invocation)] = (request["request_id"], int(request["received_ms"])) + return sources + + +def _seconds(ms: Any) -> Optional[float]: + return None if ms is None else float(ms) / 1000.0 + + +def load_replay(lines: Iterable[str], sources: Mapping[tuple[str, int], tuple[str, int]], lo_ms: float, + hi_ms: float) -> dict[str, Timing]: + """The replay's timing of each request whose source arrived in ``[lo_ms, hi_ms)``, by source request id.""" + + replay = {} + for line in lines: + if not line.strip(): + continue + event = json.loads(line) + source = sources.get((event["trajectory_id"], int(event["invocation"]))) + if source is None or not lo_ms <= source[1] < hi_ms: + continue + replay[source[0]] = Timing(_seconds(event.get("ttft_ms")), _seconds(event.get("tpot_ms")), + _seconds(event.get("e2e_ms")), event.get("completion_tokens"), + event.get("status") == "ok") + return replay + + +@dataclass +class SetTotals: + requests: int = 0 + output_tokens: int = 0 + within_slo: int = 0 + + +def production_timing(request: Mapping[str, Any], worker_sets: Mapping[str, str]) -> Timing: + worker = request.get("worker") or {} + ttft = _seconds(request.get("ttft_ms")) + return Timing(ttft, _seconds(request.get("avg_itl_ms")), _seconds(request.get("total_time_ms")), + request.get("output_tokens"), ttft is not None, + worker_sets.get(str(worker.get("decode_worker_id")), UNMAPPED)) + + +def within_slo(timing: Timing, slo_ttft_s: float, slo_tpot_s: float) -> bool: + """Succeeded, first token within the bound, and time per output token within its bound (or under two tokens).""" + + return (timing.ok and timing.ttft_s is not None and timing.ttft_s <= slo_ttft_s + and (timing.tpot_s is None or timing.tpot_s <= slo_tpot_s)) + + +def load_production(paths: Sequence[str], wanted: set[str], worker_sets: Mapping[str, str], model: Optional[str], + lo_ms: float, hi_ms: float, slo_ttft_s: float, slo_tpot_s: float, + stats: TraceStats) -> tuple[dict[str, Timing], dict[str, SetTotals]]: + """Production's timing of each wanted request; and, with ``model``, every request of the model that arrived in + ``[lo_ms, hi_ms)``, counted by the worker set that served it.""" + + found: dict[str, Timing] = {} + totals: dict[str, SetTotals] = {} + for path in trace_files(list(paths)): + for line in _trace_lines(path, stats): + if f'"{TRACE_EVENT}"' not in line: + continue + request = json.loads(line).get("request") or {} + request_id = request.get("request_id") + in_window = model is not None and request.get("model") == model and lo_ms <= (request.get("request_received_ms") or 0) < hi_ms + if request_id not in wanted and not in_window: + continue + timing = production_timing(request, worker_sets) + if request_id in wanted: + found[request_id] = timing + if in_window: + t = totals.setdefault(timing.worker_set, SetTotals()) + t.requests += 1 + t.output_tokens += timing.output_tokens or 0 + t.within_slo += within_slo(timing, slo_ttft_s, slo_tpot_s) + return found, totals + + +def percentile(values: Sequence[float], q: float) -> float: + if not values: + return math.nan + ordered = sorted(values) + return ordered[min(len(ordered) - 1, int(q * len(ordered)))] + + +def summarize(timings: Sequence[Timing], slo_ttft_s: float, slo_tpot_s: float, seconds: Optional[float] = None, + gpus: Optional[float] = None) -> dict[str, Any]: + done = [t for t in timings if t.ok] + ttft = [t.ttft_s for t in done if t.ttft_s is not None] + tpot = [t.tpot_s * 1000 for t in done if t.tpot_s is not None] + e2e = [t.e2e_s for t in done if t.e2e_s is not None] + out = { + "requests": len(timings), + "failed": len(timings) - len(done), + "within_slo": sum(within_slo(t, slo_ttft_s, slo_tpot_s) for t in timings) / max(len(timings), 1), + "ttft_s_p50": percentile(ttft, 0.5), "ttft_s_p90": percentile(ttft, 0.9), "ttft_s_p99": percentile(ttft, 0.99), + "tpot_ms_p50": percentile(tpot, 0.5), "tpot_ms_p90": percentile(tpot, 0.9), "tpot_ms_p99": percentile(tpot, 0.99), + "e2e_s_p50": percentile(e2e, 0.5), "e2e_s_p90": percentile(e2e, 0.9), + "e2e_s_mean": statistics.fmean(e2e) if e2e else math.nan, + } + if seconds and gpus: + out["requests_per_s_per_gpu"] = len(done) / seconds / gpus + out["output_tokens_per_s_per_gpu"] = sum(t.output_tokens or 0 for t in done) / seconds / gpus + out["within_slo_per_s_per_gpu"] = sum(within_slo(t, slo_ttft_s, slo_tpot_s) for t in timings) / seconds / gpus + return out + + +def compare(replay: Mapping[str, Timing], production: Mapping[str, Timing], totals: Mapping[str, SetTotals], + set_gpus: Mapping[str, float], seconds: float, replay_gpus: Optional[float], slo_ttft_s: float, + slo_tpot_s: float) -> dict[str, Any]: + both = [request_id for request_id in replay if request_id in production] + report: dict[str, Any] = { + "replayed": len(replay), + "matched": len(both), + "replay": summarize([replay[r] for r in both], slo_ttft_s, slo_tpot_s, seconds, replay_gpus), + "production": summarize([production[r] for r in both], slo_ttft_s, slo_tpot_s), + "production_by_set": {}, + "production_set_throughput": {}, + } + by_set: dict[str, list[Timing]] = {} + for request_id in both: + by_set.setdefault(production[request_id].worker_set, []).append(production[request_id]) + for name in sorted(by_set): + report["production_by_set"][name] = { + "production": summarize(by_set[name], slo_ttft_s, slo_tpot_s), + "replay": summarize([replay[r] for r in both if production[r].worker_set == name], slo_ttft_s, slo_tpot_s), + } + for name in sorted(totals): + t, gpus = totals[name], set_gpus.get(name) + row: dict[str, Any] = {"requests": t.requests, "within_slo": t.within_slo / max(t.requests, 1)} + if gpus: + row.update(gpus=gpus, requests_per_s_per_gpu=t.requests / seconds / gpus, + output_tokens_per_s_per_gpu=t.output_tokens / seconds / gpus, + within_slo_per_s_per_gpu=t.within_slo / seconds / gpus) + report["production_set_throughput"][name] = row + return report + + +def _fmt(value: Any) -> str: + return f"{value:.4g}" if isinstance(value, float) else str(value) + + +def render(report: Mapping[str, Any]) -> str: + lines = [f"replayed {report['replayed']} requests in the window, {report['matched']} matched to production rows"] + + def block(title: str, summary: Mapping[str, Any]) -> None: + lines.append(f"{title}: " + " ".join(f"{k}={_fmt(v)}" for k, v in summary.items())) + + block("replay", report["replay"]) + block("production (same requests)", report["production"]) + for name, pair in report["production_by_set"].items(): + block(f" served by {name}: production", pair["production"]) + block(f" served by {name}: replay", pair["replay"]) + for name, row in report["production_set_throughput"].items(): + block(f"production set {name}, every request in the window", row) + return "\n".join(lines) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="batchbench-compare-trace-replay", + description="Compare a replay of Dynamo request traces with production's record of the same requests.", + ) + parser.add_argument("--manifest", required=True, help="the replayed manifest (batchbench-export-plans --dynamo-traces)") + parser.add_argument("--request-events", required=True, help="the replay's batchbench-agent --request-events-jsonl") + parser.add_argument("--dynamo-traces", action="append", required=True, metavar="PATH", + help="production trace files: a directory, file or glob; repeatable") + parser.add_argument("--score-start", required=True, help="scoring window start (ISO-8601, source time)") + parser.add_argument("--score-end", required=True, help="scoring window end (ISO-8601, source time)") + parser.add_argument("--slo-ttft-s", type=float, default=15.0, help="first-token bound in seconds (default 15)") + parser.add_argument("--slo-tpot-ms", type=float, default=33.0, help="time-per-output-token bound in ms (default 33)") + parser.add_argument("--replay-gpus", type=float, help="GPUs the replay ran on, for its throughput per GPU") + parser.add_argument("--production-context", help="JSON naming worker sets and their GPUs (see module docs)") + parser.add_argument("--model", help="production model name: count every request of it in the window by set") + parser.add_argument("--json-out", help="also write the report here as JSON") + return parser + + +def main(argv: Optional[Sequence[str]] = None) -> int: + args = build_parser().parse_args(argv) + lo, hi = parse_time(args.score_start), parse_time(args.score_end) + if hi <= lo: + print("error: --score-end must be after --score-start", file=sys.stderr) + return 2 + context: dict[str, Any] = {} + if args.production_context: + with open(args.production_context, encoding="utf-8") as handle: + context = json.load(handle) + worker_sets = {str(k): v for k, v in (context.get("worker_sets") or {}).items()} + set_gpus = {k: float(v) for k, v in (context.get("set_gpus") or {}).items()} + slo_ttft, slo_tpot = args.slo_ttft_s, args.slo_tpot_ms / 1000.0 + with open(args.manifest, encoding="utf-8") as handle: + sources = load_sources(handle) + with open(args.request_events, encoding="utf-8") as handle: + replay = load_replay(handle, sources, lo, hi) + try: + production, totals = load_production(args.dynamo_traces, set(replay), worker_sets, args.model, lo, hi, + slo_ttft, slo_tpot, TraceStats()) + except ExportError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + report = compare(replay, production, totals, set_gpus, (hi - lo) / 1000.0, args.replay_gpus, + slo_ttft, slo_tpot) + print(render(report)) + if args.json_out: + with open(args.json_out, "w", encoding="utf-8") as handle: + json.dump(report, handle, indent=2) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_trace_replay.py b/tests/test_trace_replay.py new file mode 100644 index 0000000..02a2647 --- /dev/null +++ b/tests/test_trace_replay.py @@ -0,0 +1,136 @@ +"""Tests for batchbench-compare-trace-replay.""" + +from __future__ import annotations + +import io +import json +import shutil +import tempfile +import unittest +from contextlib import redirect_stdout +from pathlib import Path + +from batchbench import trace_replay + +T0 = 1_794_600_000_000 # ms +WINDOW = ("2026-11-13T20:00:00Z", "2026-11-13T20:10:00Z") # T0 is the window start + + +def production_row(request_id, received_ms, ttft_ms, itl_ms, output_tokens=100, worker=1, model="m"): + return { + "schema": "dynamo.request.trace.v1", + "event_type": "request_end", + "request": { + "request_id": request_id, + "model": model, + "request_received_ms": received_ms, + "ttft_ms": ttft_ms, + "avg_itl_ms": itl_ms, + "total_time_ms": ttft_ms + itl_ms * output_tokens, + "output_tokens": output_tokens, + "worker": {"decode_worker_id": worker}, + }, + } + + +def event(trajectory_id, invocation, ttft_ms, tpot_ms, status="ok", completion_tokens=100): + return { + "trajectory_id": trajectory_id, "agent_id": 0, "invocation": invocation, "sent_at_ms": 0, + "ttft_ms": ttft_ms if status == "ok" else None, "last_token_ms": None, + "e2e_ms": (ttft_ms + tpot_ms * completion_tokens) if status == "ok" else 50.0, + "tpot_ms": tpot_ms if status == "ok" else None, "prompt_tokens": 10, + "completion_tokens": completion_tokens if status == "ok" else None, "cached_tokens": None, "attempts": 1, + "status": status, "error": None if status == "ok" else "HTTP 529", "http_status": 200 if status == "ok" else 529, + } + + +class CompareTest(unittest.TestCase): + def setUp(self): + self.directory = Path(tempfile.mkdtemp(prefix="batchbench-compare-")) + # Trajectory a: a request before the window (link context) and two inside; trajectory b: one inside. + manifest = [ + {"trajectory_id": "a", "requests": [], "metadata": {"requests": [ + {"request_id": "a0", "x_request_id": None, "received_ms": T0 - 60_000}, + {"request_id": "a1", "x_request_id": None, "received_ms": T0 + 1_000}, + {"request_id": "a2", "x_request_id": None, "received_ms": T0 + 2_000}, + ]}}, + {"trajectory_id": "b", "requests": [], "metadata": {"requests": [ + {"request_id": "b1", "x_request_id": None, "received_ms": T0 + 3_000}, + ]}}, + ] + events = [event("a", 1, 900, 9), event("a", 2, 1000, 10), event("a", 3, 20_000, 10), + event("b", 1, 0, 0, status="error")] + rows = [production_row("a0", T0 - 60_000, 500, 10), + production_row("a1", T0 + 1_000, 2_000, 40, worker=1), + production_row("a2", T0 + 2_000, 3_000, 20, worker=2), + production_row("b1", T0 + 3_000, 1_000, 20, worker=2), + # Not replayed, but served in the window: counted for the sets' throughput. + production_row("c1", T0 + 4_000, 1_000, 20, worker=2), + production_row("other-model", T0 + 4_000, 1_000, 20, worker=2, model="n")] + self.manifest = self.write("plans.jsonl", manifest) + self.events = self.write("events.jsonl", events) + self.traces = self.write("traces.jsonl", rows) + self.context = self.directory / "context.json" + self.context.write_text(json.dumps({"worker_sets": {"1": "gh200", "2": "b300"}, + "set_gpus": {"gh200": 4, "b300": 2}}), encoding="utf-8") + + def tearDown(self): + shutil.rmtree(self.directory, ignore_errors=True) + + def write(self, name, rows): + path = self.directory / name + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + return str(path) + + def run_cli(self, *extra): + out = self.directory / "report.json" + argv = ["--manifest", self.manifest, "--request-events", self.events, "--dynamo-traces", self.traces, + "--score-start", WINDOW[0], "--score-end", WINDOW[1], "--json-out", str(out), *extra] + with redirect_stdout(io.StringIO()) as printed: + self.assertEqual(trace_replay.main(argv), 0) + return json.loads(out.read_text(encoding="utf-8")), printed.getvalue() + + def test_scores_the_same_requests_on_both_sides(self): + report, printed = self.run_cli() + # a1, a2 and b1 arrived in the window; a0 is link context. + self.assertEqual((report["replayed"], report["matched"]), (3, 3)) + replay, production = report["replay"], report["production"] + self.assertEqual((replay["requests"], replay["failed"]), (3, 1)) + # The replay: a1 within, a2 over the first-token bound, b1 failed. + self.assertAlmostEqual(replay["within_slo"], 1 / 3) + # Production: a1 over the time-per-token bound (40 ms), a2 and b1 within. + self.assertAlmostEqual(production["within_slo"], 2 / 3) + self.assertEqual(production["ttft_s_p50"], 2.0) + self.assertIn("3 matched", printed) + + def test_failed_requests_count_against_the_replay_but_not_its_latencies(self): + report, _ = self.run_cli() + self.assertEqual(report["replay"]["ttft_s_p99"], 20.0) + self.assertEqual(report["replay"]["tpot_ms_p50"], 10.0) + + def test_context_splits_production_by_set_and_prices_each_per_gpu(self): + report, _ = self.run_cli("--production-context", str(self.context), "--model", "m", "--replay-gpus", "2") + self.assertEqual(sorted(report["production_by_set"]), ["b300", "gh200"]) + self.assertEqual(report["production_by_set"]["b300"]["production"]["requests"], 2) + self.assertEqual(report["production_by_set"]["b300"]["replay"]["requests"], 2) + throughput = report["production_set_throughput"] + # Every request of model m that arrived in the window, replayed or not. + self.assertEqual(throughput["b300"]["requests"], 3) + self.assertAlmostEqual(throughput["b300"]["requests_per_s_per_gpu"], 3 / 600 / 2) + self.assertAlmostEqual(throughput["gh200"]["requests_per_s_per_gpu"], 1 / 600 / 4) + self.assertAlmostEqual(report["replay"]["requests_per_s_per_gpu"], 2 / 600 / 2) + self.assertAlmostEqual(report["replay"]["within_slo_per_s_per_gpu"], 1 / 600 / 2) + + def test_unknown_workers_are_unmapped(self): + self.context.write_text(json.dumps({"worker_sets": {"1": "gh200"}}), encoding="utf-8") + report, _ = self.run_cli("--production-context", str(self.context)) + self.assertEqual(sorted(report["production_by_set"]), ["gh200", trace_replay.UNMAPPED]) + + def test_rejects_an_empty_window(self): + argv = ["--manifest", self.manifest, "--request-events", self.events, "--dynamo-traces", self.traces, + "--score-start", WINDOW[1], "--score-end", WINDOW[0]] + self.assertEqual(trace_replay.main(argv), 2) + + +if __name__ == "__main__": + unittest.main() From 59ab41377e547e5604d80227e49fa255c66c427c Mon Sep 17 00:00:00 2001 From: Fergus Finn Date: Fri, 13 Nov 2026 23:49:58 +0000 Subject: [PATCH 2/2] feat(export): --stream marks every request streamed Trace and prompt-chain records don't say whether a request was streamed, so exported requests went out non-streamed and batchbench-agent recorded no first-token time or time per output token for them. --stream sets stream: true on every request; the comparison docs say to use it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VDPqeFzGgF3vkuiyimwMxX --- docs/export-plans.md | 3 ++- src/batchbench/export_plans.py | 5 +++++ tests/test_dynamo_traces.py | 10 ++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/export-plans.md b/docs/export-plans.md index 0ed3437..979cfa4 100644 --- a/docs/export-plans.md +++ b/docs/export-plans.md @@ -270,5 +270,6 @@ model that arrived in the window is counted by set, replayed or not, and each se GPUs gets its requests, output tokens and within-bound requests per second per GPU. Replay times are the client's and production's are the frontend's, so the replay's -include the client's round trip to its server. +include the client's round trip to its server. The client measures first-token time and +time per output token only on streamed requests, so export the manifest with `--stream`. diff --git a/src/batchbench/export_plans.py b/src/batchbench/export_plans.py index 371cf3d..3f88047 100644 --- a/src/batchbench/export_plans.py +++ b/src/batchbench/export_plans.py @@ -803,6 +803,7 @@ def build_parser() -> argparse.ArgumentParser: shaping.add_argument("--seed", type=int, default=0, help="seed for deterministic sampling") shaping.add_argument("--stratify-by-session-length", action="store_true", help="sample within power-of-two session-length buckets") shaping.add_argument("--time-scale", type=float, default=1.0, help="divide start offsets and delays by this factor") + shaping.add_argument("--stream", action="store_true", help="mark every request stream: true, so a replay measures first-token time and time per output token (the records don't say whether the original was streamed)") parser.add_argument("--output", required=True, help="manifest JSONL path") return parser @@ -835,6 +836,10 @@ def main(argv: Optional[Sequence[str]] = None) -> int: plans = _export_chains(args, window_start, window_end, link_window, stats) plans = sample_plans(plans, args.sample, args.seed, args.stratify_by_session_length) stats.sampled = len(plans) + if args.stream: + for plan in plans: + for request in plan["requests"]: + request["stream"] = True summary = ", ".join(f"{key}={value}" for key, value in {**stats.as_dict(), **extra_stats}.items()) if not plans: raise ExportError(f"no trajectories to export ({summary})") diff --git a/tests/test_dynamo_traces.py b/tests/test_dynamo_traces.py index c198956..df93bd3 100644 --- a/tests/test_dynamo_traces.py +++ b/tests/test_dynamo_traces.py @@ -510,6 +510,16 @@ def test_cli_exports_from_traces_alone_and_with_analytics_rows(self): self.assertEqual((code, len(plans)), (0, 1)) self.assertEqual(plans[0]["metadata"]["alias"], "alias-a") + def test_cli_streams_every_request_when_asked(self): + self.write("traces/frontend-a-120100-run1-0.jsonl.gz", self.session_rows()) + code, plans = self.run_cli() + self.assertEqual(code, 0) + self.assertFalse(any("stream" in request for plan in plans for request in plan["requests"])) + code, plans = self.run_cli("--stream") + self.assertEqual(code, 0) + self.assertEqual(validate_plans(plans), []) + self.assertTrue(all(request["stream"] is True for plan in plans for request in plan["requests"])) + def test_cli_rejects_mixed_sources(self): self.write("traces/a.jsonl.gz", self.session_rows()) code, _ = self.run_cli("--served-by", "upstream-a")