From 64467a8eb7cfe03827c9296128daece54b72619c Mon Sep 17 00:00:00 2001 From: aas008 Date: Tue, 6 Oct 2026 16:30:53 -0400 Subject: [PATCH 1/7] feat(manual-import): add turn_index column to manual run CSV output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add turn_index as empty string for aggregate rows so manually-imported CSVs are schema-compatible with the dashboard's per-turn logic (PR #101). Per-turn rows (turn_index=0/1/2) are generated by the Forge pipeline when request-level turn data is preserved in the benchmark JSON. Update README: 52 → 53 columns, add turn_index to column table (#7), clarify that manual import produces aggregate rows only. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- manual_runs/scripts/vllm/README.md | 97 ++++++++++--------- .../vllm/import_manual_runs_json_v2.py | 1 + 2 files changed, 50 insertions(+), 48 deletions(-) diff --git a/manual_runs/scripts/vllm/README.md b/manual_runs/scripts/vllm/README.md index e5e17d7..e20a24d 100644 --- a/manual_runs/scripts/vllm/README.md +++ b/manual_runs/scripts/vllm/README.md @@ -112,7 +112,7 @@ python import_manual_runs_json_v2.py \ --csv-file "gemma-multiturn.csv" ``` -> **Note:** `turns`, `prefix_tokens`, `prefix_count`, and `request_type` are auto-detected from the guidellm JSON. +> **Note:** `turns`, `prefix_tokens`, `prefix_count`, and `request_type` are auto-detected from the guidellm JSON. `turn_index` is always empty in manual-import output (aggregate rows only) — per-turn rows are generated by the Forge pipeline when `turn_index` data is available in the benchmark JSON. ## Appending to Consolidated Dashboard @@ -124,7 +124,7 @@ tail -n +2 my-benchmark.csv >> ../../../consolidated_dashboard.csv ## Output CSV Columns -The script outputs 52 columns compatible with the performance dashboard: +The script outputs 53 columns compatible with the performance dashboard: | # | Column | Description | | --- | ------------------------- | ---------------------------------------------------- | @@ -134,52 +134,53 @@ The script outputs 52 columns compatible with the performance dashboard: | 4 | `version` | Framework version | | 5 | `prompt toks` | Configured prompt token count | | 6 | `output toks` | Configured output token count | -| 7 | `TP` | Tensor parallelism size | -| 8 | `measured concurrency` | Actual measured concurrency | -| 9 | `intended concurrency` | Requested concurrency (streams) | -| 10 | `measured rps` | Requests per second | -| 11 | `output_tok/sec` | Output tokens per second | -| 12 | `total_tok/sec` | Total tokens per second | -| 13 | `prompt_token_count_mean` | Mean prompt token count | -| 14 | `prompt_token_count_p99` | P99 prompt token count | -| 15 | `output_token_count_mean` | Mean output token count | -| 16 | `output_token_count_p99` | P99 output token count | -| 17 | `ttft_median` | Time to first token - median (ms) | -| 18 | `ttft_p95` | Time to first token - P95 (ms) | -| 19 | `ttft_p1` | Time to first token - P1 (ms) | -| 20 | `ttft_p999` | Time to first token - P99.9 (ms) | -| 21 | `tpot_median` | Time per output token - median (ms) | -| 22 | `tpot_p95` | Time per output token - P95 (ms) | -| 23 | `tpot_p99` | Time per output token - P99 (ms) | -| 24 | `tpot_p999` | Time per output token - P99.9 (ms) | -| 25 | `tpot_p1` | Time per output token - P1 (ms) | -| 26 | `itl_median` | Inter-token latency - median (ms) | -| 27 | `itl_p95` | Inter-token latency - P95 (ms) | -| 28 | `itl_p999` | Inter-token latency - P99.9 (ms) | -| 29 | `itl_p1` | Inter-token latency - P1 (ms) | -| 30 | `request_latency_median` | End-to-end request latency - median (s) | -| 31 | `request_latency_min` | End-to-end request latency - minimum (s) | -| 32 | `request_latency_max` | End-to-end request latency - maximum (s) | -| 33 | `successful_requests` | Number of successful requests | -| 34 | `errored_requests` | Number of errored requests | -| 35 | `uuid` | Unique benchmark run ID | -| 36 | `ttft_mean` | Time to first token - mean (ms) | -| 37 | `ttft_p99` | Time to first token - P99 (ms) | -| 38 | `itl_mean` | Inter-token latency - mean (ms) | -| 39 | `itl_p99` | Inter-token latency - P99 (ms) | -| 40 | `runtime_args` | Server configuration arguments | -| 41 | `guidellm_start_time_ms` | Benchmark start time (epoch ms) | -| 42 | `guidellm_end_time_ms` | Benchmark end time (epoch ms) | -| 43 | `image_tag` | Container image used | -| 44 | `guidellm_version` | guidellm version used | -| 45 | `DP` | Data parallelism size (empty for TP runs) | -| 46 | `dataset` | Real dataset name (empty for synthetic runs) | -| 47 | `spec_decoding` | Speculative decoding method (empty if none) | -| 48 | `prefix_caching` | Prefix caching status (`yes`, `no`, or empty) | -| 49 | `turns` | Conversation turns for multiturn benchmarks | -| 50 | `prefix_tokens` | Prefix token count (auto-detected from JSON) | -| 51 | `prefix_count` | Prefix count (auto-detected from JSON) | -| 52 | `request_type` | GuideLLM API endpoint type (auto-detected from JSON) | +| 7 | `turn_index` | Per-turn breakdown index (empty for aggregate rows) | +| 9 | `TP` | Tensor parallelism size | +| 9 | `measured concurrency` | Actual measured concurrency | +| 10 | `intended concurrency` | Requested concurrency (streams) | +| 11 | `measured rps` | Requests per second | +| 12 | `output_tok/sec` | Output tokens per second | +| 13 | `total_tok/sec` | Total tokens per second | +| 14 | `prompt_token_count_mean` | Mean prompt token count | +| 15 | `prompt_token_count_p99` | P99 prompt token count | +| 16 | `output_token_count_mean` | Mean output token count | +| 17 | `output_token_count_p99` | P99 output token count | +| 18 | `ttft_median` | Time to first token - median (ms) | +| 19 | `ttft_p95` | Time to first token - P95 (ms) | +| 20 | `ttft_p1` | Time to first token - P1 (ms) | +| 21 | `ttft_p999` | Time to first token - P99.9 (ms) | +| 22 | `tpot_median` | Time per output token - median (ms) | +| 23 | `tpot_p95` | Time per output token - P95 (ms) | +| 24 | `tpot_p99` | Time per output token - P99 (ms) | +| 25 | `tpot_p999` | Time per output token - P99.9 (ms) | +| 26 | `tpot_p1` | Time per output token - P1 (ms) | +| 27 | `itl_median` | Inter-token latency - median (ms) | +| 28 | `itl_p95` | Inter-token latency - P95 (ms) | +| 29 | `itl_p999` | Inter-token latency - P99.9 (ms) | +| 30 | `itl_p1` | Inter-token latency - P1 (ms) | +| 31 | `request_latency_median` | End-to-end request latency - median (s) | +| 32 | `request_latency_min` | End-to-end request latency - minimum (s) | +| 33 | `request_latency_max` | End-to-end request latency - maximum (s) | +| 34 | `successful_requests` | Number of successful requests | +| 35 | `errored_requests` | Number of errored requests | +| 36 | `uuid` | Unique benchmark run ID | +| 37 | `ttft_mean` | Time to first token - mean (ms) | +| 38 | `ttft_p99` | Time to first token - P99 (ms) | +| 39 | `itl_mean` | Inter-token latency - mean (ms) | +| 40 | `itl_p99` | Inter-token latency - P99 (ms) | +| 41 | `runtime_args` | Server configuration arguments | +| 42 | `guidellm_start_time_ms` | Benchmark start time (epoch ms) | +| 43 | `guidellm_end_time_ms` | Benchmark end time (epoch ms) | +| 44 | `image_tag` | Container image used | +| 45 | `guidellm_version` | guidellm version used | +| 46 | `DP` | Data parallelism size (empty for TP runs) | +| 47 | `dataset` | Real dataset name (empty for synthetic runs) | +| 48 | `spec_decoding` | Speculative decoding method (empty if none) | +| 49 | `prefix_caching` | Prefix caching status (`yes`, `no`, or empty) | +| 50 | `turns` | Conversation turns for multiturn benchmarks | +| 51 | `prefix_tokens` | Prefix token count (auto-detected from JSON) | +| 52 | `prefix_count` | Prefix count (auto-detected from JSON) | +| 53 | `request_type` | GuideLLM API endpoint type (auto-detected from JSON) | ## Notes diff --git a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py index 9058c33..d30ceb6 100644 --- a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py +++ b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py @@ -205,6 +205,7 @@ def get_percentile(metrics_dict, key): "dataset": dataset, "spec_decoding": spec_decoding, "prefix_caching": prefix_caching, + "turn_index": "", "turns": turns, "prefix_tokens": detected_prefix_tokens if detected_prefix_tokens is not None From cf5d33bf487fabadea1902b12415d70d71b43410 Mon Sep 17 00:00:00 2001 From: aas008 Date: Tue, 6 Oct 2026 16:32:45 -0400 Subject: [PATCH 2/7] fix: correct turn_index position in README column table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Row numbering was corrupted by the previous renumber script — TP and measured concurrency both landed at row 9. Fix: turn_index=7, TP=8, measured concurrency=9, continuing sequentially to request_type=53. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- manual_runs/scripts/vllm/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manual_runs/scripts/vllm/README.md b/manual_runs/scripts/vllm/README.md index e20a24d..949dd45 100644 --- a/manual_runs/scripts/vllm/README.md +++ b/manual_runs/scripts/vllm/README.md @@ -135,7 +135,7 @@ The script outputs 53 columns compatible with the performance dashboard: | 5 | `prompt toks` | Configured prompt token count | | 6 | `output toks` | Configured output token count | | 7 | `turn_index` | Per-turn breakdown index (empty for aggregate rows) | -| 9 | `TP` | Tensor parallelism size | +| 8 | `TP` | Tensor parallelism size | | 9 | `measured concurrency` | Actual measured concurrency | | 10 | `intended concurrency` | Requested concurrency (streams) | | 11 | `measured rps` | Requests per second | From 89ce4ce3a581ed833816022f216eaab353302783 Mon Sep 17 00:00:00 2001 From: aas008 Date: Tue, 6 Oct 2026 16:35:09 -0400 Subject: [PATCH 3/7] fix: move turn_index adjacent to turns in README column table The script emits turn_index right before turns in the output dict. The README should reflect actual output order, not the RHAIIS schema field position (which pandas aligns by name on concat anyway). turn_index: #49, turns: #50, prefix_tokens: #51, prefix_count: #52, request_type: #53. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- manual_runs/scripts/vllm/README.md | 86 +++++++++++++++--------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/manual_runs/scripts/vllm/README.md b/manual_runs/scripts/vllm/README.md index 949dd45..11a9448 100644 --- a/manual_runs/scripts/vllm/README.md +++ b/manual_runs/scripts/vllm/README.md @@ -134,49 +134,49 @@ The script outputs 53 columns compatible with the performance dashboard: | 4 | `version` | Framework version | | 5 | `prompt toks` | Configured prompt token count | | 6 | `output toks` | Configured output token count | -| 7 | `turn_index` | Per-turn breakdown index (empty for aggregate rows) | -| 8 | `TP` | Tensor parallelism size | -| 9 | `measured concurrency` | Actual measured concurrency | -| 10 | `intended concurrency` | Requested concurrency (streams) | -| 11 | `measured rps` | Requests per second | -| 12 | `output_tok/sec` | Output tokens per second | -| 13 | `total_tok/sec` | Total tokens per second | -| 14 | `prompt_token_count_mean` | Mean prompt token count | -| 15 | `prompt_token_count_p99` | P99 prompt token count | -| 16 | `output_token_count_mean` | Mean output token count | -| 17 | `output_token_count_p99` | P99 output token count | -| 18 | `ttft_median` | Time to first token - median (ms) | -| 19 | `ttft_p95` | Time to first token - P95 (ms) | -| 20 | `ttft_p1` | Time to first token - P1 (ms) | -| 21 | `ttft_p999` | Time to first token - P99.9 (ms) | -| 22 | `tpot_median` | Time per output token - median (ms) | -| 23 | `tpot_p95` | Time per output token - P95 (ms) | -| 24 | `tpot_p99` | Time per output token - P99 (ms) | -| 25 | `tpot_p999` | Time per output token - P99.9 (ms) | -| 26 | `tpot_p1` | Time per output token - P1 (ms) | -| 27 | `itl_median` | Inter-token latency - median (ms) | -| 28 | `itl_p95` | Inter-token latency - P95 (ms) | -| 29 | `itl_p999` | Inter-token latency - P99.9 (ms) | -| 30 | `itl_p1` | Inter-token latency - P1 (ms) | -| 31 | `request_latency_median` | End-to-end request latency - median (s) | -| 32 | `request_latency_min` | End-to-end request latency - minimum (s) | -| 33 | `request_latency_max` | End-to-end request latency - maximum (s) | -| 34 | `successful_requests` | Number of successful requests | -| 35 | `errored_requests` | Number of errored requests | -| 36 | `uuid` | Unique benchmark run ID | -| 37 | `ttft_mean` | Time to first token - mean (ms) | -| 38 | `ttft_p99` | Time to first token - P99 (ms) | -| 39 | `itl_mean` | Inter-token latency - mean (ms) | -| 40 | `itl_p99` | Inter-token latency - P99 (ms) | -| 41 | `runtime_args` | Server configuration arguments | -| 42 | `guidellm_start_time_ms` | Benchmark start time (epoch ms) | -| 43 | `guidellm_end_time_ms` | Benchmark end time (epoch ms) | -| 44 | `image_tag` | Container image used | -| 45 | `guidellm_version` | guidellm version used | -| 46 | `DP` | Data parallelism size (empty for TP runs) | -| 47 | `dataset` | Real dataset name (empty for synthetic runs) | -| 48 | `spec_decoding` | Speculative decoding method (empty if none) | -| 49 | `prefix_caching` | Prefix caching status (`yes`, `no`, or empty) | +| 7 | `TP` | Tensor parallelism size | +| 8 | `measured concurrency` | Actual measured concurrency | +| 9 | `intended concurrency` | Requested concurrency (streams) | +| 10 | `measured rps` | Requests per second | +| 11 | `output_tok/sec` | Output tokens per second | +| 12 | `total_tok/sec` | Total tokens per second | +| 13 | `prompt_token_count_mean` | Mean prompt token count | +| 14 | `prompt_token_count_p99` | P99 prompt token count | +| 15 | `output_token_count_mean` | Mean output token count | +| 16 | `output_token_count_p99` | P99 output token count | +| 17 | `ttft_median` | Time to first token - median (ms) | +| 18 | `ttft_p95` | Time to first token - P95 (ms) | +| 19 | `ttft_p1` | Time to first token - P1 (ms) | +| 20 | `ttft_p999` | Time to first token - P99.9 (ms) | +| 21 | `tpot_median` | Time per output token - median (ms) | +| 22 | `tpot_p95` | Time per output token - P95 (ms) | +| 23 | `tpot_p99` | Time per output token - P99 (ms) | +| 24 | `tpot_p999` | Time per output token - P99.9 (ms) | +| 25 | `tpot_p1` | Time per output token - P1 (ms) | +| 26 | `itl_median` | Inter-token latency - median (ms) | +| 27 | `itl_p95` | Inter-token latency - P95 (ms) | +| 28 | `itl_p999` | Inter-token latency - P99.9 (ms) | +| 29 | `itl_p1` | Inter-token latency - P1 (ms) | +| 30 | `request_latency_median` | End-to-end request latency - median (s) | +| 31 | `request_latency_min` | End-to-end request latency - minimum (s) | +| 32 | `request_latency_max` | End-to-end request latency - maximum (s) | +| 33 | `successful_requests` | Number of successful requests | +| 34 | `errored_requests` | Number of errored requests | +| 35 | `uuid` | Unique benchmark run ID | +| 36 | `ttft_mean` | Time to first token - mean (ms) | +| 37 | `ttft_p99` | Time to first token - P99 (ms) | +| 38 | `itl_mean` | Inter-token latency - mean (ms) | +| 39 | `itl_p99` | Inter-token latency - P99 (ms) | +| 40 | `runtime_args` | Server configuration arguments | +| 41 | `guidellm_start_time_ms` | Benchmark start time (epoch ms) | +| 42 | `guidellm_end_time_ms` | Benchmark end time (epoch ms) | +| 43 | `image_tag` | Container image used | +| 44 | `guidellm_version` | guidellm version used | +| 45 | `DP` | Data parallelism size (empty for TP runs) | +| 46 | `dataset` | Real dataset name (empty for synthetic runs) | +| 47 | `spec_decoding` | Speculative decoding method (empty if none) | +| 48 | `prefix_caching` | Prefix caching status (`yes`, `no`, or empty) | +| 49 | `turn_index` | Per-turn breakdown index (empty for aggregate rows) | | 50 | `turns` | Conversation turns for multiturn benchmarks | | 51 | `prefix_tokens` | Prefix token count (auto-detected from JSON) | | 52 | `prefix_count` | Prefix count (auto-detected from JSON) | From 428e31f43da40bca090b358bd937f308dfe01933 Mon Sep 17 00:00:00 2001 From: aas008 Date: Tue, 6 Oct 2026 18:00:08 -0400 Subject: [PATCH 4/7] fix: add turn_index to fieldnames so it is written to CSV output The turn_index key was added to the result dict but missing from the fieldnames list passed to combined_df[fieldnames], so pandas silently dropped it. CSV output now has 53 columns as the README states. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- manual_runs/scripts/vllm/import_manual_runs_json_v2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py index d30ceb6..9712de1 100644 --- a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py +++ b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py @@ -513,6 +513,7 @@ def main(): "dataset", "spec_decoding", "prefix_caching", + "turn_index", "turns", "prefix_tokens", "prefix_count", From e03a053494fc3553e859e3cc7a8c80a9adc20273 Mon Sep 17 00:00:00 2001 From: aas008 Date: Wed, 7 Oct 2026 11:37:38 -0400 Subject: [PATCH 5/7] feat(manual-import): generate per-turn rows from request-level turn_index data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the guidellm JSON contains requests.successful with turn_index fields (multi-turn benchmarks), the script now emits: - one aggregate row per concurrency level (turn_index = empty, unchanged) - one per-turn row per turn per concurrency level (turn_index = 0, 1, 2, …) Per-turn rows carry the same metadata as the aggregate row plus per-turn stats: TTFT/ITL/TPOT median+p95+p99+p1+p999+mean, request latency, output throughput, token counts. Benchmarks without turn_index data are unaffected. Add _percentile() helper and extract_per_turn_rows() function. Update README to clarify that per-turn rows are now emitted automatically. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- manual_runs/scripts/vllm/README.md | 2 +- .../vllm/import_manual_runs_json_v2.py | 124 +++++++++++++++++- 2 files changed, 124 insertions(+), 2 deletions(-) diff --git a/manual_runs/scripts/vllm/README.md b/manual_runs/scripts/vllm/README.md index 11a9448..5d89f52 100644 --- a/manual_runs/scripts/vllm/README.md +++ b/manual_runs/scripts/vllm/README.md @@ -112,7 +112,7 @@ python import_manual_runs_json_v2.py \ --csv-file "gemma-multiturn.csv" ``` -> **Note:** `turns`, `prefix_tokens`, `prefix_count`, and `request_type` are auto-detected from the guidellm JSON. `turn_index` is always empty in manual-import output (aggregate rows only) — per-turn rows are generated by the Forge pipeline when `turn_index` data is available in the benchmark JSON. +> **Note:** `turns`, `prefix_tokens`, `prefix_count`, and `request_type` are auto-detected from the guidellm JSON. For multi-turn benchmarks, the script emits one aggregate row per concurrency level (`turn_index` = empty) **plus** one per-turn row per turn per concurrency level (`turn_index` = 0, 1, 2, …) when `requests.successful` contains `turn_index` data. Non-multi-turn benchmarks emit aggregate rows only. ## Appending to Consolidated Dashboard diff --git a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py index 9712de1..cdf56c4 100644 --- a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py +++ b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py @@ -13,6 +13,125 @@ import pandas as pd +def _percentile(values, p): + """Return the p-th percentile of a sorted list, or None if empty.""" + if not values: + return None + values = sorted(values) + idx = (len(values) - 1) * p / 100 + lo, hi = int(idx), min(int(idx) + 1, len(values) - 1) + return values[lo] + (values[hi] - values[lo]) * (idx - lo) + + +def extract_per_turn_rows(benchmark, base_row): + """Extract per-turn breakdown rows from request-level data in a benchmark section. + + For each turn_index found in requests.successful, computes per-turn stats + (TTFT, ITL, TPOT, latency, throughput) and returns one row dict per turn. + Returns an empty list if no turn_index data is present. + + Args: + benchmark: A single benchmark dict from the guidellm JSON. + base_row: The aggregate row dict produced by process_benchmark_section — + used as the metadata template for per-turn rows. + + Returns: + list[dict]: Per-turn row dicts ready to append to all_run_data. + """ + requests = benchmark.get("requests", {}) + successful = requests.get("successful", []) + errored = requests.get("errored", []) + + if not successful and not errored: + return [] + + # Discover all turn indices from both buckets + all_turns = set() + for req in successful + errored: + info = req.get("info", {}) + ti = info.get("turn_index") + if ti is not None: + try: + all_turns.add(str(int(float(ti)))) + except (ValueError, TypeError): + pass + + if not all_turns: + return [] + + # Group successful requests by turn_index + by_turn = {} + for req in successful: + info = req.get("info", {}) + ti = info.get("turn_index") + if ti is None: + continue + try: + key = str(int(float(ti))) + except (ValueError, TypeError): + continue + by_turn.setdefault(key, []).append(req) + + # Group errored requests by turn_index for error counts + err_by_turn = {} + for req in errored: + info = req.get("info", {}) + ti = info.get("turn_index") + if ti is None: + continue + try: + key = str(int(float(ti))) + except (ValueError, TypeError): + continue + err_by_turn.setdefault(key, []).append(req) + + per_turn_rows = [] + for turn_key in sorted(all_turns, key=lambda x: int(x)): + reqs = by_turn.get(turn_key, []) + n_err = len(err_by_turn.get(turn_key, [])) + + ttfts = [r.get("time_to_first_token_ms") for r in reqs if r.get("time_to_first_token_ms") is not None] + itls = [r.get("inter_token_latency_ms") for r in reqs if r.get("inter_token_latency_ms") is not None] + tpots = [r.get("time_per_output_token_ms") for r in reqs if r.get("time_per_output_token_ms") is not None] + lats = [r.get("request_latency") for r in reqs if r.get("request_latency") is not None] + out_tps = [r.get("output_tokens_per_second") for r in reqs if r.get("output_tokens_per_second") is not None] + total_tps = [r.get("tokens_per_second") for r in reqs if r.get("tokens_per_second") is not None] + out_toks = [r.get("output_tokens") for r in reqs if r.get("output_tokens") is not None] + prompt_toks = [r.get("prompt_tokens") for r in reqs if r.get("prompt_tokens") is not None] + + row = dict(base_row) + row["turn_index"] = int(turn_key) + row["successful_requests"] = len(reqs) + row["errored_requests"] = n_err + row["ttft_median"] = _percentile(ttfts, 50) + row["ttft_p95"] = _percentile(ttfts, 95) + row["ttft_p99"] = _percentile(ttfts, 99) + row["ttft_p1"] = _percentile(ttfts, 1) + row["ttft_p999"] = _percentile(ttfts, 99.9) + row["ttft_mean"] = (sum(ttfts) / len(ttfts)) if ttfts else None + row["itl_median"] = _percentile(itls, 50) + row["itl_p95"] = _percentile(itls, 95) + row["itl_p99"] = _percentile(itls, 99) + row["itl_p1"] = _percentile(itls, 1) + row["itl_p999"] = _percentile(itls, 99.9) + row["itl_mean"] = (sum(itls) / len(itls)) if itls else None + row["tpot_median"] = _percentile(tpots, 50) + row["tpot_p95"] = _percentile(tpots, 95) + row["tpot_p99"] = _percentile(tpots, 99) + row["tpot_p1"] = _percentile(tpots, 1) + row["tpot_p999"] = _percentile(tpots, 99.9) + row["request_latency_median"] = _percentile(lats, 50) + row["request_latency_min"] = min(lats) if lats else None + row["request_latency_max"] = max(lats) if lats else None + row["output_tok/sec"] = (sum(out_tps) / len(out_tps)) if out_tps else None + row["total_tok/sec"] = (sum(total_tps) / len(total_tps)) if total_tps else None + row["output_token_count_mean"] = (sum(out_toks) / len(out_toks)) if out_toks else None + row["prompt_token_count_mean"] = (sum(prompt_toks) / len(prompt_toks)) if prompt_toks else None + per_turn_rows.append(row) + + return per_turn_rows + + def process_benchmark_section( benchmark, accelerator, @@ -329,11 +448,14 @@ def parse_guidellm_json( ) if row_data: all_run_data.append(row_data) + per_turn = extract_per_turn_rows(benchmark, row_data) + all_run_data.extend(per_turn) streams = ( benchmark.get("config", {}).get("strategy", {}).get("streams", "?") ) + turn_msg = f", {len(per_turn)} per-turn rows" if per_turn else "" print( - f" Processed benchmark {i + 1}/{len(benchmarks)} (streams={streams})" + f" Processed benchmark {i + 1}/{len(benchmarks)} (streams={streams}{turn_msg})" ) if all_run_data: From 77a988ccccf616b238e0387e756a7923cf38e7d1 Mon Sep 17 00:00:00 2001 From: aas008 Date: Wed, 7 Oct 2026 11:39:58 -0400 Subject: [PATCH 6/7] refactor: align per-turn stats with Forge PR #215 implementation - _percentile(): use same k/f/c linear interpolation as Forge - Add _median(), _mean(), _sorted_field() helpers matching Forge's _turn_metric_values() helpers exactly - Add len(all_turns) <= 1 guard: skip per-turn extraction for single-turn runs (matches Forge PR #215 behavior) - Replace _percentile(50) calls with _median() for correctness Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../vllm/import_manual_runs_json_v2.py | 105 ++++++++++-------- 1 file changed, 61 insertions(+), 44 deletions(-) diff --git a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py index cdf56c4..f59fde5 100644 --- a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py +++ b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py @@ -14,13 +14,32 @@ def _percentile(values, p): - """Return the p-th percentile of a sorted list, or None if empty.""" + """Return the p-th percentile of a pre-sorted list using linear interpolation. + + Matches the implementation in Forge PR #215 _percentile(). + """ if not values: return None - values = sorted(values) - idx = (len(values) - 1) * p / 100 - lo, hi = int(idx), min(int(idx) + 1, len(values) - 1) - return values[lo] + (values[hi] - values[lo]) * (idx - lo) + k = (len(values) - 1) * p / 100.0 + f = int(k) + c = min(f + 1, len(values) - 1) + return values[f] + (k - f) * (values[c] - values[f]) + + +def _median(values): + """Return the median of a pre-sorted list. Matches Forge PR #215 _median().""" + n = len(values) + if not n: + return None + return values[n // 2] if n % 2 else (values[n // 2 - 1] + values[n // 2]) / 2 + + +def _mean(values): + return sum(values) / len(values) if values else None + + +def _sorted_field(reqs, field): + return sorted(r[field] for r in reqs if r.get(field) is not None) def extract_per_turn_rows(benchmark, base_row): @@ -56,14 +75,14 @@ def extract_per_turn_rows(benchmark, base_row): except (ValueError, TypeError): pass - if not all_turns: + # Match Forge PR #215: skip extraction for single-turn runs + if len(all_turns) <= 1: return [] - # Group successful requests by turn_index + # Group requests by turn_index by_turn = {} for req in successful: - info = req.get("info", {}) - ti = info.get("turn_index") + ti = req.get("info", {}).get("turn_index") if ti is None: continue try: @@ -72,11 +91,9 @@ def extract_per_turn_rows(benchmark, base_row): continue by_turn.setdefault(key, []).append(req) - # Group errored requests by turn_index for error counts err_by_turn = {} for req in errored: - info = req.get("info", {}) - ti = info.get("turn_index") + ti = req.get("info", {}).get("turn_index") if ti is None: continue try: @@ -90,43 +107,43 @@ def extract_per_turn_rows(benchmark, base_row): reqs = by_turn.get(turn_key, []) n_err = len(err_by_turn.get(turn_key, [])) - ttfts = [r.get("time_to_first_token_ms") for r in reqs if r.get("time_to_first_token_ms") is not None] - itls = [r.get("inter_token_latency_ms") for r in reqs if r.get("inter_token_latency_ms") is not None] - tpots = [r.get("time_per_output_token_ms") for r in reqs if r.get("time_per_output_token_ms") is not None] - lats = [r.get("request_latency") for r in reqs if r.get("request_latency") is not None] - out_tps = [r.get("output_tokens_per_second") for r in reqs if r.get("output_tokens_per_second") is not None] - total_tps = [r.get("tokens_per_second") for r in reqs if r.get("tokens_per_second") is not None] - out_toks = [r.get("output_tokens") for r in reqs if r.get("output_tokens") is not None] - prompt_toks = [r.get("prompt_tokens") for r in reqs if r.get("prompt_tokens") is not None] + ttft = _sorted_field(reqs, "time_to_first_token_ms") + itl = _sorted_field(reqs, "inter_token_latency_ms") + tpot = _sorted_field(reqs, "time_per_output_token_ms") + lat = _sorted_field(reqs, "request_latency") + out_tps = _sorted_field(reqs, "output_tokens_per_second") + total_tps = _sorted_field(reqs, "tokens_per_second") + out_toks = _sorted_field(reqs, "output_tokens") + prompt_toks = _sorted_field(reqs, "prompt_tokens") row = dict(base_row) row["turn_index"] = int(turn_key) row["successful_requests"] = len(reqs) row["errored_requests"] = n_err - row["ttft_median"] = _percentile(ttfts, 50) - row["ttft_p95"] = _percentile(ttfts, 95) - row["ttft_p99"] = _percentile(ttfts, 99) - row["ttft_p1"] = _percentile(ttfts, 1) - row["ttft_p999"] = _percentile(ttfts, 99.9) - row["ttft_mean"] = (sum(ttfts) / len(ttfts)) if ttfts else None - row["itl_median"] = _percentile(itls, 50) - row["itl_p95"] = _percentile(itls, 95) - row["itl_p99"] = _percentile(itls, 99) - row["itl_p1"] = _percentile(itls, 1) - row["itl_p999"] = _percentile(itls, 99.9) - row["itl_mean"] = (sum(itls) / len(itls)) if itls else None - row["tpot_median"] = _percentile(tpots, 50) - row["tpot_p95"] = _percentile(tpots, 95) - row["tpot_p99"] = _percentile(tpots, 99) - row["tpot_p1"] = _percentile(tpots, 1) - row["tpot_p999"] = _percentile(tpots, 99.9) - row["request_latency_median"] = _percentile(lats, 50) - row["request_latency_min"] = min(lats) if lats else None - row["request_latency_max"] = max(lats) if lats else None - row["output_tok/sec"] = (sum(out_tps) / len(out_tps)) if out_tps else None - row["total_tok/sec"] = (sum(total_tps) / len(total_tps)) if total_tps else None - row["output_token_count_mean"] = (sum(out_toks) / len(out_toks)) if out_toks else None - row["prompt_token_count_mean"] = (sum(prompt_toks) / len(prompt_toks)) if prompt_toks else None + row["ttft_median"] = _median(ttft) + row["ttft_p95"] = _percentile(ttft, 95) + row["ttft_p99"] = _percentile(ttft, 99) + row["ttft_p1"] = _percentile(ttft, 1) + row["ttft_p999"] = _percentile(ttft, 99.9) + row["ttft_mean"] = _mean(ttft) + row["itl_median"] = _median(itl) + row["itl_p95"] = _percentile(itl, 95) + row["itl_p99"] = _percentile(itl, 99) + row["itl_p1"] = _percentile(itl, 1) + row["itl_p999"] = _percentile(itl, 99.9) + row["itl_mean"] = _mean(itl) + row["tpot_median"] = _median(tpot) + row["tpot_p95"] = _percentile(tpot, 95) + row["tpot_p99"] = _percentile(tpot, 99) + row["tpot_p1"] = _percentile(tpot, 1) + row["tpot_p999"] = _percentile(tpot, 99.9) + row["request_latency_median"] = _median(lat) + row["request_latency_min"] = min(lat) if lat else None + row["request_latency_max"] = max(lat) if lat else None + row["output_tok/sec"] = _mean(out_tps) + row["total_tok/sec"] = _mean(total_tps) + row["output_token_count_mean"] = _mean(out_toks) + row["prompt_token_count_mean"] = _mean(prompt_toks) per_turn_rows.append(row) return per_turn_rows From f56017f3313c4d8163556b3fe8f5a9249719618b Mon Sep 17 00:00:00 2001 From: aas008 Date: Wed, 7 Oct 2026 14:39:16 -0400 Subject: [PATCH 7/7] feat(manual-import): add optional --mlflow-run-id and --mlflow-experiment-id flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When provided, these values are stamped on every output row and included in the fieldnames list so the dashboard's Filtered Data table renders a clickable MLflow artifact link (added to the schema in PR #83). Both flags are optional and default to empty string — no breaking change. Closes #107 Co-Authored-By: Claude Sonnet 4.6 (1M context) --- manual_runs/scripts/vllm/README.md | 6 +++++- .../vllm/import_manual_runs_json_v2.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/manual_runs/scripts/vllm/README.md b/manual_runs/scripts/vllm/README.md index 5d89f52..7c9e494 100644 --- a/manual_runs/scripts/vllm/README.md +++ b/manual_runs/scripts/vllm/README.md @@ -46,6 +46,8 @@ python import_manual_runs_json_v2.py \ | `--dataset` | No | Real dataset name (for real-dataset runs) | `mlperf-gpt-oss`, `sharegpt` | | `--spec-decoding` | No | Speculative decoding method used | `eagle3`, `ngram` | | `--prefix-caching` | No | Whether prefix caching was enabled | `yes`, `no` | +| `--mlflow-run-id` | No | MLflow run UUID — enables clickable MLflow links in dashboard | `c6aa48a0d312448380621e1bf00a8a5d` | +| `--mlflow-experiment-id` | No | MLflow experiment ID (used with `--mlflow-run-id`) | `264` | ## Examples @@ -124,7 +126,7 @@ tail -n +2 my-benchmark.csv >> ../../../consolidated_dashboard.csv ## Output CSV Columns -The script outputs 53 columns compatible with the performance dashboard: +The script outputs 55 columns compatible with the performance dashboard: | # | Column | Description | | --- | ------------------------- | ---------------------------------------------------- | @@ -181,6 +183,8 @@ The script outputs 53 columns compatible with the performance dashboard: | 51 | `prefix_tokens` | Prefix token count (auto-detected from JSON) | | 52 | `prefix_count` | Prefix count (auto-detected from JSON) | | 53 | `request_type` | GuideLLM API endpoint type (auto-detected from JSON) | +| 54 | `mlflow_run_id` | MLflow run UUID (from `--mlflow-run-id`, empty if not provided) | +| 55 | `mlflow_experiment_id` | MLflow experiment ID (from `--mlflow-experiment-id`, empty if not provided) | ## Notes diff --git a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py index f59fde5..e1ec5ea 100644 --- a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py +++ b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py @@ -559,6 +559,19 @@ def main(): help="Whether prefix caching is enabled ('yes' or 'no'). " "Leave empty if not applicable.", ) + parser.add_argument( + "--mlflow-run-id", + default="", + help="MLflow run UUID (optional). When provided, the Filtered Data table " + "in the dashboard shows a clickable MLflow artifact link. " + "Example: c6aa48a0d312448380621e1bf00a8a5d", + ) + parser.add_argument( + "--mlflow-experiment-id", + default="", + help="MLflow experiment ID (optional, used together with --mlflow-run-id). " + "Example: 264", + ) parser.add_argument( "--csv-file", default="new_benchmarks.csv", @@ -593,6 +606,10 @@ def main(): ) if new_data_df is not None and not new_data_df.empty: + # Stamp MLflow identifiers on every row (empty string when not provided) + new_data_df["mlflow_run_id"] = args.mlflow_run_id + new_data_df["mlflow_experiment_id"] = args.mlflow_experiment_id + if os.path.exists(args.csv_file): print(f"Appending {len(new_data_df)} new rows to {args.csv_file}...") existing_df = pd.read_csv(args.csv_file) @@ -657,6 +674,8 @@ def main(): "prefix_tokens", "prefix_count", "request_type", + "mlflow_run_id", + "mlflow_experiment_id", ] for col in fieldnames: