From 6b5cbb3227ed26b773faafd50a220e1989577507 Mon Sep 17 00:00:00 2001 From: Fergus Finn Date: Fri, 13 Nov 2026 14:05:48 +0000 Subject: [PATCH] feat(agent): record per-request timings with --request-events-jsonl batchbench-agent reported only aggregate latency percentiles, so a replayed request could not be compared with the latency production recorded for it. --request-events-jsonl writes one line per request, success or failure, when the run ends: when the first attempt started on the run clock, time to the first and last streamed chunks carrying content, reasoning or tool calls, end-to-end latency, time per output token, usage token counts including cached prompt tokens, attempts, status, error and HTTP status. Records are collected per worker only when the option is set. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VDPqeFzGgF3vkuiyimwMxX --- README.md | 22 ++ rust/src/agent.rs | 453 ++++++++++++++++++++++++++++++++++++++---- rust/src/agent_cli.rs | 214 ++++++++++++++++++-- rust/src/lib.rs | 2 +- 4 files changed, 632 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 9a79954..71d27bf 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,28 @@ perfect prefix caching and the end-to-end session latency. `--results-csv` persi the same figures; `--agent-events-jsonl` records per-trajectory admission and completion times. +`--request-events-jsonl` writes one line per agent request, success or failure, when +the run ends. Times are milliseconds measured by the client: + +| Field | Meaning | +|---|---| +| `trajectory_id`, `agent_id` | The trajectory the request belongs to. | +| `invocation` | The request's 1-based position in its trajectory. | +| `sent_at_ms` | Start of the first attempt, from the start of the run (the clock `--agent-events-jsonl` uses). | +| `ttft_ms` | From `sent_at_ms` to the first streamed chunk whose delta carries `content`, `reasoning_content` or `tool_calls`. Null for non-streamed requests and when no such chunk arrived. | +| `last_token_ms` | From `sent_at_ms` to the last such chunk. | +| `e2e_ms` | From `sent_at_ms` until the request succeeded or its last attempt failed. | +| `tpot_ms` | `(last_token_ms - ttft_ms) / (completion_tokens - 1)`; null below two completion tokens. | +| `prompt_tokens`, `completion_tokens` | From the response's `usage`; null on failure. | +| `cached_tokens` | `usage.prompt_tokens_details.cached_tokens`, or null when the backend omits it. | +| `attempts` | Attempts made, retries included. | +| `status`, `error` | `"ok"` with a null error, or `"error"` with the last attempt's error. | +| `http_status` | Status of the last attempt's response; null when none arrived. | + +For a retried request, `ttft_ms` and `last_token_ms` come from the attempt that +succeeded and are measured from the first attempt's start. A trajectory stops at its +first failed request, so its failure is its last line. + `batchbench-harness` drives runs on remote GPU hosts; see [HARNESS.md](HARNESS.md). ## Building from source diff --git a/rust/src/agent.rs b/rust/src/agent.rs index 6278cf0..54a86ee 100644 --- a/rust/src/agent.rs +++ b/rust/src/agent.rs @@ -142,6 +142,8 @@ pub struct AgentLoopConfig { pub replay_turn_overhead_tokens: usize, pub admission: AdmissionMode, pub time_scale: f64, + /// Keep one `AgentRequestRecord` per request in the report. + pub record_requests: bool, } /// How replayed trajectories enter the benchmark. @@ -221,6 +223,7 @@ impl AgentLoopConfig { replay_turn_overhead_tokens: 0, admission: AdmissionMode::ClosedLoop, time_scale: 1.0, + record_requests: false, }) } @@ -332,6 +335,11 @@ impl AgentLoopConfig { Ok(self) } + pub fn with_request_records(mut self, record_requests: bool) -> Self { + self.record_requests = record_requests; + self + } + pub fn add_header(mut self, name: HeaderName, value: HeaderValue) -> Self { self.headers.insert(name, value); self @@ -442,6 +450,30 @@ pub struct AgentLifecycleRecord { pub completed: bool, } +/// Client-side timing of one request, retries included. +#[derive(Clone, Debug)] +pub struct AgentRequestRecord { + pub agent_id: usize, + pub trajectory_id: String, + pub invocation: usize, + /// When the first attempt started, on the benchmark clock. + pub sent_at: Duration, + /// From `sent_at` to the first streamed chunk carrying output, in the successful attempt. + pub first_token: Option, + /// From `sent_at` to the last streamed chunk carrying output, in the successful attempt. + pub last_token: Option, + /// From `sent_at` until the request succeeded or its last attempt failed. + pub latency: Duration, + pub prompt_tokens: Option, + pub completion_tokens: Option, + /// `usage.prompt_tokens_details.cached_tokens`, when the backend reports it. + pub cached_tokens: Option, + pub attempts: usize, + /// Status of the last attempt's response; `None` when no response arrived. + pub http_status: Option, + pub error: Option, +} + #[derive(Clone, Debug)] pub struct AgentBenchmarkReport { pub total_agents: usize, @@ -469,6 +501,9 @@ pub struct AgentBenchmarkReport { pub agent_end_to_end_latency_p99: Option, pub failures: Vec, pub agent_lifecycles: Vec, + /// One record per request, ordered by agent and invocation; empty unless + /// `record_requests` is set. + pub request_records: Vec, /// Schema v2 live assistant blocks that had no previous reply to substitute. pub live_block_fallbacks: u64, /// Open-loop admissions delayed because `max_active_agents` was reached. @@ -496,6 +531,7 @@ struct AgentWorkerReport { tool_call_latency: Duration, latencies: Vec, failures: Vec, + request_records: Vec, dry_run_records: Vec, live_block_fallbacks: u64, } @@ -523,6 +559,7 @@ impl AgentWorkerReport { tool_call_latency: Duration::ZERO, latencies: Vec::new(), failures: Vec::new(), + request_records: Vec::new(), dry_run_records: Vec::new(), live_block_fallbacks: 0, } @@ -657,8 +694,24 @@ impl BlockSpec { struct RequestResult { prompt_tokens: u64, completion_tokens: u64, + cached_tokens: Option, assistant_message: Value, + /// Streamed responses: arrival of the first and last chunks carrying output. + first_token_at: Option, + last_token_at: Option, +} + +/// One request, retries included. +#[derive(Debug)] +struct RequestOutcome { + /// When the first attempt started. + sent_at: Instant, + /// From `sent_at` until the request succeeded or its last attempt failed. latency: Duration, + attempts: usize, + /// Status of the last attempt's response, if one arrived. + http_status: Option, + result: Result, } #[derive(Debug)] @@ -814,6 +867,7 @@ struct ReportAccumulator { latencies: Vec, agent_end_to_end_latencies: Vec, failures: Vec, + request_records: Vec, dry_run_records: Vec, agent_lifecycles: Vec, live_block_fallbacks: u64, @@ -861,6 +915,7 @@ impl ReportAccumulator { .saturating_add(worker.live_block_fallbacks); self.latencies.extend(worker.latencies); self.failures.extend(worker.failures); + self.request_records.extend(worker.request_records); self.dry_run_records.extend(worker.dry_run_records); } @@ -877,6 +932,8 @@ impl ReportAccumulator { self.agent_end_to_end_latencies.sort(); self.failures .sort_by_key(|failure| (failure.agent_id, failure.invocation)); + self.request_records + .sort_by_key(|record| (record.agent_id, record.invocation)); self.agent_lifecycles.sort_by_key(|record| record.agent_id); let total_requests = self .successful_requests @@ -909,6 +966,7 @@ impl ReportAccumulator { agent_end_to_end_latency_p99: percentile(&self.agent_end_to_end_latencies, 0.99), failures: self.failures, agent_lifecycles: self.agent_lifecycles, + request_records: self.request_records, live_block_fallbacks: self.live_block_fallbacks, late_admissions: outcome.late_admissions, max_admission_lag: outcome.max_admission_lag, @@ -941,6 +999,7 @@ async fn run_closed_loop( &mut join_set, plan, routing_slot, + start, Duration::ZERO, client, config, @@ -1008,6 +1067,7 @@ async fn run_closed_loop( &mut join_set, plan, routing_slot, + start, admitted_at, client, config, @@ -1213,6 +1273,7 @@ fn spawn_open_loop_agent( let result = run_agent( plan, routing_slot, + shared.start.into_std(), admitted_at, client, config, @@ -1355,10 +1416,12 @@ fn take_prepared_in_fifo_order( Ok(ready) } +#[allow(clippy::too_many_arguments)] fn spawn_agent( join_set: &mut JoinSet>, plan: AgentPlan, routing_slot: usize, + run_start: Instant, admitted_at: Duration, client: &Client, config: &std::sync::Arc, @@ -1371,6 +1434,7 @@ fn spawn_agent( run_agent( plan, routing_slot, + run_start, admitted_at, client, config, @@ -2533,6 +2597,7 @@ async fn generate_next_reset_prompt( async fn run_agent( mut plan: AgentPlan, routing_slot: usize, + run_start: Instant, admitted_at: Duration, client: Client, config: std::sync::Arc, @@ -2550,6 +2615,7 @@ async fn run_agent( plan, initial_prompt, routing_slot, + run_start, admitted_at, client, config, @@ -2562,6 +2628,7 @@ async fn run_agent( plan, cache, routing_slot, + run_start, admitted_at, client, config, @@ -2578,10 +2645,12 @@ async fn run_agent( /// Schema v1 semantics: a growing conversation where each turn appends the model reply and a /// synthetic environment response sized from the manifest. +#[allow(clippy::too_many_arguments)] async fn run_growth_agent( plan: AgentPlan, initial_prompt: String, routing_slot: usize, + run_start: Instant, admitted_at: Duration, client: Client, config: std::sync::Arc, @@ -2673,7 +2742,7 @@ async fn run_growth_agent( turn.environment_tokens, ); let next_reset = generate_next_reset_prompt(&text_generator, &plan, turn_index); - let (request_result, environment_content, next_reset_prompt) = + let (outcome, environment_content, next_reset_prompt) = tokio::join!(request, environment, next_reset); prefetched_reset_prompt = next_reset_prompt?; let environment_content = environment_content.with_context(|| { @@ -2682,10 +2751,20 @@ async fn run_growth_agent( plan.trajectory_id, invocation ) })?; + if config.record_requests { + report + .request_records + .push(request_record(&plan, invocation, run_start, &outcome)); + } - match request_result { + match outcome.result { Ok(result) => { - record_success(&mut report, &result, &mut previous_prompt_tokens); + record_success( + &mut report, + &result, + outcome.latency, + &mut previous_prompt_tokens, + ); if !turn.tool_call_latency.is_zero() { tokio::time::sleep(turn.tool_call_latency).await; @@ -2732,10 +2811,12 @@ async fn run_growth_agent( /// Schema v2 semantics: every request's prompt is assembled from its blocks. Generated block /// text is cached per trajectory, and live assistant blocks carry the model's previous reply. +#[allow(clippy::too_many_arguments)] async fn run_block_agent( plan: AgentPlan, mut cache: BlockCache, routing_slot: usize, + run_start: Instant, admitted_at: Duration, client: Client, config: std::sync::Arc, @@ -2839,7 +2920,7 @@ async fn run_block_agent( ); let prefetch = text_generator.generate_blocks(next_turn_prefetch(&plan, turn_index, &cache)); - let (request_result, prefetched_now) = tokio::join!(request, prefetch); + let (outcome, prefetched_now) = tokio::join!(request, prefetch); prefetched = prefetched_now.with_context(|| { format!( "failed to prefetch blocks for trajectory {:?} invocation {}", @@ -2847,10 +2928,20 @@ async fn run_block_agent( invocation + 1 ) })?; + if config.record_requests { + report + .request_records + .push(request_record(&plan, invocation, run_start, &outcome)); + } - match request_result { + match outcome.result { Ok(result) => { - record_success(&mut report, &result, &mut previous_prompt_tokens); + record_success( + &mut report, + &result, + outcome.latency, + &mut previous_prompt_tokens, + ); last_reply = Some(normalize_assistant_message(result.assistant_message)?); if !turn.tool_call_latency.is_zero() { @@ -2926,6 +3017,7 @@ async fn resolve_live_blocks( fn record_success( report: &mut AgentWorkerReport, result: &RequestResult, + latency: Duration, previous_prompt_tokens: &mut u64, ) { report.successful_requests += 1; @@ -2941,7 +3033,40 @@ fn record_success( result.prompt_tokens, )); *previous_prompt_tokens = result.prompt_tokens; - report.latencies.push(result.latency); + report.latencies.push(latency); +} + +fn request_record( + plan: &AgentPlan, + invocation: usize, + run_start: Instant, + outcome: &RequestOutcome, +) -> AgentRequestRecord { + let since_sent = |at: Instant| at.saturating_duration_since(outcome.sent_at); + let result = outcome.result.as_ref().ok(); + AgentRequestRecord { + agent_id: plan.agent_id, + trajectory_id: plan.trajectory_id.clone(), + invocation, + sent_at: outcome.sent_at.saturating_duration_since(run_start), + first_token: result + .and_then(|result| result.first_token_at) + .map(since_sent), + last_token: result + .and_then(|result| result.last_token_at) + .map(since_sent), + latency: outcome.latency, + prompt_tokens: result.map(|result| result.prompt_tokens), + completion_tokens: result.map(|result| result.completion_tokens), + cached_tokens: result.and_then(|result| result.cached_tokens), + attempts: outcome.attempts, + http_status: outcome.http_status, + error: outcome + .result + .as_ref() + .err() + .map(|error| format!("{error:#}")), + } } fn dry_run_record( @@ -3070,29 +3195,28 @@ async fn request_with_retries( user_tag: Option<&str>, routing_slot: usize, stream: bool, -) -> Result { - let start = Instant::now(); - let mut last_error = None; - - for attempt in 0..=config.max_retries { - match single_attempt(client, config, body, user_tag, routing_slot, stream).await { - Ok(mut result) => { - result.latency = start.elapsed(); - return Ok(result); - } - Err(err) => { - last_error = Some(err); - if attempt < config.max_retries { - let multiplier = 1u32.checked_shl(attempt.min(31) as u32).unwrap_or(u32::MAX); - tokio::time::sleep(config.retry_delay.saturating_mul(multiplier)).await; - } - } +) -> RequestOutcome { + let sent_at = Instant::now(); + let mut attempt = 0; + loop { + let (http_status, result) = + single_attempt(client, config, body, user_tag, routing_slot, stream).await; + if result.is_ok() || attempt == config.max_retries { + return RequestOutcome { + sent_at, + latency: sent_at.elapsed(), + attempts: attempt + 1, + http_status, + result, + }; } + let multiplier = 1u32.checked_shl(attempt.min(31) as u32).unwrap_or(u32::MAX); + tokio::time::sleep(config.retry_delay.saturating_mul(multiplier)).await; + attempt += 1; } - - Err(last_error.unwrap_or_else(|| anyhow!("request failed without an error"))) } +/// One attempt: the response status, once a response arrived, and the parsed result. async fn single_attempt( client: &Client, config: &AgentLoopConfig, @@ -3100,7 +3224,22 @@ async fn single_attempt( user_tag: Option<&str>, routing_slot: usize, stream: bool, -) -> Result { +) -> (Option, Result) { + let response = match send_request(client, config, body, user_tag, routing_slot).await { + Ok(response) => response, + Err(error) => return (None, Err(error)), + }; + let status = response.status().as_u16(); + (Some(status), read_response(response, config, stream).await) +} + +async fn send_request( + client: &Client, + config: &AgentLoopConfig, + body: &Bytes, + user_tag: Option<&str>, + routing_slot: usize, +) -> Result { if config.verbose { let parsed: Value = serde_json::from_slice(body) .context("failed to parse serialized request for verbose logging")?; @@ -3110,7 +3249,14 @@ async fn single_attempt( let request = client .post(config.endpoint.clone()) .headers(build_request_headers(config, user_tag, routing_slot)?); - let mut response = request.body(body.clone()).send().await?; + Ok(request.body(body.clone()).send().await?) +} + +async fn read_response( + mut response: reqwest::Response, + config: &AgentLoopConfig, + stream: bool, +) -> Result { let status = response.status(); if !status.is_success() { let bytes = response.bytes().await?; @@ -3121,17 +3267,20 @@ async fn single_attempt( )); } - let (usage, assistant_message) = if stream { + let (usage, assistant_message, first_token_at, last_token_at) = if stream { let mut lines = SseLineBuffer::default(); let mut completion = StreamedCompletion::default(); + let mut received_at = Instant::now(); while let Some(chunk) = response.chunk().await? { + received_at = Instant::now(); for payload in lines.push(&chunk) { - completion.absorb(&payload)?; + completion.absorb(&payload, received_at)?; } } if let Some(payload) = lines.finish() { - completion.absorb(&payload)?; + completion.absorb(&payload, received_at)?; } + let (first_token_at, last_token_at) = (completion.first_token_at, completion.last_token_at); let (assistant_message, usage) = completion.into_parts()?; if config.verbose { println!( @@ -3142,7 +3291,7 @@ async fn single_attempt( })) ); } - (usage, assistant_message) + (usage, assistant_message, first_token_at, last_token_at) } else { let bytes = response.bytes().await?; let payload: Value = serde_json::from_slice(&bytes)?; @@ -3157,7 +3306,7 @@ async fn single_attempt( .pointer("/choices/0/message") .cloned() .ok_or_else(|| anyhow!("response missing choices[0].message"))?; - (usage, assistant_message) + (usage, assistant_message, None, None) }; let prompt_tokens = usage .get("prompt_tokens") @@ -3171,8 +3320,12 @@ async fn single_attempt( Ok(RequestResult { prompt_tokens, completion_tokens, + cached_tokens: usage + .pointer("/prompt_tokens_details/cached_tokens") + .and_then(Value::as_u64), assistant_message, - latency: Duration::ZERO, + first_token_at, + last_token_at, }) } @@ -3215,10 +3368,13 @@ struct StreamedCompletion { reasoning_content: Option, usage: Option, chunks: usize, + /// Arrival of the first and last chunks whose delta carries output. + first_token_at: Option, + last_token_at: Option, } impl StreamedCompletion { - fn absorb(&mut self, payload: &str) -> Result<()> { + fn absorb(&mut self, payload: &str, received_at: Instant) -> Result<()> { if payload == "[DONE]" { return Ok(()); } @@ -3229,6 +3385,10 @@ impl StreamedCompletion { self.usage = Some(usage.clone()); } if let Some(delta) = event.pointer("/choices/0/delta") { + if delta_carries_output(delta) { + self.first_token_at.get_or_insert(received_at); + self.last_token_at = Some(received_at); + } if let Some(text) = delta.get("content").and_then(Value::as_str) { self.content.push_str(text); } @@ -3261,6 +3421,21 @@ impl StreamedCompletion { } } +/// Whether a streamed delta carries generated output: content, reasoning, or tool calls. +fn delta_carries_output(delta: &Value) -> bool { + let has_text = |key: &str| { + delta + .get(key) + .and_then(Value::as_str) + .is_some_and(|text| !text.is_empty()) + }; + let has_tool_calls = delta + .get("tool_calls") + .and_then(Value::as_array) + .is_some_and(|calls| !calls.is_empty()); + has_text("content") || has_text("reasoning_content") || has_tool_calls +} + fn build_request_headers( config: &AgentLoopConfig, user_tag: Option<&str>, @@ -4606,16 +4781,17 @@ mod tests { ": keep-alive\n", "data: {\"choices\":[{\"delta\":{\"content\":\"lo\"}}]}\ndata: {\"choices\":[],", ); + let now = Instant::now(); for payload in lines.push(stream.as_bytes()) { - completion.absorb(&payload).unwrap(); + completion.absorb(&payload, now).unwrap(); } for payload in lines.push(b"\"usage\":{\"prompt_tokens\":12,\"completion_tokens\":2}}\n\ndata: [DONE]") { - completion.absorb(&payload).unwrap(); + completion.absorb(&payload, now).unwrap(); } if let Some(payload) = lines.finish() { - completion.absorb(&payload).unwrap(); + completion.absorb(&payload, now).unwrap(); } let (message, usage) = completion.into_parts().unwrap(); assert_eq!(message["role"], "assistant"); @@ -4631,6 +4807,207 @@ mod tests { assert!(StreamedCompletion::default().into_parts().is_err()); } + #[test] + fn token_times_come_from_chunks_that_carry_output() { + let start = Instant::now(); + let at = |millis| start + Duration::from_millis(millis); + let events = [ + ( + 1, + r#"{"choices":[{"delta":{"role":"assistant","content":""}}]}"#, + ), + ( + 2, + r#"{"choices":[{"delta":{"reasoning_content":"think"}}]}"#, + ), + (3, r#"{"choices":[{"delta":{"content":"Hi"}}]}"#), + ( + 4, + r#"{"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{}"}}]}}]}"#, + ), + ( + 5, + r#"{"choices":[{"delta":{"tool_calls":[]},"finish_reason":"stop"}]}"#, + ), + ( + 6, + r#"{"choices":[],"usage":{"prompt_tokens":3,"completion_tokens":3}}"#, + ), + (7, "[DONE]"), + ]; + let mut completion = StreamedCompletion::default(); + for (millis, payload) in events { + completion.absorb(payload, at(millis)).unwrap(); + } + assert_eq!(completion.first_token_at, Some(at(2))); + assert_eq!(completion.last_token_at, Some(at(4))); + + let mut reasoning_only = StreamedCompletion::default(); + reasoning_only + .absorb( + r#"{"choices":[{"delta":{"reasoning_content":"a"}}]}"#, + at(1), + ) + .unwrap(); + reasoning_only + .absorb( + r#"{"choices":[{"delta":{"reasoning_content":"b"}}]}"#, + at(2), + ) + .unwrap(); + assert_eq!( + (reasoning_only.first_token_at, reasoning_only.last_token_at), + (Some(at(1)), Some(at(2))) + ); + + let mut usage_only = StreamedCompletion::default(); + usage_only + .absorb( + r#"{"choices":[],"usage":{"prompt_tokens":3,"completion_tokens":0}}"#, + at(1), + ) + .unwrap(); + assert_eq!(usage_only.first_token_at, None); + assert_eq!(usage_only.last_token_at, None); + } + + /// Serves one scripted HTTP response per connection, then stops. + fn scripted_server(responses: Vec) -> (String, std::thread::JoinHandle<()>) { + use std::io::{Read, Write}; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!( + "http://{}/v1/chat/completions", + listener.local_addr().unwrap() + ); + let server = std::thread::spawn(move || { + for response in responses { + let (mut connection, _) = listener.accept().unwrap(); + let mut request = Vec::new(); + let mut buffer = [0u8; 4096]; + loop { + let read = connection.read(&mut buffer).unwrap(); + request.extend_from_slice(&buffer[..read]); + let text = String::from_utf8_lossy(&request).to_ascii_lowercase(); + if let Some(header_end) = text.find("\r\n\r\n") { + let body_length = text[..header_end] + .lines() + .find_map(|line| line.strip_prefix("content-length:")) + .map_or(0, |value| value.trim().parse::().unwrap()); + if request.len() >= header_end + 4 + body_length { + break; + } + } + } + connection.write_all(response.as_bytes()).unwrap(); + } + }); + (url, server) + } + + fn http_response(status: &str, content_type: &str, body: &str) -> String { + format!( + "HTTP/1.1 {status}\r\ncontent-type: {content_type}\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", + body.len() + ) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn request_records_cover_retries_streams_and_the_failed_request() { + let streamed = |usage: &str| { + [ + r#"{"choices":[{"delta":{"role":"assistant"}}]}"#.to_string(), + r#"{"choices":[{"delta":{"reasoning_content":"hm"}}]}"#.to_string(), + r#"{"choices":[{"delta":{"content":"ok"}}]}"#.to_string(), + format!(r#"{{"choices":[],"usage":{usage}}}"#), + "[DONE]".to_string(), + ] + .iter() + .map(|event| format!("data: {event}\n\n")) + .collect::() + }; + let (url, server) = scripted_server(vec![ + http_response("503 Service Unavailable", "text/plain", "busy"), + http_response( + "200 OK", + "text/event-stream", + &streamed( + r#"{"prompt_tokens":8,"completion_tokens":2,"prompt_tokens_details":{"cached_tokens":5}}"#, + ), + ), + http_response( + "200 OK", + "text/event-stream", + &streamed(r#"{"prompt_tokens":16,"completion_tokens":2}"#), + ), + http_response("500 Internal Server Error", "text/plain", "down"), + http_response("500 Internal Server Error", "text/plain", "down"), + ]); + let directory = + std::env::temp_dir().join(format!("batchbench-request-records-{}", Uuid::new_v4())); + std::fs::create_dir_all(&directory).unwrap(); + let tokenizer_path = directory.join("tokenizer.json"); + word_level_tokenizer().save(&tokenizer_path, false).unwrap(); + let manifest_path = directory.join("plans.jsonl"); + std::fs::write( + &manifest_path, + concat!( + r#"{"schema_version":2,"trajectory_id":"t","requests":["#, + r#"{"prompt_tokens":8,"output_tokens":2,"stream":true},"#, + r#"{"prompt_tokens":16,"output_tokens":2,"stream":true},"#, + r#"{"prompt_tokens":24,"output_tokens":2,"stream":true},"#, + r#"{"prompt_tokens":32,"output_tokens":2,"stream":true}]}"#, + "\n" + ), + ) + .unwrap(); + let config = AgentLoopConfig::try_new( + url, + None, + "test-model", + 1, + SampleSpec::fixed(8).unwrap(), + SampleSpec::fixed(2).unwrap(), + SampleSpec::fixed(6).unwrap(), + SampleSpec::fixed(1).unwrap(), + ) + .unwrap() + .with_tokenizer_model(tokenizer_path.display().to_string()) + .with_agent_plans_jsonl(&manifest_path) + .with_retry(1, Duration::from_millis(1)) + .with_request_records(true); + + let report = run_agent_benchmark(config).await.unwrap(); + server.join().unwrap(); + std::fs::remove_dir_all(&directory).unwrap(); + + assert_eq!(report.completed_agents, 0); + assert_eq!(report.failures.len(), 1); + let records = &report.request_records; + assert_eq!(records.len(), 3); + + let retried = &records[0]; + assert_eq!((retried.invocation, retried.attempts), (1, 2)); + assert_eq!(retried.http_status, Some(200)); + assert_eq!(retried.error, None); + assert_eq!(retried.prompt_tokens, Some(8)); + assert_eq!(retried.completion_tokens, Some(2)); + assert_eq!(retried.cached_tokens, Some(5)); + let first_token = retried.first_token.unwrap(); + assert!(first_token <= retried.last_token.unwrap()); + assert!(retried.last_token.unwrap() <= retried.latency); + + assert_eq!((records[1].invocation, records[1].attempts), (2, 1)); + assert_eq!(records[1].cached_tokens, None); + assert!(records[1].sent_at >= retried.sent_at + retried.latency); + + let failed = &records[2]; + assert_eq!((failed.invocation, failed.attempts), (3, 2)); + assert_eq!(failed.http_status, Some(500)); + assert!(failed.error.as_deref().unwrap().contains("500")); + assert_eq!((failed.prompt_tokens, failed.first_token), (None, None)); + } + #[test] fn slot_pool_reuses_the_lowest_free_slot() { let mut pool = SlotPool::default(); diff --git a/rust/src/agent_cli.rs b/rust/src/agent_cli.rs index dc92c1f..63a699c 100644 --- a/rust/src/agent_cli.rs +++ b/rust/src/agent_cli.rs @@ -10,7 +10,8 @@ use serde::Serialize; use sha2::{Digest, Sha256}; use crate::{ - run_agent_benchmark, AdmissionMode, AgentBenchmarkReport, AgentLoopConfig, SampleSpec, + run_agent_benchmark, AdmissionMode, AgentBenchmarkReport, AgentLoopConfig, AgentRequestRecord, + SampleSpec, }; const DEFAULT_MODEL: &str = "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8"; @@ -241,6 +242,10 @@ struct Args { /// Optional path to write one JSONL lifecycle event per agent #[arg(long, value_name = "PATH")] agent_events_jsonl: Option, + + /// Optional path to write one JSONL timing record per request + #[arg(long, value_name = "PATH")] + request_events_jsonl: Option, } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, ValueEnum)] @@ -316,6 +321,7 @@ struct CsvResult { replay_initial_overhead_tokens: Option, replay_turn_overhead_tokens: Option, agent_events_jsonl: Option, + request_events_jsonl: Option, tool_call_latency_ms: Option, tool_call_latency_lognorm_mu: Option, tool_call_latency_lognorm_median_ms: Option, @@ -337,6 +343,67 @@ struct AgentLifecycleJson<'a> { completed: bool, } +#[derive(Debug, Serialize)] +struct RequestEventJson<'a> { + trajectory_id: &'a str, + agent_id: usize, + invocation: usize, + sent_at_ms: f64, + ttft_ms: Option, + last_token_ms: Option, + e2e_ms: f64, + tpot_ms: Option, + prompt_tokens: Option, + completion_tokens: Option, + cached_tokens: Option, + attempts: usize, + status: &'static str, + error: Option<&'a str>, + http_status: Option, +} + +impl<'a> RequestEventJson<'a> { + fn new(record: &'a AgentRequestRecord) -> Self { + let ttft_ms = record.first_token.map(duration_ms); + let last_token_ms = record.last_token.map(duration_ms); + Self { + trajectory_id: &record.trajectory_id, + agent_id: record.agent_id, + invocation: record.invocation, + sent_at_ms: duration_ms(record.sent_at), + ttft_ms, + last_token_ms, + e2e_ms: duration_ms(record.latency), + tpot_ms: time_per_output_token_ms(ttft_ms, last_token_ms, record.completion_tokens), + prompt_tokens: record.prompt_tokens, + completion_tokens: record.completion_tokens, + cached_tokens: record.cached_tokens, + attempts: record.attempts, + status: if record.error.is_none() { + "ok" + } else { + "error" + }, + error: record.error.as_deref(), + http_status: record.http_status, + } + } +} + +/// Mean gap between output tokens after the first one. +fn time_per_output_token_ms( + ttft_ms: Option, + last_token_ms: Option, + completion_tokens: Option, +) -> Option { + match (ttft_ms, last_token_ms, completion_tokens) { + (Some(first), Some(last), Some(tokens)) if tokens >= 2 => { + Some((last - first) / (tokens - 1) as f64) + } + _ => None, + } +} + enum ParsedArgs { Ready(Box), Displayed, @@ -573,7 +640,8 @@ async fn run(args: Args) -> Result<()> { .with_ignore_eos(args.ignore_eos) .with_user_tagging(!args.disable_user_tagging) .with_verbose(args.verbose) - .with_dry_run(args.dry_run); + .with_dry_run(args.dry_run) + .with_request_records(args.request_events_jsonl.is_some()); if let Some(path) = args.agent_plans_jsonl.as_deref() { config = config.with_agent_plans_jsonl(path); config = config.with_replay_prompt_overhead( @@ -609,6 +677,10 @@ async fn run(args: Args) -> Result<()> { write_agent_events_jsonl(events_path, &report, args.dp_rank_perfect_routing_num)?; println!("Wrote agent lifecycle events to {}", events_path.display()); } + if let Some(events_path) = args.request_events_jsonl.as_deref() { + write_request_events_jsonl(events_path, &report.request_records)?; + println!("Wrote request events to {}", events_path.display()); + } if let Some(csv_path) = args.results_csv.as_deref() { let record = CsvResult { @@ -668,6 +740,10 @@ async fn run(args: Args) -> Result<()> { .agent_events_jsonl .as_ref() .map(|path| path.display().to_string()), + request_events_jsonl: args + .request_events_jsonl + .as_ref() + .map(|path| path.display().to_string()), tool_call_latency_ms: args.tool_call_latency_ms, tool_call_latency_lognorm_mu: args.tool_call_latency_lognorm_mu, tool_call_latency_lognorm_median_ms: args.tool_call_latency_lognorm_median_ms, @@ -930,7 +1006,11 @@ fn print_summary(report: &AgentBenchmarkReport) { } fn milliseconds(duration: Option) -> Option { - duration.map(|value| value.as_secs_f64() * 1000.0) + duration.map(duration_ms) +} + +fn duration_ms(duration: Duration) -> f64 { + duration.as_secs_f64() * 1000.0 } fn format_latency(duration: Option) -> String { @@ -980,6 +1060,34 @@ fn write_agent_events_jsonl( report: &AgentBenchmarkReport, dp_rank_count: Option, ) -> Result<()> { + write_jsonl( + path, + report.agent_lifecycles.iter().map(|record| { + let runtime = record.finished_at.saturating_sub(record.admitted_at); + AgentLifecycleJson { + agent_id: record.agent_id, + trajectory_id: &record.trajectory_id, + routing_slot: record.routing_slot, + dp_rank: dp_rank_count.map(|ranks| record.routing_slot % ranks), + scheduled_at_seconds: record.scheduled_at.as_secs_f64(), + admitted_at_seconds: record.admitted_at.as_secs_f64(), + queue_wait_seconds: record + .admitted_at + .saturating_sub(record.scheduled_at) + .as_secs_f64(), + finished_at_seconds: record.finished_at.as_secs_f64(), + runtime_seconds: runtime.as_secs_f64(), + completed: record.completed, + } + }), + ) +} + +fn write_request_events_jsonl(path: &Path, records: &[AgentRequestRecord]) -> Result<()> { + write_jsonl(path, records.iter().map(RequestEventJson::new)) +} + +fn write_jsonl(path: &Path, events: impl IntoIterator) -> Result<()> { if let Some(parent) = path.parent() { if !parent.as_os_str().is_empty() { fs::create_dir_all(parent) @@ -989,23 +1097,7 @@ fn write_agent_events_jsonl( let file = fs::File::create(path).with_context(|| format!("failed to create {}", path.display()))?; let mut writer = BufWriter::new(file); - for record in &report.agent_lifecycles { - let runtime = record.finished_at.saturating_sub(record.admitted_at); - let event = AgentLifecycleJson { - agent_id: record.agent_id, - trajectory_id: &record.trajectory_id, - routing_slot: record.routing_slot, - dp_rank: dp_rank_count.map(|ranks| record.routing_slot % ranks), - scheduled_at_seconds: record.scheduled_at.as_secs_f64(), - admitted_at_seconds: record.admitted_at.as_secs_f64(), - queue_wait_seconds: record - .admitted_at - .saturating_sub(record.scheduled_at) - .as_secs_f64(), - finished_at_seconds: record.finished_at.as_secs_f64(), - runtime_seconds: runtime.as_secs_f64(), - completed: record.completed, - }; + for event in events { serde_json::to_writer(&mut writer, &event) .with_context(|| format!("failed to serialize {}", path.display()))?; writer @@ -1244,4 +1336,86 @@ mod tests { .unwrap_err(); assert!(error.to_string().contains("--agent-plans-jsonl")); } + + #[test] + fn time_per_output_token_spans_the_tokens_after_the_first() { + assert_eq!( + time_per_output_token_ms(Some(250.0), Some(1250.0), Some(5)), + Some(250.0) + ); + assert_eq!( + time_per_output_token_ms(Some(250.0), Some(250.0), Some(1)), + None + ); + assert_eq!(time_per_output_token_ms(None, None, Some(5)), None); + assert_eq!( + time_per_output_token_ms(Some(250.0), Some(1250.0), None), + None + ); + } + + #[test] + fn request_events_jsonl_writes_one_record_per_request() { + let ok = AgentRequestRecord { + agent_id: 3, + trajectory_id: "session-a".to_string(), + invocation: 1, + sent_at: Duration::from_millis(1500), + first_token: Some(Duration::from_millis(250)), + last_token: Some(Duration::from_millis(1250)), + latency: Duration::from_millis(1500), + prompt_tokens: Some(100), + completion_tokens: Some(5), + cached_tokens: Some(64), + attempts: 2, + http_status: Some(200), + error: None, + }; + let failed = AgentRequestRecord { + invocation: 2, + sent_at: Duration::from_millis(4000), + first_token: None, + last_token: None, + latency: Duration::from_millis(750), + prompt_tokens: None, + completion_tokens: None, + cached_tokens: None, + attempts: 3, + http_status: Some(503), + error: Some("request failed (503 Service Unavailable) busy".to_string()), + ..ok.clone() + }; + let path = std::env::temp_dir() + .join(format!("batchbench-requests-{}", uuid::Uuid::new_v4())) + .join("requests.jsonl"); + + write_request_events_jsonl(&path, &[ok, failed]).unwrap(); + let written = fs::read_to_string(&path).unwrap(); + fs::remove_dir_all(path.parent().unwrap()).unwrap(); + + let lines: Vec = written + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + assert_eq!( + lines, + [ + serde_json::json!({ + "trajectory_id": "session-a", "agent_id": 3, "invocation": 1, + "sent_at_ms": 1500.0, "ttft_ms": 250.0, "last_token_ms": 1250.0, + "e2e_ms": 1500.0, "tpot_ms": 250.0, + "prompt_tokens": 100, "completion_tokens": 5, "cached_tokens": 64, + "attempts": 2, "status": "ok", "error": null, "http_status": 200, + }), + serde_json::json!({ + "trajectory_id": "session-a", "agent_id": 3, "invocation": 2, + "sent_at_ms": 4000.0, "ttft_ms": null, "last_token_ms": null, + "e2e_ms": 750.0, "tpot_ms": null, + "prompt_tokens": null, "completion_tokens": null, "cached_tokens": null, + "attempts": 3, "status": "error", + "error": "request failed (503 Service Unavailable) busy", "http_status": 503, + }), + ] + ); + } } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 64a3b63..2d53fad 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -11,7 +11,7 @@ mod tokenizer_loader; pub use agent::{ run_agent_benchmark, AdmissionMode, AgentBenchmarkReport, AgentFailureRecord, - AgentLifecycleRecord, AgentLoopConfig, SampleSpec, + AgentLifecycleRecord, AgentLoopConfig, AgentRequestRecord, SampleSpec, }; pub use agent_cli::{run_from_argv as run_agent_from_argv, run_from_env as run_agent_from_env}; pub use cli::{run_from_argv, run_from_env};