diff --git a/README.md b/README.md index e49b55f..9a79954 100644 --- a/README.md +++ b/README.md @@ -1,69 +1,21 @@ # BatchBench -BatchBench ships a Rust benchmarking core with a thin Python wrapper. - -You can install it with: +Load generation for OpenAI-compatible chat endpoints. A Rust core with a thin Python +wrapper, published as prebuilt wheels: ```bash uv pip install batchbench ``` -The Python package exposes Rust functionality for request generation and benchmark execution. - -## Python API - -```python -import batchbench - -config = { - "endpoint": "https://example.com/v1/chat/completions", - "user_count": 1, - "mode": batchbench.finite_mode(requests_per_user=1), - "request_body": batchbench.request_entry( - { - "model": "gpt-4o-mini", - "messages": [{"role": "user", "content": "ping"}], - }, - line_idx=0, - input_tokens=1, - ), - "requests": [ - batchbench.request_entry( - { - "model": "gpt-4o-mini", - "messages": [{"role": "user", "content": "ping"}], - "max_tokens": 4, - }, - line_idx=0, - input_tokens=1, - ) - ], - "dry_run": True, -} - -report = batchbench.run_benchmark(config) -print(report) -``` - -Request generation: - -```python -requests = batchbench.generate_requests( - { - "count": 16, - "prefix_overlap": 0.2, - "target_tokens": 128, - "tokenizer_model": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", - "dist_mode": "fixed", - }, - model="Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", -) -``` +Three entrypoints: -## Python CLI +| Command | What it does | +|---|---| +| `batchbench` | Independent requests at a fixed concurrency with controlled token shapes | +| `batchbench-agent` | Stateful multi-turn sessions, sampled or replayed from a manifest | +| `batchbench-export-plans` | Builds a replay manifest from prompt-chain records in ClickHouse | -The package installs `batchbench`, which forwards directly to the Rust CLI implementation. -Use the same flags as the Rust binary: +## Independent requests ```bash batchbench \ @@ -75,21 +27,33 @@ batchbench \ --output-vary 0 ``` -Use `--sglang` to apply output token constraints via `min_new_tokens`/`max_new_tokens` -instead of `min_tokens`/`max_tokens`. +Output length is enforced with `min_tokens`/`max_tokens`; pass `--sglang` for +backends that use `min_new_tokens`/`max_new_tokens`. `Ctrl+C` cancels active +requests and prints a partial summary. The same run is available from Python: -Press `Ctrl+C` during a run to cancel active requests and print a partial summary. +```python +import batchbench -## Agent-loop benchmark +report = batchbench.run_benchmark({ + "endpoint": "https://example.com/v1/chat/completions", + "user_count": 1, + "mode": batchbench.finite_mode(requests_per_user=1), + "request_body": batchbench.request_entry( + {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "ping"}]}, + line_idx=0, input_tokens=1), + "requests": [batchbench.request_entry( + {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "ping"}], "max_tokens": 4}, + line_idx=0, input_tokens=1)], + "dry_run": True, +}) +``` -`batchbench-agent` is a separate entrypoint for stateful agent workloads. It starts -`--agents` independent loops concurrently. Within each agent, model requests remain -sequential: the returned assistant message and a synthetic tool/environment response -are appended to `messages`, then the complete growing conversation is sent again. -This makes every request after the first one reuse that agent's previous prompt as a -prefix and exercises server-side KV caching. +## Multi-turn sessions -Fixed-length example: +`batchbench-agent` runs concurrent agent loops. Within a loop, each request appends +the model's reply and a synthetic tool result to the conversation and sends it again, +so every turn reuses the previous prompt as a prefix. Token shapes are fixed or +log-normal: ```bash batchbench-agent \ @@ -103,321 +67,75 @@ batchbench-agent \ --tool-call-latency-ms 250 ``` -Log-normal example: - -```bash -batchbench-agent \ - --model Qwen/Qwen3-8B \ - --host http://localhost:8000 \ - --agents 16 \ - --input-lognorm-median 256 \ - --input-lognorm-sigma 0.5 \ - --input-lognorm-max 2048 \ - --output-lognorm-median 64 \ - --output-lognorm-sigma 0.4 \ - --output-lognorm-max 512 \ - --environment-lognorm-median 128 \ - --environment-lognorm-sigma 0.6 \ - --environment-lognorm-max 1024 \ - --tool-invocations-lognorm-median 8 \ - --tool-invocations-lognorm-sigma 0.3 \ - --tool-invocations-lognorm-max 32 \ - --tool-call-latency-lognorm-median-ms 250 \ - --tool-call-latency-lognorm-sigma 0.5 \ - --tool-call-latency-lognorm-max-ms 2000 \ - --seed 42 -``` - -Each log-normal family accepts either `*-lognorm-median` or `*-lognorm-mu`, -requires `*-lognorm-sigma`, and optionally accepts `*-lognorm-max`. Samples are -independent between agents and turns; `--seed` makes the sampled workload -reproducible. `--tokenizer-model` can be supplied when the endpoint's model name is -not also a Hugging Face tokenizer identifier. It accepts a Hugging Face model ID, -a local `tokenizer.json` file, or a local directory containing `tokenizer.json`. - -### Trajectory replay and rolling admission - -Use `--agent-plans-jsonl` to replay complete empirical token shapes instead of -sampling initial prompts, outputs, environment results, and invocation counts -independently. Each non-empty JSONL line is one trajectory, and file order is the -FIFO admission order: - -```json -{"schema_version":1,"trajectory_id":"agent-001","requests":[{"prompt_tokens":13381,"output_tokens":288},{"prompt_tokens":14800,"output_tokens":512,"delay_after_ms":25},{"prompt_tokens":7000,"output_tokens":128,"reset_before":true}]} -``` - -`prompt_tokens` and `output_tokens` are required and must be positive. -`delay_after_ms` defaults to zero. For a normal transition, BatchBench infers the -synthetic environment growth as: - -```text -next prompt - current prompt - current output -``` - -If a trajectory compacts or resets such that this value would be negative, mark the -next request with `"reset_before": true`. BatchBench then starts that request from a -fresh synthetic prompt while preserving the trajectory's user/routing identity. A -reset deliberately claims no prefix-cache reuse from the preceding request. +Each `--*-tokens` flag has a `--*-lognorm-median` / `--*-lognorm-sigma` / +`--*-lognorm-max` family; `--seed` makes sampling reproducible. `--tokenizer-model` +names the tokenizer when the endpoint's model name is not a Hugging Face identifier +(a model id, a `tokenizer.json`, or a directory containing one). Every request carries a +per-agent `user` field and routing header so a gateway can keep a session on one +worker; `--user-prefix`, `--disable-user-tagging`, and +`--dp-rank-perfect-routing` control that. -Absolute prompt counts commonly include chat-template and tool-envelope tokens that -are not part of the synthetic message content. After calibrating those values against -the target backend, use `--replay-initial-overhead-tokens` for a first request or -reset, and `--replay-turn-overhead-tokens` for each normal appended turn. BatchBench -subtracts these values when generating content while retaining the manifest's -absolute prompt targets for reporting. +## Replaying a manifest -`--max-active-agents` bounds simultaneous trajectories while retaining every plan -in the manifest. BatchBench initially admits up to that limit. Whenever an active -trajectory terminates, the next queued trajectory prepares its initial prompt and -then inherits the freed scheduler/routing slot. Multiple replacements prepare -concurrently through the same bounded worker pool while preserving FIFO admission -order, keeping active load balanced across data-parallel ranks when perfect routing -is enabled. +A manifest is a JSONL file with one trajectory per line: its requests, their prompt +and output token counts, the gaps between them, and optionally the prompt's content +blocks. The agent binary replays it instead of sampling: ```bash batchbench-agent \ --model Qwen/Qwen3-8B \ --host http://localhost:8000 \ - --agent-plans-jsonl examples/trajectory-replay/plans.jsonl \ - --max-active-agents 2 \ - --replay-initial-overhead-tokens 0 \ - --replay-turn-overhead-tokens 0 \ - --agent-events-jsonl agent-events.jsonl \ - --seed 42 \ - --dry-run -``` - -Do not combine `--agent-plans-jsonl` with `--agents` or the synthetic workload-shape -flags. `--seed` remains useful because it makes generated placeholder text -reproducible. Generated content is accepted only when it re-encodes to the requested -token count; an unconstructable field fails with its trajectory context instead of -silently changing the workload. - -Replay manifests remain compact in memory: BatchBench materializes the initially -active prompts before the benchmark clock starts, then generates only the current -environment response and the next reset prompt while the current HTTP request is in -flight. Token generation uses a bounded worker pool, and its per-agent/per-turn random -streams are independent of asynchronous scheduling. Request JSON is serialized from -the live conversation without first cloning the complete message tree. - -`--agent-events-jsonl` writes each trajectory's scheduled and actual admission time, -queue wait, reusable routing slot, optional DP rank, finish time, runtime, and -completion status. The summary separately reports the final admission time and final -drain duration. - -### Schema version 2: content blocks and open-loop admission - -Schema version 1 manifests keep working unchanged. Version 2 adds per-request -content structure, per-request overrides, and a start offset per trajectory. Lines -of both versions may be mixed in one file. - -```json -{"schema_version":2,"trajectory_id":"session-7f3a-0","start_after_ms":1250,"requests":[ - {"prompt_tokens":1340,"output_tokens":96,"overhead_tokens":41,"stream":true,"max_tokens":512,"delay_after_ms":830, - "blocks":[{"seed":"9c1d…","tokens":611,"role":"tool_definition"},{"seed":"2b77…","tokens":420,"role":"system"},{"seed":"e04a…","tokens":268,"role":"user"}]}, - {"prompt_tokens":1612,"output_tokens":40,"overhead_tokens":52,"stream":true,"max_tokens":512, - "blocks":[{"seed":"9c1d…","tokens":611,"role":"tool_definition"},{"seed":"2b77…","tokens":420,"role":"system"},{"seed":"e04a…","tokens":268,"role":"user"}, - {"seed":"51f0…","tokens":96,"role":"assistant","live":true},{"seed":"c9aa…","tokens":80,"role":"tool_call"},{"seed":"77d2…","tokens":55,"role":"tool"}]}]} -``` - -(Shown wrapped for readability; each trajectory is one JSONL line. See -`examples/trajectory-replay/plans-v2.jsonl` for two trajectories that share their -tool-definition and system blocks.) - -`blocks` describes the prompt as ordered content blocks. `role` is one of -`tool_definition`, `system`, `user`, `assistant`, `tool`, and `tool_call`. -Tool definitions become entries in the request's `tools` array (synthetic function -schemas whose serialized JSON re-encodes to `tokens`); `system`, `user`, `assistant`, -and `tool` blocks become messages of that role; a `tool_call` block becomes a -synthetic `tool_calls` entry on the preceding assistant message (or on a new -assistant message when none precedes it), and the next `tool` block references its -id. Block text is generated from the seed alone, at exactly `tokens` tokens, so -equal seeds produce identical bytes in every trajectory, request, and run. Shared -prefixes across sessions (the same system prompt, the same tool set) therefore -replay as identical bytes and exercise cross-session prefix caching. Within a -trajectory, a seed that was already sent reuses the text already sent. - -A block with `"live": true` and role `assistant` is the model's own previous reply in -this conversation: BatchBench substitutes the assistant message returned by the -previous request instead of generating text. When there is no previous reply (the -first request of a trajectory, a second live block in the same request, or a -request after a reset), the block is generated from its seed and counted in the -report as a live block fallback. - -When `blocks` is present the environment-growth inference of version 1 is not used: -each request's blocks define its prompt, so there is no negative-growth case. -`reset_before` still starts a fresh conversation (the per-trajectory block cache is -dropped) and, in version 2, is accepted on the first request to mark a session -whose earlier turns predate the exported window. `sum(blocks.tokens)` is the -content target and `prompt_tokens` remains the reporting target; they should agree -with `prompt_tokens == sum(blocks.tokens) + overhead_tokens`. A mismatch is logged -once per trajectory and the blocks are replayed as written. - -`overhead_tokens` overrides `--replay-initial-overhead-tokens` / -`--replay-turn-overhead-tokens` for that request: it is the number of chat-template -and tool-envelope tokens the backend adds on top of the content, so the content -target is `prompt_tokens - overhead_tokens`. Requests without it fall back to the -global flags (initial overhead for a first or reset request, plus the turn overhead -for each appended turn). - -`stream` sends `"stream": true` with `stream_options.include_usage` and consumes the -server-sent events into the assistant message. `max_tokens` sets the output cap the -same way the run-level output-token fields do (honouring `--sglang`); the planned -`output_tokens` remains the floor, clamped to the cap. - -`start_after_ms` is the trajectory's start offset from benchmark start. It is used by -`--admission open-loop`, which admits every trajectory at its offset regardless of -free slots; `--max-active-agents` then becomes a hard cap that delays admission -when reached, and each delayed trajectory is counted as a late admission. The -default `--admission closed-loop` is the existing behaviour (manifest-order -admission into `--max-active-agents` slots), where offsets are ignored. -`--time-scale ` divides every `start_after_ms` and `delay_after_ms` value, -so `--time-scale 4` replays a recorded hour in fifteen minutes. Under open-loop -admission the first requests of the earliest trajectories are prepared before the -clock starts, and later trajectories are prepared ahead of their offsets through a -bounded lookahead; the report's maximum admission lag shows how far any admission -slipped behind its schedule for any reason (cap or preparation). - -```bash -batchbench-agent \ - --model Qwen/Qwen3-8B \ - --host http://localhost:8000 \ - --agent-plans-jsonl examples/trajectory-replay/plans-v2.jsonl \ + --agent-plans-jsonl plans.jsonl \ --admission open-loop \ --time-scale 2 \ - --max-active-agents 64 \ - --agent-events-jsonl agent-events.jsonl \ - --dry-run + --agent-events-jsonl events.jsonl \ + --results-csv results.csv ``` -The summary, `--results-csv`, and `--agent-events-jsonl` gain `live_block_fallbacks`, -`late_admissions`, `max_admission_lag_ms`, `admission_mode`, `time_scale`, and the -per-trajectory `scheduled_at_seconds`. Under open-loop admission the reported -maximum active agents is the peak concurrency actually observed. Unknown fields are -rejected in both schema versions, and version 1 lines reject the version 2 fields. - -Trajectory replay reproduces the joint request-count and nominal token-shape plan, -not the original text. Prompt targets include workload-specific framing, while -BatchBench generates its own chat/tool framing and exact-length synthetic message -content. Uncalibrated framing can therefore make live prompts drift systematically -from their targets over a long trajectory. Live `usage.prompt_tokens` is -authoritative and should be compared with the plan before a capacity result is -accepted. Likewise, exact output lengths require a backend that honors the requested -minimum/maximum token constraints. - -Every request includes a `user` field and an `X-SMG-Routing-Key` header containing -the same UUID, which remains stable for the life of that agent and differs between -agents. Use `--disable-user-tagging` to omit both, or `--user-prefix ` to -use deterministic `-` values instead of UUIDs. - -For deterministic data-parallel routing, pass both -`--dp-rank-perfect-routing` and `--dp-rank-perfect-routing-num `. -Each request then includes `X-SMG-Target-Worker: `. Initial -agents occupy consecutive slots; under rolling admission, a replacement inherits -the slot freed by the trajectory that just terminated. This preserves the active -per-rank balance and is independent of user tagging. - -Tool-call latency defaults to zero. Set a fixed delay with -`--tool-call-latency-ms`, or sample milliseconds independently for every invocation -with `--tool-call-latency-lognorm-median-ms` (or -`--tool-call-latency-lognorm-mu`), `--tool-call-latency-lognorm-sigma`, and the -optional `--tool-call-latency-lognorm-max-ms`. After a model response succeeds, the -agent sleeps for the sampled duration before making the environment result -available and submitting its next model request. - -One tool invocation means one unconstrained model request followed by one synthetic -environment response. BatchBench treats generated assistant output as opaque state: -it preserves content and reasoning output but ignores any model-generated tool calls. -It then adds its own valid synthetic `environment` tool call and appends the sampled -environment response. This keeps the trajectory protocol-valid without assuming -anything about the generated output. - -The final report includes: - -- total input tokens sent, from successful responses' `usage.prompt_tokens`; -- total output tokens generated, from `usage.completion_tokens`; -- estimated cached input tokens under perfect prefix caching; -- total simulated tool-call latency across all agents; -- request-latency p50, p90, and p99 across successful requests; -- end-to-end p50, p90, and p99 across completed agents, measured from the start - of each agent loop through its final synthetic tool delay. - -Failed agents are excluded from the end-to-end latency distribution because their -lifetimes end early. Their request failures are still included in the failure report. +`--admission open-loop` starts each trajectory at its recorded offset; the default +closed-loop admission fills `--max-active-agents` slots in file order. `--time-scale N` +replays a recorded hour in `1/N` hours. Block text is generated from each block's seed +at exactly the requested token count, so blocks with equal seeds send identical bytes +and shared prefixes exercise prefix caching. `--dry-run` checks a manifest without +sending requests. The format is specified in [docs/manifest.md](docs/manifest.md); +`examples/trajectory-replay/` has one file per schema version. -For each successful request after an agent's first, the cache estimate adds that -same agent's preceding prompt-token count (capped by the current prompt count). -Retries and failed requests are excluded because they do not provide reliable usage -data. Use `--results-csv ` to persist the same totals, or `--dry-run` to inspect -the sampled or replayed plans without sending requests. +## Building a manifest from records -## Exporting plans from production records - -`batchbench export-plans` (also `python -m batchbench.export_plans`) turns a window of -ClickHouse prompt-chain records into a schema version 2 manifest. Each -`clay.prompt_chains` row is one chat-completions request as a chain of keyed content -hashes, one per prompt block, joined with `clay.http_analytics` on -`(instance_id, correlation_id)` for its token counts and request scalars. The chains -are read through the deduplicating `clay.prompt_chains_current` view by default -(`--chains-table` / `--chains-final` select the base table with `FINAL` instead), and -both sides of the join are bounded to the window before joining. Install the -ClickHouse client with `uv pip install "batchbench[export]"`. +`batchbench-export-plans` reads a window of prompt-chain records from ClickHouse, +rebuilds sessions from the chains, and writes a schema version 2 manifest: ```bash -CLICKHOUSE_URL=https://user:password@warehouse.example.com:8443 \ -batchbench export-plans \ +uv pip install "batchbench[export]" + +CLICKHOUSE_URL=https://warehouse.example.com:8443 CLICKHOUSE_USER=... CLICKHOUSE_PASSWORD=... \ +batchbench-export-plans \ + --chains-table chains_current --analytics-table requests \ --start 2026-09-01T09:00:00Z --end 2026-09-01T10:00:00Z \ --model vendor/model-a \ --sample 0.1 --seed 7 --stratify-by-session-length \ - --time-scale 4 \ --output plans.jsonl ``` -Sessions are reconstructed from the chains themselves: within a principal, a request -continues the latest earlier request (within `--link-window-hours`, default 24) whose -full chain is a strict prefix of its own chain, and each root with its descendants -forms one trajectory ordered by timestamp. The query starts one link window before -`--start` so that sessions which began earlier are recognised; their first in-window -request carries `reset_before`. Per request, `prompt_tokens` and `output_tokens` come -from the analytics row, `delay_after_ms` is the gap between this request's end and -the next request's start, `blocks` zips the chain hashes (as seeds) with the block -roles and token counts, `overhead_tokens` is `prompt_tokens - sum(block_tokens)`, and -the block that is the reply to the previous request in the trajectory is marked -`live`. Requests whose block token counts are missing become a single `user` block of -`prompt_tokens`. `stream` and `max_tokens` are copied when recorded. Requests without -a positive prompt or completion count are dropped and counted in the summary. +The input tables it expects, and how sessions are reconstructed, are described in +[docs/export-plans.md](docs/export-plans.md). -`--sample` keeps a deterministic fraction of trajectories (seeded by `--seed`); -`--stratify-by-session-length` applies the fraction within power-of-two -session-length buckets so short and long sessions are both represented. -`--time-scale` divides the start offsets and delays. Filters: `--principal-id`, -`--model`, `--served-by`. `--dump-rows-jsonl` saves the fetched rows and -`--rows-jsonl` re-exports from such a file without ClickHouse. An export with no -trajectories is an error, and the output is validated against the schema version 2 -rules before it is written. It replays with: +## Report -```bash -batchbench-agent --agent-plans-jsonl plans.jsonl --admission open-loop --model --host -``` +Each run prints total prompt and completion tokens from the responses' `usage` and +request latency p50/p90/p99. Agent runs add the estimated cached prompt tokens under +perfect prefix caching and the end-to-end session latency. `--results-csv` persists +the same figures; `--agent-events-jsonl` records per-trajectory admission and +completion times. -## Rust CLI +`batchbench-harness` drives runs on remote GPU hosts; see [HARNESS.md](HARNESS.md). -The existing Rust CLI is unchanged: +## Building from source ```bash cargo build --release --manifest-path rust/Cargo.toml --bin batchbench -./rust/target/release/batchbench --help -``` - -The agent-loop binary can likewise be run directly: - -```bash cargo build --release --manifest-path rust/Cargo.toml --bin batchbench-agent -./rust/target/release/batchbench-agent --help ``` -## Releases and PyPI - -- CI (`.github/workflows/ci.yaml`) checks Rust build/test, builds a wheel, and runs smoke tests. -- Release Please (`.github/workflows/release-please.yaml`) opens/updates release PRs and, on merge, creates `v*` tags/releases. -- Python release workflow (`.github/workflows/python-release.yaml`) builds and publishes prebuilt platform wheels to PyPI on `v*` tag pushes. +Releases are cut by Release Please; a `v*` tag builds platform wheels and publishes +them to PyPI. diff --git a/docs/export-plans.md b/docs/export-plans.md new file mode 100644 index 0000000..7949444 --- /dev/null +++ b/docs/export-plans.md @@ -0,0 +1,81 @@ +# Exporting a manifest from prompt-chain records + +`batchbench-export-plans` reads prompt-chain records from ClickHouse and writes a +[schema version 2 manifest](manifest.md). It needs the `export` extra: + +```bash +uv pip install "batchbench[export]" +``` + +## Input + +A prompt-chain record describes one chat-completions request without its content: +one cumulative hash per prompt block, with the block's role and token count. Because +the hashes are cumulative, two requests whose prompts share a prefix share a chain +prefix, which is enough to rebuild sessions. + +`--chains-table` names a table or view with these columns: + +| Column | Type | Meaning | +|---|---|---| +| `ts` | DateTime64 | When the request was recorded | +| `instance_id` | UUID | Gateway instance that served it | +| `correlation_id` | Int64 | Request id within that instance | +| `principal_id` | UUID | Whose request it was; sessions never cross principals | +| `model` | String | Model alias | +| `chain` | Array(FixedString(32)) | Cumulative hash after each block | +| `block_roles` | Array(String) | Role per block: `tool_definition`, `system`, `user`, `assistant`, `tool`, `tool_call` | +| `block_tokens` | Array(UInt32) | Untemplated token count per block | + +`--analytics-table` names a per-request usage table joined on +`(instance_id, correlation_id)`, with `prompt_tokens`, `completion_tokens`, `stream`, +`max_tokens`, `finish_reason`, `served_by`, `request_origin`, `user_id`, `api_key_id`, +`duration_ms`, and `duration_to_first_byte_ms`. `--analytics-ts-column` (default +`timestamp`) is the column used to bound that side of the join; it is widened by a day +on each end of the window because a request's start time and its chain's capture time +differ. + +If the chains table is a ReplacingMergeTree with duplicates, read it through a +deduplicating view, or pass `--chains-final` to read the base table with `FINAL`. + +## Output + +```bash +CLICKHOUSE_URL=https://warehouse.example.com:8443 CLICKHOUSE_USER=... CLICKHOUSE_PASSWORD=... \ +batchbench-export-plans \ + --chains-table chains_current --analytics-table requests \ + --start 2026-09-01T09:00:00Z --end 2026-09-01T10:00:00Z \ + --model vendor/model-a \ + --sample 0.1 --seed 7 --stratify-by-session-length \ + --time-scale 4 \ + --output plans.jsonl +``` + +Sessions are reconstructed from the chains: within a principal, a request continues +the latest earlier request (within `--link-window-hours`, default 24) whose full chain +is a strict prefix of its own. Each root and its descendants form one trajectory in +timestamp order. The query starts one link window before `--start` so that sessions +which began earlier are recognised; their first in-window request carries +`reset_before`. + +Per request, `prompt_tokens` and `output_tokens` come from the usage row, +`delay_after_ms` is the gap from this request's end to the next request's start, +`blocks` pairs each chain hash (as the seed) with its role and token count, +`overhead_tokens` is `prompt_tokens - sum(block_tokens)`, and the block that is the +reply to the previous request in the trajectory is marked `live`. `stream` and +`max_tokens` are copied when recorded. A request with no block counts becomes a single +`user` block of `prompt_tokens`. Requests without a positive prompt or completion +count are dropped and counted in the summary line. + +Filters: `--principal-id`, `--model`, `--served-by`. `--sample` keeps a deterministic +fraction of trajectories (`--seed`); `--stratify-by-session-length` applies it within +power-of-two session-length buckets so short and long sessions are both kept. +`--time-scale` divides offsets and delays. `--dump-rows-jsonl` saves the fetched rows +and `--rows-jsonl` re-exports from such a file without ClickHouse. An export with no +trajectories is an error, and the manifest is validated before it is written. + +Replay with: + +```bash +batchbench-agent --agent-plans-jsonl plans.jsonl --admission open-loop --model --host +``` diff --git a/docs/manifest.md b/docs/manifest.md new file mode 100644 index 0000000..c9125ec --- /dev/null +++ b/docs/manifest.md @@ -0,0 +1,108 @@ +# Replay manifest format + +A manifest is a JSONL file. Each non-empty line is one trajectory: a sequence of +requests sent one after another on the same conversation. File order is the admission +order under closed-loop admission. Lines of both schema versions may be mixed in one +file. Unknown fields are rejected. + +## Schema version 1: token counts only + +```json +{"schema_version":1,"trajectory_id":"agent-001","requests":[ + {"prompt_tokens":13381,"output_tokens":288}, + {"prompt_tokens":14800,"output_tokens":512,"delay_after_ms":25}, + {"prompt_tokens":7000,"output_tokens":128,"reset_before":true}]} +``` + +| Field | Meaning | +|---|---| +| `prompt_tokens` | Prompt length the request should reach. Required, positive. | +| `output_tokens` | Completion length to request. Required, positive. | +| `delay_after_ms` | Pause before the next request. Default 0. | +| `reset_before` | Start this request from a fresh conversation instead of appending. | + +Between two appended requests the synthetic tool result is sized so the conversation +grows to the next prompt target: `next prompt - current prompt - current output`. If +that would be negative, mark the next request `reset_before`. A reset claims no +prefix-cache reuse from the previous request. + +Prompt counts from a real backend include chat-template and tool-envelope tokens that +are not message content. `--replay-initial-overhead-tokens` (first or reset request) +and `--replay-turn-overhead-tokens` (each appended turn) are subtracted when generating +content; the manifest's absolute counts remain the reporting target. + +## Schema version 2: content blocks and start offsets + +```json +{"schema_version":2,"trajectory_id":"session-7f3a-0","start_after_ms":1250,"requests":[ + {"prompt_tokens":1340,"output_tokens":96,"overhead_tokens":41,"stream":true,"max_tokens":512,"delay_after_ms":830, + "blocks":[{"seed":"9c1d…","tokens":611,"role":"tool_definition"}, + {"seed":"2b77…","tokens":420,"role":"system"}, + {"seed":"e04a…","tokens":268,"role":"user"}]}, + {"prompt_tokens":1612,"output_tokens":40,"overhead_tokens":52,"stream":true,"max_tokens":512, + "blocks":[{"seed":"9c1d…","tokens":611,"role":"tool_definition"}, + {"seed":"2b77…","tokens":420,"role":"system"}, + {"seed":"e04a…","tokens":268,"role":"user"}, + {"seed":"51f0…","tokens":96,"role":"assistant","live":true}, + {"seed":"c9aa…","tokens":80,"role":"tool_call"}, + {"seed":"77d2…","tokens":55,"role":"tool"}]}]} +``` + +Version 2 adds, per trajectory, `start_after_ms`, and per request: + +| Field | Meaning | +|---|---| +| `blocks` | Ordered content blocks that make up the prompt. | +| `overhead_tokens` | Template and envelope tokens the backend adds on top of the content. Overrides the global overhead flags for this request. | +| `stream` | Send `"stream": true` with usage in the final chunk. | +| `max_tokens` | Output cap; `output_tokens` remains the floor, clamped to the cap. | + +### Blocks + +Each block is `{seed, tokens, role}` plus an optional `live` flag. `role` is one of +`tool_definition`, `system`, `user`, `assistant`, `tool`, `tool_call`. + +- Text is generated from `seed` alone at exactly `tokens` tokens, so equal seeds + produce identical bytes across trajectories, requests, and runs. Two sessions with + the same system prompt replay with the same system-prompt bytes. Within a trajectory + a seed that was already sent reuses the bytes already sent. Generation fails + loudly if a block cannot be constructed at the requested count. +- `tool_definition` blocks become entries in the request's `tools` array. `system`, + `user`, `assistant`, and `tool` blocks become messages of that role. A `tool_call` + block becomes a `tool_calls` entry on the preceding assistant message, and the + following `tool` block references its id. +- An `assistant` block with `"live": true` is the model's own reply from the previous + request in this trajectory, and replays with the text the model actually returned. + When no previous reply exists (first request, second live block in one request, + request after a reset) the block is generated from its seed and counted as a live + block fallback in the report. + +`sum(blocks.tokens) + overhead_tokens` should equal `prompt_tokens`. A mismatch is +logged once per trajectory and the blocks are replayed as written. `reset_before` on +the first request marks a session whose earlier turns predate the exported window. + +### Admission + +`--admission closed-loop` (default) ignores `start_after_ms` and admits trajectories +in file order into `--max-active-agents` slots; a finishing trajectory frees its slot, +and the next one inherits it, which keeps per-rank balance stable under perfect +routing. + +`--admission open-loop` admits every trajectory at `start_after_ms` regardless of free +slots. `--max-active-agents` then becomes a hard cap; a trajectory held back by it is +counted as a late admission, and the report's maximum admission lag shows how far any +admission slipped behind its schedule. `--time-scale N` divides every +`start_after_ms` and `delay_after_ms`. + +The first requests of the earliest trajectories are generated before the clock starts; +later ones are generated ahead of their offsets, and the current tool result and the +next reset prompt are generated while the previous request is in flight. + +### Fidelity + +Replay reproduces the request count, the token shape of every prompt and completion, +prefix sharing, and arrival timing. It does not reproduce the original text: block +content is placeholder text, so the model's replies, tokenization of real content, +and content-dependent effects such as expert routing are not reproduced. Compare live +`usage.prompt_tokens` against the plan before accepting a capacity result, and use a +backend that honours the requested minimum and maximum output tokens. diff --git a/src/batchbench/export_plans.py b/src/batchbench/export_plans.py index 0372c43..87c3732 100644 --- a/src/batchbench/export_plans.py +++ b/src/batchbench/export_plans.py @@ -1,8 +1,10 @@ -"""Export batchbench trajectory manifests (schema version 2) from ClickHouse prompt-chain records. +"""Export batchbench trajectory manifests (schema version 2) from prompt-chain records. -One ``clay.prompt_chains`` row describes one chat-completions request as a chain of keyed -content hashes, one per prompt block, joined with ``clay.http_analytics`` for its token counts -and request scalars. This module turns a time window of those rows into trajectories that +A prompt-chain record describes one chat-completions request as a chain of content hashes, one +per prompt block (tool definition, system prompt, message, tool call), with a role and a token +count per block. Because the hashes are cumulative, requests that share a prefix share a chain +prefix, and a session can be rebuilt from the chains alone. This module turns a time window of +such records, joined with per-request usage rows, into trajectories that ``batchbench-agent --agent-plans-jsonl`` can replay: * Within a principal, request B continues request A when A is the latest earlier request @@ -15,8 +17,9 @@ * ``start_after_ms`` and ``delay_after_ms`` reproduce the recorded arrival pattern, divided by ``--time-scale``. -The ClickHouse client is confined to :func:`fetch_rows`; everything else works on plain row -dictionaries so it can be tested with fixtures. +The expected input schema is documented in ``docs/export-plans.md``. The ClickHouse client is +confined to :func:`fetch_rows`; everything else works on plain row dictionaries so it can be +tested with fixtures. """ from __future__ import annotations @@ -33,8 +36,6 @@ SCHEMA_VERSION = 2 DEFAULT_LINK_WINDOW = timedelta(hours=24) -DEFAULT_CHAINS_TABLE = "clay.prompt_chains_current" -DEFAULT_ANALYTICS_TABLE = "clay.http_analytics" DEFAULT_ANALYTICS_TS_COLUMN = "timestamp" BLOCK_ROLES = ("tool_definition", "system", "user", "assistant", "tool", "tool_call") @@ -768,9 +769,9 @@ def build_parser() -> argparse.ArgumentParser: source.add_argument("--clickhouse-url", default=os.environ.get("CLICKHOUSE_URL"), help="ClickHouse HTTP(S) URL (env CLICKHOUSE_URL)") source.add_argument("--clickhouse-user", default=os.environ.get("CLICKHOUSE_USER"), help="ClickHouse user (env CLICKHOUSE_USER)") source.add_argument("--clickhouse-password", default=os.environ.get("CLICKHOUSE_PASSWORD"), help="ClickHouse password (env CLICKHOUSE_PASSWORD)") - source.add_argument("--chains-table", default=DEFAULT_CHAINS_TABLE, help=f"prompt-chain table or view (default {DEFAULT_CHAINS_TABLE})") + source.add_argument("--chains-table", help="prompt-chain table or view (see docs/export-plans.md for the expected columns)") source.add_argument("--chains-final", action="store_true", help="read the chains table with FINAL (when reading the ReplacingMergeTree base table instead of its deduplicating view)") - source.add_argument("--analytics-table", default=DEFAULT_ANALYTICS_TABLE, help=f"request analytics table (default {DEFAULT_ANALYTICS_TABLE})") + source.add_argument("--analytics-table", help="per-request usage table joined on (instance_id, correlation_id)") source.add_argument("--analytics-ts-column", default=DEFAULT_ANALYTICS_TS_COLUMN, help=f"timestamp column used to bound the analytics side of the join (default {DEFAULT_ANALYTICS_TS_COLUMN})") source.add_argument("--rows-jsonl", help="read previously fetched rows from this JSONL file instead of ClickHouse") source.add_argument("--dump-rows-jsonl", help="also write the fetched rows to this JSONL file") @@ -817,6 +818,8 @@ def main(argv: Optional[Sequence[str]] = None) -> int: else: if not args.clickhouse_url: raise ExportError("--clickhouse-url (or CLICKHOUSE_URL) is required without --rows-jsonl") + if not args.chains_table or not args.analytics_table: + raise ExportError("--chains-table and --analytics-table are required without --rows-jsonl") sql, parameters = build_query( args.chains_table, args.analytics_table, diff --git a/tests/test_export_plans.py b/tests/test_export_plans.py index e5fbc4f..04d1742 100644 --- a/tests/test_export_plans.py +++ b/tests/test_export_plans.py @@ -48,8 +48,8 @@ def row( principal=PRINCIPAL, stream=None, max_tokens=None, - served_by="dynamo", - model="glm-5.2", + served_by="upstream-a", + model="vendor/model-a", ): _counter[0] += 1 if tokens is None: @@ -311,16 +311,16 @@ def test_validator_mirrors_the_rust_rules(self): class QueryTest(unittest.TestCase): def test_query_applies_filters_as_parameters(self): - sql, parameters = build_query("clay.prompt_chains", "clay.http_analytics", PRINCIPAL, "glm-5.2", "dynamo", chains_final=True) - self.assertIn("FROM clay.prompt_chains FINAL", sql) + sql, parameters = build_query("chains", "analytics", PRINCIPAL, "vendor/model-a", "upstream-a", chains_final=True) + self.assertIn("FROM chains FINAL", sql) self.assertIn("principal_id = {principal_id:UUID}", sql) self.assertIn("model = {model:String}", sql) self.assertIn("h.served_by = {served_by:String}", sql) # Both join sides are bounded by the window before joining. self.assertIn("timestamp >= {lookback_start:DateTime64(3)} - INTERVAL 1 DAY", sql) - self.assertEqual(parameters, {"principal_id": PRINCIPAL, "model": "glm-5.2", "served_by": "dynamo"}) - sql, parameters = build_query(export_plans.DEFAULT_CHAINS_TABLE, "clay.http_analytics", None, None, None, analytics_ts_column="ts") - self.assertIn("FROM clay.prompt_chains_current\n", sql) + self.assertEqual(parameters, {"principal_id": PRINCIPAL, "model": "vendor/model-a", "served_by": "upstream-a"}) + sql, parameters = build_query("chains_current", "analytics", None, None, None, analytics_ts_column="ts") + self.assertIn("FROM chains_current\n", sql) self.assertNotIn("FINAL", sql) self.assertNotIn("WHERE h.", sql) self.assertIn("ts >= {lookback_start:DateTime64(3)} - INTERVAL 1 DAY", sql) @@ -374,7 +374,7 @@ def test_cli_exports_from_rows_and_applies_filters(self): "--rows-jsonl", str(rows_path), "--start", WINDOW_START.isoformat(), "--end", WINDOW_END.isoformat(), - "--served-by", "dynamo", + "--served-by", "upstream-a", "--time-scale", "2", "--output", str(output), ])