Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions docs/export-plans.md
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,18 @@ only serve as link targets. Rows with the same `x_request_id` are attempts of on
request (a gateway retry or failover reuses the header); the last attempt that finished
is kept and the others are counted as `duplicate_attempts`.

Every loaded row stays in memory until the manifest is written, and a long prompt's hash
list is large, so a busy model over a long window can need more memory than `--sample`,
which thins trajectories after they are built, can save. `--sample-sessions FRACTION`
keeps that fraction of sessions instead, deciding as each row is read and before its
hashes are parsed; dropped rows count as `sampled_out`. A row is kept or dropped with its
agent session id, so sessions stay whole. A row without one goes with its `x_request_id`
(its retried attempts together), else its `request_id`, else is drawn on its own; such a
row can lose the earlier request it extends, and then replays as the start of a new
trajectory. Each key's draw comes
from `--seed`, so for one seed a smaller fraction keeps a subset of what a larger one keeps,
which makes manifests at several loads comparable.

### Sessions and blocks

Requests link with the same strict-prefix rule as prompt chains, applied to the block
Expand Down
47 changes: 46 additions & 1 deletion src/batchbench/dynamo_traces.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

import glob
import gzip
import hashlib
import json
import os
import uuid
Expand Down Expand Up @@ -92,6 +93,7 @@ class TraceStats:
other_model: int = 0
outside_window: int = 0
malformed_hashes: int = 0
sampled_out: int = 0
loaded: int = 0
with_x_request_id: int = 0
duplicate_attempts: int = 0
Expand Down Expand Up @@ -252,18 +254,56 @@ def _trace_lines(path: str, stats: TraceStats) -> Iterator[str]:
stats.unreadable_files += 1


def session_sample_key(row: Mapping[str, Any]) -> Optional[str]:
"""What session sampling keeps or drops a row by: its agent session id, else its request.

Rows of one session share the key, so a session is kept or dropped whole. A row without a
session id is keyed by its ``x_request_id`` (so a request's retried attempts go together),
else by its ``request_id``. A session id is what :func:`record_from_trace` keeps as one.
"""

agent = row.get("agent_context")
session_id = agent.get("session_id") if isinstance(agent, Mapping) else None
if session_id not in (None, ""):
return f"session:{session_id}"
request = row.get("request")
if not isinstance(request, Mapping):
return None
x_request_id = normalize_request_id(request.get("x_request_id"))
if x_request_id:
return f"x-request:{x_request_id}"
request_id = request.get("request_id")
return f"request:{request_id}" if request_id else None


def keeps_session(key: str, fraction: float, seed: int) -> bool:
"""Whether session sampling at ``fraction`` keeps ``key``.

Each key draws one uniform number from its seeded hash and is kept when the number falls
below ``fraction``, so for one seed the keys kept at a smaller fraction are a subset of those
kept at a larger one.
"""

digest = hashlib.blake2b(f"{seed}:{key}".encode(), digest_size=8).digest()
return int.from_bytes(digest, "big") / 2.0**64 < fraction


def load_trace_records(
paths: Sequence[str],
window_start: datetime,
window_end: datetime,
link_window: timedelta = DEFAULT_LINK_WINDOW,
model: Optional[str] = None,
stats: Optional[TraceStats] = None,
sample_sessions: Optional[float] = None,
seed: int = 0,
) -> list[TraceRecord]:
"""Read ``request_end`` rows received in ``[window_start - link_window, window_end)``.

Files are named by upload time and can hold rows received long before it, so every file is
read and rows are windowed by ``request_received_ms``.
read and rows are windowed by ``request_received_ms``. With ``sample_sessions``, a row whose
session (see :func:`session_sample_key`) the sample leaves out is dropped as it is read,
before its hashes are parsed, so memory follows the sample rather than the traffic.
"""

stats = stats if stats is not None else TraceStats()
Expand Down Expand Up @@ -293,6 +333,11 @@ def load_trace_records(
if not isinstance(row, Mapping):
stats.bad_lines += 1
continue
if sample_sessions is not None:
key = session_sample_key(row) or f"line:{line}"
if not keeps_session(key, sample_sessions, seed):
stats.sampled_out += 1
continue
record = record_from_trace(row, stats, model, window_ms)
if record is not None:
records.append(record)
Expand Down
9 changes: 8 additions & 1 deletion src/batchbench/export_plans.py
Original file line number Diff line number Diff line change
Expand Up @@ -801,6 +801,7 @@ def build_parser() -> argparse.ArgumentParser:
shaping = parser.add_argument_group("shaping")
shaping.add_argument("--sample", type=float, help="fraction of trajectories to keep (0-1)")
shaping.add_argument("--seed", type=int, default=0, help="seed for deterministic sampling")
shaping.add_argument("--sample-sessions", type=float, metavar="FRACTION", help="with --dynamo-traces: keep this fraction (0-1] of sessions, chosen as rows are read; nested across fractions for one --seed")
shaping.add_argument("--stratify-by-session-length", action="store_true", help="sample within power-of-two session-length buckets")
shaping.add_argument("--time-scale", type=float, default=1.0, help="divide start offsets and delays by this factor")

Expand All @@ -826,6 +827,11 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
if args.link_window_hours < 0:
raise ExportError("--link-window-hours must be non-negative")
link_window = timedelta(hours=args.link_window_hours)
if args.sample_sessions is not None:
if not args.dynamo_traces:
raise ExportError("--sample-sessions needs --dynamo-traces")
if not 0.0 < args.sample_sessions <= 1.0:
raise ExportError("--sample-sessions must be in (0, 1]")

stats = ExportStats()
extra_stats: dict[str, Any] = {}
Expand Down Expand Up @@ -876,7 +882,8 @@ def _export_dynamo_traces(
raise ExportError("use --analytics-rows-jsonl or --analytics-table, not both")
trace_stats = dynamo_traces.TraceStats()
records = dynamo_traces.load_trace_records(
args.dynamo_traces, window_start, window_end, link_window, args.model, trace_stats
args.dynamo_traces, window_start, window_end, link_window, args.model, trace_stats,
sample_sessions=args.sample_sessions, seed=args.seed,
)
records = dynamo_traces.drop_duplicate_attempts(records, trace_stats)
analytics_rows: Optional[list[dict[str, Any]]] = None
Expand Down
71 changes: 71 additions & 0 deletions tests/test_dynamo_traces.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
build_trace_plans,
drop_duplicate_attempts,
hash_seed,
keeps_session,
session_sample_key,
load_trace_records,
normalize_request_id,
record_from_trace,
Expand Down Expand Up @@ -548,6 +550,75 @@ def test_exported_manifest_dry_runs_in_batchbench_agent(self):
self.assertNotIn("Live block fallbacks", result.stdout)


class SessionSampleTest(unittest.TestCase):
def setUp(self):
self.directory = Path(tempfile.mkdtemp(prefix="batchbench-sample-"))
rows = []
for index in range(200):
start = T0 + timedelta(seconds=index)
rows.append(trace(start, [f"s{index}", "u1"], session_id=f"sess-{index}"))
rows.append(trace(start + timedelta(seconds=1), [f"s{index}", "u1", "u2"], session_id=f"sess-{index}"))
# Two attempts of one request without a session id share its x-request-id.
rows.append(trace(T0, ["lone"], x_request_id="00000000-0000-4000-8000-0000000000aa"))
rows.append(trace(T0 + timedelta(seconds=2), ["lone"], x_request_id="00000000-0000-4000-8000-0000000000AA"))
path = self.directory / "traces.jsonl"
path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8")
self.path = str(path)

def tearDown(self):
shutil.rmtree(self.directory, ignore_errors=True)

def load(self, fraction, seed=7):
stats = TraceStats()
records = load_trace_records([self.path], WINDOW_START, WINDOW_END, sample_sessions=fraction, seed=seed, stats=stats)
return records, stats

def load_rows(self, rows, fraction, seed=7):
path = self.directory / "rows.jsonl"
path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8")
return load_trace_records([str(path)], WINDOW_START, WINDOW_END, sample_sessions=fraction, seed=seed, stats=TraceStats())

def test_a_row_without_any_id_is_drawn_on_its_own(self):
row = trace(T0, ["anon", "u1"])
del row["request"]["request_id"]
self.assertIsNone(session_sample_key(row))
self.assertEqual(len(self.load_rows([row], 1.0)), 1)

def test_a_session_id_the_records_keep_is_a_session(self):
rows = [trace(T0, ["z", "u1"], session_id=0), trace(T0 + timedelta(seconds=1), ["z", "u1", "u2"], session_id=0)]
self.assertEqual(session_sample_key(rows[0]), "session:0")
self.assertEqual(len(self.load_rows(rows, 0.5)), 2 if keeps_session("session:0", 0.5, 7) else 0)

def test_a_session_is_kept_or_dropped_whole(self):
records, stats = self.load(0.5)
by_session = {}
for record in records:
by_session.setdefault(record.session_id, []).append(record)
sessions = [key for key in by_session if key]
self.assertTrue(0 < len(sessions) < 200)
self.assertTrue(all(len(by_session[key]) == 2 for key in sessions))
self.assertEqual(stats.sampled_out + stats.loaded, 402)

def test_smaller_fractions_keep_subsets_for_one_seed(self):
kept = {fraction: {record.request_id for record in self.load(fraction)[0]} for fraction in (0.1, 0.3, 0.6, 1.0)}
self.assertLessEqual(kept[0.1], kept[0.3])
self.assertLessEqual(kept[0.3], kept[0.6])
self.assertEqual(len(kept[1.0]), 402)
self.assertNotEqual(kept[0.3], {record.request_id for record in self.load(0.3, seed=8)[0]})

def test_attempts_without_a_session_go_together(self):
key = "x-request:00000000-0000-4000-8000-0000000000aa"
records, _ = self.load(0.5)
lone = [record for record in records if record.session_id is None]
self.assertEqual(len(lone), 2 if keeps_session(key, 0.5, 7) else 0)

def test_cli_checks_the_fraction(self):
common = ["--start", "2026-09-01T12:00:00Z", "--end", "2026-09-01T13:00:00Z", "--output", str(self.directory / "plans.jsonl")]
self.assertEqual(export_plans.main(["--rows-jsonl", self.path, "--sample-sessions", "0.5", *common]), 1)
self.assertEqual(export_plans.main(["--dynamo-traces", self.path, "--sample-sessions", "0", *common]), 1)
self.assertEqual(export_plans.main(["--dynamo-traces", self.path, "--sample-sessions", "0.5", *common]), 0)


class AnalyticsQueryTest(unittest.TestCase):
def test_query_and_window(self):
sql = build_analytics_query("requests", "x_request_id", "timestamp")
Expand Down