From 9929f921cabee9689d0d4fb8734c6355136f3a2b Mon Sep 17 00:00:00 2001 From: Fergus Finn Date: Fri, 13 Nov 2026 15:33:16 +0000 Subject: [PATCH 1/2] feat(export): sample trace sessions as rows are read `--sample-sessions FRACTION` (with `--dynamo-traces`) keeps that fraction of sessions and drops the other rows as they are read, before their block hashes are parsed. Memory then follows the sample, not the traffic. `--sample` thins trajectories only after every row is loaded, and a busy model's long prompts carry hundreds of hashes each. A row goes with its agent session id, else its x-request-id (retried attempts together), else its request id. Each key's draw comes from `--seed`, so for one seed a smaller fraction keeps a subset of a larger one, and manifests at several loads stay comparable. Dropped rows count as `sampled_out`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VDPqeFzGgF3vkuiyimwMxX --- docs/export-plans.md | 11 +++++++ src/batchbench/dynamo_traces.py | 47 +++++++++++++++++++++++++++- src/batchbench/export_plans.py | 9 +++++- tests/test_dynamo_traces.py | 54 +++++++++++++++++++++++++++++++++ 4 files changed, 119 insertions(+), 2 deletions(-) diff --git a/docs/export-plans.md b/docs/export-plans.md index 7767123..c64f0d3 100644 --- a/docs/export-plans.md +++ b/docs/export-plans.md @@ -127,6 +127,17 @@ only serve as link targets. Rows with the same `x_request_id` are attempts of on request (a gateway retry or failover reuses the header); the last attempt that finished is kept and the others are counted as `duplicate_attempts`. +Every loaded row stays in memory until the manifest is written, and a long prompt's hash +list is large, so a busy model over a long window can need more memory than `--sample`, +which thins trajectories after they are built, can save. `--sample-sessions FRACTION` +keeps that fraction of sessions instead, deciding as each row is read and before its +hashes are parsed; dropped rows count as `sampled_out`. A row is kept or dropped with its +agent session id, so sessions stay whole. A row without one goes with its `x_request_id` +(its retried attempts together), else its `request_id`; such a row can lose the earlier +request it extends, and then replays as the start of a new trajectory. Each key's draw comes +from `--seed`, so for one seed a smaller fraction keeps a subset of what a larger one keeps, +which makes manifests at several loads comparable. + ### Sessions and blocks Requests link with the same strict-prefix rule as prompt chains, applied to the block diff --git a/src/batchbench/dynamo_traces.py b/src/batchbench/dynamo_traces.py index b26c8cd..18124d9 100644 --- a/src/batchbench/dynamo_traces.py +++ b/src/batchbench/dynamo_traces.py @@ -23,6 +23,7 @@ import glob import gzip +import hashlib import json import os import uuid @@ -92,6 +93,7 @@ class TraceStats: other_model: int = 0 outside_window: int = 0 malformed_hashes: int = 0 + sampled_out: int = 0 loaded: int = 0 with_x_request_id: int = 0 duplicate_attempts: int = 0 @@ -252,6 +254,40 @@ def _trace_lines(path: str, stats: TraceStats) -> Iterator[str]: stats.unreadable_files += 1 +def session_sample_key(row: Mapping[str, Any]) -> Optional[str]: + """What session sampling keeps or drops a row by: its agent session id, else its request. + + Rows of one session share the key, so a session is kept or dropped whole. A row without a + session id is keyed by its ``x_request_id`` (so a request's retried attempts go together), + else by its ``request_id``. + """ + + agent = row.get("agent_context") + session_id = agent.get("session_id") if isinstance(agent, Mapping) else None + if session_id: + return f"session:{session_id}" + request = row.get("request") + if not isinstance(request, Mapping): + return None + x_request_id = normalize_request_id(request.get("x_request_id")) + if x_request_id: + return f"x-request:{x_request_id}" + request_id = request.get("request_id") + return f"request:{request_id}" if request_id else None + + +def keeps_session(key: str, fraction: float, seed: int) -> bool: + """Whether session sampling at ``fraction`` keeps ``key``. + + Each key draws one uniform number from its seeded hash and is kept when the number falls + below ``fraction``, so for one seed the keys kept at a smaller fraction are a subset of those + kept at a larger one. + """ + + digest = hashlib.blake2b(f"{seed}:{key}".encode(), digest_size=8).digest() + return int.from_bytes(digest, "big") / 2.0**64 < fraction + + def load_trace_records( paths: Sequence[str], window_start: datetime, @@ -259,11 +295,15 @@ def load_trace_records( link_window: timedelta = DEFAULT_LINK_WINDOW, model: Optional[str] = None, stats: Optional[TraceStats] = None, + sample_sessions: Optional[float] = None, + seed: int = 0, ) -> list[TraceRecord]: """Read ``request_end`` rows received in ``[window_start - link_window, window_end)``. Files are named by upload time and can hold rows received long before it, so every file is - read and rows are windowed by ``request_received_ms``. + read and rows are windowed by ``request_received_ms``. With ``sample_sessions``, a row whose + session (see :func:`session_sample_key`) the sample leaves out is dropped as it is read, + before its hashes are parsed, so memory follows the sample rather than the traffic. """ stats = stats if stats is not None else TraceStats() @@ -293,6 +333,11 @@ def load_trace_records( if not isinstance(row, Mapping): stats.bad_lines += 1 continue + if sample_sessions is not None: + key = session_sample_key(row) + if key is None or not keeps_session(key, sample_sessions, seed): + stats.sampled_out += 1 + continue record = record_from_trace(row, stats, model, window_ms) if record is not None: records.append(record) diff --git a/src/batchbench/export_plans.py b/src/batchbench/export_plans.py index 371cf3d..dace2bb 100644 --- a/src/batchbench/export_plans.py +++ b/src/batchbench/export_plans.py @@ -801,6 +801,7 @@ def build_parser() -> argparse.ArgumentParser: shaping = parser.add_argument_group("shaping") shaping.add_argument("--sample", type=float, help="fraction of trajectories to keep (0-1)") shaping.add_argument("--seed", type=int, default=0, help="seed for deterministic sampling") + shaping.add_argument("--sample-sessions", type=float, metavar="FRACTION", help="with --dynamo-traces: keep this fraction (0-1] of sessions, chosen as rows are read; nested across fractions for one --seed") shaping.add_argument("--stratify-by-session-length", action="store_true", help="sample within power-of-two session-length buckets") shaping.add_argument("--time-scale", type=float, default=1.0, help="divide start offsets and delays by this factor") @@ -826,6 +827,11 @@ def main(argv: Optional[Sequence[str]] = None) -> int: if args.link_window_hours < 0: raise ExportError("--link-window-hours must be non-negative") link_window = timedelta(hours=args.link_window_hours) + if args.sample_sessions is not None: + if not args.dynamo_traces: + raise ExportError("--sample-sessions needs --dynamo-traces") + if not 0.0 < args.sample_sessions <= 1.0: + raise ExportError("--sample-sessions must be in (0, 1]") stats = ExportStats() extra_stats: dict[str, Any] = {} @@ -876,7 +882,8 @@ def _export_dynamo_traces( raise ExportError("use --analytics-rows-jsonl or --analytics-table, not both") trace_stats = dynamo_traces.TraceStats() records = dynamo_traces.load_trace_records( - args.dynamo_traces, window_start, window_end, link_window, args.model, trace_stats + args.dynamo_traces, window_start, window_end, link_window, args.model, trace_stats, + sample_sessions=args.sample_sessions, seed=args.seed, ) records = dynamo_traces.drop_duplicate_attempts(records, trace_stats) analytics_rows: Optional[list[dict[str, Any]]] = None diff --git a/tests/test_dynamo_traces.py b/tests/test_dynamo_traces.py index 460a110..0d78c10 100644 --- a/tests/test_dynamo_traces.py +++ b/tests/test_dynamo_traces.py @@ -21,6 +21,7 @@ build_trace_plans, drop_duplicate_attempts, hash_seed, + keeps_session, load_trace_records, normalize_request_id, record_from_trace, @@ -542,6 +543,59 @@ def test_exported_manifest_dry_runs_in_batchbench_agent(self): self.assertNotIn("Live block fallbacks", result.stdout) +class SessionSampleTest(unittest.TestCase): + def setUp(self): + self.directory = Path(tempfile.mkdtemp(prefix="batchbench-sample-")) + rows = [] + for index in range(200): + start = T0 + timedelta(seconds=index) + rows.append(trace(start, [f"s{index}", "u1"], session_id=f"sess-{index}")) + rows.append(trace(start + timedelta(seconds=1), [f"s{index}", "u1", "u2"], session_id=f"sess-{index}")) + # Two attempts of one request without a session id share its x-request-id. + rows.append(trace(T0, ["lone"], x_request_id="00000000-0000-4000-8000-0000000000aa")) + rows.append(trace(T0 + timedelta(seconds=2), ["lone"], x_request_id="00000000-0000-4000-8000-0000000000AA")) + path = self.directory / "traces.jsonl" + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + self.path = str(path) + + def tearDown(self): + shutil.rmtree(self.directory, ignore_errors=True) + + def load(self, fraction, seed=7): + stats = TraceStats() + records = load_trace_records([self.path], WINDOW_START, WINDOW_END, sample_sessions=fraction, seed=seed, stats=stats) + return records, stats + + def test_a_session_is_kept_or_dropped_whole(self): + records, stats = self.load(0.5) + by_session = {} + for record in records: + by_session.setdefault(record.session_id, []).append(record) + sessions = [key for key in by_session if key] + self.assertTrue(0 < len(sessions) < 200) + self.assertTrue(all(len(by_session[key]) == 2 for key in sessions)) + self.assertEqual(stats.sampled_out + stats.loaded, 402) + + def test_smaller_fractions_keep_subsets_for_one_seed(self): + kept = {fraction: {record.request_id for record in self.load(fraction)[0]} for fraction in (0.1, 0.3, 0.6, 1.0)} + self.assertLessEqual(kept[0.1], kept[0.3]) + self.assertLessEqual(kept[0.3], kept[0.6]) + self.assertEqual(len(kept[1.0]), 402) + self.assertNotEqual(kept[0.3], {record.request_id for record in self.load(0.3, seed=8)[0]}) + + def test_attempts_without_a_session_go_together(self): + key = "x-request:00000000-0000-4000-8000-0000000000aa" + records, _ = self.load(0.5) + lone = [record for record in records if record.session_id is None] + self.assertEqual(len(lone), 2 if keeps_session(key, 0.5, 7) else 0) + + def test_cli_checks_the_fraction(self): + common = ["--start", "2026-09-01T12:00:00Z", "--end", "2026-09-01T13:00:00Z", "--output", str(self.directory / "plans.jsonl")] + self.assertEqual(export_plans.main(["--rows-jsonl", self.path, "--sample-sessions", "0.5", *common]), 1) + self.assertEqual(export_plans.main(["--dynamo-traces", self.path, "--sample-sessions", "0", *common]), 1) + self.assertEqual(export_plans.main(["--dynamo-traces", self.path, "--sample-sessions", "0.5", *common]), 0) + + class AnalyticsQueryTest(unittest.TestCase): def test_query_and_window(self): sql = build_analytics_query("requests", "x_request_id", "timestamp") From 5cd6c038cd3d4759ada7954e0f84fcde4e403e89 Mon Sep 17 00:00:00 2001 From: Fergus Finn Date: Fri, 13 Nov 2026 17:38:29 +0000 Subject: [PATCH 2/2] fix(export): session sampling keeps what the records keep A session id is one wherever record_from_trace keeps it (anything but missing or empty), so a session of id 0 stays whole. A row with no session, x_request_id or request_id is drawn on its own line instead of being dropped at every fraction, so --sample-sessions 1 keeps every row. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VDPqeFzGgF3vkuiyimwMxX --- docs/export-plans.md | 5 +++-- src/batchbench/dynamo_traces.py | 8 ++++---- tests/test_dynamo_traces.py | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/docs/export-plans.md b/docs/export-plans.md index e8d58b9..f0506a5 100644 --- a/docs/export-plans.md +++ b/docs/export-plans.md @@ -133,8 +133,9 @@ which thins trajectories after they are built, can save. `--sample-sessions FRAC keeps that fraction of sessions instead, deciding as each row is read and before its hashes are parsed; dropped rows count as `sampled_out`. A row is kept or dropped with its agent session id, so sessions stay whole. A row without one goes with its `x_request_id` -(its retried attempts together), else its `request_id`; such a row can lose the earlier -request it extends, and then replays as the start of a new trajectory. Each key's draw comes +(its retried attempts together), else its `request_id`, else is drawn on its own; such a +row can lose the earlier request it extends, and then replays as the start of a new +trajectory. Each key's draw comes from `--seed`, so for one seed a smaller fraction keeps a subset of what a larger one keeps, which makes manifests at several loads comparable. diff --git a/src/batchbench/dynamo_traces.py b/src/batchbench/dynamo_traces.py index eaec6f8..f161618 100644 --- a/src/batchbench/dynamo_traces.py +++ b/src/batchbench/dynamo_traces.py @@ -259,12 +259,12 @@ def session_sample_key(row: Mapping[str, Any]) -> Optional[str]: Rows of one session share the key, so a session is kept or dropped whole. A row without a session id is keyed by its ``x_request_id`` (so a request's retried attempts go together), - else by its ``request_id``. + else by its ``request_id``. A session id is what :func:`record_from_trace` keeps as one. """ agent = row.get("agent_context") session_id = agent.get("session_id") if isinstance(agent, Mapping) else None - if session_id: + if session_id not in (None, ""): return f"session:{session_id}" request = row.get("request") if not isinstance(request, Mapping): @@ -334,8 +334,8 @@ def load_trace_records( stats.bad_lines += 1 continue if sample_sessions is not None: - key = session_sample_key(row) - if key is None or not keeps_session(key, sample_sessions, seed): + key = session_sample_key(row) or f"line:{line}" + if not keeps_session(key, sample_sessions, seed): stats.sampled_out += 1 continue record = record_from_trace(row, stats, model, window_ms) diff --git a/tests/test_dynamo_traces.py b/tests/test_dynamo_traces.py index 6a6cac2..b04ad7c 100644 --- a/tests/test_dynamo_traces.py +++ b/tests/test_dynamo_traces.py @@ -22,6 +22,7 @@ drop_duplicate_attempts, hash_seed, keeps_session, + session_sample_key, load_trace_records, normalize_request_id, record_from_trace, @@ -572,6 +573,22 @@ def load(self, fraction, seed=7): records = load_trace_records([self.path], WINDOW_START, WINDOW_END, sample_sessions=fraction, seed=seed, stats=stats) return records, stats + def load_rows(self, rows, fraction, seed=7): + path = self.directory / "rows.jsonl" + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + return load_trace_records([str(path)], WINDOW_START, WINDOW_END, sample_sessions=fraction, seed=seed, stats=TraceStats()) + + def test_a_row_without_any_id_is_drawn_on_its_own(self): + row = trace(T0, ["anon", "u1"]) + del row["request"]["request_id"] + self.assertIsNone(session_sample_key(row)) + self.assertEqual(len(self.load_rows([row], 1.0)), 1) + + def test_a_session_id_the_records_keep_is_a_session(self): + rows = [trace(T0, ["z", "u1"], session_id=0), trace(T0 + timedelta(seconds=1), ["z", "u1", "u2"], session_id=0)] + self.assertEqual(session_sample_key(rows[0]), "session:0") + self.assertEqual(len(self.load_rows(rows, 0.5)), 2 if keeps_session("session:0", 0.5, 7) else 0) + def test_a_session_is_kept_or_dropped_whole(self): records, stats = self.load(0.5) by_session = {}