diff --git a/docs/export-plans.md b/docs/export-plans.md index 8b161e2..f0506a5 100644 --- a/docs/export-plans.md +++ b/docs/export-plans.md @@ -127,6 +127,18 @@ only serve as link targets. Rows with the same `x_request_id` are attempts of on request (a gateway retry or failover reuses the header); the last attempt that finished is kept and the others are counted as `duplicate_attempts`. +Every loaded row stays in memory until the manifest is written, and a long prompt's hash +list is large, so a busy model over a long window can need more memory than `--sample`, +which thins trajectories after they are built, can save. `--sample-sessions FRACTION` +keeps that fraction of sessions instead, deciding as each row is read and before its +hashes are parsed; dropped rows count as `sampled_out`. A row is kept or dropped with its +agent session id, so sessions stay whole. A row without one goes with its `x_request_id` +(its retried attempts together), else its `request_id`, else is drawn on its own; such a +row can lose the earlier request it extends, and then replays as the start of a new +trajectory. Each key's draw comes +from `--seed`, so for one seed a smaller fraction keeps a subset of what a larger one keeps, +which makes manifests at several loads comparable. + ### Sessions and blocks Requests link with the same strict-prefix rule as prompt chains, applied to the block diff --git a/src/batchbench/dynamo_traces.py b/src/batchbench/dynamo_traces.py index 7d6c786..f161618 100644 --- a/src/batchbench/dynamo_traces.py +++ b/src/batchbench/dynamo_traces.py @@ -23,6 +23,7 @@ import glob import gzip +import hashlib import json import os import uuid @@ -92,6 +93,7 @@ class TraceStats: other_model: int = 0 outside_window: int = 0 malformed_hashes: int = 0 + sampled_out: int = 0 loaded: int = 0 with_x_request_id: int = 0 duplicate_attempts: int = 0 @@ -252,6 +254,40 @@ def _trace_lines(path: str, stats: TraceStats) -> Iterator[str]: stats.unreadable_files += 1 +def session_sample_key(row: Mapping[str, Any]) -> Optional[str]: + """What session sampling keeps or drops a row by: its agent session id, else its request. + + Rows of one session share the key, so a session is kept or dropped whole. A row without a + session id is keyed by its ``x_request_id`` (so a request's retried attempts go together), + else by its ``request_id``. A session id is what :func:`record_from_trace` keeps as one. + """ + + agent = row.get("agent_context") + session_id = agent.get("session_id") if isinstance(agent, Mapping) else None + if session_id not in (None, ""): + return f"session:{session_id}" + request = row.get("request") + if not isinstance(request, Mapping): + return None + x_request_id = normalize_request_id(request.get("x_request_id")) + if x_request_id: + return f"x-request:{x_request_id}" + request_id = request.get("request_id") + return f"request:{request_id}" if request_id else None + + +def keeps_session(key: str, fraction: float, seed: int) -> bool: + """Whether session sampling at ``fraction`` keeps ``key``. + + Each key draws one uniform number from its seeded hash and is kept when the number falls + below ``fraction``, so for one seed the keys kept at a smaller fraction are a subset of those + kept at a larger one. + """ + + digest = hashlib.blake2b(f"{seed}:{key}".encode(), digest_size=8).digest() + return int.from_bytes(digest, "big") / 2.0**64 < fraction + + def load_trace_records( paths: Sequence[str], window_start: datetime, @@ -259,11 +295,15 @@ def load_trace_records( link_window: timedelta = DEFAULT_LINK_WINDOW, model: Optional[str] = None, stats: Optional[TraceStats] = None, + sample_sessions: Optional[float] = None, + seed: int = 0, ) -> list[TraceRecord]: """Read ``request_end`` rows received in ``[window_start - link_window, window_end)``. Files are named by upload time and can hold rows received long before it, so every file is - read and rows are windowed by ``request_received_ms``. + read and rows are windowed by ``request_received_ms``. With ``sample_sessions``, a row whose + session (see :func:`session_sample_key`) the sample leaves out is dropped as it is read, + before its hashes are parsed, so memory follows the sample rather than the traffic. """ stats = stats if stats is not None else TraceStats() @@ -293,6 +333,11 @@ def load_trace_records( if not isinstance(row, Mapping): stats.bad_lines += 1 continue + if sample_sessions is not None: + key = session_sample_key(row) or f"line:{line}" + if not keeps_session(key, sample_sessions, seed): + stats.sampled_out += 1 + continue record = record_from_trace(row, stats, model, window_ms) if record is not None: records.append(record) diff --git a/src/batchbench/export_plans.py b/src/batchbench/export_plans.py index 371cf3d..dace2bb 100644 --- a/src/batchbench/export_plans.py +++ b/src/batchbench/export_plans.py @@ -801,6 +801,7 @@ def build_parser() -> argparse.ArgumentParser: shaping = parser.add_argument_group("shaping") shaping.add_argument("--sample", type=float, help="fraction of trajectories to keep (0-1)") shaping.add_argument("--seed", type=int, default=0, help="seed for deterministic sampling") + shaping.add_argument("--sample-sessions", type=float, metavar="FRACTION", help="with --dynamo-traces: keep this fraction (0-1] of sessions, chosen as rows are read; nested across fractions for one --seed") shaping.add_argument("--stratify-by-session-length", action="store_true", help="sample within power-of-two session-length buckets") shaping.add_argument("--time-scale", type=float, default=1.0, help="divide start offsets and delays by this factor") @@ -826,6 +827,11 @@ def main(argv: Optional[Sequence[str]] = None) -> int: if args.link_window_hours < 0: raise ExportError("--link-window-hours must be non-negative") link_window = timedelta(hours=args.link_window_hours) + if args.sample_sessions is not None: + if not args.dynamo_traces: + raise ExportError("--sample-sessions needs --dynamo-traces") + if not 0.0 < args.sample_sessions <= 1.0: + raise ExportError("--sample-sessions must be in (0, 1]") stats = ExportStats() extra_stats: dict[str, Any] = {} @@ -876,7 +882,8 @@ def _export_dynamo_traces( raise ExportError("use --analytics-rows-jsonl or --analytics-table, not both") trace_stats = dynamo_traces.TraceStats() records = dynamo_traces.load_trace_records( - args.dynamo_traces, window_start, window_end, link_window, args.model, trace_stats + args.dynamo_traces, window_start, window_end, link_window, args.model, trace_stats, + sample_sessions=args.sample_sessions, seed=args.seed, ) records = dynamo_traces.drop_duplicate_attempts(records, trace_stats) analytics_rows: Optional[list[dict[str, Any]]] = None diff --git a/tests/test_dynamo_traces.py b/tests/test_dynamo_traces.py index c198956..b04ad7c 100644 --- a/tests/test_dynamo_traces.py +++ b/tests/test_dynamo_traces.py @@ -21,6 +21,8 @@ build_trace_plans, drop_duplicate_attempts, hash_seed, + keeps_session, + session_sample_key, load_trace_records, normalize_request_id, record_from_trace, @@ -548,6 +550,75 @@ def test_exported_manifest_dry_runs_in_batchbench_agent(self): self.assertNotIn("Live block fallbacks", result.stdout) +class SessionSampleTest(unittest.TestCase): + def setUp(self): + self.directory = Path(tempfile.mkdtemp(prefix="batchbench-sample-")) + rows = [] + for index in range(200): + start = T0 + timedelta(seconds=index) + rows.append(trace(start, [f"s{index}", "u1"], session_id=f"sess-{index}")) + rows.append(trace(start + timedelta(seconds=1), [f"s{index}", "u1", "u2"], session_id=f"sess-{index}")) + # Two attempts of one request without a session id share its x-request-id. + rows.append(trace(T0, ["lone"], x_request_id="00000000-0000-4000-8000-0000000000aa")) + rows.append(trace(T0 + timedelta(seconds=2), ["lone"], x_request_id="00000000-0000-4000-8000-0000000000AA")) + path = self.directory / "traces.jsonl" + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + self.path = str(path) + + def tearDown(self): + shutil.rmtree(self.directory, ignore_errors=True) + + def load(self, fraction, seed=7): + stats = TraceStats() + records = load_trace_records([self.path], WINDOW_START, WINDOW_END, sample_sessions=fraction, seed=seed, stats=stats) + return records, stats + + def load_rows(self, rows, fraction, seed=7): + path = self.directory / "rows.jsonl" + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + return load_trace_records([str(path)], WINDOW_START, WINDOW_END, sample_sessions=fraction, seed=seed, stats=TraceStats()) + + def test_a_row_without_any_id_is_drawn_on_its_own(self): + row = trace(T0, ["anon", "u1"]) + del row["request"]["request_id"] + self.assertIsNone(session_sample_key(row)) + self.assertEqual(len(self.load_rows([row], 1.0)), 1) + + def test_a_session_id_the_records_keep_is_a_session(self): + rows = [trace(T0, ["z", "u1"], session_id=0), trace(T0 + timedelta(seconds=1), ["z", "u1", "u2"], session_id=0)] + self.assertEqual(session_sample_key(rows[0]), "session:0") + self.assertEqual(len(self.load_rows(rows, 0.5)), 2 if keeps_session("session:0", 0.5, 7) else 0) + + def test_a_session_is_kept_or_dropped_whole(self): + records, stats = self.load(0.5) + by_session = {} + for record in records: + by_session.setdefault(record.session_id, []).append(record) + sessions = [key for key in by_session if key] + self.assertTrue(0 < len(sessions) < 200) + self.assertTrue(all(len(by_session[key]) == 2 for key in sessions)) + self.assertEqual(stats.sampled_out + stats.loaded, 402) + + def test_smaller_fractions_keep_subsets_for_one_seed(self): + kept = {fraction: {record.request_id for record in self.load(fraction)[0]} for fraction in (0.1, 0.3, 0.6, 1.0)} + self.assertLessEqual(kept[0.1], kept[0.3]) + self.assertLessEqual(kept[0.3], kept[0.6]) + self.assertEqual(len(kept[1.0]), 402) + self.assertNotEqual(kept[0.3], {record.request_id for record in self.load(0.3, seed=8)[0]}) + + def test_attempts_without_a_session_go_together(self): + key = "x-request:00000000-0000-4000-8000-0000000000aa" + records, _ = self.load(0.5) + lone = [record for record in records if record.session_id is None] + self.assertEqual(len(lone), 2 if keeps_session(key, 0.5, 7) else 0) + + def test_cli_checks_the_fraction(self): + common = ["--start", "2026-09-01T12:00:00Z", "--end", "2026-09-01T13:00:00Z", "--output", str(self.directory / "plans.jsonl")] + self.assertEqual(export_plans.main(["--rows-jsonl", self.path, "--sample-sessions", "0.5", *common]), 1) + self.assertEqual(export_plans.main(["--dynamo-traces", self.path, "--sample-sessions", "0", *common]), 1) + self.assertEqual(export_plans.main(["--dynamo-traces", self.path, "--sample-sessions", "0.5", *common]), 0) + + class AnalyticsQueryTest(unittest.TestCase): def test_query_and_window(self): sql = build_analytics_query("requests", "x_request_id", "timestamp")