Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/export-plans.md
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ Fields read from each row:
| `request.replay.trace_block_size` | Tokens per hashed block; varies by deployment |
| `request.replay.input_length` | Hashed prompt length |
| `request.replay.input_sequence_hashes` | Cumulative hash of each full block of the prompt, then of the partial last block when one remains |
| `request.request_id` | Dynamo's request id, kept in trajectory `metadata` |
| `request.x_request_id` | The caller's `x-request-id` header, when it sent one |
| `agent_context.session_id` | Agent session, when present |
| `request.finish_reason_metadata.finish_reason` | Picks among retried attempts |
Expand Down Expand Up @@ -158,6 +159,10 @@ blocks come out the same from their seeds, but its copies of earlier replies are
generated rather than the model's real text. A live reply cannot be split, so a
branch that diverges inside one reuses the blocks only up to the reply's start. Each trajectory carries
`metadata` with the model name, block size and, when joined, the alias and principal.
Its `requests` list names the source row of each request, in request order:
`request_id`, `x_request_id` (lower-cased as for the join), each null when the row has
none, and `received_ms` (the row's `request_received_ms`), so replay results can be joined
back to the trace rows.

### Joining analytics on `x_request_id`

Expand Down
8 changes: 8 additions & 0 deletions src/batchbench/dynamo_traces.py
Original file line number Diff line number Diff line change
Expand Up @@ -673,6 +673,14 @@ def trace_trajectory_plan(
metadata["alias"] = root.alias
if root.principal_id:
metadata["principal"] = root.principal_id
metadata["requests"] = [
{
"request_id": record.request_id or None,
"x_request_id": record.x_request_id,
"received_ms": round(record.ts.timestamp() * 1000),
}
for record in requests
]
return {
"schema_version": SCHEMA_VERSION,
"trajectory_id": trajectory_id,
Expand Down
31 changes: 30 additions & 1 deletion tests/test_dynamo_traces.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,15 @@ def test_prompts_are_cut_where_chains_diverge_or_end(self):

# The other branch shares the system segment's seed, so its bytes replay identically.
self.assertEqual([block["seed"] for block in other["requests"][0]["blocks"]], [s2, v1])
self.assertEqual(session["metadata"], {"source": "dynamo-request-trace", "model": MODEL, "block_size": 64})
self.assertEqual(session["metadata"], {
"source": "dynamo-request-trace",
"model": MODEL,
"block_size": 64,
"requests": [
{"request_id": row["request"]["request_id"], "x_request_id": None, "received_ms": row["request"]["request_received_ms"]}
for row in (first, second)
],
})

def test_cuts_are_per_model_and_block_size(self):
records = records_for([
Expand Down Expand Up @@ -380,6 +388,27 @@ def test_retried_attempts_keep_the_last_one_that_finished(self):
self.assertEqual((kept.completion_tokens, kept.x_request_id), (33, request_id))
self.assertEqual([record.index for record in records], [0, 1])

def test_metadata_names_the_source_row_of_every_request(self):
request_id = "00000000-0000-7000-8000-0000000000D1"
failed = trace(T0, ["s1", "u1"], output_tokens=0, finish_reason=None, x_request_id=request_id, session_id="sess-1")
first = trace(T0 + timedelta(seconds=1), ["s1", "u1"], x_request_id=request_id, session_id="sess-1")
second = trace(T0 + timedelta(seconds=5), ["s1", "u1", "x1"], session_id="sess-1")
plans = plans_for([failed, first, second])
self.assertEqual(validate_plans(plans), [])
self.assertEqual(len(plans), 1)
sources = plans[0]["metadata"]["requests"]
self.assertEqual(len(sources), len(plans[0]["requests"]))
self.assertEqual(sources, [
{"request_id": first["request"]["request_id"], "x_request_id": request_id.lower(), "received_ms": first["request"]["request_received_ms"]},
{"request_id": second["request"]["request_id"], "x_request_id": None, "received_ms": second["request"]["request_received_ms"]},
])

def test_metadata_names_a_missing_request_id_null(self):
row = trace(T0, ["s1", "u1"])
del row["request"]["request_id"]
plans = plans_for([row])
self.assertEqual(plans[0]["metadata"]["requests"][0]["request_id"], None)


class SourceTest(unittest.TestCase):
def setUp(self):
Expand Down