From 0195829426e1161b730f7c7e9cf08946ce85a286 Mon Sep 17 00:00:00 2001 From: Dipesh Babu Date: Wed, 18 Nov 2026 20:12:14 -0500 Subject: [PATCH] Add a Claude Code runner for the guided workflow examples/claude_code_runner.py runs `claude -p --output-format stream-json --verbose` with only Read, Grep and Glob pre-approved and converts the stream into a native trace: one model span per API response (content blocks of one message.id count once), per-request provider usage including cache reads and writes, and one tool span per tool_use ending at its tool_result. docs/CLAUDE_CODE.md documents the mapping. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 + docs/CLAUDE_CODE.md | 54 ++++++ docs/INTEGRATIONS.md | 3 + examples/claude_code_runner.py | 288 +++++++++++++++++++++++++++++++ mkdocs.yml | 1 + tests/test_claude_code_runner.py | 287 ++++++++++++++++++++++++++++++ 6 files changed, 638 insertions(+) create mode 100644 docs/CLAUDE_CODE.md create mode 100644 examples/claude_code_runner.py create mode 100644 tests/test_claude_code_runner.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 04ec3ac..c4ace5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ trace, study, receipt, API and database schema versions. ### Added +- `examples/claude_code_runner.py` runs Claude Code (`claude -p --output-format + stream-json`) on guided tasks with only read-only tools pre-approved. It + records one model span per API response with per-request usage including + cache tokens, and one tool span per tool use. `docs/CLAUDE_CODE.md` explains + the mapping and a one-command comparison. - `agentloop improve compare` runs a whole guided comparison in one command. It captures the baseline with the unchanged runner (`--before-arg`), freezes the session, captures the changed runner alternating with the unchanged one diff --git a/docs/CLAUDE_CODE.md b/docs/CLAUDE_CODE.md new file mode 100644 index 0000000..7a4d0ac --- /dev/null +++ b/docs/CLAUDE_CODE.md @@ -0,0 +1,54 @@ +# Claude Code + +AgentLoop can profile and compare [Claude Code](https://docs.claude.com/en/docs/claude-code/overview) +runs through the guided workflow. The example +[runner](https://github.com/dipeshbabu/agentloop/blob/main/examples/claude_code_runner.py) +runs `claude -p --output-format stream-json --verbose` for each task and +converts its event stream into a native AgentLoop trace. Claude Code is not +modified and no plugin is installed. + +Each task attempt is a real Claude Code run on your Claude plan or API key and +counts against that usage. + +## How events become a trace + +The runner starts Claude Code in the directory given by `--cd`, sends the task +prompt on stdin and pre-approves only the read-only `Read`, `Grep` and `Glob` +tools with `--allowedTools`. A request for any other tool is denied in +non-interactive mode and recorded as a failed tool call. Stream events carry no +timestamps, so the runner timestamps each JSON line as it arrives. + +| Claude Code event | AgentLoop event | +| --- | --- | +| `assistant` events sharing one `message.id` | One `model_call` named `claude_response`, from the session start or the last tool result until the last part of that response arrived | +| `tool_use` block | `tool_call` named after the tool, ending when its `tool_result` arrives; `is_error` results are errors | +| `message.usage` | Provider token counts with `usage_scope: request`. Input tokens include cache reads and writes, which are also listed in metadata | +| `result` | The answer scored by the frozen task checks. Turns, durations and reported cost are kept on the root `claude_code` span | +| `result` with `is_error` or a non-success subtype | Error on the root span | + +Claude Code may emit each content block of a response as its own `assistant` +event with repeated usage. The runner counts each response once. + +## Run the guided workflow + +Tasks use a prompt string or an object with `prompt`. The Codex +[task set](https://github.com/dipeshbabu/agentloop/blob/main/examples/codex_repository_tasks.json) +works unchanged: it asks three metadata questions about this repository with +exact expected values. + +AgentLoop 0.2.5 and newer compare an unchanged and a changed configuration in +one command. Arguments after a second `--` go to `claude` unchanged, so the +candidate below only lowers the turn limit: + +```bash +agentloop improve compare --tasks examples/codex_repository_tasks.json --out runs/claude --timeout 600 --change "Limit lookups to four turns" --before-arg python --before-arg examples/claude_code_runner.py --before-arg --cd --before-arg . -- python examples/claude_code_runner.py --cd . -- --max-turns 4 +``` + +Runner options: `--cd` sets the directory Claude Code works in, +`--instructions FILE` prepends shared instructions to each prompt, `--model` +passes `--model` and records the label, `--condition` labels the trace, and +`--claude` selects the executable. + +The conversion is tested against the documented stream shape and an end-to-end +run through a stand-in executable. A recorded run against a live Claude Code +installation has not yet been published here. diff --git a/docs/INTEGRATIONS.md b/docs/INTEGRATIONS.md index 1025b27..5344fc0 100644 --- a/docs/INTEGRATIONS.md +++ b/docs/INTEGRATIONS.md @@ -10,6 +10,9 @@ deterministic-provider validation scope. For OpenAI Codex CLI, see [the Codex guide](CODEX.md). Its example runner converts `codex exec --json` events into native traces for the guided workflow. +For Claude Code, see [the Claude Code guide](CLAUDE_CODE.md). Its example runner +converts `claude -p --output-format stream-json` events into native traces. + For framework-neutral pipeline and stage annotations, see [workflow tracing](WORKFLOW_TRACING.md). diff --git a/examples/claude_code_runner.py b/examples/claude_code_runner.py new file mode 100644 index 0000000..4972754 --- /dev/null +++ b/examples/claude_code_runner.py @@ -0,0 +1,288 @@ +"""Run Claude Code on guided tasks and record a native AgentLoop trace. + +The runner receives the standard `agentloop improve capture` JSON request on +stdin. It runs ``claude -p --output-format stream-json --verbose`` with only +read-only tools allowed, timestamps each JSON line as it arrives, and converts +the stream into AgentLoop events: + +- each Claude API response (one ``message.id``) is one model span. It starts at + the session start or when the last tool result arrived, and ends when the last + part of that response arrives, so text and tool requests in one response are + one model call; +- each ``tool_use`` block is a tool span that ends when its ``tool_result`` + arrives, keeping errors reported by the tool; +- Claude reports usage per response. Input tokens include cache reads and cache + writes, which are also recorded separately, with ``usage_scope = "request"``. + +The ``result`` event's text is the task answer that the frozen checks score. +Arguments after ``--`` are passed to ``claude`` unchanged, so a candidate can +change the model, system prompt or settings without editing this file. + +Example: + + agentloop improve capture --tasks examples/codex_repository_tasks.json \ + --out runs/claude-baseline -- python examples/claude_code_runner.py --cd . +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess # nosec B404 - runs the operator's Claude Code CLI with explicit argv +import sys +import threading +import time +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path + +from agentloop.events import AgentEvent +from agentloop.tokens import PROVIDER +from agentloop.tracer import AgentTrace + +READ_ONLY_TOOLS = ("Read", "Grep", "Glob") +OUTPUT_LIMIT = 4096 + + +def task_prompt(task_input, instructions): + if isinstance(task_input, dict): + task_input = task_input.get("prompt") or task_input.get("question") + if not isinstance(task_input, str) or not task_input.strip(): + raise ValueError("Task input must be a prompt string or an object with 'prompt'") + return f"{instructions.strip()}\n\n{task_input}" if instructions else task_input + + +def clip(value): + if not isinstance(value, str): + return None + return value if len(value) <= OUTPUT_LIMIT else value[:OUTPUT_LIMIT] + "\n[truncated]" + + +def result_text(content): + """Tool results are a string or a list of content blocks.""" + if isinstance(content, str): + return content + if isinstance(content, list): + return "\n".join( + block.get("text", "") + for block in content + if isinstance(block, dict) and block.get("type") == "text" + ) + return None + + +def read_stream(process, lines): + """Timestamp each JSON line on arrival; stream events carry no timestamps.""" + for raw in process.stdout: + lines.append((time.perf_counter(), raw)) + + +def convert(lines, *, run_id, root_id, origin, clock, model): + def at(moment): + return (origin + timedelta(seconds=moment - clock)).isoformat() + + events, responses, open_tools = [], {}, {} + output, failure, details, boundary = None, None, {}, clock + for moment, raw in lines: + try: + event = json.loads(raw) + except ValueError: + continue + kind = event.get("type") + if kind == "system" and event.get("subtype") == "init": + boundary = moment + model = model or event.get("model") + details["claude_session_id"] = event.get("session_id") + elif kind == "assistant": + message = event.get("message") or {} + key = message.get("id") or f"anonymous-{len(responses)}" + span = responses.get(key) + if span is None: + span = AgentEvent( + event_id="evt_" + uuid.uuid4().hex, + run_id=run_id, + parent_id=root_id, + event_type="model_call", + name="claude_response", + model=message.get("model") or model, + started_at=at(boundary), + ended_at=at(moment), + duration_ms=(moment - boundary) * 1000, + metadata={"content_types": [], "start": boundary}, + ) + responses[key] = span + events.append(span) + span.ended_at = at(moment) + span.duration_ms = (moment - span.metadata["start"]) * 1000 + usage = message.get("usage") + if isinstance(usage, dict): + # Each part of a response repeats the same usage: set, never add. + cache_read = int(usage.get("cache_read_input_tokens") or 0) + cache_write = int(usage.get("cache_creation_input_tokens") or 0) + span.input_tokens = int(usage.get("input_tokens") or 0) + cache_read + cache_write + span.output_tokens = int(usage.get("output_tokens") or 0) + span.token_provenance = PROVIDER + span.metadata.update( + usage_scope="request", + cached_input_tokens=cache_read, + cache_creation_input_tokens=cache_write, + ) + for block in message.get("content") or []: + if not isinstance(block, dict): + continue + span.metadata["content_types"].append(block.get("type")) + if block.get("type") == "text": + span.output_text = clip(block.get("text")) + elif block.get("type") == "tool_use": + open_tools[block.get("id")] = (moment, block) + elif kind == "user": + message = event.get("message") or {} + for block in message.get("content") or []: + if not isinstance(block, dict) or block.get("type") != "tool_result": + continue + started, request = open_tools.pop(block.get("tool_use_id"), (moment, {})) + failed = block.get("is_error") is True + text = result_text(block.get("content")) + events.append( + AgentEvent( + event_id="evt_" + uuid.uuid4().hex, + run_id=run_id, + parent_id=root_id, + event_type="tool_call", + name=request.get("name") or "tool", + started_at=at(started), + ended_at=at(moment), + duration_ms=(moment - started) * 1000, + input_text=clip(json.dumps(request.get("input"), sort_keys=True)) + if "input" in request + else None, + output_text=clip(text), + status="error" if failed else "ok", + error=(clip(text) or "tool error") if failed else None, + metadata={"claude_tool_use_id": block.get("tool_use_id")}, + ) + ) + boundary = max(boundary, moment) + elif kind == "result": + if isinstance(event.get("result"), str): + output = event["result"] + if event.get("is_error") or event.get("subtype") not in (None, "success"): + failure = ( + event.get("result") + if isinstance(event.get("result"), str) and event.get("result") + else event.get("subtype") or "Claude Code reported an error" + ) + details.update( + claude_num_turns=event.get("num_turns"), + claude_duration_ms=event.get("duration_ms"), + claude_duration_api_ms=event.get("duration_api_ms"), + claude_total_cost_usd=event.get("total_cost_usd"), + ) + for span in responses.values(): + span.metadata.pop("start", None) + usage_seen = any(span.token_provenance == PROVIDER for span in responses.values()) + return events, output, usage_seen, failure, details + + +def run(args, claude_args, request): + claude = shutil.which(args.claude) + if claude is None: + raise SystemExit(f"Cannot find the Claude Code CLI executable {args.claude!r}") + instructions = args.instructions.read_text(encoding="utf-8") if args.instructions else "" + prompt = task_prompt(request["input"], instructions) + command = [ + claude, + "-p", + "--output-format", + "stream-json", + "--verbose", + "--allowedTools", + ",".join(READ_ONLY_TOOLS), + *claude_args, + ] + trace = AgentTrace( + f"claude code: {request['task_id']}", + metadata={ + "integration": "claude_code", + "condition": args.condition, + "task_id": request["task_id"], + "repetition": request["repetition"], + "claude_args": claude_args, + }, + ) + root_id = "evt_" + uuid.uuid4().hex + origin, clock = datetime.now(timezone.utc), time.perf_counter() + process = subprocess.Popen( # nosec B603 - explicit Claude Code argv, no shell + command, + cwd=args.cd.resolve(), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + ) + lines = [] + reader = threading.Thread(target=read_stream, args=(process, lines), daemon=True) + reader.start() + stderr = threading.Thread(target=lambda: process.stderr.read(), daemon=True) + stderr.start() + process.stdin.write(prompt) + process.stdin.close() + returncode = process.wait() + reader.join() + ended = time.perf_counter() + events, output, usage_seen, failure, details = convert( + lines, run_id=trace.run_id, root_id=root_id, origin=origin, clock=clock, model=args.model + ) + if returncode != 0 and failure is None: + failure = f"claude exited with code {returncode}" + trace.add_event( + AgentEvent( + event_id=root_id, + run_id=trace.run_id, + event_type="agent_step", + name="claude_code", + started_at=origin.isoformat(), + ended_at=(origin + timedelta(seconds=ended - clock)).isoformat(), + duration_ms=(ended - clock) * 1000, + input_text=prompt, + output_text=clip(output), + status="error" if failure else "ok", + error=failure, + metadata={"operation_kind": "agent", "usage_reported": usage_seen, **details}, + ) + ) + for event in events: + trace.add_event(event) + trace.metadata.update( + output=output.strip() if output is not None else None, + success=failure is None and output is not None, + ) + trace.finish() + trace.export_json(request["trace_path"]) + return 1 if failure else 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--cd", type=Path, default=Path("."), help="Directory Claude Code reads.") + parser.add_argument("--claude", default="claude", help="Claude Code CLI executable.") + parser.add_argument("--model", default=None, help="Model passed to Claude Code and recorded.") + parser.add_argument("--condition", default="unlabelled", help="Label recorded in the trace.") + parser.add_argument( + "--instructions", type=Path, default=None, help="Text file prepended to each task prompt." + ) + argv = sys.argv[1:] + split = argv.index("--") if "--" in argv else len(argv) + args = parser.parse_args(argv[:split]) + claude_args = argv[split + 1 :] + if args.model: + claude_args = ["--model", args.model, *claude_args] + request = json.loads(sys.stdin.readline()) + raise SystemExit(run(args, claude_args, request)) + + +if __name__ == "__main__": + main() diff --git a/mkdocs.yml b/mkdocs.yml index 68155a4..bd1de78 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -106,6 +106,7 @@ nav: - Framework overview: INTEGRATIONS.md - Hermes: HERMES.md - Codex CLI: CODEX.md + - Claude Code: CLAUDE_CODE.md - LangGraph: LANGGRAPH_HARNESS.md - Harbor: - Completed trials: HARBOR_TRIALS.md diff --git a/tests/test_claude_code_runner.py b/tests/test_claude_code_runner.py new file mode 100644 index 0000000..bfdc665 --- /dev/null +++ b/tests/test_claude_code_runner.py @@ -0,0 +1,287 @@ +from __future__ import annotations + +import json +import os +import stat +import subprocess +import sys +from datetime import datetime, timezone + +import pytest + +from agentloop.events import AgentEvent +from agentloop.findings import build_diagnosis +from agentloop.tracer import AgentTrace +from examples.claude_code_runner import convert, task_prompt + +USAGE = { + "input_tokens": 12, + "cache_creation_input_tokens": 3000, + "cache_read_input_tokens": 15000, + "output_tokens": 80, +} + + +def assistant(message_id, *content, usage=USAGE): + return { + "type": "assistant", + "message": { + "id": message_id, + "model": "claude-test", + "content": list(content), + "usage": usage, + }, + } + + +def tool_use(tool_id, name, value): + return {"type": "tool_use", "id": tool_id, "name": name, "input": value} + + +def tool_result(tool_id, text, is_error=False): + return { + "type": "user", + "message": { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": tool_id, + "content": text, + "is_error": is_error, + } + ], + }, + } + + +INIT = {"type": "system", "subtype": "init", "session_id": "s1", "model": "claude-test"} +RESULT = { + "type": "result", + "subtype": "success", + "is_error": False, + "result": ">=3.11", + "num_turns": 2, + "duration_ms": 3000, + "duration_api_ms": 2500, + "total_cost_usd": 0.0123, +} + +# Shape of `claude -p --output-format stream-json --verbose`, shortened. Claude +# Code may emit one assistant event per content block of the same response. +STREAM = [ + (0.5, INIT), + (1.5, assistant("msg_1", {"type": "text", "text": "Checking."})), + (2.0, assistant("msg_1", tool_use("toolu_1", "Read", {"file_path": "pyproject.toml"}))), + (2.3, tool_result("toolu_1", 'requires-python = ">=3.11"')), + (3.0, assistant("msg_2", {"type": "text", "text": ">=3.11"})), + (3.1, RESULT), +] + + +def run_convert(stream): + return convert( + [(moment, json.dumps(event) + "\n") for moment, event in stream], + run_id="run_test", + root_id="evt_root", + origin=datetime(2026, 11, 19, tzinfo=timezone.utc), + clock=0.0, + model=None, + ) + + +def test_each_response_is_one_model_span_and_each_tool_use_a_tool_span(): + events, output, usage_seen, failure, details = run_convert(STREAM) + assert (output, usage_seen, failure) == (">=3.11", True, None) + assert [(event.event_type, event.name) for event in events] == [ + ("model_call", "claude_response"), + ("tool_call", "Read"), + ("model_call", "claude_response"), + ] + decision, tool, answer = events + assert decision.duration_ms == pytest.approx(1500) + assert decision.metadata["content_types"] == ["text", "tool_use"] + assert tool.duration_ms == pytest.approx(300) + assert json.loads(tool.input_text) == {"file_path": "pyproject.toml"} + assert tool.output_text == 'requires-python = ">=3.11"' + assert answer.duration_ms == pytest.approx(700) + assert answer.output_text == ">=3.11" + assert details["claude_total_cost_usd"] == 0.0123 + assert details["claude_num_turns"] == 2 + assert details["claude_session_id"] == "s1" + + +def test_usage_is_set_once_per_response_and_includes_cache_tokens(): + events, *_ = run_convert(STREAM) + decision = events[0] + assert decision.input_tokens == 12 + 3000 + 15000 + assert decision.output_tokens == 80 + assert decision.token_provenance == "provider" + assert decision.model == "claude-test" + assert decision.metadata["usage_scope"] == "request" + assert decision.metadata["cached_input_tokens"] == 15000 + assert decision.metadata["cache_creation_input_tokens"] == 3000 + assert "start" not in decision.metadata + + +def test_parallel_tool_uses_share_one_model_span(): + stream = [ + STREAM[0], + ( + 1.0, + assistant( + "msg_1", + tool_use("a", "Grep", {"pattern": "x"}), + tool_use("b", "Glob", {"pattern": "*.py"}), + ), + ), + (1.4, tool_result("a", "hit")), + (1.6, tool_result("b", [{"type": "text", "text": "a.py"}])), + (2.6, assistant("msg_2", {"type": "text", "text": "done"})), + ] + events, *_ = run_convert(stream) + assert [event.event_type for event in events] == [ + "model_call", + "tool_call", + "tool_call", + "model_call", + ] + assert events[2].output_text == "a.py" + assert events[3].duration_ms == pytest.approx(1000) + + +def test_tool_errors_and_failed_results_are_retained(): + stream = [ + STREAM[0], + (1.0, assistant("msg_1", tool_use("t", "Bash", {"command": "rm -rf x"}))), + (1.1, tool_result("t", "Permission to use Bash has been denied.", is_error=True)), + ( + 2.0, + { + "type": "result", + "subtype": "error_max_turns", + "is_error": True, + "num_turns": 1, + }, + ), + ] + events, output, _, failure, _ = run_convert(stream) + tool = next(event for event in events if event.event_type == "tool_call") + assert tool.status == "error" and "denied" in tool.error + assert output is None + assert failure == "error_max_turns" + + +def test_converted_events_form_an_analyzable_trace(): + events, output, *_ = run_convert(STREAM) + trace = AgentTrace("claude test", metadata={"output": output}) + trace.add_event( + AgentEvent( + event_id="evt_root", + run_id=trace.run_id, + event_type="agent_step", + name="claude_code", + started_at=events[0].started_at, + ended_at=events[-1].ended_at, + duration_ms=3100, + metadata={"operation_kind": "agent"}, + ) + ) + for event in events: + event.run_id = trace.run_id + trace.add_event(event) + trace.finish() + restored = AgentTrace.from_dict(json.loads(json.dumps(trace.to_dict()))) + assert build_diagnosis(restored)["run_id"] == trace.run_id + + +@pytest.mark.parametrize( + ("value", "instructions", "expected"), + [ + ("plain", "", "plain"), + ({"prompt": "object"}, "", "object"), + ({"question": "legacy"}, "Be brief.", "Be brief.\n\nlegacy"), + ], +) +def test_task_prompt_accepts_strings_and_prompt_objects(value, instructions, expected): + assert task_prompt(value, instructions) == expected + + +FAKE_CLAUDE = """import json, os, sys +prompt = sys.stdin.read() +record = {"argv": sys.argv[1:], "cwd": os.getcwd(), "prompt": prompt} +with open(os.environ["FAKE_CLAUDE_RECORD"], "w", encoding="utf-8") as handle: + json.dump(record, handle) +for event in EVENTS: + print(json.dumps(event), flush=True) +""" + + +def test_runner_drives_the_cli_and_writes_a_successful_trace(tmp_path): + script = tmp_path / "fake_claude.py" + events = [event for _, event in STREAM] + script.write_text(f"EVENTS = {events!r}\n" + FAKE_CLAUDE, encoding="utf-8") + if os.name == "nt": + executable = tmp_path / "fake-claude.cmd" + executable.write_text(f'@"{sys.executable}" "{script}" %*\n', encoding="utf-8") + else: + executable = tmp_path / "fake-claude" + executable.write_text( + f"#!{sys.executable}\n" + script.read_text(encoding="utf-8"), encoding="utf-8" + ) + executable.chmod(executable.stat().st_mode | stat.S_IXUSR) + workdir = tmp_path / "repo" + workdir.mkdir() + record = tmp_path / "record.json" + trace_path = tmp_path / "trace.json" + request = { + "task_id": "python", + "repetition": 0, + "input": {"prompt": "Which Python?"}, + "trace_path": str(trace_path), + } + completed = subprocess.run( # noqa: S603 + [ + sys.executable, + "examples/claude_code_runner.py", + "--claude", + str(executable), + "--cd", + str(workdir), + "--model", + "claude-test", + "--condition", + "baseline", + "--", + "--max-turns", + "4", + ], + input=json.dumps(request) + "\n", + capture_output=True, + text=True, + env={**os.environ, "FAKE_CLAUDE_RECORD": str(record)}, + timeout=60, + check=False, + ) + assert completed.returncode == 0, completed.stderr + seen = json.loads(record.read_text(encoding="utf-8")) + assert seen["argv"] == [ + "-p", + "--output-format", + "stream-json", + "--verbose", + "--allowedTools", + "Read,Grep,Glob", + "--model", + "claude-test", + "--max-turns", + "4", + ] + assert os.path.samefile(seen["cwd"], workdir) + assert seen["prompt"] == "Which Python?" + trace = AgentTrace.from_dict(json.loads(trace_path.read_text(encoding="utf-8"))) + assert trace.metadata["output"] == ">=3.11" + assert trace.metadata["success"] is True + assert trace.metadata["integration"] == "claude_code" + kinds = [event.event_type for event in trace.events] + assert kinds.count("model_call") == 2 and kinds.count("tool_call") == 1