Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,15 @@ trace, study, receipt, API and database schema versions.

### Fixed

- The CLI no longer crashes with `UnicodeEncodeError` when its output is piped
or redirected on Windows and a task ID, trace name or other text falls
outside the console code page. Redirected output is written as UTF-8.
- `improve finish` rejects an interleaved control captured before the session
was frozen, matching the existing candidate check.
- In aggregate decisions, report headlines count pairs that passed correctness
and completion checks instead of implying every pair passed overall.
- The Codex example runner records one model span per model round. A message
followed by a tool call in the same round was counted as two model calls.
- `docs/CODEX.md` attributes the first run's token spikes to extra model
steps. It previously attributed them to an instruction-file read, which cost
nothing extra when it ran in the same step as the requested read.
Expand Down
27 changes: 27 additions & 0 deletions agentloop/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import json
import os
import sys
from pathlib import Path

import typer
Expand Down Expand Up @@ -66,6 +67,32 @@

app = typer.Typer(help="AgentLoop profiler CLI")
console = Console()


def _utf8_output() -> None:
"""Write UTF-8 when output is redirected on a legacy code page.

On Windows, piped or redirected output uses the ANSI code page (for example
cp1252). Task IDs, trace names and other user text outside that code page
then raise UnicodeEncodeError mid-command and hide the real result.
Interactive consoles already use UTF-8 and are left unchanged.
"""
for stream in (sys.stdout, sys.stderr):
encoding = (getattr(stream, "encoding", None) or "").lower().replace("-", "")
if encoding in {"utf8", "utf_8"} or not hasattr(stream, "reconfigure"):
continue
try:
stream.reconfigure(encoding="utf-8", errors="replace")
except (OSError, ValueError):
continue


@app.callback()
def _main() -> None:
"""AgentLoop profiler CLI"""
_utf8_output()


study_app = typer.Typer(help="Summarize offline studies and preserved intervention evidence.")
app.add_typer(study_app, name="study")
app.add_typer(aggregate_app, name="aggregate")
Expand Down
5 changes: 5 additions & 0 deletions agentloop/improvement.py
Original file line number Diff line number Diff line change
Expand Up @@ -638,6 +638,11 @@ def finish(session_path, candidate_path, root, *, change):
fingerprint(control_manifest) == reference.get("manifest_hash"),
"Interleaved control capture changed",
)
require(
datetime.fromisoformat(control_manifest["created_at"])
>= datetime.fromisoformat(specification["frozen_at"]),
"Capture the interleaved control after freezing baseline findings",
)
require(
not (baseline_ids | candidate_ids).intersection(
trace.run_id for trace in controls.values()
Expand Down
13 changes: 11 additions & 2 deletions agentloop/improvement_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,15 @@ def _metric(summary, name):
return f"{value:.3f}" if isinstance(value, float) else str(value)


def _headline(report):
if (report.get("decision") or {}).get("mode") == "aggregate":
return (
f"{report['passed_attempts']} of {report['planned_attempts']} pairs passed "
"correctness and completion checks."
)
return f"{report['passed_attempts']} of {report['planned_attempts']} planned attempts passed."


def _basis(report):
if report.get("comparison_baseline") == "interleaved_control":
return (
Expand Down Expand Up @@ -222,7 +231,7 @@ def report_markdown(report):
if report["passed"]
else "The change did not pass the declared checks.",
"",
f"{report['passed_attempts']} of {report['planned_attempts']} planned attempts passed.",
_headline(report),
"",
"Applied change",
"",
Expand Down Expand Up @@ -293,7 +302,7 @@ def export_report(root, report):
markdown = report_markdown(report)
(root / "report.md").write_text(markdown, encoding="utf-8")
sections = [
f"<header><h1>{_text(report['name'])}</h1><p>{report['passed_attempts']} of {report['planned_attempts']} planned attempts passed.</p></header>",
f"<header><h1>{_text(report['name'])}</h1><p>{_text(_headline(report))}</p></header>",
f"<h2>Applied change</h2><p>{_text(report['change'])}</p>",
f"<p>{_text(_basis(report))}</p>",
*[f"<p>{_text(line)}</p>" for line in _decision_lines(report)],
Expand Down
24 changes: 24 additions & 0 deletions docs/CODEX.md
Original file line number Diff line number Diff line change
Expand Up @@ -86,3 +86,27 @@ attempts automatically. On these 18 traces it flags exactly the three runs with
an extra model step and none of the others. More repetitions would be needed before
attributing a latency change to this configuration. These six pairs describe
one machine, one model configuration and three lookup tasks.

## Second observed run

On 2026-11-17 the maintainer repeated the task set with four repetitions per
task, using the runner's model-round accounting, an interleaved control and a
frozen `--decision aggregate`.

- **Baseline:** all twelve answers were correct. `repetition_detour` flagged one
attempt that took an extra model step and used 20,035 more input tokens than
its typical repetition. One other attempt also took an extra round, but it
added only about 480 input tokens and was not reported.
- **Candidate:** `--instructions` asked Codex to request every file read it
needed together, in one step, and to read no other files. It ran against
the unchanged runner as the interleaved control.
- **Outcome:** all 24 answers were correct, and the candidate took no costly
detours, while the control took one. But the candidate was slower in all
12 pairs, with a median latency change of −34.5%, so the aggregate decision
**rejected it**. Its first model round spent noticeably longer planning, and
it averaged 2.42 model rounds against 2.08 for the control. It still read the
user-level instruction file, though in parallel with the requested read.

Removing occasional detours did not pay for the extra planning time on these
lookup tasks. Like the first run, this describes one machine, one model
configuration and three tasks.
102 changes: 47 additions & 55 deletions examples/codex_exec_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@
each JSONL event as it arrives, and converts the stream into AgentLoop events:

- each shell command, file change, MCP tool call or web search is a tool span;
- each agent message or reasoning item is a model span ending when it arrived;
- each model round is one model span: it starts at the turn start or when the
last running tool finished, and ends when the model starts a tool or gives
its final message, so messages within a round do not count as extra calls;
- Codex reports token usage once per turn, so the turn total is attached to the
last model span of that turn with ``usage_scope = "turn_total"``.

Expand Down Expand Up @@ -66,7 +68,39 @@ def at(moment):
return (origin + timedelta(seconds=moment - clock)).isoformat()

events, open_tools, output, usage_seen = [], {}, None, False
failure, boundary = None, clock
failure, boundary, current = None, clock, None

def model_round(moment, item_type=None, text=None):
"""Open or extend the span for the current model round.

A round starts at the turn start or when the last running tool finished
and lasts until the model starts a tool or gives its final message.
Messages and reasoning inside one round belong to the same model call.
"""
nonlocal current
if current is None:
current = AgentEvent(
event_id="evt_" + uuid.uuid4().hex,
run_id=run_id,
parent_id=root_id,
event_type="model_call",
name="codex_model_round",
model=model,
started_at=at(boundary),
ended_at=at(moment),
duration_ms=(moment - boundary) * 1000,
token_provenance=UNAVAILABLE,
metadata={"codex_items": []},
)
events.append(current)
else:
current.ended_at = at(moment)
current.duration_ms = (moment - boundary) * 1000
if item_type is not None:
current.metadata["codex_items"].append(item_type)
if text is not None:
current.output_text = clip(text)

for moment, raw in lines:
try:
event = json.loads(raw)
Expand All @@ -75,25 +109,12 @@ def at(moment):
kind, item = event.get("type"), event.get("item") or {}
item_type = item.get("type")
if kind == "turn.started":
boundary = moment
boundary, current = moment, None
elif kind == "item.started" and item_type in TOOL_ITEMS:
if not open_tools and moment > boundary:
# The model chose this tool call without emitting a visible item.
events.append(
AgentEvent(
event_id="evt_" + uuid.uuid4().hex,
run_id=run_id,
parent_id=root_id,
event_type="model_call",
name="codex_tool_decision",
model=model,
started_at=at(boundary),
ended_at=at(moment),
duration_ms=(moment - boundary) * 1000,
token_provenance=UNAVAILABLE,
metadata={"codex_item_type": "tool_decision"},
)
)
if not open_tools and (current is not None or moment > boundary):
model_round(moment, "tool_call")
# The round ends when its tool calls start running.
current = None
open_tools[item.get("id")] = moment
elif kind == "item.completed" and item_type in TOOL_ITEMS:
started = open_tools.pop(item.get("id"), moment)
Expand All @@ -118,46 +139,17 @@ def at(moment):
)
boundary = max(boundary, moment)
elif kind == "item.completed" and item_type in MODEL_ITEMS:
# The model was working from the previous visible boundary until this
# item arrived, unless a tool was still running in that interval.
started = max([boundary, *open_tools.values()])
events.append(
AgentEvent(
event_id="evt_" + uuid.uuid4().hex,
run_id=run_id,
parent_id=root_id,
event_type="model_call",
name=f"codex_{item_type}",
model=model,
started_at=at(started),
ended_at=at(moment),
duration_ms=(moment - started) * 1000,
token_provenance=UNAVAILABLE,
output_text=clip(item.get("text")),
metadata={"codex_item_type": item_type},
)
)
if open_tools:
# Reported while tools still run: keep the round open after them.
boundary = max([boundary, *open_tools.values()])
model_round(moment, item_type, item.get("text"))
if item_type == "agent_message":
output = item.get("text")
boundary = moment
elif kind == "turn.completed":
usage = event.get("usage") or {}
if current is None and not any(item.event_type == "model_call" for item in events):
model_round(moment)
models = [item for item in events if item.event_type == "model_call"]
if not models:
models.append(
AgentEvent(
event_id="evt_" + uuid.uuid4().hex,
run_id=run_id,
parent_id=root_id,
event_type="model_call",
name="codex_turn",
model=model,
started_at=at(boundary),
ended_at=at(moment),
duration_ms=(moment - boundary) * 1000,
)
)
events.append(models[0])
last = models[-1]
last.input_tokens = int(usage.get("input_tokens") or 0)
last.output_tokens = int(usage.get("output_tokens") or 0)
Expand Down
57 changes: 54 additions & 3 deletions tests/test_codex_exec_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,9 @@ def test_stream_becomes_tool_and_model_spans_with_turn_usage():
assert usage_seen is True
assert failure is None
assert [(event.event_type, event.name) for event in events] == [
("model_call", "codex_tool_decision"),
("model_call", "codex_model_round"),
("tool_call", "shell"),
("model_call", "codex_agent_message"),
("model_call", "codex_model_round"),
]
decision, tool, answer = events
assert decision.duration_ms == pytest.approx(1500)
Expand All @@ -95,6 +95,57 @@ def test_stream_becomes_tool_and_model_spans_with_turn_usage():
)
assert answer.metadata["usage_scope"] == "turn_total"
assert answer.metadata["cached_input_tokens"] == 32000
assert decision.metadata["codex_items"] == ["tool_call"]
assert answer.metadata["codex_items"] == ["agent_message"]
assert answer.output_text == ">=3.11"


def test_a_message_before_a_tool_call_is_part_of_the_same_model_round():
preamble = (
1.0,
{
"type": "item.completed",
"item": {"id": "item_0", "type": "agent_message", "text": "Checking."},
},
)
events, output, _, _ = run_convert([*STREAM[:2], preamble, *STREAM[2:]])
rounds = [event for event in events if event.event_type == "model_call"]
assert len(rounds) == 2
assert rounds[0].metadata["codex_items"] == ["agent_message", "tool_call"]
assert rounds[0].duration_ms == pytest.approx(1500)
assert output == ">=3.11"


def test_sequential_tools_add_a_model_round_but_parallel_tools_do_not():
def tool(moment, item_id, phase):
item = {"id": item_id, "type": "command_execution", "command": item_id}
if phase == "completed":
item.update(exit_code=0, status="completed")
return (moment, {"type": f"item.{phase}", "item": item})

answer = (
5.0,
{"type": "item.completed", "item": {"id": "z", "type": "agent_message", "text": "ok"}},
)
parallel = [
STREAM[1],
tool(1, "a", "started"),
tool(1.1, "b", "started"),
tool(2, "a", "completed"),
tool(2.1, "b", "completed"),
answer,
]
sequential = [
STREAM[1],
tool(1, "a", "started"),
tool(2, "a", "completed"),
tool(3, "b", "started"),
tool(4, "b", "completed"),
answer,
]
for stream, expected in ((parallel, 2), (sequential, 3)):
events, _, _, _ = run_convert(stream)
assert sum(event.event_type == "model_call" for event in events) == expected


def test_failed_commands_and_turns_are_retained_as_errors():
Expand All @@ -120,7 +171,7 @@ def test_failed_commands_and_turns_are_retained_as_errors():
def test_usage_without_model_items_gets_its_own_model_span():
events, _, usage_seen, _ = run_convert([STREAM[1], STREAM[-1]])
assert usage_seen is True
assert [(event.name, event.input_tokens) for event in events] == [("codex_turn", 40000)]
assert [(event.name, event.input_tokens) for event in events] == [("codex_model_round", 40000)]


def test_converted_events_form_an_analyzable_trace():
Expand Down
Loading
Loading