diff --git a/CHANGELOG.md b/CHANGELOG.md index 089563c..dc5aea6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,15 @@ trace, study, receipt, API and database schema versions. ### Fixed +- The CLI no longer crashes with `UnicodeEncodeError` when its output is piped + or redirected on Windows and a task ID, trace name or other text falls + outside the console code page. Redirected output is written as UTF-8. +- `improve finish` rejects an interleaved control captured before the session + was frozen, matching the existing candidate check. +- In aggregate decisions, report headlines count pairs that passed correctness + and completion checks instead of implying every pair passed overall. +- The Codex example runner records one model span per model round. A message + followed by a tool call in the same round was counted as two model calls. - `docs/CODEX.md` attributes the first run's token spikes to extra model steps. It previously attributed them to an instruction-file read, which cost nothing extra when it ran in the same step as the requested read. diff --git a/agentloop/cli.py b/agentloop/cli.py index 6ff3cd6..5137d27 100644 --- a/agentloop/cli.py +++ b/agentloop/cli.py @@ -2,6 +2,7 @@ import json import os +import sys from pathlib import Path import typer @@ -66,6 +67,32 @@ app = typer.Typer(help="AgentLoop profiler CLI") console = Console() + + +def _utf8_output() -> None: + """Write UTF-8 when output is redirected on a legacy code page. + + On Windows, piped or redirected output uses the ANSI code page (for example + cp1252). Task IDs, trace names and other user text outside that code page + then raise UnicodeEncodeError mid-command and hide the real result. + Interactive consoles already use UTF-8 and are left unchanged. + """ + for stream in (sys.stdout, sys.stderr): + encoding = (getattr(stream, "encoding", None) or "").lower().replace("-", "") + if encoding in {"utf8", "utf_8"} or not hasattr(stream, "reconfigure"): + continue + try: + stream.reconfigure(encoding="utf-8", errors="replace") + except (OSError, ValueError): + continue + + +@app.callback() +def _main() -> None: + """AgentLoop profiler CLI""" + _utf8_output() + + study_app = typer.Typer(help="Summarize offline studies and preserved intervention evidence.") app.add_typer(study_app, name="study") app.add_typer(aggregate_app, name="aggregate") diff --git a/agentloop/improvement.py b/agentloop/improvement.py index 3065991..116bd51 100644 --- a/agentloop/improvement.py +++ b/agentloop/improvement.py @@ -638,6 +638,11 @@ def finish(session_path, candidate_path, root, *, change): fingerprint(control_manifest) == reference.get("manifest_hash"), "Interleaved control capture changed", ) + require( + datetime.fromisoformat(control_manifest["created_at"]) + >= datetime.fromisoformat(specification["frozen_at"]), + "Capture the interleaved control after freezing baseline findings", + ) require( not (baseline_ids | candidate_ids).intersection( trace.run_id for trace in controls.values() diff --git a/agentloop/improvement_report.py b/agentloop/improvement_report.py index 1e00c32..00557be 100644 --- a/agentloop/improvement_report.py +++ b/agentloop/improvement_report.py @@ -173,6 +173,15 @@ def _metric(summary, name): return f"{value:.3f}" if isinstance(value, float) else str(value) +def _headline(report): + if (report.get("decision") or {}).get("mode") == "aggregate": + return ( + f"{report['passed_attempts']} of {report['planned_attempts']} pairs passed " + "correctness and completion checks." + ) + return f"{report['passed_attempts']} of {report['planned_attempts']} planned attempts passed." + + def _basis(report): if report.get("comparison_baseline") == "interleaved_control": return ( @@ -222,7 +231,7 @@ def report_markdown(report): if report["passed"] else "The change did not pass the declared checks.", "", - f"{report['passed_attempts']} of {report['planned_attempts']} planned attempts passed.", + _headline(report), "", "Applied change", "", @@ -293,7 +302,7 @@ def export_report(root, report): markdown = report_markdown(report) (root / "report.md").write_text(markdown, encoding="utf-8") sections = [ - f"

{_text(report['name'])}

{report['passed_attempts']} of {report['planned_attempts']} planned attempts passed.

", + f"

{_text(report['name'])}

{_text(_headline(report))}

", f"

Applied change

{_text(report['change'])}

", f"

{_text(_basis(report))}

", *[f"

{_text(line)}

" for line in _decision_lines(report)], diff --git a/docs/CODEX.md b/docs/CODEX.md index 0e4f8b1..166f3e1 100644 --- a/docs/CODEX.md +++ b/docs/CODEX.md @@ -86,3 +86,27 @@ attempts automatically. On these 18 traces it flags exactly the three runs with an extra model step and none of the others. More repetitions would be needed before attributing a latency change to this configuration. These six pairs describe one machine, one model configuration and three lookup tasks. + +## Second observed run + +On 2026-11-17 the maintainer repeated the task set with four repetitions per +task, using the runner's model-round accounting, an interleaved control and a +frozen `--decision aggregate`. + +- **Baseline:** all twelve answers were correct. `repetition_detour` flagged one + attempt that took an extra model step and used 20,035 more input tokens than + its typical repetition. One other attempt also took an extra round, but it + added only about 480 input tokens and was not reported. +- **Candidate:** `--instructions` asked Codex to request every file read it + needed together, in one step, and to read no other files. It ran against + the unchanged runner as the interleaved control. +- **Outcome:** all 24 answers were correct, and the candidate took no costly + detours, while the control took one. But the candidate was slower in all + 12 pairs, with a median latency change of −34.5%, so the aggregate decision + **rejected it**. Its first model round spent noticeably longer planning, and + it averaged 2.42 model rounds against 2.08 for the control. It still read the + user-level instruction file, though in parallel with the requested read. + +Removing occasional detours did not pay for the extra planning time on these +lookup tasks. Like the first run, this describes one machine, one model +configuration and three tasks. diff --git a/examples/codex_exec_runner.py b/examples/codex_exec_runner.py index de7397b..7f5496a 100644 --- a/examples/codex_exec_runner.py +++ b/examples/codex_exec_runner.py @@ -5,7 +5,9 @@ each JSONL event as it arrives, and converts the stream into AgentLoop events: - each shell command, file change, MCP tool call or web search is a tool span; -- each agent message or reasoning item is a model span ending when it arrived; +- each model round is one model span: it starts at the turn start or when the + last running tool finished, and ends when the model starts a tool or gives + its final message, so messages within a round do not count as extra calls; - Codex reports token usage once per turn, so the turn total is attached to the last model span of that turn with ``usage_scope = "turn_total"``. @@ -66,7 +68,39 @@ def at(moment): return (origin + timedelta(seconds=moment - clock)).isoformat() events, open_tools, output, usage_seen = [], {}, None, False - failure, boundary = None, clock + failure, boundary, current = None, clock, None + + def model_round(moment, item_type=None, text=None): + """Open or extend the span for the current model round. + + A round starts at the turn start or when the last running tool finished + and lasts until the model starts a tool or gives its final message. + Messages and reasoning inside one round belong to the same model call. + """ + nonlocal current + if current is None: + current = AgentEvent( + event_id="evt_" + uuid.uuid4().hex, + run_id=run_id, + parent_id=root_id, + event_type="model_call", + name="codex_model_round", + model=model, + started_at=at(boundary), + ended_at=at(moment), + duration_ms=(moment - boundary) * 1000, + token_provenance=UNAVAILABLE, + metadata={"codex_items": []}, + ) + events.append(current) + else: + current.ended_at = at(moment) + current.duration_ms = (moment - boundary) * 1000 + if item_type is not None: + current.metadata["codex_items"].append(item_type) + if text is not None: + current.output_text = clip(text) + for moment, raw in lines: try: event = json.loads(raw) @@ -75,25 +109,12 @@ def at(moment): kind, item = event.get("type"), event.get("item") or {} item_type = item.get("type") if kind == "turn.started": - boundary = moment + boundary, current = moment, None elif kind == "item.started" and item_type in TOOL_ITEMS: - if not open_tools and moment > boundary: - # The model chose this tool call without emitting a visible item. - events.append( - AgentEvent( - event_id="evt_" + uuid.uuid4().hex, - run_id=run_id, - parent_id=root_id, - event_type="model_call", - name="codex_tool_decision", - model=model, - started_at=at(boundary), - ended_at=at(moment), - duration_ms=(moment - boundary) * 1000, - token_provenance=UNAVAILABLE, - metadata={"codex_item_type": "tool_decision"}, - ) - ) + if not open_tools and (current is not None or moment > boundary): + model_round(moment, "tool_call") + # The round ends when its tool calls start running. + current = None open_tools[item.get("id")] = moment elif kind == "item.completed" and item_type in TOOL_ITEMS: started = open_tools.pop(item.get("id"), moment) @@ -118,46 +139,17 @@ def at(moment): ) boundary = max(boundary, moment) elif kind == "item.completed" and item_type in MODEL_ITEMS: - # The model was working from the previous visible boundary until this - # item arrived, unless a tool was still running in that interval. - started = max([boundary, *open_tools.values()]) - events.append( - AgentEvent( - event_id="evt_" + uuid.uuid4().hex, - run_id=run_id, - parent_id=root_id, - event_type="model_call", - name=f"codex_{item_type}", - model=model, - started_at=at(started), - ended_at=at(moment), - duration_ms=(moment - started) * 1000, - token_provenance=UNAVAILABLE, - output_text=clip(item.get("text")), - metadata={"codex_item_type": item_type}, - ) - ) + if open_tools: + # Reported while tools still run: keep the round open after them. + boundary = max([boundary, *open_tools.values()]) + model_round(moment, item_type, item.get("text")) if item_type == "agent_message": output = item.get("text") - boundary = moment elif kind == "turn.completed": usage = event.get("usage") or {} + if current is None and not any(item.event_type == "model_call" for item in events): + model_round(moment) models = [item for item in events if item.event_type == "model_call"] - if not models: - models.append( - AgentEvent( - event_id="evt_" + uuid.uuid4().hex, - run_id=run_id, - parent_id=root_id, - event_type="model_call", - name="codex_turn", - model=model, - started_at=at(boundary), - ended_at=at(moment), - duration_ms=(moment - boundary) * 1000, - ) - ) - events.append(models[0]) last = models[-1] last.input_tokens = int(usage.get("input_tokens") or 0) last.output_tokens = int(usage.get("output_tokens") or 0) diff --git a/tests/test_codex_exec_runner.py b/tests/test_codex_exec_runner.py index bc662f6..3d96e1f 100644 --- a/tests/test_codex_exec_runner.py +++ b/tests/test_codex_exec_runner.py @@ -78,9 +78,9 @@ def test_stream_becomes_tool_and_model_spans_with_turn_usage(): assert usage_seen is True assert failure is None assert [(event.event_type, event.name) for event in events] == [ - ("model_call", "codex_tool_decision"), + ("model_call", "codex_model_round"), ("tool_call", "shell"), - ("model_call", "codex_agent_message"), + ("model_call", "codex_model_round"), ] decision, tool, answer = events assert decision.duration_ms == pytest.approx(1500) @@ -95,6 +95,57 @@ def test_stream_becomes_tool_and_model_spans_with_turn_usage(): ) assert answer.metadata["usage_scope"] == "turn_total" assert answer.metadata["cached_input_tokens"] == 32000 + assert decision.metadata["codex_items"] == ["tool_call"] + assert answer.metadata["codex_items"] == ["agent_message"] + assert answer.output_text == ">=3.11" + + +def test_a_message_before_a_tool_call_is_part_of_the_same_model_round(): + preamble = ( + 1.0, + { + "type": "item.completed", + "item": {"id": "item_0", "type": "agent_message", "text": "Checking."}, + }, + ) + events, output, _, _ = run_convert([*STREAM[:2], preamble, *STREAM[2:]]) + rounds = [event for event in events if event.event_type == "model_call"] + assert len(rounds) == 2 + assert rounds[0].metadata["codex_items"] == ["agent_message", "tool_call"] + assert rounds[0].duration_ms == pytest.approx(1500) + assert output == ">=3.11" + + +def test_sequential_tools_add_a_model_round_but_parallel_tools_do_not(): + def tool(moment, item_id, phase): + item = {"id": item_id, "type": "command_execution", "command": item_id} + if phase == "completed": + item.update(exit_code=0, status="completed") + return (moment, {"type": f"item.{phase}", "item": item}) + + answer = ( + 5.0, + {"type": "item.completed", "item": {"id": "z", "type": "agent_message", "text": "ok"}}, + ) + parallel = [ + STREAM[1], + tool(1, "a", "started"), + tool(1.1, "b", "started"), + tool(2, "a", "completed"), + tool(2.1, "b", "completed"), + answer, + ] + sequential = [ + STREAM[1], + tool(1, "a", "started"), + tool(2, "a", "completed"), + tool(3, "b", "started"), + tool(4, "b", "completed"), + answer, + ] + for stream, expected in ((parallel, 2), (sequential, 3)): + events, _, _, _ = run_convert(stream) + assert sum(event.event_type == "model_call" for event in events) == expected def test_failed_commands_and_turns_are_retained_as_errors(): @@ -120,7 +171,7 @@ def test_failed_commands_and_turns_are_retained_as_errors(): def test_usage_without_model_items_gets_its_own_model_span(): events, _, usage_seen, _ = run_convert([STREAM[1], STREAM[-1]]) assert usage_seen is True - assert [(event.name, event.input_tokens) for event in events] == [("codex_turn", 40000)] + assert [(event.name, event.input_tokens) for event in events] == [("codex_model_round", 40000)] def test_converted_events_form_an_analyzable_trace(): diff --git a/tests/test_improvement.py b/tests/test_improvement.py index 2e64ad7..6ec9e67 100644 --- a/tests/test_improvement.py +++ b/tests/test_improvement.py @@ -569,3 +569,72 @@ def test_cli_freezes_the_aggregate_decision(tmp_path, workload): saved = json.loads((tmp_path / "session/session.json").read_bytes()) assert saved["specification"]["decision"] == "aggregate" assert "median paired improvement" in (tmp_path / "session/plan.md").read_text(encoding="utf-8") + + +def test_interleaved_control_must_follow_the_frozen_session(tmp_path, workload): + tasks, worker = prepare(tmp_path, workload) + capture( + tasks, + tmp_path / "paired", + command(worker, "fast"), + control=command(worker, "baseline"), + ) + control_path = tmp_path / "paired/control/capture.json" + control = json.loads(control_path.read_bytes()) + control["created_at"] = "2000-01-01T00:00:00+00:00" + control_path.write_text(json.dumps(control), encoding="utf-8") + candidate_path = tmp_path / "paired/capture.json" + candidate = json.loads(candidate_path.read_bytes()) + candidate["control"]["manifest_hash"] = fingerprint(control) + candidate_path.write_text(json.dumps(candidate), encoding="utf-8") + with pytest.raises(ValueError, match="control after freezing"): + finish( + tmp_path / "session/session.json", + candidate_path, + tmp_path / "results", + change="Stale control", + ) + assert not (tmp_path / "results").exists() + + +def test_cli_output_survives_unicode_names_on_a_legacy_code_page(tmp_path): + import os + import subprocess + + tasks = tmp_path / "tasks.json" + tasks.write_text( + json.dumps( + { + "schema_version": "1.0", + "name": "Unicode ✓ 检查", + "repetitions": 1, + "tasks": [{"id": "检查-✓", "input": "one", "expected": "one"}], + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + result = subprocess.run( + [ + sys.executable, + "-c", + "from agentloop.entrypoint import app; app()", + "improve", + "capture", + "--tasks", + str(tasks), + "--out", + str(tmp_path / "capture"), + "--", + sys.executable, + str(tmp_path / "missing.py"), + ], + capture_output=True, + env={**os.environ, "PYTHONIOENCODING": "cp1252", "COLUMNS": "200"}, + timeout=120, + check=False, + ) + output = result.stdout.decode("utf-8") + assert result.returncode == 1, result.stderr.decode("utf-8", "replace") + assert "UnicodeEncodeError" not in result.stderr.decode("utf-8", "replace") + assert "检查-✓ repetition 0" in output