Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,13 @@ trace, study, receipt, API and database schema versions.
and completion checks instead of implying every pair passed overall.
- The Codex example runner records one model span per model round. A message
followed by a tool call in the same round was counted as two model calls.
- `repetition_detour` compares completed attempts only, states observed and
typical counts instead of fractional differences, and keeps diagnosis
counts in step with the merged findings.
- The Codex runner closes a model round when a tool item arrives only on
completion, and measures each round from its own start.
- `improve start` accepts `per-pair` as well as `per_pair` and names the CLI
spelling in its error.
- `docs/CODEX.md` attributes the first run's token spikes to extra model
steps. It previously attributed them to an instruction-file read, which cost
nothing extra when it ran in the same step as the requested read.
Expand Down
19 changes: 16 additions & 3 deletions agentloop/improvement.py
Original file line number Diff line number Diff line change
Expand Up @@ -374,6 +374,16 @@ def read_capture(path, workload):
return manifest, traces


def _completed(manifest, traces):
"""Traces of attempts that completed, for comparisons between repetitions."""
done = {
(row["task_id"], row["repetition"])
for row in manifest["runs"]
if row["status"] == "completed"
}
return {key: trace for key, trace in traces.items() if key in done}


def _quality(task, baseline, candidate):
return build_quality_report(
[
Expand Down Expand Up @@ -402,15 +412,17 @@ def start(
workload = read_workload(workload_path)
manifest, traces = read_capture(baseline_path, workload)
require(objective in {"latency", "cost"}, "Objective must be latency or cost")
require(decision in DECISIONS, "Decision must be per_pair or aggregate")
decision = decision.replace("-", "_") if isinstance(decision, str) else decision
require(decision in DECISIONS, "Decision must be per-pair or aggregate")
require(
type(minimum_improvement) in (int, float)
and math.isfinite(minimum_improvement)
and minimum_improvement >= 0,
"Minimum improvement must be finite and nonnegative",
)
diagnoses = add_repetition_findings(
{key: build_diagnosis(trace) for key, trace in traces.items()}, traces
{key: build_diagnosis(trace) for key, trace in traces.items()},
_completed(manifest, traces),
)
findings = [finding for diagnosis in diagnoses.values() for finding in diagnosis["findings"]]
if focus is None and findings:
Expand Down Expand Up @@ -652,7 +664,8 @@ def finish(session_path, candidate_path, root, *, change):
# Diagnose controls like the original baseline, including comparisons
# between their repetitions, so focus findings can be linked.
control_diagnoses = add_repetition_findings(
{key: build_diagnosis(trace) for key, trace in controls.items()}, controls
{key: build_diagnosis(trace) for key, trace in controls.items()},
_completed(control_manifest, controls),
)
gates = ReplayGates(**specification["gates"])
root = Path(root).resolve()
Expand Down
2 changes: 1 addition & 1 deletion agentloop/improvement_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ def start_command(
focus=focus,
objective=objective,
minimum_improvement=minimum_improvement,
decision=decision.replace("-", "_"),
decision=decision,
)
except (OSError, ValueError, TypeError, KeyError) as exc:
raise typer.BadParameter(str(exc)) from exc
Expand Down
64 changes: 37 additions & 27 deletions agentloop/repetitions.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,31 +50,46 @@ def _ratio(value, typical):
return value / typical if typical > 0 else None


def _profile(trace):
"""Scan one trace once for everything the comparison needs."""
tools = _tools(trace)
return {
"trace": trace,
"tools": tools,
"models": _models(trace),
"keys": {_key(event) for event in tools},
"tokens": _input_tokens(trace),
"runtime": float(trace.elapsed_ms or 0.0),
}


def repetition_findings(traces: dict[tuple[str, int], Any]) -> dict[tuple[str, int], list]:
"""Return detour findings keyed by ``(task_id, repetition)``.

Only tasks with at least two available repetitions are compared. An attempt
Pass only completed attempts: a failed or interrupted attempt's partial
trace would distort the typical repetition. Only tasks with at least two
available repetitions are compared. An attempt
is reported when it took more model steps or more tool calls than the
median of the other repetitions, and its input tokens or runtime are at
least 25% above the others' median.
"""
by_task: dict[str, dict[int, Any]] = {}
for (task_id, repetition), trace in traces.items():
by_task.setdefault(task_id, {})[repetition] = trace
by_task.setdefault(task_id, {})[repetition] = _profile(trace)
flagged = {}
for task_id, attempts in by_task.items():
if len(attempts) < 2:
continue
for repetition, trace in attempts.items():
for repetition, profile in attempts.items():
others = [attempts[other] for other in attempts if other != repetition]
tools, models = _tools(trace), _models(trace)
typical_tools = statistics.median(len(_tools(other)) for other in others)
typical_models = statistics.median(len(_models(other)) for other in others)
trace, tools, models = profile["trace"], profile["tools"], profile["models"]
typical_tools = statistics.median(len(other["tools"]) for other in others)
typical_models = statistics.median(len(other["models"]) for other in others)
more_tools = len(tools) > typical_tools
more_models = len(models) > typical_models
if not (more_tools or more_models):
continue
other_keys = {_key(event) for other in others for event in _tools(other)}
other_keys = set().union(*(other["keys"] for other in others))
# Tool calls with no counterpart elsewhere, and the model steps
# between the first and final ones, which each resend the context.
extra = (
Expand All @@ -83,14 +98,10 @@ def repetition_findings(traces: dict[tuple[str, int], Any]) -> dict[tuple[str, i
rounds = models[1:-1] if more_models else []
if not (extra or rounds):
continue
tokens, typical_tokens = (
_input_tokens(trace),
statistics.median(_input_tokens(other) for other in others),
)
runtime, typical_runtime = (
float(trace.elapsed_ms or 0.0),
statistics.median(float(other.elapsed_ms or 0.0) for other in others),
)
tokens = profile["tokens"]
typical_tokens = statistics.median(other["tokens"] for other in others)
runtime = profile["runtime"]
typical_runtime = statistics.median(other["runtime"] for other in others)
token_ratio = _ratio(tokens, typical_tokens)
runtime_ratio = _ratio(runtime, typical_runtime)
# Extra model steps leave a token signature. When token counts are
Expand Down Expand Up @@ -122,8 +133,11 @@ def repetition_findings(traces: dict[tuple[str, int], Any]) -> dict[tuple[str, i
}


def _plural(count, word):
return f"{count:g} {word}" + ("" if count == 1 else "s")
def _count(actual, typical, word):
# Medians of an even number of repetitions can be fractional, so state the
# observed count and the typical one instead of a fractional difference.
plural = "" if actual == 1 else "s"
return f"{actual} {word}{plural} (typically {typical:g})"


def _finding(key, item, flagged_attempts):
Expand All @@ -135,17 +149,10 @@ def _finding(key, item, flagged_attempts):
spans = [event.event_id for event in extra]
differences = []
if item["model_steps"] > item["typical_model_steps"]:
differences.append(
_plural(item["model_steps"] - item["typical_model_steps"], "more model step")
)
differences.append(_count(item["model_steps"], item["typical_model_steps"], "model step"))
if item["tool_calls"] > item["typical_tool_calls"]:
differences.append(
_plural(item["tool_calls"] - item["typical_tool_calls"], "more tool call")
)
title = (
f"Repetition {repetition} of {task_id} took {' and '.join(differences)} "
"than its typical repetition"
)
differences.append(_count(item["tool_calls"], item["typical_tool_calls"], "tool call"))
title = f"Repetition {repetition} of {task_id} took {' and '.join(differences)}"
evidence = [
{
"span_id": event.event_id,
Expand Down Expand Up @@ -241,4 +248,7 @@ def add_repetition_findings(diagnoses, traces):
diagnosis["summary"]["high_severity_count"] = sum(
finding["severity"] == "high" for finding in combined
)
diagnosis["ranking"]["ready_to_test_count"] = sum(
finding["ranking"]["status"] == "ready_to_test" for finding in diagnosis["findings"]
)
return diagnoses
10 changes: 7 additions & 3 deletions examples/codex_exec_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ def at(moment):
return (origin + timedelta(seconds=moment - clock)).isoformat()

events, open_tools, output, usage_seen = [], {}, None, False
failure, boundary, current = None, clock, None
failure, boundary, current, current_start = None, clock, None, clock

def model_round(moment, item_type=None, text=None):
"""Open or extend the span for the current model round.
Expand All @@ -77,8 +77,9 @@ def model_round(moment, item_type=None, text=None):
and lasts until the model starts a tool or gives its final message.
Messages and reasoning inside one round belong to the same model call.
"""
nonlocal current
nonlocal current, current_start
if current is None:
current_start = boundary
current = AgentEvent(
event_id="evt_" + uuid.uuid4().hex,
run_id=run_id,
Expand All @@ -95,7 +96,7 @@ def model_round(moment, item_type=None, text=None):
events.append(current)
else:
current.ended_at = at(moment)
current.duration_ms = (moment - boundary) * 1000
current.duration_ms = (moment - current_start) * 1000
if item_type is not None:
current.metadata["codex_items"].append(item_type)
if text is not None:
Expand All @@ -117,6 +118,9 @@ def model_round(moment, item_type=None, text=None):
current = None
open_tools[item.get("id")] = moment
elif kind == "item.completed" and item_type in TOOL_ITEMS:
# Some tool items arrive only on completion; either way the model
# round that requested them has ended.
current = None
started = open_tools.pop(item.get("id"), moment)
exit_code = item.get("exit_code")
failed = item.get("status") == "failed" or exit_code not in (None, 0)
Expand Down
26 changes: 26 additions & 0 deletions tests/test_codex_exec_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -212,3 +212,29 @@ def test_task_prompt_accepts_strings_and_prompt_objects(value, instructions, exp
def test_task_prompt_rejects_missing_prompts():
with pytest.raises(ValueError, match="prompt"):
task_prompt({"other": 1}, "")


def test_a_tool_reported_only_on_completion_ends_the_model_round():
message = (
1.0,
{
"type": "item.completed",
"item": {"id": "m1", "type": "agent_message", "text": "Editing."},
},
)
change = (
2.0,
{"type": "item.completed", "item": {"id": "f1", "type": "file_change", "changes": []}},
)
answer = (
3.5,
{"type": "item.completed", "item": {"id": "m2", "type": "agent_message", "text": "Done."}},
)
events, _, _, _ = run_convert([STREAM[1], message, change, answer])
rounds = [event for event in events if event.event_type == "model_call"]
assert len(rounds) == 2
for event in rounds:
started = datetime.fromisoformat(event.started_at)
ended = datetime.fromisoformat(event.ended_at)
assert event.duration_ms == pytest.approx((ended - started).total_seconds() * 1000)
assert rounds[1].duration_ms == pytest.approx(1500)
2 changes: 1 addition & 1 deletion tests/test_improvement.py
Original file line number Diff line number Diff line change
Expand Up @@ -543,7 +543,7 @@ def test_decision_is_frozen_with_the_session(tmp_path, workload):
def test_unknown_decision_is_rejected(tmp_path, workload):
tasks, worker = workload
capture(tasks, tmp_path / "baseline", command(worker, "baseline"))
with pytest.raises(ValueError, match="per_pair or aggregate"):
with pytest.raises(ValueError, match="per-pair or aggregate"):
start(tasks, tmp_path / "baseline/capture.json", tmp_path / "session", decision="any")


Expand Down
58 changes: 55 additions & 3 deletions tests/test_repetitions.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,7 @@ def test_extra_model_step_with_more_tokens_is_reported_on_the_outlier():
finding = found[("lookup", 1)][0]
assert finding["type"] == FINDING_TYPE
assert finding["title"] == (
"Repetition 1 of lookup took 1 more model step and 1 more tool call "
"than its typical repetition"
"Repetition 1 of lookup took 3 model steps (typically 2) and 2 tool calls (typically 1)"
)
assert finding["savings"]["estimated_input_tokens"] == 1000
assert finding["confidence"] == "low"
Expand Down Expand Up @@ -164,5 +163,58 @@ def test_start_offers_the_detour_as_a_focus(tmp_path):
flagged = [case for case in specification["cases"] if case["finding_ids"]]
assert [(case["task_id"], case["repetition"]) for case in flagged] == [("first", 1)]
plan = (tmp_path / "session/plan.md").read_text(encoding="utf-8")
assert "took 1 more model step and 1 more tool call" in plan
assert "took 3 model steps (typically 2) and 2 tool calls (typically 1)" in plan
assert "read notes" in plan


def test_fractional_typical_counts_are_stated_not_subtracted():
traces = {
("lookup", 0): attempt(TYPICAL),
("lookup", 1): attempt(
["model", "tool:read a", "model", "tool:read pyproject.toml", "model"]
),
("lookup", 2): attempt(DETOUR, tokens_per_step=3000),
}
finding = repetition_findings(traces)[("lookup", 2)][0]
assert "model steps (typically 2.5)" in finding["title"]
assert " 0.5 " not in finding["title"]


def test_merged_diagnosis_counts_match_its_findings():
from agentloop.findings import build_diagnosis

traces = {("lookup", 0): attempt(TYPICAL), ("lookup", 1): attempt(DETOUR)}
diagnosis = add_repetition_findings(
{key: build_diagnosis(trace) for key, trace in traces.items()}, traces
)[("lookup", 1)]
assert diagnosis["ranking"]["ready_to_test_count"] == sum(
item["ranking"]["status"] == "ready_to_test" for item in diagnosis["findings"]
)
assert diagnosis["summary"]["finding_count"] == len(diagnosis["findings"])


def test_failed_attempts_are_left_out_of_repetition_comparisons(tmp_path):
tasks = tmp_path / "tasks.json"
tasks.write_text(
json.dumps(
{
"schema_version": "1.0",
"name": "Failed detour",
"repetitions": 2,
"tasks": [{"id": "first", "input": "one", "expected": "one"}],
}
),
encoding="utf-8",
)
worker = tmp_path / "worker.py"
# The detour attempt writes its trace and then fails.
worker.write_text(WORKER + "\nif detour:\n raise SystemExit(1)\n", encoding="utf-8")
capture(tasks, tmp_path / "baseline", [sys.executable, str(worker)])
specification = start(tasks, tmp_path / "baseline/capture.json", tmp_path / "session")
assert [case["capture_status"] for case in specification["cases"]] == ["completed", "failed"]
assert not any(
finding["type"] == FINDING_TYPE
for case in specification["cases"]
if case["diagnosis"]
for finding in case["diagnosis"]["findings"]
)
Loading