From 4f319a274018abf42586f8aa92a801cc0bfb9593 Mon Sep 17 00:00:00 2001 From: Dipesh Babu Date: Tue, 17 Nov 2026 02:42:06 -0500 Subject: [PATCH] Fix review findings in repetition detours and the Codex runner - Compare only completed attempts between repetitions, so a failed or interrupted attempt's partial trace cannot distort the typical one. - State observed and typical step counts instead of fractional differences, and keep diagnosis ranking counts in step with findings. - Profile each trace once instead of rescanning every repetition. - Close a Codex model round when a tool arrives only on completion and measure rounds from their own start. - Accept per-pair in improve start and name that spelling in errors. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 ++++ agentloop/improvement.py | 19 ++++++++-- agentloop/improvement_cli.py | 2 +- agentloop/repetitions.py | 64 +++++++++++++++++++-------------- examples/codex_exec_runner.py | 10 ++++-- tests/test_codex_exec_runner.py | 26 ++++++++++++++ tests/test_improvement.py | 2 +- tests/test_repetitions.py | 58 ++++++++++++++++++++++++++++-- 8 files changed, 150 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc5aea6..21052f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,13 @@ trace, study, receipt, API and database schema versions. and completion checks instead of implying every pair passed overall. - The Codex example runner records one model span per model round. A message followed by a tool call in the same round was counted as two model calls. +- `repetition_detour` compares completed attempts only, states observed and + typical counts instead of fractional differences, and keeps diagnosis + counts in step with the merged findings. +- The Codex runner closes a model round when a tool item arrives only on + completion, and measures each round from its own start. +- `improve start` accepts `per-pair` as well as `per_pair` and names the CLI + spelling in its error. - `docs/CODEX.md` attributes the first run's token spikes to extra model steps. It previously attributed them to an instruction-file read, which cost nothing extra when it ran in the same step as the requested read. diff --git a/agentloop/improvement.py b/agentloop/improvement.py index 116bd51..91361ec 100644 --- a/agentloop/improvement.py +++ b/agentloop/improvement.py @@ -374,6 +374,16 @@ def read_capture(path, workload): return manifest, traces +def _completed(manifest, traces): + """Traces of attempts that completed, for comparisons between repetitions.""" + done = { + (row["task_id"], row["repetition"]) + for row in manifest["runs"] + if row["status"] == "completed" + } + return {key: trace for key, trace in traces.items() if key in done} + + def _quality(task, baseline, candidate): return build_quality_report( [ @@ -402,7 +412,8 @@ def start( workload = read_workload(workload_path) manifest, traces = read_capture(baseline_path, workload) require(objective in {"latency", "cost"}, "Objective must be latency or cost") - require(decision in DECISIONS, "Decision must be per_pair or aggregate") + decision = decision.replace("-", "_") if isinstance(decision, str) else decision + require(decision in DECISIONS, "Decision must be per-pair or aggregate") require( type(minimum_improvement) in (int, float) and math.isfinite(minimum_improvement) @@ -410,7 +421,8 @@ def start( "Minimum improvement must be finite and nonnegative", ) diagnoses = add_repetition_findings( - {key: build_diagnosis(trace) for key, trace in traces.items()}, traces + {key: build_diagnosis(trace) for key, trace in traces.items()}, + _completed(manifest, traces), ) findings = [finding for diagnosis in diagnoses.values() for finding in diagnosis["findings"]] if focus is None and findings: @@ -652,7 +664,8 @@ def finish(session_path, candidate_path, root, *, change): # Diagnose controls like the original baseline, including comparisons # between their repetitions, so focus findings can be linked. control_diagnoses = add_repetition_findings( - {key: build_diagnosis(trace) for key, trace in controls.items()}, controls + {key: build_diagnosis(trace) for key, trace in controls.items()}, + _completed(control_manifest, controls), ) gates = ReplayGates(**specification["gates"]) root = Path(root).resolve() diff --git a/agentloop/improvement_cli.py b/agentloop/improvement_cli.py index 7bd1d99..72ddf20 100644 --- a/agentloop/improvement_cli.py +++ b/agentloop/improvement_cli.py @@ -116,7 +116,7 @@ def start_command( focus=focus, objective=objective, minimum_improvement=minimum_improvement, - decision=decision.replace("-", "_"), + decision=decision, ) except (OSError, ValueError, TypeError, KeyError) as exc: raise typer.BadParameter(str(exc)) from exc diff --git a/agentloop/repetitions.py b/agentloop/repetitions.py index e87d590..4c1000d 100644 --- a/agentloop/repetitions.py +++ b/agentloop/repetitions.py @@ -50,31 +50,46 @@ def _ratio(value, typical): return value / typical if typical > 0 else None +def _profile(trace): + """Scan one trace once for everything the comparison needs.""" + tools = _tools(trace) + return { + "trace": trace, + "tools": tools, + "models": _models(trace), + "keys": {_key(event) for event in tools}, + "tokens": _input_tokens(trace), + "runtime": float(trace.elapsed_ms or 0.0), + } + + def repetition_findings(traces: dict[tuple[str, int], Any]) -> dict[tuple[str, int], list]: """Return detour findings keyed by ``(task_id, repetition)``. - Only tasks with at least two available repetitions are compared. An attempt + Pass only completed attempts: a failed or interrupted attempt's partial + trace would distort the typical repetition. Only tasks with at least two + available repetitions are compared. An attempt is reported when it took more model steps or more tool calls than the median of the other repetitions, and its input tokens or runtime are at least 25% above the others' median. """ by_task: dict[str, dict[int, Any]] = {} for (task_id, repetition), trace in traces.items(): - by_task.setdefault(task_id, {})[repetition] = trace + by_task.setdefault(task_id, {})[repetition] = _profile(trace) flagged = {} for task_id, attempts in by_task.items(): if len(attempts) < 2: continue - for repetition, trace in attempts.items(): + for repetition, profile in attempts.items(): others = [attempts[other] for other in attempts if other != repetition] - tools, models = _tools(trace), _models(trace) - typical_tools = statistics.median(len(_tools(other)) for other in others) - typical_models = statistics.median(len(_models(other)) for other in others) + trace, tools, models = profile["trace"], profile["tools"], profile["models"] + typical_tools = statistics.median(len(other["tools"]) for other in others) + typical_models = statistics.median(len(other["models"]) for other in others) more_tools = len(tools) > typical_tools more_models = len(models) > typical_models if not (more_tools or more_models): continue - other_keys = {_key(event) for other in others for event in _tools(other)} + other_keys = set().union(*(other["keys"] for other in others)) # Tool calls with no counterpart elsewhere, and the model steps # between the first and final ones, which each resend the context. extra = ( @@ -83,14 +98,10 @@ def repetition_findings(traces: dict[tuple[str, int], Any]) -> dict[tuple[str, i rounds = models[1:-1] if more_models else [] if not (extra or rounds): continue - tokens, typical_tokens = ( - _input_tokens(trace), - statistics.median(_input_tokens(other) for other in others), - ) - runtime, typical_runtime = ( - float(trace.elapsed_ms or 0.0), - statistics.median(float(other.elapsed_ms or 0.0) for other in others), - ) + tokens = profile["tokens"] + typical_tokens = statistics.median(other["tokens"] for other in others) + runtime = profile["runtime"] + typical_runtime = statistics.median(other["runtime"] for other in others) token_ratio = _ratio(tokens, typical_tokens) runtime_ratio = _ratio(runtime, typical_runtime) # Extra model steps leave a token signature. When token counts are @@ -122,8 +133,11 @@ def repetition_findings(traces: dict[tuple[str, int], Any]) -> dict[tuple[str, i } -def _plural(count, word): - return f"{count:g} {word}" + ("" if count == 1 else "s") +def _count(actual, typical, word): + # Medians of an even number of repetitions can be fractional, so state the + # observed count and the typical one instead of a fractional difference. + plural = "" if actual == 1 else "s" + return f"{actual} {word}{plural} (typically {typical:g})" def _finding(key, item, flagged_attempts): @@ -135,17 +149,10 @@ def _finding(key, item, flagged_attempts): spans = [event.event_id for event in extra] differences = [] if item["model_steps"] > item["typical_model_steps"]: - differences.append( - _plural(item["model_steps"] - item["typical_model_steps"], "more model step") - ) + differences.append(_count(item["model_steps"], item["typical_model_steps"], "model step")) if item["tool_calls"] > item["typical_tool_calls"]: - differences.append( - _plural(item["tool_calls"] - item["typical_tool_calls"], "more tool call") - ) - title = ( - f"Repetition {repetition} of {task_id} took {' and '.join(differences)} " - "than its typical repetition" - ) + differences.append(_count(item["tool_calls"], item["typical_tool_calls"], "tool call")) + title = f"Repetition {repetition} of {task_id} took {' and '.join(differences)}" evidence = [ { "span_id": event.event_id, @@ -241,4 +248,7 @@ def add_repetition_findings(diagnoses, traces): diagnosis["summary"]["high_severity_count"] = sum( finding["severity"] == "high" for finding in combined ) + diagnosis["ranking"]["ready_to_test_count"] = sum( + finding["ranking"]["status"] == "ready_to_test" for finding in diagnosis["findings"] + ) return diagnoses diff --git a/examples/codex_exec_runner.py b/examples/codex_exec_runner.py index 7f5496a..105b3d6 100644 --- a/examples/codex_exec_runner.py +++ b/examples/codex_exec_runner.py @@ -68,7 +68,7 @@ def at(moment): return (origin + timedelta(seconds=moment - clock)).isoformat() events, open_tools, output, usage_seen = [], {}, None, False - failure, boundary, current = None, clock, None + failure, boundary, current, current_start = None, clock, None, clock def model_round(moment, item_type=None, text=None): """Open or extend the span for the current model round. @@ -77,8 +77,9 @@ def model_round(moment, item_type=None, text=None): and lasts until the model starts a tool or gives its final message. Messages and reasoning inside one round belong to the same model call. """ - nonlocal current + nonlocal current, current_start if current is None: + current_start = boundary current = AgentEvent( event_id="evt_" + uuid.uuid4().hex, run_id=run_id, @@ -95,7 +96,7 @@ def model_round(moment, item_type=None, text=None): events.append(current) else: current.ended_at = at(moment) - current.duration_ms = (moment - boundary) * 1000 + current.duration_ms = (moment - current_start) * 1000 if item_type is not None: current.metadata["codex_items"].append(item_type) if text is not None: @@ -117,6 +118,9 @@ def model_round(moment, item_type=None, text=None): current = None open_tools[item.get("id")] = moment elif kind == "item.completed" and item_type in TOOL_ITEMS: + # Some tool items arrive only on completion; either way the model + # round that requested them has ended. + current = None started = open_tools.pop(item.get("id"), moment) exit_code = item.get("exit_code") failed = item.get("status") == "failed" or exit_code not in (None, 0) diff --git a/tests/test_codex_exec_runner.py b/tests/test_codex_exec_runner.py index 3d96e1f..de1f8da 100644 --- a/tests/test_codex_exec_runner.py +++ b/tests/test_codex_exec_runner.py @@ -212,3 +212,29 @@ def test_task_prompt_accepts_strings_and_prompt_objects(value, instructions, exp def test_task_prompt_rejects_missing_prompts(): with pytest.raises(ValueError, match="prompt"): task_prompt({"other": 1}, "") + + +def test_a_tool_reported_only_on_completion_ends_the_model_round(): + message = ( + 1.0, + { + "type": "item.completed", + "item": {"id": "m1", "type": "agent_message", "text": "Editing."}, + }, + ) + change = ( + 2.0, + {"type": "item.completed", "item": {"id": "f1", "type": "file_change", "changes": []}}, + ) + answer = ( + 3.5, + {"type": "item.completed", "item": {"id": "m2", "type": "agent_message", "text": "Done."}}, + ) + events, _, _, _ = run_convert([STREAM[1], message, change, answer]) + rounds = [event for event in events if event.event_type == "model_call"] + assert len(rounds) == 2 + for event in rounds: + started = datetime.fromisoformat(event.started_at) + ended = datetime.fromisoformat(event.ended_at) + assert event.duration_ms == pytest.approx((ended - started).total_seconds() * 1000) + assert rounds[1].duration_ms == pytest.approx(1500) diff --git a/tests/test_improvement.py b/tests/test_improvement.py index 6ec9e67..1341b2f 100644 --- a/tests/test_improvement.py +++ b/tests/test_improvement.py @@ -543,7 +543,7 @@ def test_decision_is_frozen_with_the_session(tmp_path, workload): def test_unknown_decision_is_rejected(tmp_path, workload): tasks, worker = workload capture(tasks, tmp_path / "baseline", command(worker, "baseline")) - with pytest.raises(ValueError, match="per_pair or aggregate"): + with pytest.raises(ValueError, match="per-pair or aggregate"): start(tasks, tmp_path / "baseline/capture.json", tmp_path / "session", decision="any") diff --git a/tests/test_repetitions.py b/tests/test_repetitions.py index b05f702..911a562 100644 --- a/tests/test_repetitions.py +++ b/tests/test_repetitions.py @@ -55,8 +55,7 @@ def test_extra_model_step_with_more_tokens_is_reported_on_the_outlier(): finding = found[("lookup", 1)][0] assert finding["type"] == FINDING_TYPE assert finding["title"] == ( - "Repetition 1 of lookup took 1 more model step and 1 more tool call " - "than its typical repetition" + "Repetition 1 of lookup took 3 model steps (typically 2) and 2 tool calls (typically 1)" ) assert finding["savings"]["estimated_input_tokens"] == 1000 assert finding["confidence"] == "low" @@ -164,5 +163,58 @@ def test_start_offers_the_detour_as_a_focus(tmp_path): flagged = [case for case in specification["cases"] if case["finding_ids"]] assert [(case["task_id"], case["repetition"]) for case in flagged] == [("first", 1)] plan = (tmp_path / "session/plan.md").read_text(encoding="utf-8") - assert "took 1 more model step and 1 more tool call" in plan + assert "took 3 model steps (typically 2) and 2 tool calls (typically 1)" in plan assert "read notes" in plan + + +def test_fractional_typical_counts_are_stated_not_subtracted(): + traces = { + ("lookup", 0): attempt(TYPICAL), + ("lookup", 1): attempt( + ["model", "tool:read a", "model", "tool:read pyproject.toml", "model"] + ), + ("lookup", 2): attempt(DETOUR, tokens_per_step=3000), + } + finding = repetition_findings(traces)[("lookup", 2)][0] + assert "model steps (typically 2.5)" in finding["title"] + assert " 0.5 " not in finding["title"] + + +def test_merged_diagnosis_counts_match_its_findings(): + from agentloop.findings import build_diagnosis + + traces = {("lookup", 0): attempt(TYPICAL), ("lookup", 1): attempt(DETOUR)} + diagnosis = add_repetition_findings( + {key: build_diagnosis(trace) for key, trace in traces.items()}, traces + )[("lookup", 1)] + assert diagnosis["ranking"]["ready_to_test_count"] == sum( + item["ranking"]["status"] == "ready_to_test" for item in diagnosis["findings"] + ) + assert diagnosis["summary"]["finding_count"] == len(diagnosis["findings"]) + + +def test_failed_attempts_are_left_out_of_repetition_comparisons(tmp_path): + tasks = tmp_path / "tasks.json" + tasks.write_text( + json.dumps( + { + "schema_version": "1.0", + "name": "Failed detour", + "repetitions": 2, + "tasks": [{"id": "first", "input": "one", "expected": "one"}], + } + ), + encoding="utf-8", + ) + worker = tmp_path / "worker.py" + # The detour attempt writes its trace and then fails. + worker.write_text(WORKER + "\nif detour:\n raise SystemExit(1)\n", encoding="utf-8") + capture(tasks, tmp_path / "baseline", [sys.executable, str(worker)]) + specification = start(tasks, tmp_path / "baseline/capture.json", tmp_path / "session") + assert [case["capture_status"] for case in specification["cases"]] == ["completed", "failed"] + assert not any( + finding["type"] == FINDING_TYPE + for case in specification["cases"] + if case["diagnosis"] + for finding in case["diagnosis"]["findings"] + )