From f0a5f28918df402671cc9c563dad0083763a7018 Mon Sep 17 00:00:00 2001 From: Dipesh Babu Date: Tue, 17 Nov 2026 01:33:37 -0500 Subject: [PATCH] Report detours that only some repetitions take improve start now compares repetitions of each task. An attempt that took more model steps or tool calls than its typical repetition and used at least 25% more input tokens (runtime when tokens are unrecorded) gets a repetition_detour finding listing the extra calls and intermediate model steps. Interleaved controls are diagnosed the same way so the finding can be linked in intervention records. On the 18 retained Codex traces it flags exactly the three runs that took an extra model step. docs/CODEX.md now attributes those token spikes to the extra step rather than to an instruction-file read. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 11 ++ agentloop/improvement.py | 12 +- agentloop/repetitions.py | 244 ++++++++++++++++++++++++++++++++++++ docs/CODEX.md | 18 ++- docs/GUIDED_OPTIMIZATION.md | 10 ++ tests/test_repetitions.py | 168 +++++++++++++++++++++++++ 6 files changed, 456 insertions(+), 7 deletions(-) create mode 100644 agentloop/repetitions.py create mode 100644 tests/test_repetitions.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f9f183c..089563c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,17 @@ trace, study, receipt, API and database schema versions. more pairs must improve than regress. Every pair must still complete and pass its quality and non-performance gates. Reports include the counts and a non-gating sign-test p-value. The default per-pair decision is unchanged. +- `improve start` compares repetitions of each task and reports a + `repetition_detour` finding when an attempt took more model steps or tool + calls than its typical repetition and used at least 25% more input tokens + (runtime when tokens are unrecorded). Evidence lists the extra calls and + intermediate model steps. Interleaved controls are diagnosed the same way. + +### Fixed + +- `docs/CODEX.md` attributes the first run's token spikes to extra model + steps. It previously attributed them to an instruction-file read, which cost + nothing extra when it ran in the same step as the requested read. ## [0.2.2] - 2026-11-17 diff --git a/agentloop/improvement.py b/agentloop/improvement.py index 2744783..3065991 100644 --- a/agentloop/improvement.py +++ b/agentloop/improvement.py @@ -18,6 +18,7 @@ from agentloop.findings import build_diagnosis from agentloop.interventions import build_intervention from agentloop.quality import build_quality_report, validate_quality_fixtures +from agentloop.repetitions import add_repetition_findings from agentloop.replay import ReplayGates, build_replay_report from agentloop.replay import _summary as replay_summary from agentloop.tracer import AgentTrace @@ -408,7 +409,9 @@ def start( and minimum_improvement >= 0, "Minimum improvement must be finite and nonnegative", ) - diagnoses = {key: build_diagnosis(trace) for key, trace in traces.items()} + diagnoses = add_repetition_findings( + {key: build_diagnosis(trace) for key, trace in traces.items()}, traces + ) findings = [finding for diagnosis in diagnoses.values() for finding in diagnosis["findings"]] if focus is None and findings: focus = findings[0]["type"] @@ -641,6 +644,11 @@ def finish(session_path, candidate_path, root, *, change): ), "Control, baseline and candidate must have distinct run IDs", ) + # Diagnose controls like the original baseline, including comparisons + # between their repetitions, so focus findings can be linked. + control_diagnoses = add_repetition_findings( + {key: build_diagnosis(trace) for key, trace in controls.items()}, controls + ) gates = ReplayGates(**specification["gates"]) root = Path(root).resolve() root.mkdir(parents=True, exist_ok=False) @@ -668,7 +676,7 @@ def finish(session_path, candidate_path, root, *, change): ) baseline_status = control_row["status"] baseline_diagnostic = control_row.get("diagnostic") - diagnosis = build_diagnosis(baseline) if baseline is not None else None + diagnosis = control_diagnoses.get(key) finding_ids = ( [ finding["finding_id"] diff --git a/agentloop/repetitions.py b/agentloop/repetitions.py new file mode 100644 index 0000000..e87d590 --- /dev/null +++ b/agentloop/repetitions.py @@ -0,0 +1,244 @@ +"""Findings that compare repetitions of the same task. + +Single-trace rules judge one run in isolation, so a detour that only some +repetitions take looks normal on its own. This module compares the repeated +attempts of each task and reports an attempt that took more model steps or +tool calls than its typical repetition and cost clearly more input tokens or +time. Each extra model step usually resends the whole context, so one extra +round can cost far more than the tool call it makes. The extra steps are the +observed difference, not a proven cause. +""" + +from __future__ import annotations + +import statistics +from typing import Any + +from agentloop.findings import OptimizationFinding, _stable_finding_id +from agentloop.ranking import RANKING_KEY, rank_findings + +FINDING_TYPE = "repetition_detour" +RULE_ID = "agentloop.repetition_detour" +RULE_VERSION = "1" +# An attempt must cost at least this much more than the median of the other +# repetitions, in input tokens or runtime, before its extra calls are reported. +HEAVIER_RATIO = 1.25 +EXCERPT = 240 + + +def _tools(trace): + return [event for event in trace.events if event.operation_kind == "tool"] + + +def _models(trace): + return sorted( + (event for event in trace.events if event.operation_kind == "model"), + key=lambda event: event.started_at, + ) + + +def _key(event): + text = " ".join((event.input_text or "").lower().split()) + return event.name, text[:500] + + +def _input_tokens(trace): + return sum(event.input_tokens for event in trace.events if event.operation_kind == "model") + + +def _ratio(value, typical): + return value / typical if typical > 0 else None + + +def repetition_findings(traces: dict[tuple[str, int], Any]) -> dict[tuple[str, int], list]: + """Return detour findings keyed by ``(task_id, repetition)``. + + Only tasks with at least two available repetitions are compared. An attempt + is reported when it took more model steps or more tool calls than the + median of the other repetitions, and its input tokens or runtime are at + least 25% above the others' median. + """ + by_task: dict[str, dict[int, Any]] = {} + for (task_id, repetition), trace in traces.items(): + by_task.setdefault(task_id, {})[repetition] = trace + flagged = {} + for task_id, attempts in by_task.items(): + if len(attempts) < 2: + continue + for repetition, trace in attempts.items(): + others = [attempts[other] for other in attempts if other != repetition] + tools, models = _tools(trace), _models(trace) + typical_tools = statistics.median(len(_tools(other)) for other in others) + typical_models = statistics.median(len(_models(other)) for other in others) + more_tools = len(tools) > typical_tools + more_models = len(models) > typical_models + if not (more_tools or more_models): + continue + other_keys = {_key(event) for other in others for event in _tools(other)} + # Tool calls with no counterpart elsewhere, and the model steps + # between the first and final ones, which each resend the context. + extra = ( + [event for event in tools if _key(event) not in other_keys] if more_tools else [] + ) + rounds = models[1:-1] if more_models else [] + if not (extra or rounds): + continue + tokens, typical_tokens = ( + _input_tokens(trace), + statistics.median(_input_tokens(other) for other in others), + ) + runtime, typical_runtime = ( + float(trace.elapsed_ms or 0.0), + statistics.median(float(other.elapsed_ms or 0.0) for other in others), + ) + token_ratio = _ratio(tokens, typical_tokens) + runtime_ratio = _ratio(runtime, typical_runtime) + # Extra model steps leave a token signature. When token counts are + # recorded, require it; a slower run alone may be provider latency. + # Runtime decides only for traces without token counts. + measure = token_ratio if token_ratio is not None and tokens > 0 else runtime_ratio + if measure is None or measure < HEAVIER_RATIO: + continue + flagged[(task_id, repetition)] = { + "trace": trace, + "extra": extra, + "rounds": rounds, + "tool_calls": len(tools), + "typical_tool_calls": typical_tools, + "model_steps": len(models), + "typical_model_steps": typical_models, + "repetitions_compared": len(attempts), + "input_tokens": tokens, + "typical_input_tokens": typical_tokens, + "runtime_ms": runtime, + "typical_runtime_ms": typical_runtime, + "token_ratio": token_ratio, + "runtime_ratio": runtime_ratio, + "measure": measure, + } + return { + key: [_finding(key, item, len(flagged))] + for key, item in sorted(flagged.items(), key=lambda pair: pair[0]) + } + + +def _plural(count, word): + return f"{count:g} {word}" + ("" if count == 1 else "s") + + +def _finding(key, item, flagged_attempts): + task_id, repetition = key + extra = [*item["rounds"], *item["extra"]] + ratio = item["measure"] + latency_gap = max(0.0, item["runtime_ms"] - item["typical_runtime_ms"]) + token_gap = max(0, item["input_tokens"] - round(item["typical_input_tokens"])) + spans = [event.event_id for event in extra] + differences = [] + if item["model_steps"] > item["typical_model_steps"]: + differences.append( + _plural(item["model_steps"] - item["typical_model_steps"], "more model step") + ) + if item["tool_calls"] > item["typical_tool_calls"]: + differences.append( + _plural(item["tool_calls"] - item["typical_tool_calls"], "more tool call") + ) + title = ( + f"Repetition {repetition} of {task_id} took {' and '.join(differences)} " + "than its typical repetition" + ) + evidence = [ + { + "span_id": event.event_id, + "name": event.name, + "event_type": event.event_type, + "operation_kind": event.operation_kind, + "duration_ms": event.duration_ms, + "input_tokens": event.input_tokens, + "output_tokens": event.output_tokens, + "model": event.model, + "status": event.status, + "input_excerpt": (event.input_text or "")[:EXCERPT], + } + for event in extra + ] + observations = { + "task_id": task_id, + "repetition": repetition, + "repetitions_compared": item["repetitions_compared"], + "input_tokens": item["input_tokens"], + "typical_input_tokens": item["typical_input_tokens"], + "runtime_ms": round(item["runtime_ms"], 3), + "typical_runtime_ms": round(item["typical_runtime_ms"], 3), + "model_steps": item["model_steps"], + "typical_model_steps": item["typical_model_steps"], + "tool_calls": item["tool_calls"], + "typical_tool_calls": item["typical_tool_calls"], + "attempts_with_a_detour": flagged_attempts, + } + return OptimizationFinding( + finding_id=_stable_finding_id(FINDING_TYPE, title, spans), + severity="high" if ratio >= 1.5 else "medium", + type=FINDING_TYPE, + title=title, + confidence="medium" if flagged_attempts >= 2 else "low", + affected_spans=spans, + evidence=evidence, + savings={ + "estimated_latency_savings_ms": round(latency_gap, 3), + "estimated_cost_savings_usd": None, + "estimated_input_tokens": token_gap, + "formula": "attempt runtime and input tokens minus the median of the other repetitions", + }, + rewrite={ + "kind": FINDING_TYPE, + "hint": "Compare this attempt's steps with a typical repetition. Each extra model " + "step resends the context, so ask the agent to gather what it needs in one step " + "(for example, read all required files together), and remove instructions or " + "ambiguity that trigger follow-up reads.", + "patchable": False, + }, + validation={ + "command": "agentloop improve finish", + "acceptance_criteria": "repetitions take a consistent number of model steps and " + "tool calls, and input tokens or runtime fall toward the typical repetition while " + "quality checks pass", + }, + metadata={ + "why": "Other repetitions of the same task finished in fewer steps.", + "cost_status": "unknown", + "identity": "content-v1", + }, + evidence_level="observed", + assumptions=[ + "Repetitions of a task receive identical inputs.", + "The extra steps are associated with the higher cost; they are not proven to cause it.", + ], + observations=observations, + rule_id=RULE_ID, + rule_version=RULE_VERSION, + estimate={ + "estimator_id": "repetition_median_gap", + "estimator_version": "1", + "formula": "attempt minus median of the other repetitions", + "unmodeled_metrics": ["cost"], + "assumptions": ["Savings assume the attempt can match its typical repetition."], + }, + ).to_dict() + + +def add_repetition_findings(diagnoses, traces): + """Merge detour findings into each attempt's diagnosis and re-rank it.""" + for key, extra in repetition_findings(traces).items(): + diagnosis = diagnoses.get(key) + if diagnosis is None: + continue + trace = traces[key] + combined = [*diagnosis["findings"], *extra] + diagnosis["findings"] = rank_findings( + combined, inputs=trace.metadata.get(RANKING_KEY, {}), sort_by="priority" + ) + diagnosis["summary"]["finding_count"] = len(combined) + diagnosis["summary"]["high_severity_count"] = sum( + finding["severity"] == "high" for finding in combined + ) + return diagnoses diff --git a/docs/CODEX.md b/docs/CODEX.md index d07e18f..0e4f8b1 100644 --- a/docs/CODEX.md +++ b/docs/CODEX.md @@ -71,10 +71,18 @@ the three tasks ran twice per condition. same path. Both `engines.node` pairs were slower than their controls, by 12% and 19%, and failed the latency gates. **The change was rejected.** -In three runs, two controls and one candidate, Codex made an extra shell call to -read a user-level instruction file referenced by the global Codex instructions. -Each of those runs used about 16,000–20,000 more input tokens. The traces make -that call visible as a separate `tool_call` span. This path variation is larger -than the measured context saving. More repetitions would be needed before +Three runs, two controls and one candidate, used about 18,000–20,400 more input +tokens than their other repetition. Each of them took an extra model step. Codex +ran one tool, called the model again, and then ran another tool, instead of +running both tools in one step, and each model step resent about 19,000 tokens +of context. Several runs also read a user-level instruction file referenced by +the global Codex instructions. That read cost nothing extra when Codex ran it in +the same step as the requested file read. The extra step, not the read itself, +accounted for the increase. This path variation is larger than the measured +context saving. + +The `repetition_detour` finding, available from AgentLoop 0.2.3, reports these +attempts automatically. On these 18 traces it flags exactly the three runs with +an extra model step and none of the others. More repetitions would be needed before attributing a latency change to this configuration. These six pairs describe one machine, one model configuration and three lookup tasks. diff --git a/docs/GUIDED_OPTIMIZATION.md b/docs/GUIDED_OPTIMIZATION.md index 614c665..bebac61 100644 --- a/docs/GUIDED_OPTIMIZATION.md +++ b/docs/GUIDED_OPTIMIZATION.md @@ -141,6 +141,16 @@ and missing inputs. They remain hypotheses until candidate measurements exist. The default focus is the first baseline finding family. Select another observed family with `--focus cache_context`, for example. +From AgentLoop 0.2.3, `start` also compares repetitions of each task. The +`repetition_detour` finding marks an attempt that took more model steps or tool +calls than the median of its other repetitions and used at least 25% more input +tokens. Runtime replaces tokens only when traces record no token counts. Its +evidence lists the extra tool calls and the intermediate model steps, each of +which usually resends the whole context. Confidence is medium when two or more +attempts in the task set take such a detour. Select it with +`--focus repetition_detour`. Single-run rules cannot see this pattern, because +each detour looks normal on its own. Use at least two repetitions per task. + `start` saves original diagnoses, selected finding IDs, trace snapshots, task definitions, and gate settings in a content-hashed `session.json` before any candidate capture. The default objective is latency. Use `--objective cost` to diff --git a/tests/test_repetitions.py b/tests/test_repetitions.py new file mode 100644 index 0000000..b05f702 --- /dev/null +++ b/tests/test_repetitions.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +import json +import sys +from datetime import datetime, timedelta, timezone + +from agentloop.events import AgentEvent +from agentloop.improvement import capture, start +from agentloop.repetitions import FINDING_TYPE, add_repetition_findings, repetition_findings +from agentloop.tracer import AgentTrace + +ORIGIN = datetime(2026, 11, 17, tzinfo=timezone.utc) + + +def attempt(steps, *, tokens_per_step=1000, step_ms=100.0): + """Build a trace from a step plan such as ["model", "tool:read a", "model"].""" + trace = AgentTrace("repetition fixture") + clock = 0.0 + for index, step in enumerate(steps): + kind, _, text = step.partition(":") + model = kind == "model" + trace.add_event( + AgentEvent( + event_id=f"evt_{index}", + run_id=trace.run_id, + event_type="model_call" if model else "tool_call", + name="model" if model else "shell", + started_at=(ORIGIN + timedelta(milliseconds=clock)).isoformat(), + ended_at=(ORIGIN + timedelta(milliseconds=clock + step_ms)).isoformat(), + duration_ms=step_ms, + input_tokens=tokens_per_step if model else 0, + token_provenance="provider" if model and tokens_per_step else None, + input_text=text or None, + ) + ) + clock += step_ms + trace.finish() + trace.ended_at = (ORIGIN + timedelta(milliseconds=clock)).isoformat() + trace.elapsed_ms = clock + return trace + + +TYPICAL = ["model", "tool:read pyproject.toml", "model"] +DETOUR = ["model", "tool:read notes.md", "model", "tool:read pyproject.toml", "model"] + + +def test_extra_model_step_with_more_tokens_is_reported_on_the_outlier(): + traces = { + ("lookup", 0): attempt(TYPICAL), + ("lookup", 1): attempt(DETOUR), + ("lookup", 2): attempt(TYPICAL), + } + found = repetition_findings(traces) + assert list(found) == [("lookup", 1)] + finding = found[("lookup", 1)][0] + assert finding["type"] == FINDING_TYPE + assert finding["title"] == ( + "Repetition 1 of lookup took 1 more model step and 1 more tool call " + "than its typical repetition" + ) + assert finding["savings"]["estimated_input_tokens"] == 1000 + assert finding["confidence"] == "low" + excerpts = {row["input_excerpt"] for row in finding["evidence"]} + assert "read notes.md" in excerpts + assert "read pyproject.toml" not in excerpts + + +def test_slower_run_without_extra_tokens_is_not_reported(): + traces = { + ("lookup", 0): attempt(TYPICAL), + ("lookup", 1): attempt( + ["model", "model", "tool:read pyproject.toml", "model"], step_ms=400 + ), + } + traces[("lookup", 1)].events[1].input_tokens = 0 + assert repetition_findings(traces) == {} + + +def test_runtime_decides_when_token_counts_are_unavailable(): + traces = { + ("lookup", 0): attempt(TYPICAL, tokens_per_step=0), + ("lookup", 1): attempt(DETOUR, tokens_per_step=0), + } + found = repetition_findings(traces) + assert list(found) == [("lookup", 1)] + assert found[("lookup", 1)][0]["savings"]["estimated_latency_savings_ms"] == 200 + + +def test_same_steps_or_single_repetitions_are_not_compared(): + assert repetition_findings({("a", 0): attempt(DETOUR)}) == {} + assert repetition_findings({("a", 0): attempt(TYPICAL), ("a", 1): attempt(TYPICAL)}) == {} + + +def test_recurring_detours_raise_confidence(): + traces = { + ("first", 0): attempt(TYPICAL), + ("first", 1): attempt(DETOUR), + ("second", 0): attempt(DETOUR), + ("second", 1): attempt(TYPICAL), + } + found = repetition_findings(traces) + assert {key: value[0]["confidence"] for key, value in found.items()} == { + ("first", 1): "medium", + ("second", 0): "medium", + } + + +def test_findings_merge_into_ranked_diagnoses(): + from agentloop.findings import build_diagnosis + + traces = {("lookup", 0): attempt(TYPICAL), ("lookup", 1): attempt(DETOUR)} + diagnoses = add_repetition_findings( + {key: build_diagnosis(trace) for key, trace in traces.items()}, traces + ) + merged = [item for item in diagnoses[("lookup", 1)]["findings"] if item["type"] == FINDING_TYPE] + assert len(merged) == 1 + assert merged[0]["ranking"]["priority_rank"] >= 1 + assert diagnoses[("lookup", 1)]["summary"]["finding_count"] >= 1 + assert not any(item["type"] == FINDING_TYPE for item in diagnoses[("lookup", 0)]["findings"]) + + +WORKER = """import json,sys +from datetime import datetime,timedelta,timezone +from agentloop.events import AgentEvent +from agentloop.tracer import AgentTrace +request=json.loads(sys.stdin.readline()) +detour=request["task_id"]=="first" and request["repetition"]==1 +steps=["model","tool:read source","model"] +if detour: + steps=["model","tool:read notes","model","tool:read source","model"] +trace=AgentTrace("worker",metadata={"synthetic":True,"output":request["input"]}) +origin=datetime(2026,11,17,tzinfo=timezone.utc) +for index,step in enumerate(steps): + kind,_,text=step.partition(":") + trace.add_event(AgentEvent(event_id=f"evt_{index}",run_id=trace.run_id,event_type="model_call" if kind=="model" else "tool_call",name=kind,started_at=(origin+timedelta(seconds=index)).isoformat(),ended_at=(origin+timedelta(seconds=index+1)).isoformat(),duration_ms=1000,input_tokens=1000 if kind=="model" else 0,token_provenance="provider" if kind=="model" else None,input_text=text or None)) +trace.finish() +trace.export_json(request["trace_path"]) +""" + + +def test_start_offers_the_detour_as_a_focus(tmp_path): + tasks = tmp_path / "tasks.json" + tasks.write_text( + json.dumps( + { + "schema_version": "1.0", + "name": "Repetition tasks", + "repetitions": 2, + "tasks": [ + {"id": "first", "input": "one", "expected": "one"}, + {"id": "second", "input": "two", "expected": "two"}, + ], + } + ), + encoding="utf-8", + ) + worker = tmp_path / "worker.py" + worker.write_text(WORKER, encoding="utf-8") + capture(tasks, tmp_path / "baseline", [sys.executable, str(worker)]) + specification = start( + tasks, tmp_path / "baseline/capture.json", tmp_path / "session", focus=FINDING_TYPE + ) + assert specification["focus"] == FINDING_TYPE + flagged = [case for case in specification["cases"] if case["finding_ids"]] + assert [(case["task_id"], case["repetition"]) for case in flagged] == [("first", 1)] + plan = (tmp_path / "session/plan.md").read_text(encoding="utf-8") + assert "took 1 more model step and 1 more tool call" in plan + assert "read notes" in plan