diff --git a/CHANGELOG.md b/CHANGELOG.md index f9f183c..089563c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,17 @@ trace, study, receipt, API and database schema versions. more pairs must improve than regress. Every pair must still complete and pass its quality and non-performance gates. Reports include the counts and a non-gating sign-test p-value. The default per-pair decision is unchanged. +- `improve start` compares repetitions of each task and reports a + `repetition_detour` finding when an attempt took more model steps or tool + calls than its typical repetition and used at least 25% more input tokens + (runtime when tokens are unrecorded). Evidence lists the extra calls and + intermediate model steps. Interleaved controls are diagnosed the same way. + +### Fixed + +- `docs/CODEX.md` attributes the first run's token spikes to extra model + steps. It previously attributed them to an instruction-file read, which cost + nothing extra when it ran in the same step as the requested read. ## [0.2.2] - 2026-11-17 diff --git a/agentloop/improvement.py b/agentloop/improvement.py index 2744783..3065991 100644 --- a/agentloop/improvement.py +++ b/agentloop/improvement.py @@ -18,6 +18,7 @@ from agentloop.findings import build_diagnosis from agentloop.interventions import build_intervention from agentloop.quality import build_quality_report, validate_quality_fixtures +from agentloop.repetitions import add_repetition_findings from agentloop.replay import ReplayGates, build_replay_report from agentloop.replay import _summary as replay_summary from agentloop.tracer import AgentTrace @@ -408,7 +409,9 @@ def start( and minimum_improvement >= 0, "Minimum improvement must be finite and nonnegative", ) - diagnoses = {key: build_diagnosis(trace) for key, trace in traces.items()} + diagnoses = add_repetition_findings( + {key: build_diagnosis(trace) for key, trace in traces.items()}, traces + ) findings = [finding for diagnosis in diagnoses.values() for finding in diagnosis["findings"]] if focus is None and findings: focus = findings[0]["type"] @@ -641,6 +644,11 @@ def finish(session_path, candidate_path, root, *, change): ), "Control, baseline and candidate must have distinct run IDs", ) + # Diagnose controls like the original baseline, including comparisons + # between their repetitions, so focus findings can be linked. + control_diagnoses = add_repetition_findings( + {key: build_diagnosis(trace) for key, trace in controls.items()}, controls + ) gates = ReplayGates(**specification["gates"]) root = Path(root).resolve() root.mkdir(parents=True, exist_ok=False) @@ -668,7 +676,7 @@ def finish(session_path, candidate_path, root, *, change): ) baseline_status = control_row["status"] baseline_diagnostic = control_row.get("diagnostic") - diagnosis = build_diagnosis(baseline) if baseline is not None else None + diagnosis = control_diagnoses.get(key) finding_ids = ( [ finding["finding_id"] diff --git a/agentloop/repetitions.py b/agentloop/repetitions.py new file mode 100644 index 0000000..e87d590 --- /dev/null +++ b/agentloop/repetitions.py @@ -0,0 +1,244 @@ +"""Findings that compare repetitions of the same task. + +Single-trace rules judge one run in isolation, so a detour that only some +repetitions take looks normal on its own. This module compares the repeated +attempts of each task and reports an attempt that took more model steps or +tool calls than its typical repetition and cost clearly more input tokens or +time. Each extra model step usually resends the whole context, so one extra +round can cost far more than the tool call it makes. The extra steps are the +observed difference, not a proven cause. +""" + +from __future__ import annotations + +import statistics +from typing import Any + +from agentloop.findings import OptimizationFinding, _stable_finding_id +from agentloop.ranking import RANKING_KEY, rank_findings + +FINDING_TYPE = "repetition_detour" +RULE_ID = "agentloop.repetition_detour" +RULE_VERSION = "1" +# An attempt must cost at least this much more than the median of the other +# repetitions, in input tokens or runtime, before its extra calls are reported. +HEAVIER_RATIO = 1.25 +EXCERPT = 240 + + +def _tools(trace): + return [event for event in trace.events if event.operation_kind == "tool"] + + +def _models(trace): + return sorted( + (event for event in trace.events if event.operation_kind == "model"), + key=lambda event: event.started_at, + ) + + +def _key(event): + text = " ".join((event.input_text or "").lower().split()) + return event.name, text[:500] + + +def _input_tokens(trace): + return sum(event.input_tokens for event in trace.events if event.operation_kind == "model") + + +def _ratio(value, typical): + return value / typical if typical > 0 else None + + +def repetition_findings(traces: dict[tuple[str, int], Any]) -> dict[tuple[str, int], list]: + """Return detour findings keyed by ``(task_id, repetition)``. + + Only tasks with at least two available repetitions are compared. An attempt + is reported when it took more model steps or more tool calls than the + median of the other repetitions, and its input tokens or runtime are at + least 25% above the others' median. + """ + by_task: dict[str, dict[int, Any]] = {} + for (task_id, repetition), trace in traces.items(): + by_task.setdefault(task_id, {})[repetition] = trace + flagged = {} + for task_id, attempts in by_task.items(): + if len(attempts) < 2: + continue + for repetition, trace in attempts.items(): + others = [attempts[other] for other in attempts if other != repetition] + tools, models = _tools(trace), _models(trace) + typical_tools = statistics.median(len(_tools(other)) for other in others) + typical_models = statistics.median(len(_models(other)) for other in others) + more_tools = len(tools) > typical_tools + more_models = len(models) > typical_models + if not (more_tools or more_models): + continue + other_keys = {_key(event) for other in others for event in _tools(other)} + # Tool calls with no counterpart elsewhere, and the model steps + # between the first and final ones, which each resend the context. + extra = ( + [event for event in tools if _key(event) not in other_keys] if more_tools else [] + ) + rounds = models[1:-1] if more_models else [] + if not (extra or rounds): + continue + tokens, typical_tokens = ( + _input_tokens(trace), + statistics.median(_input_tokens(other) for other in others), + ) + runtime, typical_runtime = ( + float(trace.elapsed_ms or 0.0), + statistics.median(float(other.elapsed_ms or 0.0) for other in others), + ) + token_ratio = _ratio(tokens, typical_tokens) + runtime_ratio = _ratio(runtime, typical_runtime) + # Extra model steps leave a token signature. When token counts are + # recorded, require it; a slower run alone may be provider latency. + # Runtime decides only for traces without token counts. + measure = token_ratio if token_ratio is not None and tokens > 0 else runtime_ratio + if measure is None or measure < HEAVIER_RATIO: + continue + flagged[(task_id, repetition)] = { + "trace": trace, + "extra": extra, + "rounds": rounds, + "tool_calls": len(tools), + "typical_tool_calls": typical_tools, + "model_steps": len(models), + "typical_model_steps": typical_models, + "repetitions_compared": len(attempts), + "input_tokens": tokens, + "typical_input_tokens": typical_tokens, + "runtime_ms": runtime, + "typical_runtime_ms": typical_runtime, + "token_ratio": token_ratio, + "runtime_ratio": runtime_ratio, + "measure": measure, + } + return { + key: [_finding(key, item, len(flagged))] + for key, item in sorted(flagged.items(), key=lambda pair: pair[0]) + } + + +def _plural(count, word): + return f"{count:g} {word}" + ("" if count == 1 else "s") + + +def _finding(key, item, flagged_attempts): + task_id, repetition = key + extra = [*item["rounds"], *item["extra"]] + ratio = item["measure"] + latency_gap = max(0.0, item["runtime_ms"] - item["typical_runtime_ms"]) + token_gap = max(0, item["input_tokens"] - round(item["typical_input_tokens"])) + spans = [event.event_id for event in extra] + differences = [] + if item["model_steps"] > item["typical_model_steps"]: + differences.append( + _plural(item["model_steps"] - item["typical_model_steps"], "more model step") + ) + if item["tool_calls"] > item["typical_tool_calls"]: + differences.append( + _plural(item["tool_calls"] - item["typical_tool_calls"], "more tool call") + ) + title = ( + f"Repetition {repetition} of {task_id} took {' and '.join(differences)} " + "than its typical repetition" + ) + evidence = [ + { + "span_id": event.event_id, + "name": event.name, + "event_type": event.event_type, + "operation_kind": event.operation_kind, + "duration_ms": event.duration_ms, + "input_tokens": event.input_tokens, + "output_tokens": event.output_tokens, + "model": event.model, + "status": event.status, + "input_excerpt": (event.input_text or "")[:EXCERPT], + } + for event in extra + ] + observations = { + "task_id": task_id, + "repetition": repetition, + "repetitions_compared": item["repetitions_compared"], + "input_tokens": item["input_tokens"], + "typical_input_tokens": item["typical_input_tokens"], + "runtime_ms": round(item["runtime_ms"], 3), + "typical_runtime_ms": round(item["typical_runtime_ms"], 3), + "model_steps": item["model_steps"], + "typical_model_steps": item["typical_model_steps"], + "tool_calls": item["tool_calls"], + "typical_tool_calls": item["typical_tool_calls"], + "attempts_with_a_detour": flagged_attempts, + } + return OptimizationFinding( + finding_id=_stable_finding_id(FINDING_TYPE, title, spans), + severity="high" if ratio >= 1.5 else "medium", + type=FINDING_TYPE, + title=title, + confidence="medium" if flagged_attempts >= 2 else "low", + affected_spans=spans, + evidence=evidence, + savings={ + "estimated_latency_savings_ms": round(latency_gap, 3), + "estimated_cost_savings_usd": None, + "estimated_input_tokens": token_gap, + "formula": "attempt runtime and input tokens minus the median of the other repetitions", + }, + rewrite={ + "kind": FINDING_TYPE, + "hint": "Compare this attempt's steps with a typical repetition. Each extra model " + "step resends the context, so ask the agent to gather what it needs in one step " + "(for example, read all required files together), and remove instructions or " + "ambiguity that trigger follow-up reads.", + "patchable": False, + }, + validation={ + "command": "agentloop improve finish", + "acceptance_criteria": "repetitions take a consistent number of model steps and " + "tool calls, and input tokens or runtime fall toward the typical repetition while " + "quality checks pass", + }, + metadata={ + "why": "Other repetitions of the same task finished in fewer steps.", + "cost_status": "unknown", + "identity": "content-v1", + }, + evidence_level="observed", + assumptions=[ + "Repetitions of a task receive identical inputs.", + "The extra steps are associated with the higher cost; they are not proven to cause it.", + ], + observations=observations, + rule_id=RULE_ID, + rule_version=RULE_VERSION, + estimate={ + "estimator_id": "repetition_median_gap", + "estimator_version": "1", + "formula": "attempt minus median of the other repetitions", + "unmodeled_metrics": ["cost"], + "assumptions": ["Savings assume the attempt can match its typical repetition."], + }, + ).to_dict() + + +def add_repetition_findings(diagnoses, traces): + """Merge detour findings into each attempt's diagnosis and re-rank it.""" + for key, extra in repetition_findings(traces).items(): + diagnosis = diagnoses.get(key) + if diagnosis is None: + continue + trace = traces[key] + combined = [*diagnosis["findings"], *extra] + diagnosis["findings"] = rank_findings( + combined, inputs=trace.metadata.get(RANKING_KEY, {}), sort_by="priority" + ) + diagnosis["summary"]["finding_count"] = len(combined) + diagnosis["summary"]["high_severity_count"] = sum( + finding["severity"] == "high" for finding in combined + ) + return diagnoses diff --git a/docs/CODEX.md b/docs/CODEX.md index d07e18f..0e4f8b1 100644 --- a/docs/CODEX.md +++ b/docs/CODEX.md @@ -71,10 +71,18 @@ the three tasks ran twice per condition. same path. Both `engines.node` pairs were slower than their controls, by 12% and 19%, and failed the latency gates. **The change was rejected.** -In three runs, two controls and one candidate, Codex made an extra shell call to -read a user-level instruction file referenced by the global Codex instructions. -Each of those runs used about 16,000–20,000 more input tokens. The traces make -that call visible as a separate `tool_call` span. This path variation is larger -than the measured context saving. More repetitions would be needed before +Three runs, two controls and one candidate, used about 18,000–20,400 more input +tokens than their other repetition. Each of them took an extra model step. Codex +ran one tool, called the model again, and then ran another tool, instead of +running both tools in one step, and each model step resent about 19,000 tokens +of context. Several runs also read a user-level instruction file referenced by +the global Codex instructions. That read cost nothing extra when Codex ran it in +the same step as the requested file read. The extra step, not the read itself, +accounted for the increase. This path variation is larger than the measured +context saving. + +The `repetition_detour` finding, available from AgentLoop 0.2.3, reports these +attempts automatically. On these 18 traces it flags exactly the three runs with +an extra model step and none of the others. More repetitions would be needed before attributing a latency change to this configuration. These six pairs describe one machine, one model configuration and three lookup tasks. diff --git a/docs/GUIDED_OPTIMIZATION.md b/docs/GUIDED_OPTIMIZATION.md index 614c665..bebac61 100644 --- a/docs/GUIDED_OPTIMIZATION.md +++ b/docs/GUIDED_OPTIMIZATION.md @@ -141,6 +141,16 @@ and missing inputs. They remain hypotheses until candidate measurements exist. The default focus is the first baseline finding family. Select another observed family with `--focus cache_context`, for example. +From AgentLoop 0.2.3, `start` also compares repetitions of each task. The +`repetition_detour` finding marks an attempt that took more model steps or tool +calls than the median of its other repetitions and used at least 25% more input +tokens. Runtime replaces tokens only when traces record no token counts. Its +evidence lists the extra tool calls and the intermediate model steps, each of +which usually resends the whole context. Confidence is medium when two or more +attempts in the task set take such a detour. Select it with +`--focus repetition_detour`. Single-run rules cannot see this pattern, because +each detour looks normal on its own. Use at least two repetitions per task. + `start` saves original diagnoses, selected finding IDs, trace snapshots, task definitions, and gate settings in a content-hashed `session.json` before any candidate capture. The default objective is latency. Use `--objective cost` to diff --git a/tests/test_repetitions.py b/tests/test_repetitions.py new file mode 100644 index 0000000..b05f702 --- /dev/null +++ b/tests/test_repetitions.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +import json +import sys +from datetime import datetime, timedelta, timezone + +from agentloop.events import AgentEvent +from agentloop.improvement import capture, start +from agentloop.repetitions import FINDING_TYPE, add_repetition_findings, repetition_findings +from agentloop.tracer import AgentTrace + +ORIGIN = datetime(2026, 11, 17, tzinfo=timezone.utc) + + +def attempt(steps, *, tokens_per_step=1000, step_ms=100.0): + """Build a trace from a step plan such as ["model", "tool:read a", "model"].""" + trace = AgentTrace("repetition fixture") + clock = 0.0 + for index, step in enumerate(steps): + kind, _, text = step.partition(":") + model = kind == "model" + trace.add_event( + AgentEvent( + event_id=f"evt_{index}", + run_id=trace.run_id, + event_type="model_call" if model else "tool_call", + name="model" if model else "shell", + started_at=(ORIGIN + timedelta(milliseconds=clock)).isoformat(), + ended_at=(ORIGIN + timedelta(milliseconds=clock + step_ms)).isoformat(), + duration_ms=step_ms, + input_tokens=tokens_per_step if model else 0, + token_provenance="provider" if model and tokens_per_step else None, + input_text=text or None, + ) + ) + clock += step_ms + trace.finish() + trace.ended_at = (ORIGIN + timedelta(milliseconds=clock)).isoformat() + trace.elapsed_ms = clock + return trace + + +TYPICAL = ["model", "tool:read pyproject.toml", "model"] +DETOUR = ["model", "tool:read notes.md", "model", "tool:read pyproject.toml", "model"] + + +def test_extra_model_step_with_more_tokens_is_reported_on_the_outlier(): + traces = { + ("lookup", 0): attempt(TYPICAL), + ("lookup", 1): attempt(DETOUR), + ("lookup", 2): attempt(TYPICAL), + } + found = repetition_findings(traces) + assert list(found) == [("lookup", 1)] + finding = found[("lookup", 1)][0] + assert finding["type"] == FINDING_TYPE + assert finding["title"] == ( + "Repetition 1 of lookup took 1 more model step and 1 more tool call " + "than its typical repetition" + ) + assert finding["savings"]["estimated_input_tokens"] == 1000 + assert finding["confidence"] == "low" + excerpts = {row["input_excerpt"] for row in finding["evidence"]} + assert "read notes.md" in excerpts + assert "read pyproject.toml" not in excerpts + + +def test_slower_run_without_extra_tokens_is_not_reported(): + traces = { + ("lookup", 0): attempt(TYPICAL), + ("lookup", 1): attempt( + ["model", "model", "tool:read pyproject.toml", "model"], step_ms=400 + ), + } + traces[("lookup", 1)].events[1].input_tokens = 0 + assert repetition_findings(traces) == {} + + +def test_runtime_decides_when_token_counts_are_unavailable(): + traces = { + ("lookup", 0): attempt(TYPICAL, tokens_per_step=0), + ("lookup", 1): attempt(DETOUR, tokens_per_step=0), + } + found = repetition_findings(traces) + assert list(found) == [("lookup", 1)] + assert found[("lookup", 1)][0]["savings"]["estimated_latency_savings_ms"] == 200 + + +def test_same_steps_or_single_repetitions_are_not_compared(): + assert repetition_findings({("a", 0): attempt(DETOUR)}) == {} + assert repetition_findings({("a", 0): attempt(TYPICAL), ("a", 1): attempt(TYPICAL)}) == {} + + +def test_recurring_detours_raise_confidence(): + traces = { + ("first", 0): attempt(TYPICAL), + ("first", 1): attempt(DETOUR), + ("second", 0): attempt(DETOUR), + ("second", 1): attempt(TYPICAL), + } + found = repetition_findings(traces) + assert {key: value[0]["confidence"] for key, value in found.items()} == { + ("first", 1): "medium", + ("second", 0): "medium", + } + + +def test_findings_merge_into_ranked_diagnoses(): + from agentloop.findings import build_diagnosis + + traces = {("lookup", 0): attempt(TYPICAL), ("lookup", 1): attempt(DETOUR)} + diagnoses = add_repetition_findings( + {key: build_diagnosis(trace) for key, trace in traces.items()}, traces + ) + merged = [item for item in diagnoses[("lookup", 1)]["findings"] if item["type"] == FINDING_TYPE] + assert len(merged) == 1 + assert merged[0]["ranking"]["priority_rank"] >= 1 + assert diagnoses[("lookup", 1)]["summary"]["finding_count"] >= 1 + assert not any(item["type"] == FINDING_TYPE for item in diagnoses[("lookup", 0)]["findings"]) + + +WORKER = """import json,sys +from datetime import datetime,timedelta,timezone +from agentloop.events import AgentEvent +from agentloop.tracer import AgentTrace +request=json.loads(sys.stdin.readline()) +detour=request["task_id"]=="first" and request["repetition"]==1 +steps=["model","tool:read source","model"] +if detour: + steps=["model","tool:read notes","model","tool:read source","model"] +trace=AgentTrace("worker",metadata={"synthetic":True,"output":request["input"]}) +origin=datetime(2026,11,17,tzinfo=timezone.utc) +for index,step in enumerate(steps): + kind,_,text=step.partition(":") + trace.add_event(AgentEvent(event_id=f"evt_{index}",run_id=trace.run_id,event_type="model_call" if kind=="model" else "tool_call",name=kind,started_at=(origin+timedelta(seconds=index)).isoformat(),ended_at=(origin+timedelta(seconds=index+1)).isoformat(),duration_ms=1000,input_tokens=1000 if kind=="model" else 0,token_provenance="provider" if kind=="model" else None,input_text=text or None)) +trace.finish() +trace.export_json(request["trace_path"]) +""" + + +def test_start_offers_the_detour_as_a_focus(tmp_path): + tasks = tmp_path / "tasks.json" + tasks.write_text( + json.dumps( + { + "schema_version": "1.0", + "name": "Repetition tasks", + "repetitions": 2, + "tasks": [ + {"id": "first", "input": "one", "expected": "one"}, + {"id": "second", "input": "two", "expected": "two"}, + ], + } + ), + encoding="utf-8", + ) + worker = tmp_path / "worker.py" + worker.write_text(WORKER, encoding="utf-8") + capture(tasks, tmp_path / "baseline", [sys.executable, str(worker)]) + specification = start( + tasks, tmp_path / "baseline/capture.json", tmp_path / "session", focus=FINDING_TYPE + ) + assert specification["focus"] == FINDING_TYPE + flagged = [case for case in specification["cases"] if case["finding_ids"]] + assert [(case["task_id"], case["repetition"]) for case in flagged] == [("first", 1)] + plan = (tmp_path / "session/plan.md").read_text(encoding="utf-8") + assert "took 1 more model step and 1 more tool call" in plan + assert "read notes" in plan