Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@ trace, study, receipt, API and database schema versions.
as an interleaved control, and writes the verdict. It checks inputs before
any runner executes, stops if no baseline attempt completes, and defaults to
the aggregate decision. `improvement.compare()` exposes the same workflow.
- `improve start` and `improve compare` warn when a plan is too weak to
separate a change from noise: fewer than five pairs, one repetition per task,
or baseline latency that varies between repetitions by at least 10% and more
than the required improvement. Notes are recorded in the frozen session and
listed under "Evidence strength" in `plan.md`; they do not change decisions.

## [0.2.4] - 2026-11-18

Expand Down
67 changes: 66 additions & 1 deletion agentloop/improvement.py
Original file line number Diff line number Diff line change
Expand Up @@ -424,6 +424,66 @@ def _runner_budget(value):
) from error


MIN_SIGN_TEST_PAIRS = 5
NOISY_SPREAD_PCT = 10.0


def planning_guidance(workload, manifest, traces, objective, minimum_improvement):
"""Warn when the planned comparison cannot separate a change from noise.

The sign test needs at least five pairs before even a unanimous result can
fall below p = 0.05. When repeated baseline attempts of the same task vary
by more than the required improvement, smaller differences may be noise.
"""
notes = []
pairs = len(workload["tasks"]) * workload["repetitions"]
if pairs < MIN_SIGN_TEST_PAIRS:
notes.append(
{
"kind": "few_pairs",
"pairs": pairs,
"message": f"Only {pairs} pair{'s' if pairs != 1 else ''} planned. Even if every "
"pair improves, a one-sided sign test cannot fall below p = 0.05 with fewer than "
f"{MIN_SIGN_TEST_PAIRS} pairs. Add tasks or raise repetitions in the task file.",
}
)
if workload["repetitions"] < 2:
notes.append(
{
"kind": "noise_unmeasured",
"message": "Each task runs once, so baseline run-to-run variation cannot be "
"measured. Use at least 2 repetitions to see how noisy the agent is.",
}
)
elif objective == "latency":
completed = _completed(manifest, traces)
spreads = []
for task in workload["tasks"]:
values = [
trace.elapsed_ms
for (task_id, _), trace in completed.items()
if task_id == task["id"] and _trace_usable(trace)
]
middle = statistics.median(values) if len(values) >= 2 else 0
if middle > 0:
spreads.append((max(values) - min(values)) / middle * 100)
if spreads:
spread = statistics.median(spreads)
if spread >= NOISY_SPREAD_PCT and spread > minimum_improvement:
notes.append(
{
"kind": "noisy_baseline",
"spread_pct": round(spread, 2),
"message": f"Baseline latency of the same task varied by a median "
f"{spread:.1f}% between repetitions, more than the "
f"{minimum_improvement:g}% required improvement. Smaller differences "
"may be noise: require a larger improvement, add repetitions, or "
"compare against an interleaved control with the aggregate decision.",
}
)
return notes


def start(
workload_path,
baseline_path,
Expand Down Expand Up @@ -515,6 +575,9 @@ def start(
"gates": asdict(gates),
"cases": cases,
"baseline_capture": manifest,
"planning_notes": planning_guidance(
workload, manifest, traces, objective, minimum_improvement
),
}
if verifier is not None:
specification["verifier"] = verifier.declaration()
Expand Down Expand Up @@ -1139,7 +1202,7 @@ def compare(
notify(
f"Baseline captured {completed} of {len(baseline['runs'])} attempts; freezing the session"
)
start(
specification = start(
workload_path,
paths["baseline"] / "capture.json",
paths["session"],
Expand All @@ -1151,6 +1214,8 @@ def compare(
verifier_timeout=verifier_timeout,
max_runner_regression_ms=max_runner_regression_ms,
)
for note in specification["planning_notes"]:
notify(f"Note: {note['message']}")
notify("Capturing the after runner, alternating with the before runner as a control")
capture(
workload_path,
Expand Down
6 changes: 6 additions & 0 deletions agentloop/improvement_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ def start_command(
console.print(
f"Investigation focus is {specification['focus'] or 'unavailable'}.", markup=False
)
_print_planning_notes(specification)
console.print(
f"Review {out / 'plan.md'}, apply your change, then capture the same tasks again.",
markup=False,
Expand Down Expand Up @@ -176,6 +177,11 @@ def finish_command(
_print_report(report, out)


def _print_planning_notes(specification):
for note in specification.get("planning_notes", []):
console.print(f"Note: {note['message']}", markup=False)


def _print_report(report, out):
"""Print the verdict and timing summary; exit 1 when the change is rejected."""
verdict = report["decision"]
Expand Down
4 changes: 4 additions & 0 deletions agentloop/improvement_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ def plan_markdown(specification):
]
if specification.get("verifier"):
lines.extend([_verifier_scope(specification["verifier"]), ""])
if notes := specification.get("planning_notes"):
lines.extend(["## Evidence strength", ""])
lines.extend(f"- {markdown_text(note['message'])}" for note in notes)
lines.append("")
if (limit := specification.get("max_runner_regression_ms")) is not None:
lines.extend(
[f"Runner wall-time gate allows an increase of up to {limit:g} ms in each pair.", ""]
Expand Down
14 changes: 14 additions & 0 deletions docs/GUIDED_OPTIMIZATION.md
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,20 @@ context and does not gate the decision. Use enough repetitions for the median
to mean something, and combine aggregate decisions with an
[interleaved control](#compare-against-an-interleaved-control).

`improve start` and `improve compare` warn before the candidate runs when the
plan is too weak to separate a change from noise. AgentLoop 0.2.5 and newer
print these notes, record them in the frozen session, and list them under
**Evidence strength** in `plan.md`:

- fewer than five pairs: even if every pair improves, the sign test cannot fall
below p = 0.05;
- one repetition per task: run-to-run variation cannot be measured;
- noisy baseline latency: repetitions of the same task varied by a median of at
least 10%, more than `--minimum-improvement`.

The notes do not change the decision. Respond by adding tasks or repetitions,
requiring a larger improvement, or comparing against an interleaved control.

## Compare against an interleaved control

Interleaved controls require AgentLoop 0.2.2 or newer.
Expand Down
58 changes: 55 additions & 3 deletions tests/test_improvement.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,21 @@

import json
import sys
from types import ModuleType
from types import ModuleType, SimpleNamespace

import pytest
from typer.testing import CliRunner

from agentloop.entrypoint import app
from agentloop.experiment_types import fingerprint
from agentloop.improvement import capture, compare, finish, read_workload, start
from agentloop.improvement import (
capture,
compare,
finish,
planning_guidance,
read_workload,
start,
)

WORKER = """import json,sys,time
from pathlib import Path
Expand Down Expand Up @@ -816,7 +823,9 @@ def test_compare_runs_every_stage_with_an_interleaved_control(tmp_path, workload
assert (tmp_path / "compare" / stage).is_file()
assert (tmp_path / "compare/paired/control/capture.json").is_file()
assert (tmp_path / "compare/results/report.html").is_file()
assert len(messages) == 4
notes = [message for message in messages if message.startswith("Note: ")]
assert len(notes) == 1 and "Only 4 pairs planned" in notes[0]
assert len(messages) == 5


def test_compare_rejects_a_wrong_after_runner(tmp_path, workload):
Expand Down Expand Up @@ -898,3 +907,46 @@ def test_cli_compare_accepts_repeated_before_arguments(tmp_path, workload):
assert "Capturing the baseline with the before runner" in result.output
assert "Aggregate" in result.output
assert (tmp_path / "compare/results/report.json").is_file()


def _guidance(repetitions, elapsed, *, objective="latency", minimum=0, tasks=3):
workload = {
"tasks": [{"id": f"t{index}"} for index in range(tasks)],
"repetitions": repetitions,
}
runs, traces = [], {}
for index in range(tasks):
for repetition in range(repetitions):
key = (f"t{index}", repetition)
runs.append({"task_id": key[0], "repetition": repetition, "status": "completed"})
traces[key] = SimpleNamespace(elapsed_ms=elapsed[repetition], metadata={}, events=[])
return planning_guidance(workload, {"runs": runs}, traces, objective, minimum)


def test_planning_guidance_flags_too_few_pairs_for_a_sign_test():
notes = _guidance(1, [100], tasks=4)
assert [note["kind"] for note in notes] == ["few_pairs", "noise_unmeasured"]
assert notes[0]["pairs"] == 4
assert "fewer than 5 pairs" in notes[0]["message"]


def test_planning_guidance_flags_baseline_noise_above_the_required_gain():
notes = _guidance(2, [100, 140], minimum=5)
assert [note["kind"] for note in notes] == ["noisy_baseline"]
assert notes[0]["spread_pct"] == pytest.approx(33.33, abs=0.01)
assert "33.3%" in notes[0]["message"] and "5% required" in notes[0]["message"]


def test_planning_guidance_is_quiet_for_stable_or_well_bounded_baselines():
assert _guidance(2, [100, 104]) == []
assert _guidance(2, [100, 140], minimum=50) == []
assert _guidance(2, [100, 140], objective="cost") == []


def test_start_records_planning_notes_in_the_session_and_plan(tmp_path, workload):
tasks, _ = prepare(tmp_path, workload)
saved = json.loads((tmp_path / "session/session.json").read_text(encoding="utf-8"))
notes = saved["specification"]["planning_notes"]
assert notes[0]["kind"] == "few_pairs"
plan = (tmp_path / "session/plan.md").read_text(encoding="utf-8")
assert "## Evidence strength" in plan and "Only 4 pairs planned" in plan
Loading