From 7a41c8d351e78d69b19d39d4c52c77cee1aee01c Mon Sep 17 00:00:00 2001 From: Dipesh Babu Date: Wed, 18 Nov 2026 20:06:14 -0500 Subject: [PATCH] Warn when a guided comparison plan is too weak to detect a change improve start and improve compare now note, before the candidate runs, when fewer than five pairs are planned (a unanimous sign test cannot reach p < 0.05), when each task runs once (noise is unmeasurable), or when baseline latency varies between repetitions by at least 10% and more than the required improvement. Notes are frozen in the session, printed by the CLI and listed under Evidence strength in plan.md. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 +++ agentloop/improvement.py | 67 ++++++++++++++++++++++++++++++++- agentloop/improvement_cli.py | 6 +++ agentloop/improvement_report.py | 4 ++ docs/GUIDED_OPTIMIZATION.md | 14 +++++++ tests/test_improvement.py | 58 ++++++++++++++++++++++++++-- 6 files changed, 150 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04ec3ac..88eb147 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,11 @@ trace, study, receipt, API and database schema versions. as an interleaved control, and writes the verdict. It checks inputs before any runner executes, stops if no baseline attempt completes, and defaults to the aggregate decision. `improvement.compare()` exposes the same workflow. +- `improve start` and `improve compare` warn when a plan is too weak to + separate a change from noise: fewer than five pairs, one repetition per task, + or baseline latency that varies between repetitions by at least 10% and more + than the required improvement. Notes are recorded in the frozen session and + listed under "Evidence strength" in `plan.md`; they do not change decisions. ## [0.2.4] - 2026-11-18 diff --git a/agentloop/improvement.py b/agentloop/improvement.py index 42f9cc9..32bb854 100644 --- a/agentloop/improvement.py +++ b/agentloop/improvement.py @@ -424,6 +424,66 @@ def _runner_budget(value): ) from error +MIN_SIGN_TEST_PAIRS = 5 +NOISY_SPREAD_PCT = 10.0 + + +def planning_guidance(workload, manifest, traces, objective, minimum_improvement): + """Warn when the planned comparison cannot separate a change from noise. + + The sign test needs at least five pairs before even a unanimous result can + fall below p = 0.05. When repeated baseline attempts of the same task vary + by more than the required improvement, smaller differences may be noise. + """ + notes = [] + pairs = len(workload["tasks"]) * workload["repetitions"] + if pairs < MIN_SIGN_TEST_PAIRS: + notes.append( + { + "kind": "few_pairs", + "pairs": pairs, + "message": f"Only {pairs} pair{'s' if pairs != 1 else ''} planned. Even if every " + "pair improves, a one-sided sign test cannot fall below p = 0.05 with fewer than " + f"{MIN_SIGN_TEST_PAIRS} pairs. Add tasks or raise repetitions in the task file.", + } + ) + if workload["repetitions"] < 2: + notes.append( + { + "kind": "noise_unmeasured", + "message": "Each task runs once, so baseline run-to-run variation cannot be " + "measured. Use at least 2 repetitions to see how noisy the agent is.", + } + ) + elif objective == "latency": + completed = _completed(manifest, traces) + spreads = [] + for task in workload["tasks"]: + values = [ + trace.elapsed_ms + for (task_id, _), trace in completed.items() + if task_id == task["id"] and _trace_usable(trace) + ] + middle = statistics.median(values) if len(values) >= 2 else 0 + if middle > 0: + spreads.append((max(values) - min(values)) / middle * 100) + if spreads: + spread = statistics.median(spreads) + if spread >= NOISY_SPREAD_PCT and spread > minimum_improvement: + notes.append( + { + "kind": "noisy_baseline", + "spread_pct": round(spread, 2), + "message": f"Baseline latency of the same task varied by a median " + f"{spread:.1f}% between repetitions, more than the " + f"{minimum_improvement:g}% required improvement. Smaller differences " + "may be noise: require a larger improvement, add repetitions, or " + "compare against an interleaved control with the aggregate decision.", + } + ) + return notes + + def start( workload_path, baseline_path, @@ -515,6 +575,9 @@ def start( "gates": asdict(gates), "cases": cases, "baseline_capture": manifest, + "planning_notes": planning_guidance( + workload, manifest, traces, objective, minimum_improvement + ), } if verifier is not None: specification["verifier"] = verifier.declaration() @@ -1139,7 +1202,7 @@ def compare( notify( f"Baseline captured {completed} of {len(baseline['runs'])} attempts; freezing the session" ) - start( + specification = start( workload_path, paths["baseline"] / "capture.json", paths["session"], @@ -1151,6 +1214,8 @@ def compare( verifier_timeout=verifier_timeout, max_runner_regression_ms=max_runner_regression_ms, ) + for note in specification["planning_notes"]: + notify(f"Note: {note['message']}") notify("Capturing the after runner, alternating with the before runner as a control") capture( workload_path, diff --git a/agentloop/improvement_cli.py b/agentloop/improvement_cli.py index ee99514..6d04b77 100644 --- a/agentloop/improvement_cli.py +++ b/agentloop/improvement_cli.py @@ -145,6 +145,7 @@ def start_command( console.print( f"Investigation focus is {specification['focus'] or 'unavailable'}.", markup=False ) + _print_planning_notes(specification) console.print( f"Review {out / 'plan.md'}, apply your change, then capture the same tasks again.", markup=False, @@ -176,6 +177,11 @@ def finish_command( _print_report(report, out) +def _print_planning_notes(specification): + for note in specification.get("planning_notes", []): + console.print(f"Note: {note['message']}", markup=False) + + def _print_report(report, out): """Print the verdict and timing summary; exit 1 when the change is rejected.""" verdict = report["decision"] diff --git a/agentloop/improvement_report.py b/agentloop/improvement_report.py index 2317249..59cc0a9 100644 --- a/agentloop/improvement_report.py +++ b/agentloop/improvement_report.py @@ -45,6 +45,10 @@ def plan_markdown(specification): ] if specification.get("verifier"): lines.extend([_verifier_scope(specification["verifier"]), ""]) + if notes := specification.get("planning_notes"): + lines.extend(["## Evidence strength", ""]) + lines.extend(f"- {markdown_text(note['message'])}" for note in notes) + lines.append("") if (limit := specification.get("max_runner_regression_ms")) is not None: lines.extend( [f"Runner wall-time gate allows an increase of up to {limit:g} ms in each pair.", ""] diff --git a/docs/GUIDED_OPTIMIZATION.md b/docs/GUIDED_OPTIMIZATION.md index dd24bb5..487e9a6 100644 --- a/docs/GUIDED_OPTIMIZATION.md +++ b/docs/GUIDED_OPTIMIZATION.md @@ -278,6 +278,20 @@ context and does not gate the decision. Use enough repetitions for the median to mean something, and combine aggregate decisions with an [interleaved control](#compare-against-an-interleaved-control). +`improve start` and `improve compare` warn before the candidate runs when the +plan is too weak to separate a change from noise. AgentLoop 0.2.5 and newer +print these notes, record them in the frozen session, and list them under +**Evidence strength** in `plan.md`: + +- fewer than five pairs: even if every pair improves, the sign test cannot fall + below p = 0.05; +- one repetition per task: run-to-run variation cannot be measured; +- noisy baseline latency: repetitions of the same task varied by a median of at + least 10%, more than `--minimum-improvement`. + +The notes do not change the decision. Respond by adding tasks or repetitions, +requiring a larger improvement, or comparing against an interleaved control. + ## Compare against an interleaved control Interleaved controls require AgentLoop 0.2.2 or newer. diff --git a/tests/test_improvement.py b/tests/test_improvement.py index cd6ed58..8ce9660 100644 --- a/tests/test_improvement.py +++ b/tests/test_improvement.py @@ -2,14 +2,21 @@ import json import sys -from types import ModuleType +from types import ModuleType, SimpleNamespace import pytest from typer.testing import CliRunner from agentloop.entrypoint import app from agentloop.experiment_types import fingerprint -from agentloop.improvement import capture, compare, finish, read_workload, start +from agentloop.improvement import ( + capture, + compare, + finish, + planning_guidance, + read_workload, + start, +) WORKER = """import json,sys,time from pathlib import Path @@ -816,7 +823,9 @@ def test_compare_runs_every_stage_with_an_interleaved_control(tmp_path, workload assert (tmp_path / "compare" / stage).is_file() assert (tmp_path / "compare/paired/control/capture.json").is_file() assert (tmp_path / "compare/results/report.html").is_file() - assert len(messages) == 4 + notes = [message for message in messages if message.startswith("Note: ")] + assert len(notes) == 1 and "Only 4 pairs planned" in notes[0] + assert len(messages) == 5 def test_compare_rejects_a_wrong_after_runner(tmp_path, workload): @@ -898,3 +907,46 @@ def test_cli_compare_accepts_repeated_before_arguments(tmp_path, workload): assert "Capturing the baseline with the before runner" in result.output assert "Aggregate" in result.output assert (tmp_path / "compare/results/report.json").is_file() + + +def _guidance(repetitions, elapsed, *, objective="latency", minimum=0, tasks=3): + workload = { + "tasks": [{"id": f"t{index}"} for index in range(tasks)], + "repetitions": repetitions, + } + runs, traces = [], {} + for index in range(tasks): + for repetition in range(repetitions): + key = (f"t{index}", repetition) + runs.append({"task_id": key[0], "repetition": repetition, "status": "completed"}) + traces[key] = SimpleNamespace(elapsed_ms=elapsed[repetition], metadata={}, events=[]) + return planning_guidance(workload, {"runs": runs}, traces, objective, minimum) + + +def test_planning_guidance_flags_too_few_pairs_for_a_sign_test(): + notes = _guidance(1, [100], tasks=4) + assert [note["kind"] for note in notes] == ["few_pairs", "noise_unmeasured"] + assert notes[0]["pairs"] == 4 + assert "fewer than 5 pairs" in notes[0]["message"] + + +def test_planning_guidance_flags_baseline_noise_above_the_required_gain(): + notes = _guidance(2, [100, 140], minimum=5) + assert [note["kind"] for note in notes] == ["noisy_baseline"] + assert notes[0]["spread_pct"] == pytest.approx(33.33, abs=0.01) + assert "33.3%" in notes[0]["message"] and "5% required" in notes[0]["message"] + + +def test_planning_guidance_is_quiet_for_stable_or_well_bounded_baselines(): + assert _guidance(2, [100, 104]) == [] + assert _guidance(2, [100, 140], minimum=50) == [] + assert _guidance(2, [100, 140], objective="cost") == [] + + +def test_start_records_planning_notes_in_the_session_and_plan(tmp_path, workload): + tasks, _ = prepare(tmp_path, workload) + saved = json.loads((tmp_path / "session/session.json").read_text(encoding="utf-8")) + notes = saved["specification"]["planning_notes"] + assert notes[0]["kind"] == "few_pairs" + plan = (tmp_path / "session/plan.md").read_text(encoding="utf-8") + assert "## Evidence strength" in plan and "Only 4 pairs planned" in plan