diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9adb146..f9f183c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,14 @@ trace, study, receipt, API and database schema versions.
## [Unreleased]
+### Added
+
+- `agentloop improve start --decision aggregate` judges speed or cost across
+ all pairs: the median paired improvement must meet the frozen minimum and
+ more pairs must improve than regress. Every pair must still complete and
+ pass its quality and non-performance gates. Reports include the counts and
+ a non-gating sign-test p-value. The default per-pair decision is unchanged.
+
## [0.2.2] - 2026-11-17
### Added
diff --git a/agentloop/improvement.py b/agentloop/improvement.py
index 78431e1..2744783 100644
--- a/agentloop/improvement.py
+++ b/agentloop/improvement.py
@@ -7,6 +7,7 @@
import math
import os
import signal
+import statistics
import subprocess # nosec B404 - execution requires explicit CLI argv
from dataclasses import asdict
from datetime import datetime, timezone
@@ -22,6 +23,14 @@
from agentloop.tracer import AgentTrace
VERSION = "1.0"
+DECISIONS = {"per_pair", "aggregate"}
+# Speed and cost gates; an aggregate decision judges these across all pairs.
+PERFORMANCE_GATES = {
+ "latency_regression",
+ "latency_improvement",
+ "cost_regression",
+ "cost_improvement",
+}
SCORERS = {"exact_match", "contains", "json_subset", "required_fields", "glob"}
@@ -380,11 +389,19 @@ def _quality(task, baseline, candidate):
def start(
- workload_path, baseline_path, root, *, focus=None, objective="latency", minimum_improvement=0
+ workload_path,
+ baseline_path,
+ root,
+ *,
+ focus=None,
+ objective="latency",
+ minimum_improvement=0,
+ decision="per_pair",
):
workload = read_workload(workload_path)
manifest, traces = read_capture(baseline_path, workload)
require(objective in {"latency", "cost"}, "Objective must be latency or cost")
+ require(decision in DECISIONS, "Decision must be per_pair or aggregate")
require(
type(minimum_improvement) in (int, float)
and math.isfinite(minimum_improvement)
@@ -442,6 +459,7 @@ def start(
"workload": workload,
"focus": focus,
"objective": objective,
+ "decision": decision,
"gates": asdict(gates),
"cases": cases,
"baseline_capture": manifest,
@@ -473,6 +491,83 @@ def _trace_usable(trace):
return not any(event.parent_id is None and event.status == "error" for event in trace.events)
+def _sign_test_p_value(improved, regressed):
+ """One-sided exact sign test: chance of at least this many improvements."""
+ trials = improved + regressed
+ if trials == 0:
+ return None
+ return sum(math.comb(trials, k) for k in range(improved, trials + 1)) / 2**trials
+
+
+def _aggregate_decision(rows, objective, gates):
+ """Judge speed or cost across all planned pairs.
+
+ Correctness, completion and non-performance gates were already enforced on
+ every pair. Here the median paired improvement in the selected objective
+ must be positive and meet the frozen minimum, and more pairs must improve
+ than regress. The other metric's median regression must stay within its
+ allowed regression when every pair can measure it.
+ """
+ secondary = "cost" if objective == "latency" else "latency"
+ required = (
+ gates.min_latency_improvement_pct
+ if objective == "latency"
+ else gates.min_cost_improvement_pct
+ )
+ allowed = (
+ gates.max_cost_regression_pct if secondary == "cost" else gates.max_latency_regression_pct
+ )
+ gains = [row.get("objective_gain_pct") for row in rows]
+ regressions = [
+ (row["replay"] or {}).get("deltas", {}).get(f"{secondary}_regression_pct")
+ if row.get("replay")
+ else None
+ for row in rows
+ ]
+ reasons = []
+ measured = [gain for gain in gains if gain is not None]
+ improved = sum(gain > 0 for gain in measured)
+ regressed = sum(gain < 0 for gain in measured)
+ median = statistics.median(measured) if measured else None
+ if len(measured) != len(rows):
+ reasons.append(
+ f"{len(rows) - len(measured)} of {len(rows)} pairs have no measured {objective} improvement"
+ )
+ if median is None or median <= 0 or median < required:
+ detail = "unavailable" if median is None else f"{median:.2f}%"
+ reasons.append(
+ f"Median {objective} improvement {detail} must be positive and at least {required:.2f}%"
+ )
+ if improved <= regressed:
+ reasons.append(
+ f"{improved} pairs improved and {regressed} regressed; more pairs must improve than regress"
+ )
+ secondary_median = None
+ if all(value is not None for value in regressions):
+ secondary_median = statistics.median(regressions)
+ if secondary_median > allowed:
+ reasons.append(
+ f"Median {secondary} regression {secondary_median:.2f}% exceeds {allowed:.2f}% allowed"
+ )
+ return {
+ "mode": "aggregate",
+ "objective": objective,
+ "pairs": len(rows),
+ "median_improvement_pct": median,
+ "required_improvement_pct": required,
+ "improved_pairs": improved,
+ "regressed_pairs": regressed,
+ "unchanged_pairs": len(measured) - improved - regressed,
+ "sign_test_p_value": _sign_test_p_value(improved, regressed),
+ "secondary_metric": secondary,
+ "secondary_median_regression_pct": secondary_median,
+ "secondary_allowed_regression_pct": allowed,
+ "secondary_judged": secondary_median is not None,
+ "passed": not reasons,
+ "reasons": reasons,
+ }
+
+
def finish(session_path, candidate_path, root, *, change):
session_path = Path(session_path).resolve()
saved = load_json(session_path)
@@ -484,6 +579,9 @@ def finish(session_path, candidate_path, root, *, change):
require(specification.get("schema_version") == VERSION, "Unsupported frozen session version")
workload = validate_workload(specification["workload"])
require(specification.get("objective") in {"latency", "cost"}, "Unsupported frozen objective")
+ # Sessions frozen before 0.2.3 have no decision field and keep per-pair judgment.
+ decision = specification.get("decision", "per_pair")
+ require(decision in DECISIONS, "Unsupported frozen decision")
planned = {(task["id"], repetition) for task, repetition in slots(workload)}
cases = specification.get("cases")
require(
@@ -629,13 +727,17 @@ def finish(session_path, candidate_path, root, *, change):
row["reasons"].append(
f"{label if side == 'baseline' else 'Candidate'} answer failed the declared {task.get('scorer', {}).get('type', 'exact_match')} criterion: {result[side]['detail']}"
)
- if not replay["gates"]["passed"]:
+ # Aggregate decisions judge speed and cost across all pairs after
+ # this loop; every other gate still applies to each pair.
+ judged = [
+ gate
+ for gate in replay["gates"]["results"]
+ if not gate["passed"]
+ and (decision == "per_pair" or gate["name"] not in PERFORMANCE_GATES)
+ ]
+ if judged:
row["reasons"].append("Native replay gates failed")
- row["reasons"].extend(
- f"{gate['name']}: {gate['detail']}"
- for gate in replay["gates"]["results"]
- if not gate["passed"]
- )
+ row["reasons"].extend(f"{gate['name']}: {gate['detail']}" for gate in judged)
if not diagnosis.get("analysis_complete", True) or not candidate.report().get(
"analysis_complete", True
):
@@ -645,14 +747,20 @@ def finish(session_path, candidate_path, root, *, change):
if specification["objective"] == "latency"
else "cost_improvement_pct"
]
- if gain is None or gain <= 0:
+ if decision == "per_pair" and (gain is None or gain <= 0):
row["reasons"].append("The selected objective has no measured improvement")
- row.update(quality=quality, replay=replay, passed=not row["reasons"])
+ row.update(
+ quality=quality, replay=replay, passed=not row["reasons"], objective_gain_pct=gain
+ )
replay["gates"]["results"].append(
{
"name": "guided_evidence",
"passed": row["passed"],
- "detail": "All guided execution, quality and measured-objective checks passed"
+ "detail": (
+ "All guided execution, quality and measured-objective checks passed"
+ if decision == "per_pair"
+ else "All guided execution and quality checks passed; performance is judged across all pairs"
+ )
if row["passed"]
else " ".join(row["reasons"]),
"indeterminate": False,
@@ -712,6 +820,11 @@ def finish(session_path, candidate_path, root, *, change):
}
study_file = write_once(root, "study.json", study_manifest)
study = summarize_study(study_file)
+ verdict = (
+ {"mode": "per_pair", "passed": True}
+ if decision == "per_pair"
+ else _aggregate_decision(rows, specification["objective"], gates)
+ )
report = {
"schema_version": VERSION,
"name": workload["name"],
@@ -729,7 +842,8 @@ def finish(session_path, candidate_path, root, *, change):
"control_runner_hash": None if controls is None else control_manifest.get("runner_hash"),
"planned_attempts": len(rows),
"passed_attempts": sum(row["passed"] for row in rows),
- "passed": all(row["passed"] for row in rows),
+ "decision": verdict,
+ "passed": all(row["passed"] for row in rows) and verdict["passed"],
"synthetic": any(
trace.metadata.get("synthetic") is True
for trace in [
diff --git a/agentloop/improvement_cli.py b/agentloop/improvement_cli.py
index f1f52db..7bd1d99 100644
--- a/agentloop/improvement_cli.py
+++ b/agentloop/improvement_cli.py
@@ -101,6 +101,11 @@ def start_command(
minimum_improvement: float = typer.Option(
0, min=0, help="Required percentage gain in the selected objective."
),
+ decision: str = typer.Option(
+ "per-pair",
+ help="per-pair requires every pair to improve; aggregate judges the median "
+ "paired improvement for noisy agents. Correctness is checked per pair in both.",
+ ),
):
"""Freeze task checks and original findings before editing the agent."""
try:
@@ -111,6 +116,7 @@ def start_command(
focus=focus,
objective=objective,
minimum_improvement=minimum_improvement,
+ decision=decision.replace("-", "_"),
)
except (OSError, ValueError, TypeError, KeyError) as exc:
raise typer.BadParameter(str(exc)) from exc
@@ -136,10 +142,28 @@ def finish_command(
report = improvement.finish(session, candidate, out, change=change)
except (OSError, ValueError, TypeError, KeyError) as exc:
raise typer.BadParameter(str(exc)) from exc
- console.print(
- f"{report['passed_attempts']} of {report['planned_attempts']} planned comparisons passed.",
- markup=False,
- )
+ verdict = report["decision"]
+ if verdict["mode"] == "aggregate":
+ console.print(
+ f"{report['passed_attempts']} of {report['planned_attempts']} pairs passed "
+ "correctness and completion checks.",
+ markup=False,
+ )
+ median = verdict["median_improvement_pct"]
+ console.print(
+ f"Aggregate {verdict['objective']} decision "
+ + ("passed" if verdict["passed"] else "failed")
+ + f": median improvement {'unavailable' if median is None else f'{median:.2f}%'}, "
+ f"{verdict['improved_pairs']} improved, {verdict['regressed_pairs']} regressed.",
+ markup=False,
+ )
+ for reason in verdict["reasons"]:
+ console.print(f"Aggregate check failed: {reason}", markup=False)
+ else:
+ console.print(
+ f"{report['passed_attempts']} of {report['planned_attempts']} planned comparisons passed.",
+ markup=False,
+ )
console.print(f"Read {out / 'report.html'}", markup=False)
for case in report["cases"]:
if not case["passed"]:
diff --git a/agentloop/improvement_report.py b/agentloop/improvement_report.py
index 74e4da9..1e00c32 100644
--- a/agentloop/improvement_report.py
+++ b/agentloop/improvement_report.py
@@ -22,6 +22,9 @@ def plan_markdown(specification):
"",
f"Investigation focus is {markdown_text(specification['focus'] or 'unavailable')}.",
f"The measured objective is {specification['objective']}. Every baseline and candidate output must pass its declared quality checks.",
+ "The frozen decision judges the objective across all pairs by median paired improvement."
+ if specification.get("decision") == "aggregate"
+ else "The frozen decision requires every pair to improve the objective.",
"",
]
for case in specification["cases"]:
@@ -182,6 +185,35 @@ def _basis(report):
)
+def _decision_lines(report):
+ verdict = report.get("decision") or {"mode": "per_pair"}
+ if verdict["mode"] != "aggregate":
+ return ["Decision: every pair must pass correctness, completion and performance checks."]
+ median = verdict["median_improvement_pct"]
+ p_value = verdict["sign_test_p_value"]
+ lines = [
+ "Decision: every pair must pass correctness and completion checks; "
+ f"{verdict['objective']} is judged across all pairs.",
+ f"Aggregate {verdict['objective']} decision "
+ + ("passed" if verdict["passed"] else "failed")
+ + ": median paired improvement "
+ + ("unavailable" if median is None else f"{median:.2f}%")
+ + f" (required at least {verdict['required_improvement_pct']:.2f}%), "
+ f"{verdict['improved_pairs']} improved, {verdict['regressed_pairs']} regressed, "
+ f"{verdict['unchanged_pairs']} unchanged.",
+ ]
+ if p_value is not None:
+ lines.append(
+ f"One-sided sign test p-value {p_value:.3f}. It is reported for context and does not gate the decision."
+ )
+ if not verdict["secondary_judged"]:
+ lines.append(
+ f"Median {verdict['secondary_metric']} regression was not judged because it was unavailable for some pairs."
+ )
+ lines.extend(f"Aggregate check failed: {reason}" for reason in verdict["reasons"])
+ return lines
+
+
def report_markdown(report):
lines = [
f"# {markdown_heading(report['name'])}",
@@ -198,6 +230,7 @@ def report_markdown(report):
"",
_basis(report),
"",
+ *[item for line in _decision_lines(report) for item in (markdown_text(line), "")],
]
if report["synthetic"]:
lines.extend(
@@ -263,6 +296,7 @@ def export_report(root, report):
f" {report['passed_attempts']} of {report['planned_attempts']} planned attempts passed.{_text(report['name'])}
{_text(report['change'])}
", f"{_text(_basis(report))}
", + *[f"{_text(line)}
" for line in _decision_lines(report)], ] if report["synthetic"]: sections.append( diff --git a/docs/GUIDED_OPTIMIZATION.md b/docs/GUIDED_OPTIMIZATION.md index 2ed9223..614c665 100644 --- a/docs/GUIDED_OPTIMIZATION.md +++ b/docs/GUIDED_OPTIMIZATION.md @@ -169,6 +169,37 @@ cost stays unavailable. A latency comparison can pass with an indeterminate, non-gating cost estimate. A required cost improvement fails when pricing or usable usage is missing. +## Judge noisy agents across all pairs + +The default `--decision per-pair` rejects a change when any single pair is slower +or costlier. Agents that call hosted models vary from run to run, so one noisy +pair can reject a change that helps on almost every attempt. For those agents, +freeze an aggregate decision when starting the session: + +```bash +agentloop improve start --tasks tasks.json --baseline runs/baseline/capture.json --decision aggregate --out runs/optimization +``` + +Aggregate decisions require AgentLoop 0.2.3 or newer. The decision is part of +the hashed session, so it cannot be changed after candidate results are known. + +Every pair must still complete, pass its quality checks on both sides, have +complete analysis, and pass the replay gates that are not about speed or cost, +such as retry and quality gates. The speed and cost gates are judged across all +pairs instead: + +- the median paired improvement in the selected objective must be positive and + at least `--minimum-improvement`; +- more pairs must improve than regress; +- the other metric's median regression must stay within its allowed regression + when every pair can measure it. + +The report lists each pair's measurements and failed performance gates for +review. It also gives a one-sided sign-test p-value, which is reported for +context and does not gate the decision. Use enough repetitions for the median +to mean something, and combine aggregate decisions with an +[interleaved control](#compare-against-an-interleaved-control). + ## Compare against an interleaved control Interleaved controls require AgentLoop 0.2.2 or newer. diff --git a/tests/test_improvement.py b/tests/test_improvement.py index 69d1f65..2e64ad7 100644 --- a/tests/test_improvement.py +++ b/tests/test_improvement.py @@ -441,3 +441,131 @@ def test_cli_accepts_repeated_control_arguments(tmp_path, workload): assert result.exit_code == 0, result.output assert "Captured 8 of 8 planned attempts." in result.output assert (tmp_path / "paired/control/capture.json").is_file() + + +NOISY = """import json,sys,time +from agentloop.tracer import AgentTrace +request=json.loads(sys.stdin.readline()) +delays={0:0.12,1:0.12,2:0.12,3:0.12} +fast={0:0.02,1:0.02,2:0.02,3:0.2} +index={"first":0,"second":2}[request["task_id"]]+request["repetition"] +trace=AgentTrace("noisy worker",metadata={"synthetic":True,"output":request["input"]}) +time.sleep((fast if sys.argv[1]=="candidate" else delays)[index]) +trace.finish() +trace.export_json(request["trace_path"]) +""" + + +def noisy(tmp_path, workload, decision): + tasks, _ = workload + worker = tmp_path / "noisy.py" + worker.write_text(NOISY, encoding="utf-8") + capture(tasks, tmp_path / "baseline", [sys.executable, str(worker), "baseline"]) + start(tasks, tmp_path / "baseline/capture.json", tmp_path / "session", decision=decision) + capture(tasks, tmp_path / "candidate", [sys.executable, str(worker), "candidate"]) + return finish( + tmp_path / "session/session.json", + tmp_path / "candidate/capture.json", + tmp_path / "results", + change="Mostly faster with one slow outlier", + ) + + +def test_per_pair_decision_rejects_a_single_slower_pair(tmp_path, workload): + report = noisy(tmp_path, workload, "per_pair") + assert report["decision"] == {"mode": "per_pair", "passed": True} + assert report["passed"] is False + assert report["passed_attempts"] == 3 + + +def test_aggregate_decision_accepts_a_median_improvement(tmp_path, workload): + report = noisy(tmp_path, workload, "aggregate") + verdict = report["decision"] + assert report["passed"] is True, verdict + assert report["passed_attempts"] == 4 + assert verdict["mode"] == "aggregate" + assert (verdict["improved_pairs"], verdict["regressed_pairs"]) == (3, 1) + assert verdict["median_improvement_pct"] > 0 + assert verdict["sign_test_p_value"] == pytest.approx(5 / 16) + assert "judged across all pairs" in (tmp_path / "results/report.md").read_text(encoding="utf-8") + + +def test_aggregate_decision_still_requires_every_answer_to_pass(tmp_path, workload): + tasks, worker = workload + capture(tasks, tmp_path / "baseline", command(worker, "baseline")) + start(tasks, tmp_path / "baseline/capture.json", tmp_path / "session", decision="aggregate") + capture(tasks, tmp_path / "candidate", command(worker, "wrong")) + report = finish( + tmp_path / "session/session.json", + tmp_path / "candidate/capture.json", + tmp_path / "results", + change="Faster but wrong", + ) + assert report["passed"] is False + assert report["passed_attempts"] == 0 + assert report["decision"]["median_improvement_pct"] > 0 + + +def test_aggregate_decision_rejects_a_median_slowdown(tmp_path, workload): + tasks, _ = workload + worker = tmp_path / "noisy.py" + worker.write_text(NOISY, encoding="utf-8") + # Reverse the roles: three pairs slow down and one speeds up. + capture(tasks, tmp_path / "baseline", [sys.executable, str(worker), "candidate"]) + start(tasks, tmp_path / "baseline/capture.json", tmp_path / "session", decision="aggregate") + capture(tasks, tmp_path / "candidate", [sys.executable, str(worker), "baseline"]) + report = finish( + tmp_path / "session/session.json", + tmp_path / "candidate/capture.json", + tmp_path / "results", + change="Mostly slower", + ) + verdict = report["decision"] + assert report["passed_attempts"] == 4 + assert report["passed"] is False + assert (verdict["improved_pairs"], verdict["regressed_pairs"]) == (1, 3) + assert any("Median latency" in reason for reason in verdict["reasons"]) + assert any("more pairs must improve" in reason for reason in verdict["reasons"]) + + +def test_decision_is_frozen_with_the_session(tmp_path, workload): + tasks, worker = prepare(tmp_path, workload) + path = tmp_path / "session/session.json" + saved = json.loads(path.read_bytes()) + assert saved["specification"]["decision"] == "per_pair" + saved["specification"]["decision"] = "aggregate" + path.write_text(json.dumps(saved), encoding="utf-8") + capture(tasks, tmp_path / "candidate", command(worker, "fast")) + with pytest.raises(ValueError, match="Frozen session was modified"): + finish(path, tmp_path / "candidate/capture.json", tmp_path / "results", change="Late") + + +def test_unknown_decision_is_rejected(tmp_path, workload): + tasks, worker = workload + capture(tasks, tmp_path / "baseline", command(worker, "baseline")) + with pytest.raises(ValueError, match="per_pair or aggregate"): + start(tasks, tmp_path / "baseline/capture.json", tmp_path / "session", decision="any") + + +def test_cli_freezes_the_aggregate_decision(tmp_path, workload): + tasks, worker = workload + capture(tasks, tmp_path / "baseline", command(worker, "baseline")) + result = CliRunner().invoke( + app, + [ + "improve", + "start", + "--tasks", + str(tasks), + "--baseline", + str(tmp_path / "baseline/capture.json"), + "--decision", + "aggregate", + "--out", + str(tmp_path / "session"), + ], + ) + assert result.exit_code == 0, result.output + saved = json.loads((tmp_path / "session/session.json").read_bytes()) + assert saved["specification"]["decision"] == "aggregate" + assert "median paired improvement" in (tmp_path / "session/plan.md").read_text(encoding="utf-8")