Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,14 @@ trace, study, receipt, API and database schema versions.

## [Unreleased]

### Added

- `agentloop improve start --decision aggregate` judges speed or cost across
all pairs: the median paired improvement must meet the frozen minimum and
more pairs must improve than regress. Every pair must still complete and
pass its quality and non-performance gates. Reports include the counts and
a non-gating sign-test p-value. The default per-pair decision is unchanged.

## [0.2.2] - 2026-11-17

### Added
Expand Down
136 changes: 125 additions & 11 deletions agentloop/improvement.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import math
import os
import signal
import statistics
import subprocess # nosec B404 - execution requires explicit CLI argv
from dataclasses import asdict
from datetime import datetime, timezone
Expand All @@ -22,6 +23,14 @@
from agentloop.tracer import AgentTrace

VERSION = "1.0"
DECISIONS = {"per_pair", "aggregate"}
# Speed and cost gates; an aggregate decision judges these across all pairs.
PERFORMANCE_GATES = {
"latency_regression",
"latency_improvement",
"cost_regression",
"cost_improvement",
}
SCORERS = {"exact_match", "contains", "json_subset", "required_fields", "glob"}


Expand Down Expand Up @@ -380,11 +389,19 @@ def _quality(task, baseline, candidate):


def start(
workload_path, baseline_path, root, *, focus=None, objective="latency", minimum_improvement=0
workload_path,
baseline_path,
root,
*,
focus=None,
objective="latency",
minimum_improvement=0,
decision="per_pair",
):
workload = read_workload(workload_path)
manifest, traces = read_capture(baseline_path, workload)
require(objective in {"latency", "cost"}, "Objective must be latency or cost")
require(decision in DECISIONS, "Decision must be per_pair or aggregate")
require(
type(minimum_improvement) in (int, float)
and math.isfinite(minimum_improvement)
Expand Down Expand Up @@ -442,6 +459,7 @@ def start(
"workload": workload,
"focus": focus,
"objective": objective,
"decision": decision,
"gates": asdict(gates),
"cases": cases,
"baseline_capture": manifest,
Expand Down Expand Up @@ -473,6 +491,83 @@ def _trace_usable(trace):
return not any(event.parent_id is None and event.status == "error" for event in trace.events)


def _sign_test_p_value(improved, regressed):
"""One-sided exact sign test: chance of at least this many improvements."""
trials = improved + regressed
if trials == 0:
return None
return sum(math.comb(trials, k) for k in range(improved, trials + 1)) / 2**trials


def _aggregate_decision(rows, objective, gates):
"""Judge speed or cost across all planned pairs.

Correctness, completion and non-performance gates were already enforced on
every pair. Here the median paired improvement in the selected objective
must be positive and meet the frozen minimum, and more pairs must improve
than regress. The other metric's median regression must stay within its
allowed regression when every pair can measure it.
"""
secondary = "cost" if objective == "latency" else "latency"
required = (
gates.min_latency_improvement_pct
if objective == "latency"
else gates.min_cost_improvement_pct
)
allowed = (
gates.max_cost_regression_pct if secondary == "cost" else gates.max_latency_regression_pct
)
gains = [row.get("objective_gain_pct") for row in rows]
regressions = [
(row["replay"] or {}).get("deltas", {}).get(f"{secondary}_regression_pct")
if row.get("replay")
else None
for row in rows
]
reasons = []
measured = [gain for gain in gains if gain is not None]
improved = sum(gain > 0 for gain in measured)
regressed = sum(gain < 0 for gain in measured)
median = statistics.median(measured) if measured else None
if len(measured) != len(rows):
reasons.append(
f"{len(rows) - len(measured)} of {len(rows)} pairs have no measured {objective} improvement"
)
if median is None or median <= 0 or median < required:
detail = "unavailable" if median is None else f"{median:.2f}%"
reasons.append(
f"Median {objective} improvement {detail} must be positive and at least {required:.2f}%"
)
if improved <= regressed:
reasons.append(
f"{improved} pairs improved and {regressed} regressed; more pairs must improve than regress"
)
secondary_median = None
if all(value is not None for value in regressions):
secondary_median = statistics.median(regressions)
if secondary_median > allowed:
reasons.append(
f"Median {secondary} regression {secondary_median:.2f}% exceeds {allowed:.2f}% allowed"
)
return {
"mode": "aggregate",
"objective": objective,
"pairs": len(rows),
"median_improvement_pct": median,
"required_improvement_pct": required,
"improved_pairs": improved,
"regressed_pairs": regressed,
"unchanged_pairs": len(measured) - improved - regressed,
"sign_test_p_value": _sign_test_p_value(improved, regressed),
"secondary_metric": secondary,
"secondary_median_regression_pct": secondary_median,
"secondary_allowed_regression_pct": allowed,
"secondary_judged": secondary_median is not None,
"passed": not reasons,
"reasons": reasons,
}


def finish(session_path, candidate_path, root, *, change):
session_path = Path(session_path).resolve()
saved = load_json(session_path)
Expand All @@ -484,6 +579,9 @@ def finish(session_path, candidate_path, root, *, change):
require(specification.get("schema_version") == VERSION, "Unsupported frozen session version")
workload = validate_workload(specification["workload"])
require(specification.get("objective") in {"latency", "cost"}, "Unsupported frozen objective")
# Sessions frozen before 0.2.3 have no decision field and keep per-pair judgment.
decision = specification.get("decision", "per_pair")
require(decision in DECISIONS, "Unsupported frozen decision")
planned = {(task["id"], repetition) for task, repetition in slots(workload)}
cases = specification.get("cases")
require(
Expand Down Expand Up @@ -629,13 +727,17 @@ def finish(session_path, candidate_path, root, *, change):
row["reasons"].append(
f"{label if side == 'baseline' else 'Candidate'} answer failed the declared {task.get('scorer', {}).get('type', 'exact_match')} criterion: {result[side]['detail']}"
)
if not replay["gates"]["passed"]:
# Aggregate decisions judge speed and cost across all pairs after
# this loop; every other gate still applies to each pair.
judged = [
gate
for gate in replay["gates"]["results"]
if not gate["passed"]
and (decision == "per_pair" or gate["name"] not in PERFORMANCE_GATES)
]
if judged:
row["reasons"].append("Native replay gates failed")
row["reasons"].extend(
f"{gate['name']}: {gate['detail']}"
for gate in replay["gates"]["results"]
if not gate["passed"]
)
row["reasons"].extend(f"{gate['name']}: {gate['detail']}" for gate in judged)
if not diagnosis.get("analysis_complete", True) or not candidate.report().get(
"analysis_complete", True
):
Expand All @@ -645,14 +747,20 @@ def finish(session_path, candidate_path, root, *, change):
if specification["objective"] == "latency"
else "cost_improvement_pct"
]
if gain is None or gain <= 0:
if decision == "per_pair" and (gain is None or gain <= 0):
row["reasons"].append("The selected objective has no measured improvement")
row.update(quality=quality, replay=replay, passed=not row["reasons"])
row.update(
quality=quality, replay=replay, passed=not row["reasons"], objective_gain_pct=gain
)
replay["gates"]["results"].append(
{
"name": "guided_evidence",
"passed": row["passed"],
"detail": "All guided execution, quality and measured-objective checks passed"
"detail": (
"All guided execution, quality and measured-objective checks passed"
if decision == "per_pair"
else "All guided execution and quality checks passed; performance is judged across all pairs"
)
if row["passed"]
else " ".join(row["reasons"]),
"indeterminate": False,
Expand Down Expand Up @@ -712,6 +820,11 @@ def finish(session_path, candidate_path, root, *, change):
}
study_file = write_once(root, "study.json", study_manifest)
study = summarize_study(study_file)
verdict = (
{"mode": "per_pair", "passed": True}
if decision == "per_pair"
else _aggregate_decision(rows, specification["objective"], gates)
)
report = {
"schema_version": VERSION,
"name": workload["name"],
Expand All @@ -729,7 +842,8 @@ def finish(session_path, candidate_path, root, *, change):
"control_runner_hash": None if controls is None else control_manifest.get("runner_hash"),
"planned_attempts": len(rows),
"passed_attempts": sum(row["passed"] for row in rows),
"passed": all(row["passed"] for row in rows),
"decision": verdict,
"passed": all(row["passed"] for row in rows) and verdict["passed"],
"synthetic": any(
trace.metadata.get("synthetic") is True
for trace in [
Expand Down
32 changes: 28 additions & 4 deletions agentloop/improvement_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,11 @@ def start_command(
minimum_improvement: float = typer.Option(
0, min=0, help="Required percentage gain in the selected objective."
),
decision: str = typer.Option(
"per-pair",
help="per-pair requires every pair to improve; aggregate judges the median "
"paired improvement for noisy agents. Correctness is checked per pair in both.",
),
):
"""Freeze task checks and original findings before editing the agent."""
try:
Expand All @@ -111,6 +116,7 @@ def start_command(
focus=focus,
objective=objective,
minimum_improvement=minimum_improvement,
decision=decision.replace("-", "_"),
)
except (OSError, ValueError, TypeError, KeyError) as exc:
raise typer.BadParameter(str(exc)) from exc
Expand All @@ -136,10 +142,28 @@ def finish_command(
report = improvement.finish(session, candidate, out, change=change)
except (OSError, ValueError, TypeError, KeyError) as exc:
raise typer.BadParameter(str(exc)) from exc
console.print(
f"{report['passed_attempts']} of {report['planned_attempts']} planned comparisons passed.",
markup=False,
)
verdict = report["decision"]
if verdict["mode"] == "aggregate":
console.print(
f"{report['passed_attempts']} of {report['planned_attempts']} pairs passed "
"correctness and completion checks.",
markup=False,
)
median = verdict["median_improvement_pct"]
console.print(
f"Aggregate {verdict['objective']} decision "
+ ("passed" if verdict["passed"] else "failed")
+ f": median improvement {'unavailable' if median is None else f'{median:.2f}%'}, "
f"{verdict['improved_pairs']} improved, {verdict['regressed_pairs']} regressed.",
markup=False,
)
for reason in verdict["reasons"]:
console.print(f"Aggregate check failed: {reason}", markup=False)
else:
console.print(
f"{report['passed_attempts']} of {report['planned_attempts']} planned comparisons passed.",
markup=False,
)
console.print(f"Read {out / 'report.html'}", markup=False)
for case in report["cases"]:
if not case["passed"]:
Expand Down
34 changes: 34 additions & 0 deletions agentloop/improvement_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ def plan_markdown(specification):
"",
f"Investigation focus is {markdown_text(specification['focus'] or 'unavailable')}.",
f"The measured objective is {specification['objective']}. Every baseline and candidate output must pass its declared quality checks.",
"The frozen decision judges the objective across all pairs by median paired improvement."
if specification.get("decision") == "aggregate"
else "The frozen decision requires every pair to improve the objective.",
"",
]
for case in specification["cases"]:
Expand Down Expand Up @@ -182,6 +185,35 @@ def _basis(report):
)


def _decision_lines(report):
verdict = report.get("decision") or {"mode": "per_pair"}
if verdict["mode"] != "aggregate":
return ["Decision: every pair must pass correctness, completion and performance checks."]
median = verdict["median_improvement_pct"]
p_value = verdict["sign_test_p_value"]
lines = [
"Decision: every pair must pass correctness and completion checks; "
f"{verdict['objective']} is judged across all pairs.",
f"Aggregate {verdict['objective']} decision "
+ ("passed" if verdict["passed"] else "failed")
+ ": median paired improvement "
+ ("unavailable" if median is None else f"{median:.2f}%")
+ f" (required at least {verdict['required_improvement_pct']:.2f}%), "
f"{verdict['improved_pairs']} improved, {verdict['regressed_pairs']} regressed, "
f"{verdict['unchanged_pairs']} unchanged.",
]
if p_value is not None:
lines.append(
f"One-sided sign test p-value {p_value:.3f}. It is reported for context and does not gate the decision."
)
if not verdict["secondary_judged"]:
lines.append(
f"Median {verdict['secondary_metric']} regression was not judged because it was unavailable for some pairs."
)
lines.extend(f"Aggregate check failed: {reason}" for reason in verdict["reasons"])
return lines


def report_markdown(report):
lines = [
f"# {markdown_heading(report['name'])}",
Expand All @@ -198,6 +230,7 @@ def report_markdown(report):
"",
_basis(report),
"",
*[item for line in _decision_lines(report) for item in (markdown_text(line), "")],
]
if report["synthetic"]:
lines.extend(
Expand Down Expand Up @@ -263,6 +296,7 @@ def export_report(root, report):
f"<header><h1>{_text(report['name'])}</h1><p>{report['passed_attempts']} of {report['planned_attempts']} planned attempts passed.</p></header>",
f"<h2>Applied change</h2><p>{_text(report['change'])}</p>",
f"<p>{_text(_basis(report))}</p>",
*[f"<p>{_text(line)}</p>" for line in _decision_lines(report)],
]
if report["synthetic"]:
sections.append(
Expand Down
31 changes: 31 additions & 0 deletions docs/GUIDED_OPTIMIZATION.md
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,37 @@ cost stays unavailable. A latency comparison can pass with an indeterminate,
non-gating cost estimate. A required cost improvement fails when pricing or
usable usage is missing.

## Judge noisy agents across all pairs

The default `--decision per-pair` rejects a change when any single pair is slower
or costlier. Agents that call hosted models vary from run to run, so one noisy
pair can reject a change that helps on almost every attempt. For those agents,
freeze an aggregate decision when starting the session:

```bash
agentloop improve start --tasks tasks.json --baseline runs/baseline/capture.json --decision aggregate --out runs/optimization
```

Aggregate decisions require AgentLoop 0.2.3 or newer. The decision is part of
the hashed session, so it cannot be changed after candidate results are known.

Every pair must still complete, pass its quality checks on both sides, have
complete analysis, and pass the replay gates that are not about speed or cost,
such as retry and quality gates. The speed and cost gates are judged across all
pairs instead:

- the median paired improvement in the selected objective must be positive and
at least `--minimum-improvement`;
- more pairs must improve than regress;
- the other metric's median regression must stay within its allowed regression
when every pair can measure it.

The report lists each pair's measurements and failed performance gates for
review. It also gives a one-sided sign-test p-value, which is reported for
context and does not gate the decision. Use enough repetitions for the median
to mean something, and combine aggregate decisions with an
[interleaved control](#compare-against-an-interleaved-control).

## Compare against an interleaved control

Interleaved controls require AgentLoop 0.2.2 or newer.
Expand Down
Loading
Loading