diff --git a/CHANGELOG.md b/CHANGELOG.md index 9adb146..f9f183c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ trace, study, receipt, API and database schema versions. ## [Unreleased] +### Added + +- `agentloop improve start --decision aggregate` judges speed or cost across + all pairs: the median paired improvement must meet the frozen minimum and + more pairs must improve than regress. Every pair must still complete and + pass its quality and non-performance gates. Reports include the counts and + a non-gating sign-test p-value. The default per-pair decision is unchanged. + ## [0.2.2] - 2026-11-17 ### Added diff --git a/agentloop/improvement.py b/agentloop/improvement.py index 78431e1..2744783 100644 --- a/agentloop/improvement.py +++ b/agentloop/improvement.py @@ -7,6 +7,7 @@ import math import os import signal +import statistics import subprocess # nosec B404 - execution requires explicit CLI argv from dataclasses import asdict from datetime import datetime, timezone @@ -22,6 +23,14 @@ from agentloop.tracer import AgentTrace VERSION = "1.0" +DECISIONS = {"per_pair", "aggregate"} +# Speed and cost gates; an aggregate decision judges these across all pairs. +PERFORMANCE_GATES = { + "latency_regression", + "latency_improvement", + "cost_regression", + "cost_improvement", +} SCORERS = {"exact_match", "contains", "json_subset", "required_fields", "glob"} @@ -380,11 +389,19 @@ def _quality(task, baseline, candidate): def start( - workload_path, baseline_path, root, *, focus=None, objective="latency", minimum_improvement=0 + workload_path, + baseline_path, + root, + *, + focus=None, + objective="latency", + minimum_improvement=0, + decision="per_pair", ): workload = read_workload(workload_path) manifest, traces = read_capture(baseline_path, workload) require(objective in {"latency", "cost"}, "Objective must be latency or cost") + require(decision in DECISIONS, "Decision must be per_pair or aggregate") require( type(minimum_improvement) in (int, float) and math.isfinite(minimum_improvement) @@ -442,6 +459,7 @@ def start( "workload": workload, "focus": focus, "objective": objective, + "decision": decision, "gates": asdict(gates), "cases": cases, "baseline_capture": manifest, @@ -473,6 +491,83 @@ def _trace_usable(trace): return not any(event.parent_id is None and event.status == "error" for event in trace.events) +def _sign_test_p_value(improved, regressed): + """One-sided exact sign test: chance of at least this many improvements.""" + trials = improved + regressed + if trials == 0: + return None + return sum(math.comb(trials, k) for k in range(improved, trials + 1)) / 2**trials + + +def _aggregate_decision(rows, objective, gates): + """Judge speed or cost across all planned pairs. + + Correctness, completion and non-performance gates were already enforced on + every pair. Here the median paired improvement in the selected objective + must be positive and meet the frozen minimum, and more pairs must improve + than regress. The other metric's median regression must stay within its + allowed regression when every pair can measure it. + """ + secondary = "cost" if objective == "latency" else "latency" + required = ( + gates.min_latency_improvement_pct + if objective == "latency" + else gates.min_cost_improvement_pct + ) + allowed = ( + gates.max_cost_regression_pct if secondary == "cost" else gates.max_latency_regression_pct + ) + gains = [row.get("objective_gain_pct") for row in rows] + regressions = [ + (row["replay"] or {}).get("deltas", {}).get(f"{secondary}_regression_pct") + if row.get("replay") + else None + for row in rows + ] + reasons = [] + measured = [gain for gain in gains if gain is not None] + improved = sum(gain > 0 for gain in measured) + regressed = sum(gain < 0 for gain in measured) + median = statistics.median(measured) if measured else None + if len(measured) != len(rows): + reasons.append( + f"{len(rows) - len(measured)} of {len(rows)} pairs have no measured {objective} improvement" + ) + if median is None or median <= 0 or median < required: + detail = "unavailable" if median is None else f"{median:.2f}%" + reasons.append( + f"Median {objective} improvement {detail} must be positive and at least {required:.2f}%" + ) + if improved <= regressed: + reasons.append( + f"{improved} pairs improved and {regressed} regressed; more pairs must improve than regress" + ) + secondary_median = None + if all(value is not None for value in regressions): + secondary_median = statistics.median(regressions) + if secondary_median > allowed: + reasons.append( + f"Median {secondary} regression {secondary_median:.2f}% exceeds {allowed:.2f}% allowed" + ) + return { + "mode": "aggregate", + "objective": objective, + "pairs": len(rows), + "median_improvement_pct": median, + "required_improvement_pct": required, + "improved_pairs": improved, + "regressed_pairs": regressed, + "unchanged_pairs": len(measured) - improved - regressed, + "sign_test_p_value": _sign_test_p_value(improved, regressed), + "secondary_metric": secondary, + "secondary_median_regression_pct": secondary_median, + "secondary_allowed_regression_pct": allowed, + "secondary_judged": secondary_median is not None, + "passed": not reasons, + "reasons": reasons, + } + + def finish(session_path, candidate_path, root, *, change): session_path = Path(session_path).resolve() saved = load_json(session_path) @@ -484,6 +579,9 @@ def finish(session_path, candidate_path, root, *, change): require(specification.get("schema_version") == VERSION, "Unsupported frozen session version") workload = validate_workload(specification["workload"]) require(specification.get("objective") in {"latency", "cost"}, "Unsupported frozen objective") + # Sessions frozen before 0.2.3 have no decision field and keep per-pair judgment. + decision = specification.get("decision", "per_pair") + require(decision in DECISIONS, "Unsupported frozen decision") planned = {(task["id"], repetition) for task, repetition in slots(workload)} cases = specification.get("cases") require( @@ -629,13 +727,17 @@ def finish(session_path, candidate_path, root, *, change): row["reasons"].append( f"{label if side == 'baseline' else 'Candidate'} answer failed the declared {task.get('scorer', {}).get('type', 'exact_match')} criterion: {result[side]['detail']}" ) - if not replay["gates"]["passed"]: + # Aggregate decisions judge speed and cost across all pairs after + # this loop; every other gate still applies to each pair. + judged = [ + gate + for gate in replay["gates"]["results"] + if not gate["passed"] + and (decision == "per_pair" or gate["name"] not in PERFORMANCE_GATES) + ] + if judged: row["reasons"].append("Native replay gates failed") - row["reasons"].extend( - f"{gate['name']}: {gate['detail']}" - for gate in replay["gates"]["results"] - if not gate["passed"] - ) + row["reasons"].extend(f"{gate['name']}: {gate['detail']}" for gate in judged) if not diagnosis.get("analysis_complete", True) or not candidate.report().get( "analysis_complete", True ): @@ -645,14 +747,20 @@ def finish(session_path, candidate_path, root, *, change): if specification["objective"] == "latency" else "cost_improvement_pct" ] - if gain is None or gain <= 0: + if decision == "per_pair" and (gain is None or gain <= 0): row["reasons"].append("The selected objective has no measured improvement") - row.update(quality=quality, replay=replay, passed=not row["reasons"]) + row.update( + quality=quality, replay=replay, passed=not row["reasons"], objective_gain_pct=gain + ) replay["gates"]["results"].append( { "name": "guided_evidence", "passed": row["passed"], - "detail": "All guided execution, quality and measured-objective checks passed" + "detail": ( + "All guided execution, quality and measured-objective checks passed" + if decision == "per_pair" + else "All guided execution and quality checks passed; performance is judged across all pairs" + ) if row["passed"] else " ".join(row["reasons"]), "indeterminate": False, @@ -712,6 +820,11 @@ def finish(session_path, candidate_path, root, *, change): } study_file = write_once(root, "study.json", study_manifest) study = summarize_study(study_file) + verdict = ( + {"mode": "per_pair", "passed": True} + if decision == "per_pair" + else _aggregate_decision(rows, specification["objective"], gates) + ) report = { "schema_version": VERSION, "name": workload["name"], @@ -729,7 +842,8 @@ def finish(session_path, candidate_path, root, *, change): "control_runner_hash": None if controls is None else control_manifest.get("runner_hash"), "planned_attempts": len(rows), "passed_attempts": sum(row["passed"] for row in rows), - "passed": all(row["passed"] for row in rows), + "decision": verdict, + "passed": all(row["passed"] for row in rows) and verdict["passed"], "synthetic": any( trace.metadata.get("synthetic") is True for trace in [ diff --git a/agentloop/improvement_cli.py b/agentloop/improvement_cli.py index f1f52db..7bd1d99 100644 --- a/agentloop/improvement_cli.py +++ b/agentloop/improvement_cli.py @@ -101,6 +101,11 @@ def start_command( minimum_improvement: float = typer.Option( 0, min=0, help="Required percentage gain in the selected objective." ), + decision: str = typer.Option( + "per-pair", + help="per-pair requires every pair to improve; aggregate judges the median " + "paired improvement for noisy agents. Correctness is checked per pair in both.", + ), ): """Freeze task checks and original findings before editing the agent.""" try: @@ -111,6 +116,7 @@ def start_command( focus=focus, objective=objective, minimum_improvement=minimum_improvement, + decision=decision.replace("-", "_"), ) except (OSError, ValueError, TypeError, KeyError) as exc: raise typer.BadParameter(str(exc)) from exc @@ -136,10 +142,28 @@ def finish_command( report = improvement.finish(session, candidate, out, change=change) except (OSError, ValueError, TypeError, KeyError) as exc: raise typer.BadParameter(str(exc)) from exc - console.print( - f"{report['passed_attempts']} of {report['planned_attempts']} planned comparisons passed.", - markup=False, - ) + verdict = report["decision"] + if verdict["mode"] == "aggregate": + console.print( + f"{report['passed_attempts']} of {report['planned_attempts']} pairs passed " + "correctness and completion checks.", + markup=False, + ) + median = verdict["median_improvement_pct"] + console.print( + f"Aggregate {verdict['objective']} decision " + + ("passed" if verdict["passed"] else "failed") + + f": median improvement {'unavailable' if median is None else f'{median:.2f}%'}, " + f"{verdict['improved_pairs']} improved, {verdict['regressed_pairs']} regressed.", + markup=False, + ) + for reason in verdict["reasons"]: + console.print(f"Aggregate check failed: {reason}", markup=False) + else: + console.print( + f"{report['passed_attempts']} of {report['planned_attempts']} planned comparisons passed.", + markup=False, + ) console.print(f"Read {out / 'report.html'}", markup=False) for case in report["cases"]: if not case["passed"]: diff --git a/agentloop/improvement_report.py b/agentloop/improvement_report.py index 74e4da9..1e00c32 100644 --- a/agentloop/improvement_report.py +++ b/agentloop/improvement_report.py @@ -22,6 +22,9 @@ def plan_markdown(specification): "", f"Investigation focus is {markdown_text(specification['focus'] or 'unavailable')}.", f"The measured objective is {specification['objective']}. Every baseline and candidate output must pass its declared quality checks.", + "The frozen decision judges the objective across all pairs by median paired improvement." + if specification.get("decision") == "aggregate" + else "The frozen decision requires every pair to improve the objective.", "", ] for case in specification["cases"]: @@ -182,6 +185,35 @@ def _basis(report): ) +def _decision_lines(report): + verdict = report.get("decision") or {"mode": "per_pair"} + if verdict["mode"] != "aggregate": + return ["Decision: every pair must pass correctness, completion and performance checks."] + median = verdict["median_improvement_pct"] + p_value = verdict["sign_test_p_value"] + lines = [ + "Decision: every pair must pass correctness and completion checks; " + f"{verdict['objective']} is judged across all pairs.", + f"Aggregate {verdict['objective']} decision " + + ("passed" if verdict["passed"] else "failed") + + ": median paired improvement " + + ("unavailable" if median is None else f"{median:.2f}%") + + f" (required at least {verdict['required_improvement_pct']:.2f}%), " + f"{verdict['improved_pairs']} improved, {verdict['regressed_pairs']} regressed, " + f"{verdict['unchanged_pairs']} unchanged.", + ] + if p_value is not None: + lines.append( + f"One-sided sign test p-value {p_value:.3f}. It is reported for context and does not gate the decision." + ) + if not verdict["secondary_judged"]: + lines.append( + f"Median {verdict['secondary_metric']} regression was not judged because it was unavailable for some pairs." + ) + lines.extend(f"Aggregate check failed: {reason}" for reason in verdict["reasons"]) + return lines + + def report_markdown(report): lines = [ f"# {markdown_heading(report['name'])}", @@ -198,6 +230,7 @@ def report_markdown(report): "", _basis(report), "", + *[item for line in _decision_lines(report) for item in (markdown_text(line), "")], ] if report["synthetic"]: lines.extend( @@ -263,6 +296,7 @@ def export_report(root, report): f"

{_text(report['name'])}

{report['passed_attempts']} of {report['planned_attempts']} planned attempts passed.

", f"

Applied change

{_text(report['change'])}

", f"

{_text(_basis(report))}

", + *[f"

{_text(line)}

" for line in _decision_lines(report)], ] if report["synthetic"]: sections.append( diff --git a/docs/GUIDED_OPTIMIZATION.md b/docs/GUIDED_OPTIMIZATION.md index 2ed9223..614c665 100644 --- a/docs/GUIDED_OPTIMIZATION.md +++ b/docs/GUIDED_OPTIMIZATION.md @@ -169,6 +169,37 @@ cost stays unavailable. A latency comparison can pass with an indeterminate, non-gating cost estimate. A required cost improvement fails when pricing or usable usage is missing. +## Judge noisy agents across all pairs + +The default `--decision per-pair` rejects a change when any single pair is slower +or costlier. Agents that call hosted models vary from run to run, so one noisy +pair can reject a change that helps on almost every attempt. For those agents, +freeze an aggregate decision when starting the session: + +```bash +agentloop improve start --tasks tasks.json --baseline runs/baseline/capture.json --decision aggregate --out runs/optimization +``` + +Aggregate decisions require AgentLoop 0.2.3 or newer. The decision is part of +the hashed session, so it cannot be changed after candidate results are known. + +Every pair must still complete, pass its quality checks on both sides, have +complete analysis, and pass the replay gates that are not about speed or cost, +such as retry and quality gates. The speed and cost gates are judged across all +pairs instead: + +- the median paired improvement in the selected objective must be positive and + at least `--minimum-improvement`; +- more pairs must improve than regress; +- the other metric's median regression must stay within its allowed regression + when every pair can measure it. + +The report lists each pair's measurements and failed performance gates for +review. It also gives a one-sided sign-test p-value, which is reported for +context and does not gate the decision. Use enough repetitions for the median +to mean something, and combine aggregate decisions with an +[interleaved control](#compare-against-an-interleaved-control). + ## Compare against an interleaved control Interleaved controls require AgentLoop 0.2.2 or newer. diff --git a/tests/test_improvement.py b/tests/test_improvement.py index 69d1f65..2e64ad7 100644 --- a/tests/test_improvement.py +++ b/tests/test_improvement.py @@ -441,3 +441,131 @@ def test_cli_accepts_repeated_control_arguments(tmp_path, workload): assert result.exit_code == 0, result.output assert "Captured 8 of 8 planned attempts." in result.output assert (tmp_path / "paired/control/capture.json").is_file() + + +NOISY = """import json,sys,time +from agentloop.tracer import AgentTrace +request=json.loads(sys.stdin.readline()) +delays={0:0.12,1:0.12,2:0.12,3:0.12} +fast={0:0.02,1:0.02,2:0.02,3:0.2} +index={"first":0,"second":2}[request["task_id"]]+request["repetition"] +trace=AgentTrace("noisy worker",metadata={"synthetic":True,"output":request["input"]}) +time.sleep((fast if sys.argv[1]=="candidate" else delays)[index]) +trace.finish() +trace.export_json(request["trace_path"]) +""" + + +def noisy(tmp_path, workload, decision): + tasks, _ = workload + worker = tmp_path / "noisy.py" + worker.write_text(NOISY, encoding="utf-8") + capture(tasks, tmp_path / "baseline", [sys.executable, str(worker), "baseline"]) + start(tasks, tmp_path / "baseline/capture.json", tmp_path / "session", decision=decision) + capture(tasks, tmp_path / "candidate", [sys.executable, str(worker), "candidate"]) + return finish( + tmp_path / "session/session.json", + tmp_path / "candidate/capture.json", + tmp_path / "results", + change="Mostly faster with one slow outlier", + ) + + +def test_per_pair_decision_rejects_a_single_slower_pair(tmp_path, workload): + report = noisy(tmp_path, workload, "per_pair") + assert report["decision"] == {"mode": "per_pair", "passed": True} + assert report["passed"] is False + assert report["passed_attempts"] == 3 + + +def test_aggregate_decision_accepts_a_median_improvement(tmp_path, workload): + report = noisy(tmp_path, workload, "aggregate") + verdict = report["decision"] + assert report["passed"] is True, verdict + assert report["passed_attempts"] == 4 + assert verdict["mode"] == "aggregate" + assert (verdict["improved_pairs"], verdict["regressed_pairs"]) == (3, 1) + assert verdict["median_improvement_pct"] > 0 + assert verdict["sign_test_p_value"] == pytest.approx(5 / 16) + assert "judged across all pairs" in (tmp_path / "results/report.md").read_text(encoding="utf-8") + + +def test_aggregate_decision_still_requires_every_answer_to_pass(tmp_path, workload): + tasks, worker = workload + capture(tasks, tmp_path / "baseline", command(worker, "baseline")) + start(tasks, tmp_path / "baseline/capture.json", tmp_path / "session", decision="aggregate") + capture(tasks, tmp_path / "candidate", command(worker, "wrong")) + report = finish( + tmp_path / "session/session.json", + tmp_path / "candidate/capture.json", + tmp_path / "results", + change="Faster but wrong", + ) + assert report["passed"] is False + assert report["passed_attempts"] == 0 + assert report["decision"]["median_improvement_pct"] > 0 + + +def test_aggregate_decision_rejects_a_median_slowdown(tmp_path, workload): + tasks, _ = workload + worker = tmp_path / "noisy.py" + worker.write_text(NOISY, encoding="utf-8") + # Reverse the roles: three pairs slow down and one speeds up. + capture(tasks, tmp_path / "baseline", [sys.executable, str(worker), "candidate"]) + start(tasks, tmp_path / "baseline/capture.json", tmp_path / "session", decision="aggregate") + capture(tasks, tmp_path / "candidate", [sys.executable, str(worker), "baseline"]) + report = finish( + tmp_path / "session/session.json", + tmp_path / "candidate/capture.json", + tmp_path / "results", + change="Mostly slower", + ) + verdict = report["decision"] + assert report["passed_attempts"] == 4 + assert report["passed"] is False + assert (verdict["improved_pairs"], verdict["regressed_pairs"]) == (1, 3) + assert any("Median latency" in reason for reason in verdict["reasons"]) + assert any("more pairs must improve" in reason for reason in verdict["reasons"]) + + +def test_decision_is_frozen_with_the_session(tmp_path, workload): + tasks, worker = prepare(tmp_path, workload) + path = tmp_path / "session/session.json" + saved = json.loads(path.read_bytes()) + assert saved["specification"]["decision"] == "per_pair" + saved["specification"]["decision"] = "aggregate" + path.write_text(json.dumps(saved), encoding="utf-8") + capture(tasks, tmp_path / "candidate", command(worker, "fast")) + with pytest.raises(ValueError, match="Frozen session was modified"): + finish(path, tmp_path / "candidate/capture.json", tmp_path / "results", change="Late") + + +def test_unknown_decision_is_rejected(tmp_path, workload): + tasks, worker = workload + capture(tasks, tmp_path / "baseline", command(worker, "baseline")) + with pytest.raises(ValueError, match="per_pair or aggregate"): + start(tasks, tmp_path / "baseline/capture.json", tmp_path / "session", decision="any") + + +def test_cli_freezes_the_aggregate_decision(tmp_path, workload): + tasks, worker = workload + capture(tasks, tmp_path / "baseline", command(worker, "baseline")) + result = CliRunner().invoke( + app, + [ + "improve", + "start", + "--tasks", + str(tasks), + "--baseline", + str(tmp_path / "baseline/capture.json"), + "--decision", + "aggregate", + "--out", + str(tmp_path / "session"), + ], + ) + assert result.exit_code == 0, result.output + saved = json.loads((tmp_path / "session/session.json").read_bytes()) + assert saved["specification"]["decision"] == "aggregate" + assert "median paired improvement" in (tmp_path / "session/plan.md").read_text(encoding="utf-8")