diff --git a/CHANGELOG.md b/CHANGELOG.md index a9270c4..4b66f99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,9 @@ trace, study, receipt, API and database schema versions. - Guided comparisons can require a trusted local Python verifier selected by the developer. The frozen session binds the script and timeout, and both sides must pass the verifier and their built-in quality checks. +- `improve start --max-runner-regression-ms` freezes an optional per-pair limit + on runner wall-time increases. A configured gate requires measured monotonic + timings and works with interleaved controls and either decision mode. ## [0.2.3] - 2026-11-17 diff --git a/agentloop/improvement.py b/agentloop/improvement.py index 18a9140..f627147 100644 --- a/agentloop/improvement.py +++ b/agentloop/improvement.py @@ -415,6 +415,15 @@ def _quality(task, baseline, candidate, verification=None): ) +def _runner_budget(value): + try: + return validate_elapsed_ms(value) + except TraceValidationError as error: + raise ValueError( + "Runner regression budget must be a finite nonnegative number in ms" + ) from error + + def start( workload_path, baseline_path, @@ -426,6 +435,7 @@ def start( decision="per_pair", verifier=None, verifier_timeout=DEFAULT_TIMEOUT, + max_runner_regression_ms=None, ): workload = read_workload(workload_path) manifest, traces = read_capture(baseline_path, workload) @@ -433,6 +443,7 @@ def start( require(objective in {"latency", "cost"}, "Objective must be latency or cost") decision = decision.replace("-", "_") if isinstance(decision, str) else decision require(decision in DECISIONS, "Decision must be per-pair or aggregate") + runner_budget = _runner_budget(max_runner_regression_ms) require( type(minimum_improvement) in (int, float) and math.isfinite(minimum_improvement) @@ -507,6 +518,8 @@ def start( } if verifier is not None: specification["verifier"] = verifier.declaration() + if runner_budget is not None: + specification["max_runner_regression_ms"] = runner_budget write_once( root, "session.json", @@ -565,6 +578,26 @@ def _attempt_timing(row, trace): return timing +def _runner_wall_gate(row, limit_ms, comparison): + before, after = (row["timing"][side] for side in ("baseline", "candidate")) + complete = row["baseline_status"] == row["candidate_status"] == "completed" + if complete and before["runner_clock"] == after["runner_clock"] == "monotonic": + delta = after["runner_elapsed_ms"] - before["runner_elapsed_ms"] + passed = delta <= limit_ms + detail = f"Candidate minus {comparison.lower()} runner time was {delta:+.3f} ms. Limit {limit_ms:g} ms." + else: + delta, passed = None, False + detail = "Both completed attempts need monotonic runner times. Recapture the missing side, or capture a fresh interleaved control and candidate." + return { + "name": "runner_wall_time_regression", + "passed": passed, + "detail": detail, + "indeterminate": False, + "delta_ms": delta, + "limit_ms": limit_ms, + } + + def _median_duration(values): if not values: return None @@ -681,6 +714,7 @@ def finish(session_path, candidate_path, root, *, change, verifier=None): require(specification.get("schema_version") == VERSION, "Unsupported frozen session version") workload = validate_workload(specification["workload"]) verifier = resume_verifier(specification.get("verifier"), verifier) + runner_budget = _runner_budget(specification.get("max_runner_regression_ms")) require(specification.get("objective") in {"latency", "cost"}, "Unsupported frozen objective") # Sessions frozen before 0.2.3 have no decision field and keep per-pair judgment. decision = specification.get("decision", "per_pair") @@ -815,6 +849,11 @@ def finish(session_path, candidate_path, root, *, change, verifier=None): frozen_baseline_status=case["capture_status"], control_ran_first=baseline_row.get("order") == 0, ) + if runner_budget is not None: + gate = _runner_wall_gate(row, runner_budget, label) + row["runner_gate"] = gate + if not gate["passed"]: + row["reasons"].append(f"Runner wall-time gate failed. {gate['detail']}") verification = None if verifier is not None: verification = { @@ -896,6 +935,8 @@ def finish(session_path, candidate_path, root, *, change, verifier=None): row.update( quality=quality, replay=replay, passed=not row["reasons"], objective_gain_pct=gain ) + if runner_budget is not None: + replay["gates"]["results"].append(row["runner_gate"]) replay["gates"]["results"].append( { "name": "guided_evidence", @@ -903,7 +944,7 @@ def finish(session_path, candidate_path, root, *, change, verifier=None): "detail": ( "All guided execution, quality and measured-objective checks passed" if decision == "per_pair" - else "All guided execution and quality checks passed; performance is judged across all pairs" + else "All guided per-pair checks passed. The selected objective is judged across all pairs" ) if row["passed"] else " ".join(row["reasons"]), @@ -923,7 +964,14 @@ def finish(session_path, candidate_path, root, *, change, verifier=None): candidate, target_finding_ids=finding_ids, intervention_type=specification["focus"], - configuration={"change": change}, + configuration={ + "change": change, + **( + {"max_runner_regression_ms": runner_budget} + if runner_budget is not None + else {} + ), + }, diagnosis=diagnosis, gates=gates, quality_report=quality, @@ -1002,6 +1050,12 @@ def finish(session_path, candidate_path, root, *, change, verifier=None): } if verifier is not None: report["verifier"] = verifier.declaration() + if runner_budget is not None: + report["runner_gate"] = { + "limit_ms": runner_budget, + "passed_pairs": sum(row["runner_gate"]["passed"] for row in rows), + "unavailable_pairs": sum(row["runner_gate"]["delta_ms"] is None for row in rows), + } from agentloop.improvement_report import export_report export_report(root, report) diff --git a/agentloop/improvement_cli.py b/agentloop/improvement_cli.py index 3aaac7e..efbfe9c 100644 --- a/agentloop/improvement_cli.py +++ b/agentloop/improvement_cli.py @@ -10,7 +10,13 @@ from agentloop import improvement from agentloop.improvement_init import initialize, project_tasks -from agentloop.improvement_report import comparison_label, decision_objective, timing_summary_rows +from agentloop.improvement_report import ( + _headline, + comparison_label, + decision_objective, + runner_gate_summary, + timing_summary_rows, +) from agentloop.improvement_verifier import DEFAULT_TIMEOUT improve_app = typer.Typer( @@ -104,6 +110,9 @@ def start_command( minimum_improvement: float = typer.Option( 0, min=0, help="Required percentage gain in the selected objective." ), + max_runner_regression_ms: float | None = typer.Option( + None, min=0, help="Optional maximum runner wall-time increase in ms for each pair." + ), decision: str = typer.Option( "per-pair", help="per-pair requires every pair to improve; aggregate judges the median " @@ -126,6 +135,7 @@ def start_command( focus=focus, objective=objective, minimum_improvement=minimum_improvement, + max_runner_regression_ms=max_runner_regression_ms, decision=decision, verifier=verifier, verifier_timeout=verifier_timeout, @@ -165,11 +175,7 @@ def finish_command( raise typer.BadParameter(str(exc)) from exc verdict = report["decision"] if verdict["mode"] == "aggregate": - console.print( - f"{report['passed_attempts']} of {report['planned_attempts']} pairs passed " - "correctness and completion checks.", - markup=False, - ) + console.print(_headline(report), markup=False) median = verdict["median_improvement_pct"] console.print( f"Aggregate {decision_objective(verdict)} check " @@ -185,12 +191,14 @@ def finish_command( f"{report['passed_attempts']} of {report['planned_attempts']} planned comparisons passed.", markup=False, ) + if summary := runner_gate_summary(report): + console.print(summary, markup=False) timing = Table( "Timing", f"{comparison_label(report)} median", "Candidate median", "Measured pairs", - caption="Completed, correct pairs. Latency gates use trace latency.", + caption="Completed, correct pairs. The latency objective uses trace time.", ) for label, before, after, _, pairs in timing_summary_rows(report): timing.add_row(label, before, after, pairs) diff --git a/agentloop/improvement_report.py b/agentloop/improvement_report.py index a23191b..2317249 100644 --- a/agentloop/improvement_report.py +++ b/agentloop/improvement_report.py @@ -26,7 +26,7 @@ _TIMING_SCOPE = ( "Trace latency covers the instrumented execution. Runner wall time spans launch " "through exit, including startup, trace export, and cleanup after interruption. " - "Latency gates use trace latency." + "The latency objective uses trace time." ) @@ -45,6 +45,10 @@ def plan_markdown(specification): ] if specification.get("verifier"): lines.extend([_verifier_scope(specification["verifier"]), ""]) + if (limit := specification.get("max_runner_regression_ms")) is not None: + lines.extend( + [f"Runner wall-time gate allows an increase of up to {limit:g} ms in each pair.", ""] + ) for case in specification["cases"]: lines.extend( [f"## {markdown_heading(case['task_id'])}, repetition {case['repetition']}", ""] @@ -195,10 +199,10 @@ def _metric(summary, name): def _headline(report): if (report.get("decision") or {}).get("mode") == "aggregate": - return ( - f"{report['passed_attempts']} of {report['planned_attempts']} pairs passed " - "correctness and completion checks." + checks = ( + "per-pair checks" if report.get("runner_gate") else "correctness and completion checks" ) + return f"{report['passed_attempts']} of {report['planned_attempts']} pairs passed {checks}." return f"{report['passed_attempts']} of {report['planned_attempts']} planned attempts passed." @@ -286,6 +290,16 @@ def decision_objective(verdict): return "trace latency" if verdict["objective"] == "latency" else verdict["objective"] +def runner_gate_summary(report): + if gate := report.get("runner_gate"): + return ( + f"Runner wall-time gate passed on {gate['passed_pairs']} of {report['planned_attempts']} pairs. " + f"Limit {gate['limit_ms']:g} ms more per pair. " + f"{gate['unavailable_pairs']} pairs lacked completed monotonic timing." + ) + return None + + def _verifier_scope(verifier): return ( f"Local verifier SHA-256 {verifier['sha256']}. Each execution has a " @@ -351,6 +365,8 @@ def report_markdown(report): ) if report.get("verifier"): lines.extend([_verifier_scope(report["verifier"]), ""]) + if summary := runner_gate_summary(report): + lines.extend([summary, ""]) timing_rows = timing_summary_rows(report) if timing_rows: headers = _timing_headers(comparison) @@ -424,6 +440,8 @@ def export_report(root, report): ) if report.get("verifier"): sections.append(f"

{_text(_verifier_scope(report['verifier']))}

") + if summary := runner_gate_summary(report): + sections.append(f"

{_text(summary)}

") timing_rows = timing_summary_rows(report) if timing_rows: sections.extend( diff --git a/docs/GUIDED_OPTIMIZATION.md b/docs/GUIDED_OPTIMIZATION.md index 3ffe10b..184160a 100644 --- a/docs/GUIDED_OPTIMIZATION.md +++ b/docs/GUIDED_OPTIMIZATION.md @@ -173,7 +173,8 @@ HTML, and Markdown. Trace latency covers the instrumented execution and drives latency gates. Runner wall time covers process launch through exit, including startup, trace export, and cleanup after interruption. It can help explain a small end-to-end gain when -the traced change is much faster. Runner wall time is descriptive. +the traced change is much faster. Set a runner regression budget to make this +measurement part of acceptance. Timing medians use completed, correct pairs with available measurements on both sides. Each metric lists how many pairs it could measure out of the full planned @@ -193,6 +194,24 @@ cost stays unavailable. A latency comparison can pass with an indeterminate, non-gating cost estimate. A required cost improvement fails when pricing or usable usage is missing. +## Limit runner wall-time regressions + +In AgentLoop 0.2.4 and newer, freeze an optional budget with +`agentloop improve start --max-runner-regression-ms 25` alongside the baseline +and output options. Each candidate attempt may take at most that many additional +milliseconds from process launch to exit compared with its matching baseline +or interleaved control. A limit of `0` requires each candidate runner to be at +least as fast as its comparison attempt. Choose a tolerance that reflects +run-to-run variability on your workload. + +Every pair must meet the budget, including with `--decision aggregate`. The +latency objective still judges trace time. Both sides need completed attempts +with monotonic runner measurements. Legacy timestamp-derived timings remain +visible in reports. For a gated comparison with an older baseline, capture a +fresh interleaved control and candidate after freezing the session. An overage +or missing measurement makes `improve finish` return exit code 1 and retain +the evidence and gate reason. + ## Judge noisy agents across all pairs The default `--decision per-pair` rejects a change when any single pair is slower diff --git a/scripts/smoke_guided_wheel.py b/scripts/smoke_guided_wheel.py index 75f323d..c17818a 100644 --- a/scripts/smoke_guided_wheel.py +++ b/scripts/smoke_guided_wheel.py @@ -129,7 +129,11 @@ def capture(mode, expected=0): objective, "--out", root / ("session-" + objective), - *(["--verifier", verifier] if objective == "latency" else []), + *( + ["--verifier", verifier, "--max-runner-regression-ms", "3600000"] + if objective == "latency" + else [] + ), ) for mode in ("candidate", "wrong", "incomplete", "missing-usage", "cancelled"): capture(mode, 1 if mode == "cancelled" else 0) @@ -157,8 +161,11 @@ def capture(mode, expected=0): raise RuntimeError("Installed guided HTML report missing") timing = report["cases"][0]["timing"]["candidate"] verified = report["cases"][0]["verifier"]["candidate"] + runner_gate = report["cases"][0]["runner_gate"] if verified["passed"] is not (mode in {"candidate", "missing-usage"}): raise RuntimeError("Installed guided local verifier lost its outcome") + if runner_gate["passed"] is not (mode != "cancelled"): + raise RuntimeError("Installed guided runner gate lost comparison coverage") if timing["runner_clock"] != "monotonic" or timing["runner_elapsed_ms"] <= 0: raise RuntimeError("Installed guided capture lost process timing") if report["timing_summary"]["runner_elapsed_ms"]["paired_count"] != int(report["passed"]): diff --git a/tests/test_improvement.py b/tests/test_improvement.py index 04b3529..9327b64 100644 --- a/tests/test_improvement.py +++ b/tests/test_improvement.py @@ -435,6 +435,7 @@ def test_timing_summary_reads_legacy_receipts_and_keeps_runner_time_out_of_gates change="Spend less time inside the trace", ) assert report["passed"] + assert "runner_gate" not in report assert report["timing_summary"]["eligible_pairs"] == 4 assert report["timing_summary"]["trace_elapsed_ms"]["paired_count"] == 4 assert report["timing_summary"]["runner_elapsed_ms"] == { @@ -448,6 +449,94 @@ def test_timing_summary_reads_legacy_receipts_and_keeps_runner_time_out_of_gates assert report["cases"][3]["timing"]["baseline"]["runner_elapsed_ms"] is None +@pytest.mark.parametrize("decision", ["per-pair", "aggregate"]) +def test_runner_wall_gate_uses_interleaved_control_and_rejects_unavailable_measurements( + tmp_path, workload, decision +): + tasks, worker = workload + baseline_path = tmp_path / "baseline/capture.json" + baseline = capture(tasks, baseline_path.parent, command(worker, "baseline")) + for row in baseline["runs"]: + row["runner_elapsed_ms"] = 1000 + baseline_path.write_text(json.dumps(baseline), encoding="utf-8") + session = tmp_path / "session" + cli = CliRunner() + initialized = cli.invoke( + app, + [ + "improve", + "start", + "--tasks", + str(tasks), + "--baseline", + str(baseline_path), + "--out", + str(session), + "--max-runner-regression-ms", + "30", + "--decision", + decision, + ], + ) + assert initialized.exit_code == 0, initialized.output + assert ( + json.loads((session / "session.json").read_bytes())["specification"][ + "max_runner_regression_ms" + ] + == 30 + ) + + paired = tmp_path / "paired" + candidate = capture(tasks, paired, command(worker, "fast"), control=command(worker, "baseline")) + control_path = paired / "control/capture.json" + control = json.loads(control_path.read_bytes()) + for row in control["runs"]: + row["runner_elapsed_ms"] = 100 + control_path.write_text(json.dumps(control), encoding="utf-8") + candidate["control"]["manifest_hash"] = fingerprint(control) + for row, elapsed in zip(candidate["runs"], (99, 130, 131), strict=False): + row["runner_elapsed_ms"] = elapsed + candidate["runs"][3].pop( + "runner_elapsed_ms" + ) # A valid legacy timestamp pair stays descriptive. + candidate_path = paired / "capture.json" + candidate_path.write_text(json.dumps(candidate), encoding="utf-8") + + output = tmp_path / "results" + result = cli.invoke( + app, + [ + "improve", + "finish", + "--session", + str(session / "session.json"), + "--candidate", + str(candidate_path), + "--change", + "Fast trace with a slow runner", + "--out", + str(output), + ], + ) + assert result.exit_code == 1, result.output + assert "Runner wall-time gate passed on 2 of 4 pairs" in result.output + report = json.loads((output / "report.json").read_bytes()) + assert report["runner_gate"] == {"limit_ms": 30, "passed_pairs": 2, "unavailable_pairs": 1} + assert report["passed"] is False + assert [row["runner_gate"]["delta_ms"] for row in report["cases"]] == [-1, 30, 31, None] + assert all(row["quality"]["passed"] for row in report["cases"]) + for row in report["cases"][2:]: + assert row["passed"] is False + assert row["intervention"]["gates_passed"] is False + assert any( + gate["name"] == "runner_wall_time_regression" and not gate["passed"] + for gate in row["replay"]["gates"]["results"] + ) + assert "Runner wall-time gate passed on 2 of 4 pairs" in (output / "report.html").read_text( + encoding="utf-8" + ) + + def test_interleaved_control_must_also_pass_quality_checks(tmp_path, workload): tasks, worker = prepare(tmp_path, workload) capture(