Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ trace, study, receipt, API and database schema versions.
- Guided comparisons can require a trusted local Python verifier selected by
the developer. The frozen session binds the script and timeout, and both
sides must pass the verifier and their built-in quality checks.
- `improve start --max-runner-regression-ms` freezes an optional per-pair limit
on runner wall-time increases. A configured gate requires measured monotonic
timings and works with interleaved controls and either decision mode.

## [0.2.3] - 2026-11-17

Expand Down
58 changes: 56 additions & 2 deletions agentloop/improvement.py
Original file line number Diff line number Diff line change
Expand Up @@ -415,6 +415,15 @@ def _quality(task, baseline, candidate, verification=None):
)


def _runner_budget(value):
try:
return validate_elapsed_ms(value)
except TraceValidationError as error:
raise ValueError(
"Runner regression budget must be a finite nonnegative number in ms"
) from error


def start(
workload_path,
baseline_path,
Expand All @@ -426,13 +435,15 @@ def start(
decision="per_pair",
verifier=None,
verifier_timeout=DEFAULT_TIMEOUT,
max_runner_regression_ms=None,
):
workload = read_workload(workload_path)
manifest, traces = read_capture(baseline_path, workload)
verifier = LocalVerifier.read(verifier, verifier_timeout) if verifier is not None else None
require(objective in {"latency", "cost"}, "Objective must be latency or cost")
decision = decision.replace("-", "_") if isinstance(decision, str) else decision
require(decision in DECISIONS, "Decision must be per-pair or aggregate")
runner_budget = _runner_budget(max_runner_regression_ms)
require(
type(minimum_improvement) in (int, float)
and math.isfinite(minimum_improvement)
Expand Down Expand Up @@ -507,6 +518,8 @@ def start(
}
if verifier is not None:
specification["verifier"] = verifier.declaration()
if runner_budget is not None:
specification["max_runner_regression_ms"] = runner_budget
write_once(
root,
"session.json",
Expand Down Expand Up @@ -565,6 +578,26 @@ def _attempt_timing(row, trace):
return timing


def _runner_wall_gate(row, limit_ms, comparison):
before, after = (row["timing"][side] for side in ("baseline", "candidate"))
complete = row["baseline_status"] == row["candidate_status"] == "completed"
if complete and before["runner_clock"] == after["runner_clock"] == "monotonic":
delta = after["runner_elapsed_ms"] - before["runner_elapsed_ms"]
passed = delta <= limit_ms
detail = f"Candidate minus {comparison.lower()} runner time was {delta:+.3f} ms. Limit {limit_ms:g} ms."
else:
delta, passed = None, False
detail = "Both completed attempts need monotonic runner times. Recapture the missing side, or capture a fresh interleaved control and candidate."
return {
"name": "runner_wall_time_regression",
"passed": passed,
"detail": detail,
"indeterminate": False,
"delta_ms": delta,
"limit_ms": limit_ms,
}


def _median_duration(values):
if not values:
return None
Expand Down Expand Up @@ -681,6 +714,7 @@ def finish(session_path, candidate_path, root, *, change, verifier=None):
require(specification.get("schema_version") == VERSION, "Unsupported frozen session version")
workload = validate_workload(specification["workload"])
verifier = resume_verifier(specification.get("verifier"), verifier)
runner_budget = _runner_budget(specification.get("max_runner_regression_ms"))
require(specification.get("objective") in {"latency", "cost"}, "Unsupported frozen objective")
# Sessions frozen before 0.2.3 have no decision field and keep per-pair judgment.
decision = specification.get("decision", "per_pair")
Expand Down Expand Up @@ -815,6 +849,11 @@ def finish(session_path, candidate_path, root, *, change, verifier=None):
frozen_baseline_status=case["capture_status"],
control_ran_first=baseline_row.get("order") == 0,
)
if runner_budget is not None:
gate = _runner_wall_gate(row, runner_budget, label)
row["runner_gate"] = gate
if not gate["passed"]:
row["reasons"].append(f"Runner wall-time gate failed. {gate['detail']}")
verification = None
if verifier is not None:
verification = {
Expand Down Expand Up @@ -896,14 +935,16 @@ def finish(session_path, candidate_path, root, *, change, verifier=None):
row.update(
quality=quality, replay=replay, passed=not row["reasons"], objective_gain_pct=gain
)
if runner_budget is not None:
replay["gates"]["results"].append(row["runner_gate"])
replay["gates"]["results"].append(
{
"name": "guided_evidence",
"passed": row["passed"],
"detail": (
"All guided execution, quality and measured-objective checks passed"
if decision == "per_pair"
else "All guided execution and quality checks passed; performance is judged across all pairs"
else "All guided per-pair checks passed. The selected objective is judged across all pairs"
)
if row["passed"]
else " ".join(row["reasons"]),
Expand All @@ -923,7 +964,14 @@ def finish(session_path, candidate_path, root, *, change, verifier=None):
candidate,
target_finding_ids=finding_ids,
intervention_type=specification["focus"],
configuration={"change": change},
configuration={
"change": change,
**(
{"max_runner_regression_ms": runner_budget}
if runner_budget is not None
else {}
),
},
diagnosis=diagnosis,
gates=gates,
quality_report=quality,
Expand Down Expand Up @@ -1002,6 +1050,12 @@ def finish(session_path, candidate_path, root, *, change, verifier=None):
}
if verifier is not None:
report["verifier"] = verifier.declaration()
if runner_budget is not None:
report["runner_gate"] = {
"limit_ms": runner_budget,
"passed_pairs": sum(row["runner_gate"]["passed"] for row in rows),
"unavailable_pairs": sum(row["runner_gate"]["delta_ms"] is None for row in rows),
}
from agentloop.improvement_report import export_report

export_report(root, report)
Expand Down
22 changes: 15 additions & 7 deletions agentloop/improvement_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,13 @@

from agentloop import improvement
from agentloop.improvement_init import initialize, project_tasks
from agentloop.improvement_report import comparison_label, decision_objective, timing_summary_rows
from agentloop.improvement_report import (
_headline,
comparison_label,
decision_objective,
runner_gate_summary,
timing_summary_rows,
)
from agentloop.improvement_verifier import DEFAULT_TIMEOUT

improve_app = typer.Typer(
Expand Down Expand Up @@ -104,6 +110,9 @@ def start_command(
minimum_improvement: float = typer.Option(
0, min=0, help="Required percentage gain in the selected objective."
),
max_runner_regression_ms: float | None = typer.Option(
None, min=0, help="Optional maximum runner wall-time increase in ms for each pair."
),
decision: str = typer.Option(
"per-pair",
help="per-pair requires every pair to improve; aggregate judges the median "
Expand All @@ -126,6 +135,7 @@ def start_command(
focus=focus,
objective=objective,
minimum_improvement=minimum_improvement,
max_runner_regression_ms=max_runner_regression_ms,
decision=decision,
verifier=verifier,
verifier_timeout=verifier_timeout,
Expand Down Expand Up @@ -165,11 +175,7 @@ def finish_command(
raise typer.BadParameter(str(exc)) from exc
verdict = report["decision"]
if verdict["mode"] == "aggregate":
console.print(
f"{report['passed_attempts']} of {report['planned_attempts']} pairs passed "
"correctness and completion checks.",
markup=False,
)
console.print(_headline(report), markup=False)
median = verdict["median_improvement_pct"]
console.print(
f"Aggregate {decision_objective(verdict)} check "
Expand All @@ -185,12 +191,14 @@ def finish_command(
f"{report['passed_attempts']} of {report['planned_attempts']} planned comparisons passed.",
markup=False,
)
if summary := runner_gate_summary(report):
console.print(summary, markup=False)
timing = Table(
"Timing",
f"{comparison_label(report)} median",
"Candidate median",
"Measured pairs",
caption="Completed, correct pairs. Latency gates use trace latency.",
caption="Completed, correct pairs. The latency objective uses trace time.",
)
for label, before, after, _, pairs in timing_summary_rows(report):
timing.add_row(label, before, after, pairs)
Expand Down
26 changes: 22 additions & 4 deletions agentloop/improvement_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
_TIMING_SCOPE = (
"Trace latency covers the instrumented execution. Runner wall time spans launch "
"through exit, including startup, trace export, and cleanup after interruption. "
"Latency gates use trace latency."
"The latency objective uses trace time."
)


Expand All @@ -45,6 +45,10 @@ def plan_markdown(specification):
]
if specification.get("verifier"):
lines.extend([_verifier_scope(specification["verifier"]), ""])
if (limit := specification.get("max_runner_regression_ms")) is not None:
lines.extend(
[f"Runner wall-time gate allows an increase of up to {limit:g} ms in each pair.", ""]
)
for case in specification["cases"]:
lines.extend(
[f"## {markdown_heading(case['task_id'])}, repetition {case['repetition']}", ""]
Expand Down Expand Up @@ -195,10 +199,10 @@ def _metric(summary, name):

def _headline(report):
if (report.get("decision") or {}).get("mode") == "aggregate":
return (
f"{report['passed_attempts']} of {report['planned_attempts']} pairs passed "
"correctness and completion checks."
checks = (
"per-pair checks" if report.get("runner_gate") else "correctness and completion checks"
)
return f"{report['passed_attempts']} of {report['planned_attempts']} pairs passed {checks}."
return f"{report['passed_attempts']} of {report['planned_attempts']} planned attempts passed."


Expand Down Expand Up @@ -286,6 +290,16 @@ def decision_objective(verdict):
return "trace latency" if verdict["objective"] == "latency" else verdict["objective"]


def runner_gate_summary(report):
if gate := report.get("runner_gate"):
return (
f"Runner wall-time gate passed on {gate['passed_pairs']} of {report['planned_attempts']} pairs. "
f"Limit {gate['limit_ms']:g} ms more per pair. "
f"{gate['unavailable_pairs']} pairs lacked completed monotonic timing."
)
return None


def _verifier_scope(verifier):
return (
f"Local verifier SHA-256 {verifier['sha256']}. Each execution has a "
Expand Down Expand Up @@ -351,6 +365,8 @@ def report_markdown(report):
)
if report.get("verifier"):
lines.extend([_verifier_scope(report["verifier"]), ""])
if summary := runner_gate_summary(report):
lines.extend([summary, ""])
timing_rows = timing_summary_rows(report)
if timing_rows:
headers = _timing_headers(comparison)
Expand Down Expand Up @@ -424,6 +440,8 @@ def export_report(root, report):
)
if report.get("verifier"):
sections.append(f"<p>{_text(_verifier_scope(report['verifier']))}</p>")
if summary := runner_gate_summary(report):
sections.append(f"<p>{_text(summary)}</p>")
timing_rows = timing_summary_rows(report)
if timing_rows:
sections.extend(
Expand Down
21 changes: 20 additions & 1 deletion docs/GUIDED_OPTIMIZATION.md
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,8 @@ HTML, and Markdown.
Trace latency covers the instrumented execution and drives latency gates. Runner
wall time covers process launch through exit, including startup, trace export,
and cleanup after interruption. It can help explain a small end-to-end gain when
the traced change is much faster. Runner wall time is descriptive.
the traced change is much faster. Set a runner regression budget to make this
measurement part of acceptance.

Timing medians use completed, correct pairs with available measurements on both
sides. Each metric lists how many pairs it could measure out of the full planned
Expand All @@ -193,6 +194,24 @@ cost stays unavailable. A latency comparison can pass with an indeterminate,
non-gating cost estimate. A required cost improvement fails when pricing or
usable usage is missing.

## Limit runner wall-time regressions

In AgentLoop 0.2.4 and newer, freeze an optional budget with
`agentloop improve start --max-runner-regression-ms 25` alongside the baseline
and output options. Each candidate attempt may take at most that many additional
milliseconds from process launch to exit compared with its matching baseline
or interleaved control. A limit of `0` requires each candidate runner to be at
least as fast as its comparison attempt. Choose a tolerance that reflects
run-to-run variability on your workload.

Every pair must meet the budget, including with `--decision aggregate`. The
latency objective still judges trace time. Both sides need completed attempts
with monotonic runner measurements. Legacy timestamp-derived timings remain
visible in reports. For a gated comparison with an older baseline, capture a
fresh interleaved control and candidate after freezing the session. An overage
or missing measurement makes `improve finish` return exit code 1 and retain
the evidence and gate reason.

## Judge noisy agents across all pairs

The default `--decision per-pair` rejects a change when any single pair is slower
Expand Down
9 changes: 8 additions & 1 deletion scripts/smoke_guided_wheel.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,11 @@ def capture(mode, expected=0):
objective,
"--out",
root / ("session-" + objective),
*(["--verifier", verifier] if objective == "latency" else []),
*(
["--verifier", verifier, "--max-runner-regression-ms", "3600000"]
if objective == "latency"
else []
),
)
for mode in ("candidate", "wrong", "incomplete", "missing-usage", "cancelled"):
capture(mode, 1 if mode == "cancelled" else 0)
Expand Down Expand Up @@ -157,8 +161,11 @@ def capture(mode, expected=0):
raise RuntimeError("Installed guided HTML report missing")
timing = report["cases"][0]["timing"]["candidate"]
verified = report["cases"][0]["verifier"]["candidate"]
runner_gate = report["cases"][0]["runner_gate"]
if verified["passed"] is not (mode in {"candidate", "missing-usage"}):
raise RuntimeError("Installed guided local verifier lost its outcome")
if runner_gate["passed"] is not (mode != "cancelled"):
raise RuntimeError("Installed guided runner gate lost comparison coverage")
if timing["runner_clock"] != "monotonic" or timing["runner_elapsed_ms"] <= 0:
raise RuntimeError("Installed guided capture lost process timing")
if report["timing_summary"]["runner_elapsed_ms"]["paired_count"] != int(report["passed"]):
Expand Down
Loading
Loading