Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,15 @@ trace, study, receipt, API and database schema versions.

## [Unreleased]

### Added

- `agentloop improve compare` runs a whole guided comparison in one command.
It captures the baseline with the unchanged runner (`--before-arg`), freezes
the session, captures the changed runner alternating with the unchanged one
as an interleaved control, and writes the verdict. It checks inputs before
any runner executes, stops if no baseline attempt completes, and defaults to
the aggregate decision. `improvement.compare()` exposes the same workflow.

## [0.2.4] - 2026-11-18

### Fixed
Expand Down
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,12 @@ fictional notes; it is labeled synthetic. Its timings describe this example only
A faster answer that fails the frozen correctness checks is rejected with exit
code 1. Failed attempts and unknown costs remain in the report.

When you already have both versions, `agentloop improve compare` runs the
baseline, the frozen session, an interleaved before/after capture and the
verdict in one command. Pass the unchanged runner as repeated `--before-arg`
values and the changed runner after `--`. See
[comparing two versions in one command](https://dipeshbabu.github.io/agentloop/GUIDED_OPTIMIZATION/#compare-two-versions-in-one-command).

To connect your application, run `agentloop improve init --agent my_agent:answer`
in its project directory. The generated adapter supports synchronous and async
Python callables. Replace the example task inputs and expected outputs with
Expand Down
108 changes: 108 additions & 0 deletions agentloop/improvement.py
Original file line number Diff line number Diff line change
Expand Up @@ -1060,3 +1060,111 @@ def finish(session_path, candidate_path, root, *, change, verifier=None):

export_report(root, report)
return report


COMPARE_LAYOUT = {
"baseline": "baseline",
"session": "session",
"paired": "paired",
"results": "results",
}


def compare(
workload_path,
root,
before,
after,
*,
change,
timeout=180,
show_output=False,
focus=None,
objective="latency",
minimum_improvement=0,
decision="aggregate",
verifier=None,
verifier_timeout=DEFAULT_TIMEOUT,
max_runner_regression_ms=None,
progress=None,
):
"""Run the whole guided comparison for an unchanged and a changed runner.

The ``before`` runner is captured as the baseline, the session is frozen,
and the ``after`` runner is then captured with ``before`` as an
interleaved control. ``before`` must therefore stay runnable and unchanged,
for example from a separate git worktree. Every stage writes its usual
artifacts under ``root`` (see ``COMPARE_LAYOUT``), so a failed run can be
inspected or resumed with the individual commands.
"""
notify = progress or (lambda message: None)
# Check cheap inputs before any runner executes, so a typo cannot waste a
# full baseline capture.
require(isinstance(change, str) and bool(change.strip()), "Describe the change being compared")
for argv, side in ((before, "before"), (after, "after")):
require(
isinstance(argv, (list, tuple))
and argv
and all(isinstance(a, str) and a for a in argv),
f"Supply an explicit {side} runner command",
)
require(list(before) != list(after), "The before and after runners must differ")
require(objective in {"latency", "cost"}, "Objective must be latency or cost")
decision = decision.replace("-", "_") if isinstance(decision, str) else decision
require(decision in DECISIONS, "Decision must be per-pair or aggregate")
_runner_budget(max_runner_regression_ms)
require(
type(minimum_improvement) in (int, float)
and math.isfinite(minimum_improvement)
and minimum_improvement >= 0,
"Minimum improvement must be finite and nonnegative",
)
if verifier is not None:
LocalVerifier.read(verifier, verifier_timeout)
read_workload(workload_path)
root = Path(root).resolve()
root.mkdir(parents=True, exist_ok=False)
paths = {name: local_path(root, relative) for name, relative in COMPARE_LAYOUT.items()}

notify("Capturing the baseline with the before runner")
baseline = capture(
workload_path, paths["baseline"], list(before), timeout=timeout, show_output=show_output
)
completed = sum(row["status"] == "completed" for row in baseline["runs"])
require(
completed > 0,
"No baseline attempt completed. Fix the before runner, using "
f"{paths['baseline'] / 'capture.json'} and --show-runner-output, then rerun.",
)
notify(
f"Baseline captured {completed} of {len(baseline['runs'])} attempts; freezing the session"
)
start(
workload_path,
paths["baseline"] / "capture.json",
paths["session"],
focus=focus,
objective=objective,
minimum_improvement=minimum_improvement,
decision=decision,
verifier=verifier,
verifier_timeout=verifier_timeout,
max_runner_regression_ms=max_runner_regression_ms,
)
notify("Capturing the after runner, alternating with the before runner as a control")
capture(
workload_path,
paths["paired"],
list(after),
timeout=timeout,
show_output=show_output,
control=list(before),
)
notify("Comparing every planned pair")
return finish(
paths["session"] / "session.json",
paths["paired"] / "capture.json",
paths["results"],
change=change,
verifier=verifier,
)
72 changes: 72 additions & 0 deletions agentloop/improvement_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,11 @@ def finish_command(
report = improvement.finish(session, candidate, out, change=change, verifier=verifier)
except (OSError, ValueError, TypeError, KeyError) as exc:
raise typer.BadParameter(str(exc)) from exc
_print_report(report, out)


def _print_report(report, out):
"""Print the verdict and timing summary; exit 1 when the change is rejected."""
verdict = report["decision"]
if verdict["mode"] == "aggregate":
console.print(_headline(report), markup=False)
Expand Down Expand Up @@ -213,3 +218,70 @@ def finish_command(
)
if not report["passed"]:
raise typer.Exit(1)


@improve_app.command("compare")
def compare_command(
tasks: Path | None = typer.Option(None, help="Fixed tasks, overriding the project config."),
project: Path = typer.Option(
Path("agentloop.toml"), help="Data-only project config used when --tasks is omitted."
),
out: Path = typer.Option(..., help="Fresh directory for every stage of the comparison."),
before_arg: list[str] = typer.Option(
...,
"--before-arg",
help="One argument of the unchanged before runner; repeat for each argument. "
"It runs the baseline and then alternates with the after runner as a control.",
),
change: str = typer.Option(..., help="Describe the change the after runner applies."),
timeout: float = typer.Option(180, min=0.01, max=3600, help="Seconds per task attempt."),
show_runner_output: bool = typer.Option(
False, help="Stream the runners' stdout and stderr for diagnosis."
),
focus: str | None = typer.Option(
None, help="Finding type to investigate. Defaults to the first baseline finding."
),
objective: str = typer.Option("latency", help="latency or cost."),
minimum_improvement: float = typer.Option(
0, min=0, help="Required percentage gain in the selected objective."
),
max_runner_regression_ms: float | None = typer.Option(
None, min=0, help="Optional maximum runner wall-time increase in ms for each pair."
),
decision: str = typer.Option(
"aggregate",
help="aggregate (default here) judges the median paired improvement; per-pair "
"requires every pair to improve. Correctness is checked per pair in both.",
),
verifier: Path | None = typer.Option(None, help="Explicit trusted local Python verifier file."),
verifier_timeout: float = typer.Option(
DEFAULT_TIMEOUT, min=0.01, max=3600, help="Seconds per verifier execution."
),
command: list[str] = typer.Argument(
..., help="Explicit after-runner argv after --. Receives a JSON request on stdin."
),
):
"""Capture, freeze, interleave and compare an unchanged and a changed runner."""
try:
report = improvement.compare(
tasks_path(tasks, project),
out,
before_arg,
command,
change=change,
timeout=timeout,
show_output=show_runner_output,
focus=focus,
objective=objective,
minimum_improvement=minimum_improvement,
decision=decision,
verifier=verifier,
verifier_timeout=verifier_timeout,
max_runner_regression_ms=max_runner_regression_ms,
progress=lambda message: console.print(message, markup=False),
)
except (OSError, ValueError, TypeError, KeyError) as exc:
raise typer.BadParameter(str(exc)) from exc
session = out / improvement.COMPARE_LAYOUT["session"]
console.print(f"Baseline findings and plan: {session / 'plan.md'}", markup=False)
_print_report(report, out / improvement.COMPARE_LAYOUT["results"])
35 changes: 35 additions & 0 deletions docs/GUIDED_OPTIMIZATION.md
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,41 @@ The request contains `task_id`, zero-based `repetition`, `input`, and an absolut
in `metadata.output` so the quality scorer can evaluate it. Existing native model
outputs are also supported through the usual quality-report lookup.

## Compare two versions in one command

When you already have the change, `improve compare` runs the whole workflow
below in one command. It needs AgentLoop 0.2.5 or newer. Keep the unchanged
agent runnable, for example in a separate `git worktree`, and pass its command
as repeated `--before-arg` values. Put the changed agent's command after `--`:

```bash
git worktree add ../agent-before HEAD
agentloop improve compare --tasks tasks.json --out runs/compare --change "Describe the change" --before-arg python --before-arg ../agent-before/your_agent.py -- python your_agent.py
```

`compare` runs these stages in order:

1. It captures the baseline with the before runner in `runs/compare/baseline/`.
2. It freezes the findings and checks in `runs/compare/session/`. Review its
`plan.md` afterwards.
3. It captures the after runner, alternating with the before runner as an
[interleaved control](#compare-against-an-interleaved-control), in
`runs/compare/paired/`.
4. It writes the verdict to `runs/compare/results/`.

It checks the change description, decision, objective and verifier before any
runner executes. It stops after the baseline when no baseline attempt completed.
Unlike `start`, it defaults to `--decision aggregate`, which suits agents whose
timing varies between runs. Pass `--decision per-pair` to require every pair to
improve. `--focus`, `--objective`, `--minimum-improvement`,
`--max-runner-regression-ms`, `--verifier` and `--timeout` work as they do for
the individual commands. Use at least two repetitions per task, and an even
number so that each runner goes first equally often.

Because the session is frozen after the baseline runs, `compare` cannot let the
plan inform the change. To investigate findings before writing a change, use
the individual commands below.

## Capture and review the baseline

```bash
Expand Down
104 changes: 103 additions & 1 deletion tests/test_improvement.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

from agentloop.entrypoint import app
from agentloop.experiment_types import fingerprint
from agentloop.improvement import capture, finish, read_workload, start
from agentloop.improvement import capture, compare, finish, read_workload, start

WORKER = """import json,sys,time
from pathlib import Path
Expand Down Expand Up @@ -796,3 +796,105 @@ def test_cli_output_survives_unicode_names_on_a_legacy_code_page(tmp_path):
assert result.returncode == 1, result.stderr.decode("utf-8", "replace")
assert "UnicodeEncodeError" not in result.stderr.decode("utf-8", "replace")
assert "检查-✓ repetition 0" in output


def test_compare_runs_every_stage_with_an_interleaved_control(tmp_path, workload):
tasks, worker = workload
messages = []
report = compare(
tasks,
tmp_path / "compare",
command(worker, "baseline"),
command(worker, "fast"),
change="Remove repeated work",
progress=messages.append,
)
assert report["passed"] is True
assert report["comparison_baseline"] == "interleaved_control"
assert report["decision"]["mode"] == "aggregate"
for stage in ("baseline/capture.json", "session/session.json", "session/plan.md"):
assert (tmp_path / "compare" / stage).is_file()
assert (tmp_path / "compare/paired/control/capture.json").is_file()
assert (tmp_path / "compare/results/report.html").is_file()
assert len(messages) == 4


def test_compare_rejects_a_wrong_after_runner(tmp_path, workload):
tasks, worker = workload
report = compare(
tasks,
tmp_path / "compare",
command(worker, "baseline"),
command(worker, "wrong"),
change="Faster but wrong",
decision="per-pair",
)
assert report["passed"] is False
assert report["decision"]["mode"] == "per_pair"


@pytest.mark.parametrize(
("override", "message"),
[
({"change": " "}, "Describe the change"),
({"after": None}, "before and after runners must differ"),
({"decision": "median"}, "per-pair or aggregate"),
({"objective": "speed"}, "latency or cost"),
({"minimum_improvement": -1}, "nonnegative"),
],
)
def test_compare_checks_inputs_before_running_anything(tmp_path, workload, override, message):
tasks, worker = workload
before = command(worker, "baseline")
arguments = {
"change": "Change",
"after": command(worker, "fast"),
"decision": "aggregate",
"objective": "latency",
"minimum_improvement": 0,
}
arguments.update(override)
if arguments["after"] is None:
arguments["after"] = list(before)
after = arguments.pop("after")
with pytest.raises(ValueError, match=message):
compare(tasks, tmp_path / "compare", before, after, **arguments)
assert not (tmp_path / "compare").exists()


def test_compare_stops_when_no_baseline_attempt_completes(tmp_path, workload):
tasks, worker = workload
with pytest.raises(ValueError, match="No baseline attempt completed"):
compare(
tasks,
tmp_path / "compare",
command(worker, "failure"),
command(worker, "fast"),
change="Change",
)
assert (tmp_path / "compare/baseline/capture.json").is_file()
assert not (tmp_path / "compare/session").exists()


def test_cli_compare_accepts_repeated_before_arguments(tmp_path, workload):
tasks, worker = workload
result = CliRunner().invoke(
app,
[
"improve",
"compare",
"--tasks",
str(tasks),
"--out",
str(tmp_path / "compare"),
*[item for arg in command(worker, "baseline") for item in ("--before-arg", arg)],
"--change",
"Remove repeated work",
"--",
*command(worker, "fast"),
],
)
assert result.exit_code == 0, result.output
assert "Capturing the baseline with the before runner" in result.output
assert "Aggregate" in result.output
assert (tmp_path / "compare/results/report.json").is_file()
Loading