From 7a59282075b7f94157e0e7770c0531930eee15a7 Mon Sep 17 00:00:00 2001 From: Dipesh Babu Date: Wed, 18 Nov 2026 14:32:21 -0500 Subject: [PATCH] Add improve compare to run a guided comparison in one command improve compare captures the baseline with the unchanged before runner, freezes the session, captures the changed runner alternating with the before runner as an interleaved control, and writes the verdict. It checks inputs before any runner executes, stops when no baseline attempt completes, and defaults to the aggregate decision. finish's result printing moves into a shared helper so both commands print the same verdict. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 9 +++ README.md | 6 ++ agentloop/improvement.py | 108 +++++++++++++++++++++++++++++++++++ agentloop/improvement_cli.py | 72 +++++++++++++++++++++++ docs/GUIDED_OPTIMIZATION.md | 35 ++++++++++++ tests/test_improvement.py | 104 ++++++++++++++++++++++++++++++++- 6 files changed, 333 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7567154..04ec3ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ trace, study, receipt, API and database schema versions. ## [Unreleased] +### Added + +- `agentloop improve compare` runs a whole guided comparison in one command. + It captures the baseline with the unchanged runner (`--before-arg`), freezes + the session, captures the changed runner alternating with the unchanged one + as an interleaved control, and writes the verdict. It checks inputs before + any runner executes, stops if no baseline attempt completes, and defaults to + the aggregate decision. `improvement.compare()` exposes the same workflow. + ## [0.2.4] - 2026-11-18 ### Fixed diff --git a/README.md b/README.md index d7fa650..cb57564 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,12 @@ fictional notes; it is labeled synthetic. Its timings describe this example only A faster answer that fails the frozen correctness checks is rejected with exit code 1. Failed attempts and unknown costs remain in the report. +When you already have both versions, `agentloop improve compare` runs the +baseline, the frozen session, an interleaved before/after capture and the +verdict in one command. Pass the unchanged runner as repeated `--before-arg` +values and the changed runner after `--`. See +[comparing two versions in one command](https://dipeshbabu.github.io/agentloop/GUIDED_OPTIMIZATION/#compare-two-versions-in-one-command). + To connect your application, run `agentloop improve init --agent my_agent:answer` in its project directory. The generated adapter supports synchronous and async Python callables. Replace the example task inputs and expected outputs with diff --git a/agentloop/improvement.py b/agentloop/improvement.py index f627147..42f9cc9 100644 --- a/agentloop/improvement.py +++ b/agentloop/improvement.py @@ -1060,3 +1060,111 @@ def finish(session_path, candidate_path, root, *, change, verifier=None): export_report(root, report) return report + + +COMPARE_LAYOUT = { + "baseline": "baseline", + "session": "session", + "paired": "paired", + "results": "results", +} + + +def compare( + workload_path, + root, + before, + after, + *, + change, + timeout=180, + show_output=False, + focus=None, + objective="latency", + minimum_improvement=0, + decision="aggregate", + verifier=None, + verifier_timeout=DEFAULT_TIMEOUT, + max_runner_regression_ms=None, + progress=None, +): + """Run the whole guided comparison for an unchanged and a changed runner. + + The ``before`` runner is captured as the baseline, the session is frozen, + and the ``after`` runner is then captured with ``before`` as an + interleaved control. ``before`` must therefore stay runnable and unchanged, + for example from a separate git worktree. Every stage writes its usual + artifacts under ``root`` (see ``COMPARE_LAYOUT``), so a failed run can be + inspected or resumed with the individual commands. + """ + notify = progress or (lambda message: None) + # Check cheap inputs before any runner executes, so a typo cannot waste a + # full baseline capture. + require(isinstance(change, str) and bool(change.strip()), "Describe the change being compared") + for argv, side in ((before, "before"), (after, "after")): + require( + isinstance(argv, (list, tuple)) + and argv + and all(isinstance(a, str) and a for a in argv), + f"Supply an explicit {side} runner command", + ) + require(list(before) != list(after), "The before and after runners must differ") + require(objective in {"latency", "cost"}, "Objective must be latency or cost") + decision = decision.replace("-", "_") if isinstance(decision, str) else decision + require(decision in DECISIONS, "Decision must be per-pair or aggregate") + _runner_budget(max_runner_regression_ms) + require( + type(minimum_improvement) in (int, float) + and math.isfinite(minimum_improvement) + and minimum_improvement >= 0, + "Minimum improvement must be finite and nonnegative", + ) + if verifier is not None: + LocalVerifier.read(verifier, verifier_timeout) + read_workload(workload_path) + root = Path(root).resolve() + root.mkdir(parents=True, exist_ok=False) + paths = {name: local_path(root, relative) for name, relative in COMPARE_LAYOUT.items()} + + notify("Capturing the baseline with the before runner") + baseline = capture( + workload_path, paths["baseline"], list(before), timeout=timeout, show_output=show_output + ) + completed = sum(row["status"] == "completed" for row in baseline["runs"]) + require( + completed > 0, + "No baseline attempt completed. Fix the before runner, using " + f"{paths['baseline'] / 'capture.json'} and --show-runner-output, then rerun.", + ) + notify( + f"Baseline captured {completed} of {len(baseline['runs'])} attempts; freezing the session" + ) + start( + workload_path, + paths["baseline"] / "capture.json", + paths["session"], + focus=focus, + objective=objective, + minimum_improvement=minimum_improvement, + decision=decision, + verifier=verifier, + verifier_timeout=verifier_timeout, + max_runner_regression_ms=max_runner_regression_ms, + ) + notify("Capturing the after runner, alternating with the before runner as a control") + capture( + workload_path, + paths["paired"], + list(after), + timeout=timeout, + show_output=show_output, + control=list(before), + ) + notify("Comparing every planned pair") + return finish( + paths["session"] / "session.json", + paths["paired"] / "capture.json", + paths["results"], + change=change, + verifier=verifier, + ) diff --git a/agentloop/improvement_cli.py b/agentloop/improvement_cli.py index efbfe9c..ee99514 100644 --- a/agentloop/improvement_cli.py +++ b/agentloop/improvement_cli.py @@ -173,6 +173,11 @@ def finish_command( report = improvement.finish(session, candidate, out, change=change, verifier=verifier) except (OSError, ValueError, TypeError, KeyError) as exc: raise typer.BadParameter(str(exc)) from exc + _print_report(report, out) + + +def _print_report(report, out): + """Print the verdict and timing summary; exit 1 when the change is rejected.""" verdict = report["decision"] if verdict["mode"] == "aggregate": console.print(_headline(report), markup=False) @@ -213,3 +218,70 @@ def finish_command( ) if not report["passed"]: raise typer.Exit(1) + + +@improve_app.command("compare") +def compare_command( + tasks: Path | None = typer.Option(None, help="Fixed tasks, overriding the project config."), + project: Path = typer.Option( + Path("agentloop.toml"), help="Data-only project config used when --tasks is omitted." + ), + out: Path = typer.Option(..., help="Fresh directory for every stage of the comparison."), + before_arg: list[str] = typer.Option( + ..., + "--before-arg", + help="One argument of the unchanged before runner; repeat for each argument. " + "It runs the baseline and then alternates with the after runner as a control.", + ), + change: str = typer.Option(..., help="Describe the change the after runner applies."), + timeout: float = typer.Option(180, min=0.01, max=3600, help="Seconds per task attempt."), + show_runner_output: bool = typer.Option( + False, help="Stream the runners' stdout and stderr for diagnosis." + ), + focus: str | None = typer.Option( + None, help="Finding type to investigate. Defaults to the first baseline finding." + ), + objective: str = typer.Option("latency", help="latency or cost."), + minimum_improvement: float = typer.Option( + 0, min=0, help="Required percentage gain in the selected objective." + ), + max_runner_regression_ms: float | None = typer.Option( + None, min=0, help="Optional maximum runner wall-time increase in ms for each pair." + ), + decision: str = typer.Option( + "aggregate", + help="aggregate (default here) judges the median paired improvement; per-pair " + "requires every pair to improve. Correctness is checked per pair in both.", + ), + verifier: Path | None = typer.Option(None, help="Explicit trusted local Python verifier file."), + verifier_timeout: float = typer.Option( + DEFAULT_TIMEOUT, min=0.01, max=3600, help="Seconds per verifier execution." + ), + command: list[str] = typer.Argument( + ..., help="Explicit after-runner argv after --. Receives a JSON request on stdin." + ), +): + """Capture, freeze, interleave and compare an unchanged and a changed runner.""" + try: + report = improvement.compare( + tasks_path(tasks, project), + out, + before_arg, + command, + change=change, + timeout=timeout, + show_output=show_runner_output, + focus=focus, + objective=objective, + minimum_improvement=minimum_improvement, + decision=decision, + verifier=verifier, + verifier_timeout=verifier_timeout, + max_runner_regression_ms=max_runner_regression_ms, + progress=lambda message: console.print(message, markup=False), + ) + except (OSError, ValueError, TypeError, KeyError) as exc: + raise typer.BadParameter(str(exc)) from exc + session = out / improvement.COMPARE_LAYOUT["session"] + console.print(f"Baseline findings and plan: {session / 'plan.md'}", markup=False) + _print_report(report, out / improvement.COMPARE_LAYOUT["results"]) diff --git a/docs/GUIDED_OPTIMIZATION.md b/docs/GUIDED_OPTIMIZATION.md index 184160a..dd24bb5 100644 --- a/docs/GUIDED_OPTIMIZATION.md +++ b/docs/GUIDED_OPTIMIZATION.md @@ -126,6 +126,41 @@ The request contains `task_id`, zero-based `repetition`, `input`, and an absolut in `metadata.output` so the quality scorer can evaluate it. Existing native model outputs are also supported through the usual quality-report lookup. +## Compare two versions in one command + +When you already have the change, `improve compare` runs the whole workflow +below in one command. It needs AgentLoop 0.2.5 or newer. Keep the unchanged +agent runnable, for example in a separate `git worktree`, and pass its command +as repeated `--before-arg` values. Put the changed agent's command after `--`: + +```bash +git worktree add ../agent-before HEAD +agentloop improve compare --tasks tasks.json --out runs/compare --change "Describe the change" --before-arg python --before-arg ../agent-before/your_agent.py -- python your_agent.py +``` + +`compare` runs these stages in order: + +1. It captures the baseline with the before runner in `runs/compare/baseline/`. +2. It freezes the findings and checks in `runs/compare/session/`. Review its + `plan.md` afterwards. +3. It captures the after runner, alternating with the before runner as an + [interleaved control](#compare-against-an-interleaved-control), in + `runs/compare/paired/`. +4. It writes the verdict to `runs/compare/results/`. + +It checks the change description, decision, objective and verifier before any +runner executes. It stops after the baseline when no baseline attempt completed. +Unlike `start`, it defaults to `--decision aggregate`, which suits agents whose +timing varies between runs. Pass `--decision per-pair` to require every pair to +improve. `--focus`, `--objective`, `--minimum-improvement`, +`--max-runner-regression-ms`, `--verifier` and `--timeout` work as they do for +the individual commands. Use at least two repetitions per task, and an even +number so that each runner goes first equally often. + +Because the session is frozen after the baseline runs, `compare` cannot let the +plan inform the change. To investigate findings before writing a change, use +the individual commands below. + ## Capture and review the baseline ```bash diff --git a/tests/test_improvement.py b/tests/test_improvement.py index 9327b64..cd6ed58 100644 --- a/tests/test_improvement.py +++ b/tests/test_improvement.py @@ -9,7 +9,7 @@ from agentloop.entrypoint import app from agentloop.experiment_types import fingerprint -from agentloop.improvement import capture, finish, read_workload, start +from agentloop.improvement import capture, compare, finish, read_workload, start WORKER = """import json,sys,time from pathlib import Path @@ -796,3 +796,105 @@ def test_cli_output_survives_unicode_names_on_a_legacy_code_page(tmp_path): assert result.returncode == 1, result.stderr.decode("utf-8", "replace") assert "UnicodeEncodeError" not in result.stderr.decode("utf-8", "replace") assert "检查-✓ repetition 0" in output + + +def test_compare_runs_every_stage_with_an_interleaved_control(tmp_path, workload): + tasks, worker = workload + messages = [] + report = compare( + tasks, + tmp_path / "compare", + command(worker, "baseline"), + command(worker, "fast"), + change="Remove repeated work", + progress=messages.append, + ) + assert report["passed"] is True + assert report["comparison_baseline"] == "interleaved_control" + assert report["decision"]["mode"] == "aggregate" + for stage in ("baseline/capture.json", "session/session.json", "session/plan.md"): + assert (tmp_path / "compare" / stage).is_file() + assert (tmp_path / "compare/paired/control/capture.json").is_file() + assert (tmp_path / "compare/results/report.html").is_file() + assert len(messages) == 4 + + +def test_compare_rejects_a_wrong_after_runner(tmp_path, workload): + tasks, worker = workload + report = compare( + tasks, + tmp_path / "compare", + command(worker, "baseline"), + command(worker, "wrong"), + change="Faster but wrong", + decision="per-pair", + ) + assert report["passed"] is False + assert report["decision"]["mode"] == "per_pair" + + +@pytest.mark.parametrize( + ("override", "message"), + [ + ({"change": " "}, "Describe the change"), + ({"after": None}, "before and after runners must differ"), + ({"decision": "median"}, "per-pair or aggregate"), + ({"objective": "speed"}, "latency or cost"), + ({"minimum_improvement": -1}, "nonnegative"), + ], +) +def test_compare_checks_inputs_before_running_anything(tmp_path, workload, override, message): + tasks, worker = workload + before = command(worker, "baseline") + arguments = { + "change": "Change", + "after": command(worker, "fast"), + "decision": "aggregate", + "objective": "latency", + "minimum_improvement": 0, + } + arguments.update(override) + if arguments["after"] is None: + arguments["after"] = list(before) + after = arguments.pop("after") + with pytest.raises(ValueError, match=message): + compare(tasks, tmp_path / "compare", before, after, **arguments) + assert not (tmp_path / "compare").exists() + + +def test_compare_stops_when_no_baseline_attempt_completes(tmp_path, workload): + tasks, worker = workload + with pytest.raises(ValueError, match="No baseline attempt completed"): + compare( + tasks, + tmp_path / "compare", + command(worker, "failure"), + command(worker, "fast"), + change="Change", + ) + assert (tmp_path / "compare/baseline/capture.json").is_file() + assert not (tmp_path / "compare/session").exists() + + +def test_cli_compare_accepts_repeated_before_arguments(tmp_path, workload): + tasks, worker = workload + result = CliRunner().invoke( + app, + [ + "improve", + "compare", + "--tasks", + str(tasks), + "--out", + str(tmp_path / "compare"), + *[item for arg in command(worker, "baseline") for item in ("--before-arg", arg)], + "--change", + "Remove repeated work", + "--", + *command(worker, "fast"), + ], + ) + assert result.exit_code == 0, result.output + assert "Capturing the baseline with the before runner" in result.output + assert "Aggregate" in result.output + assert (tmp_path / "compare/results/report.json").is_file()