diff --git a/CHANGELOG.md b/CHANGELOG.md index 7567154..04ec3ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ trace, study, receipt, API and database schema versions. ## [Unreleased] +### Added + +- `agentloop improve compare` runs a whole guided comparison in one command. + It captures the baseline with the unchanged runner (`--before-arg`), freezes + the session, captures the changed runner alternating with the unchanged one + as an interleaved control, and writes the verdict. It checks inputs before + any runner executes, stops if no baseline attempt completes, and defaults to + the aggregate decision. `improvement.compare()` exposes the same workflow. + ## [0.2.4] - 2026-11-18 ### Fixed diff --git a/README.md b/README.md index d7fa650..cb57564 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,12 @@ fictional notes; it is labeled synthetic. Its timings describe this example only A faster answer that fails the frozen correctness checks is rejected with exit code 1. Failed attempts and unknown costs remain in the report. +When you already have both versions, `agentloop improve compare` runs the +baseline, the frozen session, an interleaved before/after capture and the +verdict in one command. Pass the unchanged runner as repeated `--before-arg` +values and the changed runner after `--`. See +[comparing two versions in one command](https://dipeshbabu.github.io/agentloop/GUIDED_OPTIMIZATION/#compare-two-versions-in-one-command). + To connect your application, run `agentloop improve init --agent my_agent:answer` in its project directory. The generated adapter supports synchronous and async Python callables. Replace the example task inputs and expected outputs with diff --git a/agentloop/improvement.py b/agentloop/improvement.py index f627147..42f9cc9 100644 --- a/agentloop/improvement.py +++ b/agentloop/improvement.py @@ -1060,3 +1060,111 @@ def finish(session_path, candidate_path, root, *, change, verifier=None): export_report(root, report) return report + + +COMPARE_LAYOUT = { + "baseline": "baseline", + "session": "session", + "paired": "paired", + "results": "results", +} + + +def compare( + workload_path, + root, + before, + after, + *, + change, + timeout=180, + show_output=False, + focus=None, + objective="latency", + minimum_improvement=0, + decision="aggregate", + verifier=None, + verifier_timeout=DEFAULT_TIMEOUT, + max_runner_regression_ms=None, + progress=None, +): + """Run the whole guided comparison for an unchanged and a changed runner. + + The ``before`` runner is captured as the baseline, the session is frozen, + and the ``after`` runner is then captured with ``before`` as an + interleaved control. ``before`` must therefore stay runnable and unchanged, + for example from a separate git worktree. Every stage writes its usual + artifacts under ``root`` (see ``COMPARE_LAYOUT``), so a failed run can be + inspected or resumed with the individual commands. + """ + notify = progress or (lambda message: None) + # Check cheap inputs before any runner executes, so a typo cannot waste a + # full baseline capture. + require(isinstance(change, str) and bool(change.strip()), "Describe the change being compared") + for argv, side in ((before, "before"), (after, "after")): + require( + isinstance(argv, (list, tuple)) + and argv + and all(isinstance(a, str) and a for a in argv), + f"Supply an explicit {side} runner command", + ) + require(list(before) != list(after), "The before and after runners must differ") + require(objective in {"latency", "cost"}, "Objective must be latency or cost") + decision = decision.replace("-", "_") if isinstance(decision, str) else decision + require(decision in DECISIONS, "Decision must be per-pair or aggregate") + _runner_budget(max_runner_regression_ms) + require( + type(minimum_improvement) in (int, float) + and math.isfinite(minimum_improvement) + and minimum_improvement >= 0, + "Minimum improvement must be finite and nonnegative", + ) + if verifier is not None: + LocalVerifier.read(verifier, verifier_timeout) + read_workload(workload_path) + root = Path(root).resolve() + root.mkdir(parents=True, exist_ok=False) + paths = {name: local_path(root, relative) for name, relative in COMPARE_LAYOUT.items()} + + notify("Capturing the baseline with the before runner") + baseline = capture( + workload_path, paths["baseline"], list(before), timeout=timeout, show_output=show_output + ) + completed = sum(row["status"] == "completed" for row in baseline["runs"]) + require( + completed > 0, + "No baseline attempt completed. Fix the before runner, using " + f"{paths['baseline'] / 'capture.json'} and --show-runner-output, then rerun.", + ) + notify( + f"Baseline captured {completed} of {len(baseline['runs'])} attempts; freezing the session" + ) + start( + workload_path, + paths["baseline"] / "capture.json", + paths["session"], + focus=focus, + objective=objective, + minimum_improvement=minimum_improvement, + decision=decision, + verifier=verifier, + verifier_timeout=verifier_timeout, + max_runner_regression_ms=max_runner_regression_ms, + ) + notify("Capturing the after runner, alternating with the before runner as a control") + capture( + workload_path, + paths["paired"], + list(after), + timeout=timeout, + show_output=show_output, + control=list(before), + ) + notify("Comparing every planned pair") + return finish( + paths["session"] / "session.json", + paths["paired"] / "capture.json", + paths["results"], + change=change, + verifier=verifier, + ) diff --git a/agentloop/improvement_cli.py b/agentloop/improvement_cli.py index efbfe9c..ee99514 100644 --- a/agentloop/improvement_cli.py +++ b/agentloop/improvement_cli.py @@ -173,6 +173,11 @@ def finish_command( report = improvement.finish(session, candidate, out, change=change, verifier=verifier) except (OSError, ValueError, TypeError, KeyError) as exc: raise typer.BadParameter(str(exc)) from exc + _print_report(report, out) + + +def _print_report(report, out): + """Print the verdict and timing summary; exit 1 when the change is rejected.""" verdict = report["decision"] if verdict["mode"] == "aggregate": console.print(_headline(report), markup=False) @@ -213,3 +218,70 @@ def finish_command( ) if not report["passed"]: raise typer.Exit(1) + + +@improve_app.command("compare") +def compare_command( + tasks: Path | None = typer.Option(None, help="Fixed tasks, overriding the project config."), + project: Path = typer.Option( + Path("agentloop.toml"), help="Data-only project config used when --tasks is omitted." + ), + out: Path = typer.Option(..., help="Fresh directory for every stage of the comparison."), + before_arg: list[str] = typer.Option( + ..., + "--before-arg", + help="One argument of the unchanged before runner; repeat for each argument. " + "It runs the baseline and then alternates with the after runner as a control.", + ), + change: str = typer.Option(..., help="Describe the change the after runner applies."), + timeout: float = typer.Option(180, min=0.01, max=3600, help="Seconds per task attempt."), + show_runner_output: bool = typer.Option( + False, help="Stream the runners' stdout and stderr for diagnosis." + ), + focus: str | None = typer.Option( + None, help="Finding type to investigate. Defaults to the first baseline finding." + ), + objective: str = typer.Option("latency", help="latency or cost."), + minimum_improvement: float = typer.Option( + 0, min=0, help="Required percentage gain in the selected objective." + ), + max_runner_regression_ms: float | None = typer.Option( + None, min=0, help="Optional maximum runner wall-time increase in ms for each pair." + ), + decision: str = typer.Option( + "aggregate", + help="aggregate (default here) judges the median paired improvement; per-pair " + "requires every pair to improve. Correctness is checked per pair in both.", + ), + verifier: Path | None = typer.Option(None, help="Explicit trusted local Python verifier file."), + verifier_timeout: float = typer.Option( + DEFAULT_TIMEOUT, min=0.01, max=3600, help="Seconds per verifier execution." + ), + command: list[str] = typer.Argument( + ..., help="Explicit after-runner argv after --. Receives a JSON request on stdin." + ), +): + """Capture, freeze, interleave and compare an unchanged and a changed runner.""" + try: + report = improvement.compare( + tasks_path(tasks, project), + out, + before_arg, + command, + change=change, + timeout=timeout, + show_output=show_runner_output, + focus=focus, + objective=objective, + minimum_improvement=minimum_improvement, + decision=decision, + verifier=verifier, + verifier_timeout=verifier_timeout, + max_runner_regression_ms=max_runner_regression_ms, + progress=lambda message: console.print(message, markup=False), + ) + except (OSError, ValueError, TypeError, KeyError) as exc: + raise typer.BadParameter(str(exc)) from exc + session = out / improvement.COMPARE_LAYOUT["session"] + console.print(f"Baseline findings and plan: {session / 'plan.md'}", markup=False) + _print_report(report, out / improvement.COMPARE_LAYOUT["results"]) diff --git a/docs/GUIDED_OPTIMIZATION.md b/docs/GUIDED_OPTIMIZATION.md index 184160a..dd24bb5 100644 --- a/docs/GUIDED_OPTIMIZATION.md +++ b/docs/GUIDED_OPTIMIZATION.md @@ -126,6 +126,41 @@ The request contains `task_id`, zero-based `repetition`, `input`, and an absolut in `metadata.output` so the quality scorer can evaluate it. Existing native model outputs are also supported through the usual quality-report lookup. +## Compare two versions in one command + +When you already have the change, `improve compare` runs the whole workflow +below in one command. It needs AgentLoop 0.2.5 or newer. Keep the unchanged +agent runnable, for example in a separate `git worktree`, and pass its command +as repeated `--before-arg` values. Put the changed agent's command after `--`: + +```bash +git worktree add ../agent-before HEAD +agentloop improve compare --tasks tasks.json --out runs/compare --change "Describe the change" --before-arg python --before-arg ../agent-before/your_agent.py -- python your_agent.py +``` + +`compare` runs these stages in order: + +1. It captures the baseline with the before runner in `runs/compare/baseline/`. +2. It freezes the findings and checks in `runs/compare/session/`. Review its + `plan.md` afterwards. +3. It captures the after runner, alternating with the before runner as an + [interleaved control](#compare-against-an-interleaved-control), in + `runs/compare/paired/`. +4. It writes the verdict to `runs/compare/results/`. + +It checks the change description, decision, objective and verifier before any +runner executes. It stops after the baseline when no baseline attempt completed. +Unlike `start`, it defaults to `--decision aggregate`, which suits agents whose +timing varies between runs. Pass `--decision per-pair` to require every pair to +improve. `--focus`, `--objective`, `--minimum-improvement`, +`--max-runner-regression-ms`, `--verifier` and `--timeout` work as they do for +the individual commands. Use at least two repetitions per task, and an even +number so that each runner goes first equally often. + +Because the session is frozen after the baseline runs, `compare` cannot let the +plan inform the change. To investigate findings before writing a change, use +the individual commands below. + ## Capture and review the baseline ```bash diff --git a/tests/test_improvement.py b/tests/test_improvement.py index 9327b64..cd6ed58 100644 --- a/tests/test_improvement.py +++ b/tests/test_improvement.py @@ -9,7 +9,7 @@ from agentloop.entrypoint import app from agentloop.experiment_types import fingerprint -from agentloop.improvement import capture, finish, read_workload, start +from agentloop.improvement import capture, compare, finish, read_workload, start WORKER = """import json,sys,time from pathlib import Path @@ -796,3 +796,105 @@ def test_cli_output_survives_unicode_names_on_a_legacy_code_page(tmp_path): assert result.returncode == 1, result.stderr.decode("utf-8", "replace") assert "UnicodeEncodeError" not in result.stderr.decode("utf-8", "replace") assert "检查-✓ repetition 0" in output + + +def test_compare_runs_every_stage_with_an_interleaved_control(tmp_path, workload): + tasks, worker = workload + messages = [] + report = compare( + tasks, + tmp_path / "compare", + command(worker, "baseline"), + command(worker, "fast"), + change="Remove repeated work", + progress=messages.append, + ) + assert report["passed"] is True + assert report["comparison_baseline"] == "interleaved_control" + assert report["decision"]["mode"] == "aggregate" + for stage in ("baseline/capture.json", "session/session.json", "session/plan.md"): + assert (tmp_path / "compare" / stage).is_file() + assert (tmp_path / "compare/paired/control/capture.json").is_file() + assert (tmp_path / "compare/results/report.html").is_file() + assert len(messages) == 4 + + +def test_compare_rejects_a_wrong_after_runner(tmp_path, workload): + tasks, worker = workload + report = compare( + tasks, + tmp_path / "compare", + command(worker, "baseline"), + command(worker, "wrong"), + change="Faster but wrong", + decision="per-pair", + ) + assert report["passed"] is False + assert report["decision"]["mode"] == "per_pair" + + +@pytest.mark.parametrize( + ("override", "message"), + [ + ({"change": " "}, "Describe the change"), + ({"after": None}, "before and after runners must differ"), + ({"decision": "median"}, "per-pair or aggregate"), + ({"objective": "speed"}, "latency or cost"), + ({"minimum_improvement": -1}, "nonnegative"), + ], +) +def test_compare_checks_inputs_before_running_anything(tmp_path, workload, override, message): + tasks, worker = workload + before = command(worker, "baseline") + arguments = { + "change": "Change", + "after": command(worker, "fast"), + "decision": "aggregate", + "objective": "latency", + "minimum_improvement": 0, + } + arguments.update(override) + if arguments["after"] is None: + arguments["after"] = list(before) + after = arguments.pop("after") + with pytest.raises(ValueError, match=message): + compare(tasks, tmp_path / "compare", before, after, **arguments) + assert not (tmp_path / "compare").exists() + + +def test_compare_stops_when_no_baseline_attempt_completes(tmp_path, workload): + tasks, worker = workload + with pytest.raises(ValueError, match="No baseline attempt completed"): + compare( + tasks, + tmp_path / "compare", + command(worker, "failure"), + command(worker, "fast"), + change="Change", + ) + assert (tmp_path / "compare/baseline/capture.json").is_file() + assert not (tmp_path / "compare/session").exists() + + +def test_cli_compare_accepts_repeated_before_arguments(tmp_path, workload): + tasks, worker = workload + result = CliRunner().invoke( + app, + [ + "improve", + "compare", + "--tasks", + str(tasks), + "--out", + str(tmp_path / "compare"), + *[item for arg in command(worker, "baseline") for item in ("--before-arg", arg)], + "--change", + "Remove repeated work", + "--", + *command(worker, "fast"), + ], + ) + assert result.exit_code == 0, result.output + assert "Capturing the baseline with the before runner" in result.output + assert "Aggregate" in result.output + assert (tmp_path / "compare/results/report.json").is_file()