From 1e4c102ae730579b7bc7097d1ad627b3e473b6c0 Mon Sep 17 00:00:00 2001 From: Dipesh Babu Date: Sat, 14 Nov 2026 18:45:32 -0500 Subject: [PATCH] Add interleaved control capture to the guided workflow improve capture --control-arg runs an unchanged control runner alongside the candidate, alternating which condition runs first on each attempt. improve finish then measures the candidate against that control while keeping findings, focus and gates frozen from the original baseline, so cache warm-up and host drift affect both conditions instead of favouring the later candidate run. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 + agentloop/improvement.py | 239 +++++++++++++++++++++++--------- agentloop/improvement_cli.py | 21 ++- agentloop/improvement_report.py | 15 ++ docs/ADOPTION_GAPS.md | 2 +- docs/GUIDED_OPTIMIZATION.md | 31 ++++- tests/test_improvement.py | 100 +++++++++++++ 7 files changed, 338 insertions(+), 75 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 091dac8..5812c66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ trace, study, receipt, API and database schema versions. `typescript-vX.Y.Z` tags on main, after Node.js 22/24 build and tests, using trusted publishing with provenance. The SDK installs with `npm install agentloop-profiler`. +- `agentloop improve capture --control-arg ...` captures an unchanged control + runner alongside the candidate, alternating which runs first on each attempt. + `improve finish` then measures the candidate against that control, keeping + findings frozen from the original baseline, so cache warm-up and host drift + affect both conditions. ## [0.2.1] - 2026-11-12 diff --git a/agentloop/improvement.py b/agentloop/improvement.py index d48b893..78431e1 100644 --- a/agentloop/improvement.py +++ b/agentloop/improvement.py @@ -185,33 +185,21 @@ def _invoke(command, request, timeout, *, show_output=False): raise -def capture(workload_path, root, command, *, timeout=180, show_output=False): - workload = read_workload(workload_path) - require( - command and all(isinstance(arg, str) and arg for arg in command), - "Supply an explicit runner command after --", - ) - require( - type(timeout) in (int, float) and math.isfinite(timeout) and 0 < timeout <= 3600, - "Task timeout must be positive and at most 3600 seconds", - ) - root = Path(root).resolve() - root.mkdir(parents=True, exist_ok=False) - write_once(root, "workload.json", workload) +CONTROL_CAPTURE = "control/capture.json" + + +def _new_manifest(workload, command): scripts = [ {"argument_index": index, "sha256": hashlib.sha256(Path(argument).read_bytes()).hexdigest()} for index, argument in enumerate(command) if Path(argument).suffix in {".py", ".js", ".mjs", ".ts"} and Path(argument).is_file() ] - manifest = { + return { "schema_version": VERSION, "created_at": now(), "workload_hash": fingerprint(workload), "runner_hash": fingerprint({"argv": command, "scripts": scripts}), - "runs": [], - } - for index, (task, repetition) in enumerate(slots(workload)): - manifest["runs"].append( + "runs": [ { "task_id": task["id"], "repetition": repetition, @@ -220,46 +208,104 @@ def capture(workload_path, root, command, *, timeout=180, show_output=False): "trace": None, "trace_sha256": None, } - ) + for task, repetition in slots(workload) + ], + } + + +def _attempt(root, index, row, task, command, timeout, show_output): + relative = f"traces/{index:04d}.json" + path = local_path(root, relative) + path.parent.mkdir(exist_ok=True) + request = { + "task_id": row["task_id"], + "repetition": row["repetition"], + "input": task["input"], + "trace_path": str(path), + } + row["started_at"] = now() + try: + row.update(_invoke(command, request, timeout, show_output=show_output)) + except KeyboardInterrupt: + row["status"] = "cancelled" + raise + finally: + row["ended_at"] = now() + if path.exists(): + try: + _, digest = trace_receipt(root, relative) + row.update(trace=relative, trace_sha256=digest) + except (ValueError, OSError, TypeError, KeyError, AttributeError) as error: + # Keep the more specific failed, timed-out or cancelled outcome. + if row["status"] == "completed": + row["status"] = "invalid_trace" + row["diagnostic"] = ( + (row.get("diagnostic") or "") + + f" Trace {relative} is invalid: {error}. Export a finalized native trace to request['trace_path']." + ) + elif row["status"] == "completed": + row["status"] = "missing_trace" + row["diagnostic"] = ( + "Runner exited successfully without a trace. Export the finalized trace to request['trace_path']." + ) + write_once(root, f"receipts/{index:04d}.json", row) + + +def capture(workload_path, root, command, *, timeout=180, show_output=False, control=None): + """Run every planned attempt, optionally alternating with an unchanged control. + + With ``control``, each task attempt runs both the control and the candidate + back to back, and the condition that goes first alternates between attempts. + Cache warm-up, host load and other drift then affect both conditions instead + of favouring whichever condition ran later. + """ + workload = read_workload(workload_path) + require( + command and all(isinstance(arg, str) and arg for arg in command), + "Supply an explicit runner command after --", + ) + require( + control is None or (control and all(isinstance(arg, str) and arg for arg in control)), + "Supply the unchanged control runner as repeated --control-arg values", + ) + require( + type(timeout) in (int, float) and math.isfinite(timeout) and 0 < timeout <= 3600, + "Task timeout must be positive and at most 3600 seconds", + ) + root = Path(root).resolve() + root.mkdir(parents=True, exist_ok=False) + write_once(root, "workload.json", workload) + manifest = _new_manifest(workload, command) + conditions = [(root, manifest, command)] + control_manifest = None + if control is not None: + control_root = local_path(root, "control") + control_root.mkdir() + write_once(control_root, "workload.json", workload) + control_manifest = _new_manifest(workload, control) + control_manifest["schedule"] = {"mode": "interleaved", "role": "control"} + manifest["schedule"] = { + "mode": "interleaved", + "role": "candidate", + "pattern": "Control runs first on even attempt indexes and second on odd ones", + } + conditions.append((control_root, control_manifest, control)) try: for index, row in enumerate(manifest["runs"]): task = next(task for task in workload["tasks"] if task["id"] == row["task_id"]) - relative = f"traces/{index:04d}.json" - path = local_path(root, relative) - path.parent.mkdir(exist_ok=True) - request = { - "task_id": row["task_id"], - "repetition": row["repetition"], - "input": task["input"], - "trace_path": str(path), - } - row["started_at"] = now() - try: - row.update(_invoke(command, request, timeout, show_output=show_output)) - except KeyboardInterrupt: - row["status"] = "cancelled" - raise - finally: - row["ended_at"] = now() - if path.exists(): - try: - _, digest = trace_receipt(root, relative) - row.update(trace=relative, trace_sha256=digest) - except (ValueError, OSError, TypeError, KeyError, AttributeError) as error: - # Keep the more specific failed, timed-out or cancelled outcome. - if row["status"] == "completed": - row["status"] = "invalid_trace" - row["diagnostic"] = ( - (row.get("diagnostic") or "") - + f" Trace {relative} is invalid: {error}. Export a finalized native trace to request['trace_path']." - ) - elif row["status"] == "completed": - row["status"] = "missing_trace" - row["diagnostic"] = ( - "Runner exited successfully without a trace. Export the finalized trace to request['trace_path']." - ) - write_once(root, f"receipts/{index:04d}.json", row) + order = conditions if index % 2 else conditions[::-1] + for position, (condition_root, condition, argv) in enumerate(order): + attempt = condition["runs"][index] + if control is not None: + attempt["order"] = position + _attempt(condition_root, index, attempt, task, argv, timeout, show_output) finally: + if control_manifest is not None: + write_once(root, CONTROL_CAPTURE, control_manifest) + manifest["control"] = { + "capture": CONTROL_CAPTURE, + "manifest_hash": fingerprint(control_manifest), + } write_once(root, "capture.json", manifest) return manifest @@ -472,10 +518,31 @@ def finish(session_path, candidate_path, root, *, change): ) baseline_ids.add(trace.run_id) baselines[key] = trace + candidate_ids = {trace.run_id for trace in candidates.values()} require( - not baseline_ids.intersection(trace.run_id for trace in candidates.values()), + not baseline_ids.intersection(candidate_ids), "Baseline and candidate must have distinct run IDs", ) + controls = control_manifest = None + if "control" in candidate_manifest: + reference = candidate_manifest["control"] + require( + isinstance(reference, dict) and reference.get("capture") == CONTROL_CAPTURE, + "Invalid interleaved control reference", + ) + control_manifest, controls = read_capture( + Path(candidate_path).resolve().parent / CONTROL_CAPTURE, workload + ) + require( + fingerprint(control_manifest) == reference.get("manifest_hash"), + "Interleaved control capture changed", + ) + require( + not (baseline_ids | candidate_ids).intersection( + trace.run_id for trace in controls.values() + ), + "Control, baseline and candidate must have distinct run IDs", + ) gates = ReplayGates(**specification["gates"]) root = Path(root).resolve() root.mkdir(parents=True, exist_ok=False) @@ -483,14 +550,40 @@ def finish(session_path, candidate_path, root, *, change): for index, case in enumerate(specification["cases"]): key = (case["task_id"], case["repetition"]) task = next(task for task in workload["tasks"] if task["id"] == key[0]) - baseline, candidate = baselines.get(key), candidates.get(key) + candidate = candidates.get(key) candidate_row = next( row for row in candidate_manifest["runs"] if (row["task_id"], row["repetition"]) == key ) + if controls is None: + label, baseline = "Baseline", baselines.get(key) + baseline_status = case["capture_status"] + baseline_diagnostic = case.get("capture_diagnostic") + diagnosis, finding_ids = case["diagnosis"], case["finding_ids"] + else: + # Findings stay frozen from the original baseline. Measurements compare + # the candidate with the unchanged control run alongside it. + label, baseline = "Control", controls.get(key) + control_row = next( + row + for row in control_manifest["runs"] + if (row["task_id"], row["repetition"]) == key + ) + baseline_status = control_row["status"] + baseline_diagnostic = control_row.get("diagnostic") + diagnosis = build_diagnosis(baseline) if baseline is not None else None + finding_ids = ( + [ + finding["finding_id"] + for finding in diagnosis["findings"] + if finding["type"] == specification["focus"] + ] + if diagnosis and case["finding_ids"] + else [] + ) row = { "task_id": key[0], "repetition": key[1], - "baseline_status": case["capture_status"], + "baseline_status": baseline_status, "candidate_status": candidate_row["status"], "passed": False, "reasons": [], @@ -498,10 +591,15 @@ def finish(session_path, candidate_path, root, *, change): "replay": None, "intervention": None, } + if controls is not None: + row.update( + frozen_baseline_status=case["capture_status"], + control_ran_first=control_row.get("order") == 0, + ) if baseline is None or candidate is None: - row["reasons"].append("A planned baseline or candidate trace is unavailable") + row["reasons"].append(f"A planned {label.lower()} or candidate trace is unavailable") for side, status, diagnostic in ( - ("Baseline", case["capture_status"], case.get("capture_diagnostic")), + (label, baseline_status, baseline_diagnostic), ("Candidate", candidate_row["status"], candidate_row.get("diagnostic")), ): if status != "completed": @@ -512,7 +610,7 @@ def finish(session_path, candidate_path, root, *, change): quality = _quality(task, baseline, candidate) replay = build_replay_report(baseline, candidate, gates=gates, quality_report=quality) if ( - case["capture_status"] != "completed" + baseline_status != "completed" or candidate_row["status"] != "completed" or not all(_trace_usable(trace) for trace in (baseline, candidate)) ): @@ -523,13 +621,13 @@ def finish(session_path, candidate_path, root, *, change): for side in ("baseline", "candidate") ): row["reasons"].append( - "Baseline and candidate must both pass the declared quality checks" + f"{label} and candidate must both pass the declared quality checks" ) for result in quality["cases"]: for side in ("baseline", "candidate"): if not result[side]["passed"]: row["reasons"].append( - f"{side.title()} answer failed the declared {task.get('scorer', {}).get('type', 'exact_match')} criterion: {result[side]['detail']}" + f"{label if side == 'baseline' else 'Candidate'} answer failed the declared {task.get('scorer', {}).get('type', 'exact_match')} criterion: {result[side]['detail']}" ) if not replay["gates"]["passed"]: row["reasons"].append("Native replay gates failed") @@ -538,7 +636,7 @@ def finish(session_path, candidate_path, root, *, change): for gate in replay["gates"]["results"] if not gate["passed"] ) - if not case["diagnosis"].get("analysis_complete", True) or not candidate.report().get( + if not diagnosis.get("analysis_complete", True) or not candidate.report().get( "analysis_complete", True ): row["reasons"].append("Finding analysis is incomplete") @@ -567,14 +665,14 @@ def finish(session_path, candidate_path, root, *, change): cost_evaluable=replay["gates"]["cost_evaluable"], pricing_known=replay["gates"]["pricing_known"], ) - if case["finding_ids"]: + if finding_ids: record = build_intervention( baseline, candidate, - target_finding_ids=case["finding_ids"], + target_finding_ids=finding_ids, intervention_type=specification["focus"], configuration={"change": change}, - diagnosis=case["diagnosis"], + diagnosis=diagnosis, gates=gates, quality_report=quality, replay_report=replay, @@ -626,12 +724,19 @@ def finish(session_path, candidate_path, root, *, change): "candidate_capture_hash": fingerprint(candidate_manifest), "baseline_runner_hash": specification["baseline_capture"].get("runner_hash"), "candidate_runner_hash": candidate_manifest.get("runner_hash"), + "comparison_baseline": "frozen_baseline" if controls is None else "interleaved_control", + "control_capture_hash": None if controls is None else fingerprint(control_manifest), + "control_runner_hash": None if controls is None else control_manifest.get("runner_hash"), "planned_attempts": len(rows), "passed_attempts": sum(row["passed"] for row in rows), "passed": all(row["passed"] for row in rows), "synthetic": any( trace.metadata.get("synthetic") is True - for trace in [*baselines.values(), *candidates.values()] + for trace in [ + *baselines.values(), + *candidates.values(), + *(controls or {}).values(), + ] ), "cases": rows, "study": study, diff --git a/agentloop/improvement_cli.py b/agentloop/improvement_cli.py index 25845b6..f1f52db 100644 --- a/agentloop/improvement_cli.py +++ b/agentloop/improvement_cli.py @@ -49,6 +49,12 @@ def capture_command( show_runner_output: bool = typer.Option( False, help="Stream the runner's stdout and stderr for diagnosis." ), + control_arg: list[str] | None = typer.Option( + None, + "--control-arg", + help="One argument of the unchanged control runner; repeat for each argument. " + "Alternates control and candidate attempts so drift affects both.", + ), command: list[str] = typer.Argument( ..., help="Explicit runner argv after --. Receives a JSON request on stdin." ), @@ -61,19 +67,24 @@ def capture_command( command, timeout=timeout, show_output=show_runner_output, + control=control_arg or None, ) except (OSError, ValueError, TypeError, KeyError) as exc: raise typer.BadParameter(str(exc)) from exc - complete = sum(row["status"] == "completed" for row in manifest["runs"]) - console.print(f"Captured {complete} of {len(manifest['runs'])} planned attempts.", markup=False) + attempts = [("", row) for row in manifest["runs"]] + if control_arg: + control = improvement.load_json(out.resolve() / improvement.CONTROL_CAPTURE) + attempts += [("Control: ", row) for row in control["runs"]] + complete = sum(row["status"] == "completed" for _, row in attempts) + console.print(f"Captured {complete} of {len(attempts)} planned attempts.", markup=False) console.print(f"Capture manifest written to {out / 'capture.json'}", markup=False) - for attempt in manifest["runs"]: + for prefix, attempt in attempts: if attempt["status"] != "completed": console.print( - f"{attempt['task_id']} repetition {attempt['repetition']} — {attempt['status']}. {attempt.get('diagnostic') or 'Inspect the runner with --show-runner-output.'}", + f"{prefix}{attempt['task_id']} repetition {attempt['repetition']} — {attempt['status']}. {attempt.get('diagnostic') or 'Inspect the runner with --show-runner-output.'}", markup=False, ) - if complete != len(manifest["runs"]): + if complete != len(attempts): raise typer.Exit(1) diff --git a/agentloop/improvement_report.py b/agentloop/improvement_report.py index f3f8cfc..74e4da9 100644 --- a/agentloop/improvement_report.py +++ b/agentloop/improvement_report.py @@ -170,6 +170,18 @@ def _metric(summary, name): return f"{value:.3f}" if isinstance(value, float) else str(value) +def _basis(report): + if report.get("comparison_baseline") == "interleaved_control": + return ( + "Measurements compare the candidate with an unchanged control captured in " + "alternating order. Findings remain frozen from the original baseline." + ) + return ( + "Measurements compare the candidate with the frozen baseline captured before " + "the change. Run order is not counterbalanced." + ) + + def report_markdown(report): lines = [ f"# {markdown_heading(report['name'])}", @@ -184,6 +196,8 @@ def report_markdown(report): "", markdown_text(report["change"]), "", + _basis(report), + "", ] if report["synthetic"]: lines.extend( @@ -248,6 +262,7 @@ def export_report(root, report): sections = [ f"

{_text(report['name'])}

{report['passed_attempts']} of {report['planned_attempts']} planned attempts passed.

", f"

Applied change

{_text(report['change'])}

", + f"

{_text(_basis(report))}

", ] if report["synthetic"]: sections.append( diff --git a/docs/ADOPTION_GAPS.md b/docs/ADOPTION_GAPS.md index d5c2eb3..f6d0b0e 100644 --- a/docs/ADOPTION_GAPS.md +++ b/docs/ADOPTION_GAPS.md @@ -10,7 +10,7 @@ work prioritizes this path over adding another control or experiment system. | Representative agent work | Small Hermes repository questions, older model/tool studies, and a reproducible three-workload guided suite | Collect coding, retrieval and tool-heavy task sets from actual applications, with owners' independent acceptance criteria | | Recommendation precision | Native evidence/risk rankings, explicit assumptions, retained false positives and rejected changes | Review recommendations against independent labels; report unsupported recommendations and abstentions alongside accepted ones | | Quality coverage | Frozen task inputs and declared per-task checks; a wrong fast candidate cannot pass those checks | Expand adversarial and held-out tasks; measure quality dimensions beyond exact structured answers | -| Causal performance | Per-attempt measurements and disclosed fixed-order confounds | Counterbalance conditions, control cache/load effects, and increase independent task/host/model coverage | +| Causal performance | Per-attempt measurements, disclosed fixed-order confounds, and an optional interleaved control that alternates condition order | Use interleaved controls in real studies, control cache/load effects further, and increase independent task/host/model coverage | | Instrumentation overhead | Counterbalanced CPU controls separate recording, JSON export and native analysis at declared span volumes; model controls retain quality failures | Repeat controls around actual application execution, across hosts and realistic payload sizes | | Cost | Actual token provenance and qualified model-price estimates | Obtain authorized billing data; retain unknown local hardware/energy cost instead of inventing dollars | | Hermes compatibility | 0.21.6 contract coverage and real 0.22.7/0.23.1 conformance | Revalidate future hook changes against the pinned runtime scenarios before extending the supported range | diff --git a/docs/GUIDED_OPTIMIZATION.md b/docs/GUIDED_OPTIMIZATION.md index db88aa5..574996b 100644 --- a/docs/GUIDED_OPTIMIZATION.md +++ b/docs/GUIDED_OPTIMIZATION.md @@ -169,6 +169,32 @@ cost stays unavailable. A latency comparison can pass with an indeterminate, non-gating cost estimate. A required cost improvement fails when pricing or usable usage is missing. +## Compare against an interleaved control + +A plain candidate capture runs after the baseline, so cache warm-up, host load or +provider drift can look like an improvement. When you can keep the unchanged +agent runnable, for example in a separate `git worktree`, capture it as a control +alongside the candidate: + +```bash +git worktree add ../agent-before HEAD +agentloop improve capture --tasks tasks.json --out runs/paired --control-arg python --control-arg ../agent-before/your_agent.py -- python your_agent.py +agentloop improve finish --session runs/optimization/session.json --candidate runs/paired/capture.json --change "Describe the change applied to the agent" --out runs/results +``` + +Repeat `--control-arg` once for each argument of the control runner. Each +planned attempt runs the control and the candidate back to back. The control +runs first on even attempt indexes and second on odd ones. Use an even number +of attempts, through `repetitions`, so each condition runs first equally often. +The control capture is stored in `runs/paired/control/` and its manifest hash is +recorded in the candidate capture. + +`finish` then measures the candidate against the control instead of the frozen +baseline. Findings, the focus and the gates stay frozen from the original +baseline. The control must pass the same quality checks. Intervention records +link the control's matching findings. The report states which comparison basis +it used, and each case records whether the control ran first. + ## Failures, evidence and execution `capture` retains every planned attempt, including failed, timed-out, cancelled, @@ -194,8 +220,9 @@ default. Pass `--show-runner-output` to stream them when diagnosing a failing ru Raw task inputs, expected answers, traces, and outputs are retained in the local artifacts. Review them before sharing. HTML escapes supplied text and shows metrics and decisions. A fixed-order comparison describes the supplied tasks and -run conditions. Repeat across independent tasks and inspect variability before -generalizing a result. +run conditions. An interleaved control reduces order effects but does not remove +other differences between conditions. Repeat across independent tasks and +inspect variability before generalizing a result. ## Reproduce the Hermes example with local inference diff --git a/tests/test_improvement.py b/tests/test_improvement.py index f895da4..69d1f65 100644 --- a/tests/test_improvement.py +++ b/tests/test_improvement.py @@ -341,3 +341,103 @@ def test_rehashed_sessions_still_validate_the_full_attempt_inventory(tmp_path, w change="Invalid inventory", ) assert not (tmp_path / "results").exists() + + +def test_interleaved_control_alternates_order_and_becomes_the_comparison(tmp_path, workload): + tasks, worker = prepare(tmp_path, workload) + manifest = capture( + tasks, + tmp_path / "paired", + command(worker, "fast"), + control=command(worker, "baseline"), + ) + control = json.loads((tmp_path / "paired/control/capture.json").read_bytes()) + assert manifest["control"]["manifest_hash"] == fingerprint(control) + assert [row["order"] for row in control["runs"]] == [0, 1, 0, 1] + assert [row["order"] for row in manifest["runs"]] == [1, 0, 1, 0] + for index, (first, second) in enumerate(zip(control["runs"], manifest["runs"])): + ordered = (first, second) if index % 2 == 0 else (second, first) + assert ordered[0]["ended_at"] <= ordered[1]["started_at"] + report = finish( + tmp_path / "session/session.json", + tmp_path / "paired/capture.json", + tmp_path / "results", + change="Remove repeated work", + ) + assert report["passed"] is True + assert report["comparison_baseline"] == "interleaved_control" + assert report["control_capture_hash"] == fingerprint(control) + control_ids = { + json.loads((tmp_path / "paired/control" / row["trace"]).read_bytes())["run_id"] + for row in control["runs"] + } + for row in report["cases"]: + assert row["replay"]["baseline"]["run_id"] in control_ids + assert row["intervention"]["baseline_run_id"] in control_ids + assert row["frozen_baseline_status"] == "completed" + assert [row["control_ran_first"] for row in report["cases"]] == [True, False, True, False] + assert "alternating order" in (tmp_path / "results/report.md").read_text(encoding="utf-8") + + +def test_interleaved_control_must_also_pass_quality_checks(tmp_path, workload): + tasks, worker = prepare(tmp_path, workload) + capture( + tasks, + tmp_path / "paired", + command(worker, "fast"), + control=command(worker, "wrong"), + ) + report = finish( + tmp_path / "session/session.json", + tmp_path / "paired/capture.json", + tmp_path / "results", + change="Wrong control", + ) + assert report["passed"] is False + assert all( + any(reason.startswith("Control answer failed") for reason in row["reasons"]) + for row in report["cases"] + ) + + +def test_modified_interleaved_control_is_rejected(tmp_path, workload): + tasks, worker = prepare(tmp_path, workload) + capture( + tasks, + tmp_path / "paired", + command(worker, "fast"), + control=command(worker, "baseline"), + ) + path = tmp_path / "paired/control/capture.json" + control = json.loads(path.read_bytes()) + control["runner_hash"] = "changed" + path.write_text(json.dumps(control), encoding="utf-8") + with pytest.raises(ValueError, match="control capture changed"): + finish( + tmp_path / "session/session.json", + tmp_path / "paired/capture.json", + tmp_path / "results", + change="Tampered control", + ) + assert not (tmp_path / "results").exists() + + +def test_cli_accepts_repeated_control_arguments(tmp_path, workload): + tasks, worker = workload + result = CliRunner().invoke( + app, + [ + "improve", + "capture", + "--tasks", + str(tasks), + "--out", + str(tmp_path / "paired"), + *[item for arg in command(worker, "baseline") for item in ("--control-arg", arg)], + "--", + *command(worker, "fast"), + ], + ) + assert result.exit_code == 0, result.output + assert "Captured 8 of 8 planned attempts." in result.output + assert (tmp_path / "paired/control/capture.json").is_file()