diff --git a/CHANGELOG.md b/CHANGELOG.md
index 091dac8..5812c66 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,11 @@ trace, study, receipt, API and database schema versions.
`typescript-vX.Y.Z` tags on main, after Node.js 22/24 build and tests, using
trusted publishing with provenance. The SDK installs with
`npm install agentloop-profiler`.
+- `agentloop improve capture --control-arg ...` captures an unchanged control
+ runner alongside the candidate, alternating which runs first on each attempt.
+ `improve finish` then measures the candidate against that control, keeping
+ findings frozen from the original baseline, so cache warm-up and host drift
+ affect both conditions.
## [0.2.1] - 2026-11-12
diff --git a/agentloop/improvement.py b/agentloop/improvement.py
index d48b893..78431e1 100644
--- a/agentloop/improvement.py
+++ b/agentloop/improvement.py
@@ -185,33 +185,21 @@ def _invoke(command, request, timeout, *, show_output=False):
raise
-def capture(workload_path, root, command, *, timeout=180, show_output=False):
- workload = read_workload(workload_path)
- require(
- command and all(isinstance(arg, str) and arg for arg in command),
- "Supply an explicit runner command after --",
- )
- require(
- type(timeout) in (int, float) and math.isfinite(timeout) and 0 < timeout <= 3600,
- "Task timeout must be positive and at most 3600 seconds",
- )
- root = Path(root).resolve()
- root.mkdir(parents=True, exist_ok=False)
- write_once(root, "workload.json", workload)
+CONTROL_CAPTURE = "control/capture.json"
+
+
+def _new_manifest(workload, command):
scripts = [
{"argument_index": index, "sha256": hashlib.sha256(Path(argument).read_bytes()).hexdigest()}
for index, argument in enumerate(command)
if Path(argument).suffix in {".py", ".js", ".mjs", ".ts"} and Path(argument).is_file()
]
- manifest = {
+ return {
"schema_version": VERSION,
"created_at": now(),
"workload_hash": fingerprint(workload),
"runner_hash": fingerprint({"argv": command, "scripts": scripts}),
- "runs": [],
- }
- for index, (task, repetition) in enumerate(slots(workload)):
- manifest["runs"].append(
+ "runs": [
{
"task_id": task["id"],
"repetition": repetition,
@@ -220,46 +208,104 @@ def capture(workload_path, root, command, *, timeout=180, show_output=False):
"trace": None,
"trace_sha256": None,
}
- )
+ for task, repetition in slots(workload)
+ ],
+ }
+
+
+def _attempt(root, index, row, task, command, timeout, show_output):
+ relative = f"traces/{index:04d}.json"
+ path = local_path(root, relative)
+ path.parent.mkdir(exist_ok=True)
+ request = {
+ "task_id": row["task_id"],
+ "repetition": row["repetition"],
+ "input": task["input"],
+ "trace_path": str(path),
+ }
+ row["started_at"] = now()
+ try:
+ row.update(_invoke(command, request, timeout, show_output=show_output))
+ except KeyboardInterrupt:
+ row["status"] = "cancelled"
+ raise
+ finally:
+ row["ended_at"] = now()
+ if path.exists():
+ try:
+ _, digest = trace_receipt(root, relative)
+ row.update(trace=relative, trace_sha256=digest)
+ except (ValueError, OSError, TypeError, KeyError, AttributeError) as error:
+ # Keep the more specific failed, timed-out or cancelled outcome.
+ if row["status"] == "completed":
+ row["status"] = "invalid_trace"
+ row["diagnostic"] = (
+ (row.get("diagnostic") or "")
+ + f" Trace {relative} is invalid: {error}. Export a finalized native trace to request['trace_path']."
+ )
+ elif row["status"] == "completed":
+ row["status"] = "missing_trace"
+ row["diagnostic"] = (
+ "Runner exited successfully without a trace. Export the finalized trace to request['trace_path']."
+ )
+ write_once(root, f"receipts/{index:04d}.json", row)
+
+
+def capture(workload_path, root, command, *, timeout=180, show_output=False, control=None):
+ """Run every planned attempt, optionally alternating with an unchanged control.
+
+ With ``control``, each task attempt runs both the control and the candidate
+ back to back, and the condition that goes first alternates between attempts.
+ Cache warm-up, host load and other drift then affect both conditions instead
+ of favouring whichever condition ran later.
+ """
+ workload = read_workload(workload_path)
+ require(
+ command and all(isinstance(arg, str) and arg for arg in command),
+ "Supply an explicit runner command after --",
+ )
+ require(
+ control is None or (control and all(isinstance(arg, str) and arg for arg in control)),
+ "Supply the unchanged control runner as repeated --control-arg values",
+ )
+ require(
+ type(timeout) in (int, float) and math.isfinite(timeout) and 0 < timeout <= 3600,
+ "Task timeout must be positive and at most 3600 seconds",
+ )
+ root = Path(root).resolve()
+ root.mkdir(parents=True, exist_ok=False)
+ write_once(root, "workload.json", workload)
+ manifest = _new_manifest(workload, command)
+ conditions = [(root, manifest, command)]
+ control_manifest = None
+ if control is not None:
+ control_root = local_path(root, "control")
+ control_root.mkdir()
+ write_once(control_root, "workload.json", workload)
+ control_manifest = _new_manifest(workload, control)
+ control_manifest["schedule"] = {"mode": "interleaved", "role": "control"}
+ manifest["schedule"] = {
+ "mode": "interleaved",
+ "role": "candidate",
+ "pattern": "Control runs first on even attempt indexes and second on odd ones",
+ }
+ conditions.append((control_root, control_manifest, control))
try:
for index, row in enumerate(manifest["runs"]):
task = next(task for task in workload["tasks"] if task["id"] == row["task_id"])
- relative = f"traces/{index:04d}.json"
- path = local_path(root, relative)
- path.parent.mkdir(exist_ok=True)
- request = {
- "task_id": row["task_id"],
- "repetition": row["repetition"],
- "input": task["input"],
- "trace_path": str(path),
- }
- row["started_at"] = now()
- try:
- row.update(_invoke(command, request, timeout, show_output=show_output))
- except KeyboardInterrupt:
- row["status"] = "cancelled"
- raise
- finally:
- row["ended_at"] = now()
- if path.exists():
- try:
- _, digest = trace_receipt(root, relative)
- row.update(trace=relative, trace_sha256=digest)
- except (ValueError, OSError, TypeError, KeyError, AttributeError) as error:
- # Keep the more specific failed, timed-out or cancelled outcome.
- if row["status"] == "completed":
- row["status"] = "invalid_trace"
- row["diagnostic"] = (
- (row.get("diagnostic") or "")
- + f" Trace {relative} is invalid: {error}. Export a finalized native trace to request['trace_path']."
- )
- elif row["status"] == "completed":
- row["status"] = "missing_trace"
- row["diagnostic"] = (
- "Runner exited successfully without a trace. Export the finalized trace to request['trace_path']."
- )
- write_once(root, f"receipts/{index:04d}.json", row)
+ order = conditions if index % 2 else conditions[::-1]
+ for position, (condition_root, condition, argv) in enumerate(order):
+ attempt = condition["runs"][index]
+ if control is not None:
+ attempt["order"] = position
+ _attempt(condition_root, index, attempt, task, argv, timeout, show_output)
finally:
+ if control_manifest is not None:
+ write_once(root, CONTROL_CAPTURE, control_manifest)
+ manifest["control"] = {
+ "capture": CONTROL_CAPTURE,
+ "manifest_hash": fingerprint(control_manifest),
+ }
write_once(root, "capture.json", manifest)
return manifest
@@ -472,10 +518,31 @@ def finish(session_path, candidate_path, root, *, change):
)
baseline_ids.add(trace.run_id)
baselines[key] = trace
+ candidate_ids = {trace.run_id for trace in candidates.values()}
require(
- not baseline_ids.intersection(trace.run_id for trace in candidates.values()),
+ not baseline_ids.intersection(candidate_ids),
"Baseline and candidate must have distinct run IDs",
)
+ controls = control_manifest = None
+ if "control" in candidate_manifest:
+ reference = candidate_manifest["control"]
+ require(
+ isinstance(reference, dict) and reference.get("capture") == CONTROL_CAPTURE,
+ "Invalid interleaved control reference",
+ )
+ control_manifest, controls = read_capture(
+ Path(candidate_path).resolve().parent / CONTROL_CAPTURE, workload
+ )
+ require(
+ fingerprint(control_manifest) == reference.get("manifest_hash"),
+ "Interleaved control capture changed",
+ )
+ require(
+ not (baseline_ids | candidate_ids).intersection(
+ trace.run_id for trace in controls.values()
+ ),
+ "Control, baseline and candidate must have distinct run IDs",
+ )
gates = ReplayGates(**specification["gates"])
root = Path(root).resolve()
root.mkdir(parents=True, exist_ok=False)
@@ -483,14 +550,40 @@ def finish(session_path, candidate_path, root, *, change):
for index, case in enumerate(specification["cases"]):
key = (case["task_id"], case["repetition"])
task = next(task for task in workload["tasks"] if task["id"] == key[0])
- baseline, candidate = baselines.get(key), candidates.get(key)
+ candidate = candidates.get(key)
candidate_row = next(
row for row in candidate_manifest["runs"] if (row["task_id"], row["repetition"]) == key
)
+ if controls is None:
+ label, baseline = "Baseline", baselines.get(key)
+ baseline_status = case["capture_status"]
+ baseline_diagnostic = case.get("capture_diagnostic")
+ diagnosis, finding_ids = case["diagnosis"], case["finding_ids"]
+ else:
+ # Findings stay frozen from the original baseline. Measurements compare
+ # the candidate with the unchanged control run alongside it.
+ label, baseline = "Control", controls.get(key)
+ control_row = next(
+ row
+ for row in control_manifest["runs"]
+ if (row["task_id"], row["repetition"]) == key
+ )
+ baseline_status = control_row["status"]
+ baseline_diagnostic = control_row.get("diagnostic")
+ diagnosis = build_diagnosis(baseline) if baseline is not None else None
+ finding_ids = (
+ [
+ finding["finding_id"]
+ for finding in diagnosis["findings"]
+ if finding["type"] == specification["focus"]
+ ]
+ if diagnosis and case["finding_ids"]
+ else []
+ )
row = {
"task_id": key[0],
"repetition": key[1],
- "baseline_status": case["capture_status"],
+ "baseline_status": baseline_status,
"candidate_status": candidate_row["status"],
"passed": False,
"reasons": [],
@@ -498,10 +591,15 @@ def finish(session_path, candidate_path, root, *, change):
"replay": None,
"intervention": None,
}
+ if controls is not None:
+ row.update(
+ frozen_baseline_status=case["capture_status"],
+ control_ran_first=control_row.get("order") == 0,
+ )
if baseline is None or candidate is None:
- row["reasons"].append("A planned baseline or candidate trace is unavailable")
+ row["reasons"].append(f"A planned {label.lower()} or candidate trace is unavailable")
for side, status, diagnostic in (
- ("Baseline", case["capture_status"], case.get("capture_diagnostic")),
+ (label, baseline_status, baseline_diagnostic),
("Candidate", candidate_row["status"], candidate_row.get("diagnostic")),
):
if status != "completed":
@@ -512,7 +610,7 @@ def finish(session_path, candidate_path, root, *, change):
quality = _quality(task, baseline, candidate)
replay = build_replay_report(baseline, candidate, gates=gates, quality_report=quality)
if (
- case["capture_status"] != "completed"
+ baseline_status != "completed"
or candidate_row["status"] != "completed"
or not all(_trace_usable(trace) for trace in (baseline, candidate))
):
@@ -523,13 +621,13 @@ def finish(session_path, candidate_path, root, *, change):
for side in ("baseline", "candidate")
):
row["reasons"].append(
- "Baseline and candidate must both pass the declared quality checks"
+ f"{label} and candidate must both pass the declared quality checks"
)
for result in quality["cases"]:
for side in ("baseline", "candidate"):
if not result[side]["passed"]:
row["reasons"].append(
- f"{side.title()} answer failed the declared {task.get('scorer', {}).get('type', 'exact_match')} criterion: {result[side]['detail']}"
+ f"{label if side == 'baseline' else 'Candidate'} answer failed the declared {task.get('scorer', {}).get('type', 'exact_match')} criterion: {result[side]['detail']}"
)
if not replay["gates"]["passed"]:
row["reasons"].append("Native replay gates failed")
@@ -538,7 +636,7 @@ def finish(session_path, candidate_path, root, *, change):
for gate in replay["gates"]["results"]
if not gate["passed"]
)
- if not case["diagnosis"].get("analysis_complete", True) or not candidate.report().get(
+ if not diagnosis.get("analysis_complete", True) or not candidate.report().get(
"analysis_complete", True
):
row["reasons"].append("Finding analysis is incomplete")
@@ -567,14 +665,14 @@ def finish(session_path, candidate_path, root, *, change):
cost_evaluable=replay["gates"]["cost_evaluable"],
pricing_known=replay["gates"]["pricing_known"],
)
- if case["finding_ids"]:
+ if finding_ids:
record = build_intervention(
baseline,
candidate,
- target_finding_ids=case["finding_ids"],
+ target_finding_ids=finding_ids,
intervention_type=specification["focus"],
configuration={"change": change},
- diagnosis=case["diagnosis"],
+ diagnosis=diagnosis,
gates=gates,
quality_report=quality,
replay_report=replay,
@@ -626,12 +724,19 @@ def finish(session_path, candidate_path, root, *, change):
"candidate_capture_hash": fingerprint(candidate_manifest),
"baseline_runner_hash": specification["baseline_capture"].get("runner_hash"),
"candidate_runner_hash": candidate_manifest.get("runner_hash"),
+ "comparison_baseline": "frozen_baseline" if controls is None else "interleaved_control",
+ "control_capture_hash": None if controls is None else fingerprint(control_manifest),
+ "control_runner_hash": None if controls is None else control_manifest.get("runner_hash"),
"planned_attempts": len(rows),
"passed_attempts": sum(row["passed"] for row in rows),
"passed": all(row["passed"] for row in rows),
"synthetic": any(
trace.metadata.get("synthetic") is True
- for trace in [*baselines.values(), *candidates.values()]
+ for trace in [
+ *baselines.values(),
+ *candidates.values(),
+ *(controls or {}).values(),
+ ]
),
"cases": rows,
"study": study,
diff --git a/agentloop/improvement_cli.py b/agentloop/improvement_cli.py
index 25845b6..f1f52db 100644
--- a/agentloop/improvement_cli.py
+++ b/agentloop/improvement_cli.py
@@ -49,6 +49,12 @@ def capture_command(
show_runner_output: bool = typer.Option(
False, help="Stream the runner's stdout and stderr for diagnosis."
),
+ control_arg: list[str] | None = typer.Option(
+ None,
+ "--control-arg",
+ help="One argument of the unchanged control runner; repeat for each argument. "
+ "Alternates control and candidate attempts so drift affects both.",
+ ),
command: list[str] = typer.Argument(
..., help="Explicit runner argv after --. Receives a JSON request on stdin."
),
@@ -61,19 +67,24 @@ def capture_command(
command,
timeout=timeout,
show_output=show_runner_output,
+ control=control_arg or None,
)
except (OSError, ValueError, TypeError, KeyError) as exc:
raise typer.BadParameter(str(exc)) from exc
- complete = sum(row["status"] == "completed" for row in manifest["runs"])
- console.print(f"Captured {complete} of {len(manifest['runs'])} planned attempts.", markup=False)
+ attempts = [("", row) for row in manifest["runs"]]
+ if control_arg:
+ control = improvement.load_json(out.resolve() / improvement.CONTROL_CAPTURE)
+ attempts += [("Control: ", row) for row in control["runs"]]
+ complete = sum(row["status"] == "completed" for _, row in attempts)
+ console.print(f"Captured {complete} of {len(attempts)} planned attempts.", markup=False)
console.print(f"Capture manifest written to {out / 'capture.json'}", markup=False)
- for attempt in manifest["runs"]:
+ for prefix, attempt in attempts:
if attempt["status"] != "completed":
console.print(
- f"{attempt['task_id']} repetition {attempt['repetition']} — {attempt['status']}. {attempt.get('diagnostic') or 'Inspect the runner with --show-runner-output.'}",
+ f"{prefix}{attempt['task_id']} repetition {attempt['repetition']} — {attempt['status']}. {attempt.get('diagnostic') or 'Inspect the runner with --show-runner-output.'}",
markup=False,
)
- if complete != len(manifest["runs"]):
+ if complete != len(attempts):
raise typer.Exit(1)
diff --git a/agentloop/improvement_report.py b/agentloop/improvement_report.py
index f3f8cfc..74e4da9 100644
--- a/agentloop/improvement_report.py
+++ b/agentloop/improvement_report.py
@@ -170,6 +170,18 @@ def _metric(summary, name):
return f"{value:.3f}" if isinstance(value, float) else str(value)
+def _basis(report):
+ if report.get("comparison_baseline") == "interleaved_control":
+ return (
+ "Measurements compare the candidate with an unchanged control captured in "
+ "alternating order. Findings remain frozen from the original baseline."
+ )
+ return (
+ "Measurements compare the candidate with the frozen baseline captured before "
+ "the change. Run order is not counterbalanced."
+ )
+
+
def report_markdown(report):
lines = [
f"# {markdown_heading(report['name'])}",
@@ -184,6 +196,8 @@ def report_markdown(report):
"",
markdown_text(report["change"]),
"",
+ _basis(report),
+ "",
]
if report["synthetic"]:
lines.extend(
@@ -248,6 +262,7 @@ def export_report(root, report):
sections = [
f" {report['passed_attempts']} of {report['planned_attempts']} planned attempts passed.{_text(report['name'])}
{_text(report['change'])}
", + f"{_text(_basis(report))}
", ] if report["synthetic"]: sections.append( diff --git a/docs/ADOPTION_GAPS.md b/docs/ADOPTION_GAPS.md index d5c2eb3..f6d0b0e 100644 --- a/docs/ADOPTION_GAPS.md +++ b/docs/ADOPTION_GAPS.md @@ -10,7 +10,7 @@ work prioritizes this path over adding another control or experiment system. | Representative agent work | Small Hermes repository questions, older model/tool studies, and a reproducible three-workload guided suite | Collect coding, retrieval and tool-heavy task sets from actual applications, with owners' independent acceptance criteria | | Recommendation precision | Native evidence/risk rankings, explicit assumptions, retained false positives and rejected changes | Review recommendations against independent labels; report unsupported recommendations and abstentions alongside accepted ones | | Quality coverage | Frozen task inputs and declared per-task checks; a wrong fast candidate cannot pass those checks | Expand adversarial and held-out tasks; measure quality dimensions beyond exact structured answers | -| Causal performance | Per-attempt measurements and disclosed fixed-order confounds | Counterbalance conditions, control cache/load effects, and increase independent task/host/model coverage | +| Causal performance | Per-attempt measurements, disclosed fixed-order confounds, and an optional interleaved control that alternates condition order | Use interleaved controls in real studies, control cache/load effects further, and increase independent task/host/model coverage | | Instrumentation overhead | Counterbalanced CPU controls separate recording, JSON export and native analysis at declared span volumes; model controls retain quality failures | Repeat controls around actual application execution, across hosts and realistic payload sizes | | Cost | Actual token provenance and qualified model-price estimates | Obtain authorized billing data; retain unknown local hardware/energy cost instead of inventing dollars | | Hermes compatibility | 0.21.6 contract coverage and real 0.22.7/0.23.1 conformance | Revalidate future hook changes against the pinned runtime scenarios before extending the supported range | diff --git a/docs/GUIDED_OPTIMIZATION.md b/docs/GUIDED_OPTIMIZATION.md index db88aa5..574996b 100644 --- a/docs/GUIDED_OPTIMIZATION.md +++ b/docs/GUIDED_OPTIMIZATION.md @@ -169,6 +169,32 @@ cost stays unavailable. A latency comparison can pass with an indeterminate, non-gating cost estimate. A required cost improvement fails when pricing or usable usage is missing. +## Compare against an interleaved control + +A plain candidate capture runs after the baseline, so cache warm-up, host load or +provider drift can look like an improvement. When you can keep the unchanged +agent runnable, for example in a separate `git worktree`, capture it as a control +alongside the candidate: + +```bash +git worktree add ../agent-before HEAD +agentloop improve capture --tasks tasks.json --out runs/paired --control-arg python --control-arg ../agent-before/your_agent.py -- python your_agent.py +agentloop improve finish --session runs/optimization/session.json --candidate runs/paired/capture.json --change "Describe the change applied to the agent" --out runs/results +``` + +Repeat `--control-arg` once for each argument of the control runner. Each +planned attempt runs the control and the candidate back to back. The control +runs first on even attempt indexes and second on odd ones. Use an even number +of attempts, through `repetitions`, so each condition runs first equally often. +The control capture is stored in `runs/paired/control/` and its manifest hash is +recorded in the candidate capture. + +`finish` then measures the candidate against the control instead of the frozen +baseline. Findings, the focus and the gates stay frozen from the original +baseline. The control must pass the same quality checks. Intervention records +link the control's matching findings. The report states which comparison basis +it used, and each case records whether the control ran first. + ## Failures, evidence and execution `capture` retains every planned attempt, including failed, timed-out, cancelled, @@ -194,8 +220,9 @@ default. Pass `--show-runner-output` to stream them when diagnosing a failing ru Raw task inputs, expected answers, traces, and outputs are retained in the local artifacts. Review them before sharing. HTML escapes supplied text and shows metrics and decisions. A fixed-order comparison describes the supplied tasks and -run conditions. Repeat across independent tasks and inspect variability before -generalizing a result. +run conditions. An interleaved control reduces order effects but does not remove +other differences between conditions. Repeat across independent tasks and +inspect variability before generalizing a result. ## Reproduce the Hermes example with local inference diff --git a/tests/test_improvement.py b/tests/test_improvement.py index f895da4..69d1f65 100644 --- a/tests/test_improvement.py +++ b/tests/test_improvement.py @@ -341,3 +341,103 @@ def test_rehashed_sessions_still_validate_the_full_attempt_inventory(tmp_path, w change="Invalid inventory", ) assert not (tmp_path / "results").exists() + + +def test_interleaved_control_alternates_order_and_becomes_the_comparison(tmp_path, workload): + tasks, worker = prepare(tmp_path, workload) + manifest = capture( + tasks, + tmp_path / "paired", + command(worker, "fast"), + control=command(worker, "baseline"), + ) + control = json.loads((tmp_path / "paired/control/capture.json").read_bytes()) + assert manifest["control"]["manifest_hash"] == fingerprint(control) + assert [row["order"] for row in control["runs"]] == [0, 1, 0, 1] + assert [row["order"] for row in manifest["runs"]] == [1, 0, 1, 0] + for index, (first, second) in enumerate(zip(control["runs"], manifest["runs"])): + ordered = (first, second) if index % 2 == 0 else (second, first) + assert ordered[0]["ended_at"] <= ordered[1]["started_at"] + report = finish( + tmp_path / "session/session.json", + tmp_path / "paired/capture.json", + tmp_path / "results", + change="Remove repeated work", + ) + assert report["passed"] is True + assert report["comparison_baseline"] == "interleaved_control" + assert report["control_capture_hash"] == fingerprint(control) + control_ids = { + json.loads((tmp_path / "paired/control" / row["trace"]).read_bytes())["run_id"] + for row in control["runs"] + } + for row in report["cases"]: + assert row["replay"]["baseline"]["run_id"] in control_ids + assert row["intervention"]["baseline_run_id"] in control_ids + assert row["frozen_baseline_status"] == "completed" + assert [row["control_ran_first"] for row in report["cases"]] == [True, False, True, False] + assert "alternating order" in (tmp_path / "results/report.md").read_text(encoding="utf-8") + + +def test_interleaved_control_must_also_pass_quality_checks(tmp_path, workload): + tasks, worker = prepare(tmp_path, workload) + capture( + tasks, + tmp_path / "paired", + command(worker, "fast"), + control=command(worker, "wrong"), + ) + report = finish( + tmp_path / "session/session.json", + tmp_path / "paired/capture.json", + tmp_path / "results", + change="Wrong control", + ) + assert report["passed"] is False + assert all( + any(reason.startswith("Control answer failed") for reason in row["reasons"]) + for row in report["cases"] + ) + + +def test_modified_interleaved_control_is_rejected(tmp_path, workload): + tasks, worker = prepare(tmp_path, workload) + capture( + tasks, + tmp_path / "paired", + command(worker, "fast"), + control=command(worker, "baseline"), + ) + path = tmp_path / "paired/control/capture.json" + control = json.loads(path.read_bytes()) + control["runner_hash"] = "changed" + path.write_text(json.dumps(control), encoding="utf-8") + with pytest.raises(ValueError, match="control capture changed"): + finish( + tmp_path / "session/session.json", + tmp_path / "paired/capture.json", + tmp_path / "results", + change="Tampered control", + ) + assert not (tmp_path / "results").exists() + + +def test_cli_accepts_repeated_control_arguments(tmp_path, workload): + tasks, worker = workload + result = CliRunner().invoke( + app, + [ + "improve", + "capture", + "--tasks", + str(tasks), + "--out", + str(tmp_path / "paired"), + *[item for arg in command(worker, "baseline") for item in ("--control-arg", arg)], + "--", + *command(worker, "fast"), + ], + ) + assert result.exit_code == 0, result.output + assert "Captured 8 of 8 planned attempts." in result.output + assert (tmp_path / "paired/control/capture.json").is_file()