diff --git a/CHANGELOG.md b/CHANGELOG.md index 091dac8..5812c66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ trace, study, receipt, API and database schema versions. `typescript-vX.Y.Z` tags on main, after Node.js 22/24 build and tests, using trusted publishing with provenance. The SDK installs with `npm install agentloop-profiler`. +- `agentloop improve capture --control-arg ...` captures an unchanged control + runner alongside the candidate, alternating which runs first on each attempt. + `improve finish` then measures the candidate against that control, keeping + findings frozen from the original baseline, so cache warm-up and host drift + affect both conditions. ## [0.2.1] - 2026-11-12 diff --git a/agentloop/improvement.py b/agentloop/improvement.py index d48b893..78431e1 100644 --- a/agentloop/improvement.py +++ b/agentloop/improvement.py @@ -185,33 +185,21 @@ def _invoke(command, request, timeout, *, show_output=False): raise -def capture(workload_path, root, command, *, timeout=180, show_output=False): - workload = read_workload(workload_path) - require( - command and all(isinstance(arg, str) and arg for arg in command), - "Supply an explicit runner command after --", - ) - require( - type(timeout) in (int, float) and math.isfinite(timeout) and 0 < timeout <= 3600, - "Task timeout must be positive and at most 3600 seconds", - ) - root = Path(root).resolve() - root.mkdir(parents=True, exist_ok=False) - write_once(root, "workload.json", workload) +CONTROL_CAPTURE = "control/capture.json" + + +def _new_manifest(workload, command): scripts = [ {"argument_index": index, "sha256": hashlib.sha256(Path(argument).read_bytes()).hexdigest()} for index, argument in enumerate(command) if Path(argument).suffix in {".py", ".js", ".mjs", ".ts"} and Path(argument).is_file() ] - manifest = { + return { "schema_version": VERSION, "created_at": now(), "workload_hash": fingerprint(workload), "runner_hash": fingerprint({"argv": command, "scripts": scripts}), - "runs": [], - } - for index, (task, repetition) in enumerate(slots(workload)): - manifest["runs"].append( + "runs": [ { "task_id": task["id"], "repetition": repetition, @@ -220,46 +208,104 @@ def capture(workload_path, root, command, *, timeout=180, show_output=False): "trace": None, "trace_sha256": None, } - ) + for task, repetition in slots(workload) + ], + } + + +def _attempt(root, index, row, task, command, timeout, show_output): + relative = f"traces/{index:04d}.json" + path = local_path(root, relative) + path.parent.mkdir(exist_ok=True) + request = { + "task_id": row["task_id"], + "repetition": row["repetition"], + "input": task["input"], + "trace_path": str(path), + } + row["started_at"] = now() + try: + row.update(_invoke(command, request, timeout, show_output=show_output)) + except KeyboardInterrupt: + row["status"] = "cancelled" + raise + finally: + row["ended_at"] = now() + if path.exists(): + try: + _, digest = trace_receipt(root, relative) + row.update(trace=relative, trace_sha256=digest) + except (ValueError, OSError, TypeError, KeyError, AttributeError) as error: + # Keep the more specific failed, timed-out or cancelled outcome. + if row["status"] == "completed": + row["status"] = "invalid_trace" + row["diagnostic"] = ( + (row.get("diagnostic") or "") + + f" Trace {relative} is invalid: {error}. Export a finalized native trace to request['trace_path']." + ) + elif row["status"] == "completed": + row["status"] = "missing_trace" + row["diagnostic"] = ( + "Runner exited successfully without a trace. Export the finalized trace to request['trace_path']." + ) + write_once(root, f"receipts/{index:04d}.json", row) + + +def capture(workload_path, root, command, *, timeout=180, show_output=False, control=None): + """Run every planned attempt, optionally alternating with an unchanged control. + + With ``control``, each task attempt runs both the control and the candidate + back to back, and the condition that goes first alternates between attempts. + Cache warm-up, host load and other drift then affect both conditions instead + of favouring whichever condition ran later. + """ + workload = read_workload(workload_path) + require( + command and all(isinstance(arg, str) and arg for arg in command), + "Supply an explicit runner command after --", + ) + require( + control is None or (control and all(isinstance(arg, str) and arg for arg in control)), + "Supply the unchanged control runner as repeated --control-arg values", + ) + require( + type(timeout) in (int, float) and math.isfinite(timeout) and 0 < timeout <= 3600, + "Task timeout must be positive and at most 3600 seconds", + ) + root = Path(root).resolve() + root.mkdir(parents=True, exist_ok=False) + write_once(root, "workload.json", workload) + manifest = _new_manifest(workload, command) + conditions = [(root, manifest, command)] + control_manifest = None + if control is not None: + control_root = local_path(root, "control") + control_root.mkdir() + write_once(control_root, "workload.json", workload) + control_manifest = _new_manifest(workload, control) + control_manifest["schedule"] = {"mode": "interleaved", "role": "control"} + manifest["schedule"] = { + "mode": "interleaved", + "role": "candidate", + "pattern": "Control runs first on even attempt indexes and second on odd ones", + } + conditions.append((control_root, control_manifest, control)) try: for index, row in enumerate(manifest["runs"]): task = next(task for task in workload["tasks"] if task["id"] == row["task_id"]) - relative = f"traces/{index:04d}.json" - path = local_path(root, relative) - path.parent.mkdir(exist_ok=True) - request = { - "task_id": row["task_id"], - "repetition": row["repetition"], - "input": task["input"], - "trace_path": str(path), - } - row["started_at"] = now() - try: - row.update(_invoke(command, request, timeout, show_output=show_output)) - except KeyboardInterrupt: - row["status"] = "cancelled" - raise - finally: - row["ended_at"] = now() - if path.exists(): - try: - _, digest = trace_receipt(root, relative) - row.update(trace=relative, trace_sha256=digest) - except (ValueError, OSError, TypeError, KeyError, AttributeError) as error: - # Keep the more specific failed, timed-out or cancelled outcome. - if row["status"] == "completed": - row["status"] = "invalid_trace" - row["diagnostic"] = ( - (row.get("diagnostic") or "") - + f" Trace {relative} is invalid: {error}. Export a finalized native trace to request['trace_path']." - ) - elif row["status"] == "completed": - row["status"] = "missing_trace" - row["diagnostic"] = ( - "Runner exited successfully without a trace. Export the finalized trace to request['trace_path']." - ) - write_once(root, f"receipts/{index:04d}.json", row) + order = conditions if index % 2 else conditions[::-1] + for position, (condition_root, condition, argv) in enumerate(order): + attempt = condition["runs"][index] + if control is not None: + attempt["order"] = position + _attempt(condition_root, index, attempt, task, argv, timeout, show_output) finally: + if control_manifest is not None: + write_once(root, CONTROL_CAPTURE, control_manifest) + manifest["control"] = { + "capture": CONTROL_CAPTURE, + "manifest_hash": fingerprint(control_manifest), + } write_once(root, "capture.json", manifest) return manifest @@ -472,10 +518,31 @@ def finish(session_path, candidate_path, root, *, change): ) baseline_ids.add(trace.run_id) baselines[key] = trace + candidate_ids = {trace.run_id for trace in candidates.values()} require( - not baseline_ids.intersection(trace.run_id for trace in candidates.values()), + not baseline_ids.intersection(candidate_ids), "Baseline and candidate must have distinct run IDs", ) + controls = control_manifest = None + if "control" in candidate_manifest: + reference = candidate_manifest["control"] + require( + isinstance(reference, dict) and reference.get("capture") == CONTROL_CAPTURE, + "Invalid interleaved control reference", + ) + control_manifest, controls = read_capture( + Path(candidate_path).resolve().parent / CONTROL_CAPTURE, workload + ) + require( + fingerprint(control_manifest) == reference.get("manifest_hash"), + "Interleaved control capture changed", + ) + require( + not (baseline_ids | candidate_ids).intersection( + trace.run_id for trace in controls.values() + ), + "Control, baseline and candidate must have distinct run IDs", + ) gates = ReplayGates(**specification["gates"]) root = Path(root).resolve() root.mkdir(parents=True, exist_ok=False) @@ -483,14 +550,40 @@ def finish(session_path, candidate_path, root, *, change): for index, case in enumerate(specification["cases"]): key = (case["task_id"], case["repetition"]) task = next(task for task in workload["tasks"] if task["id"] == key[0]) - baseline, candidate = baselines.get(key), candidates.get(key) + candidate = candidates.get(key) candidate_row = next( row for row in candidate_manifest["runs"] if (row["task_id"], row["repetition"]) == key ) + if controls is None: + label, baseline = "Baseline", baselines.get(key) + baseline_status = case["capture_status"] + baseline_diagnostic = case.get("capture_diagnostic") + diagnosis, finding_ids = case["diagnosis"], case["finding_ids"] + else: + # Findings stay frozen from the original baseline. Measurements compare + # the candidate with the unchanged control run alongside it. + label, baseline = "Control", controls.get(key) + control_row = next( + row + for row in control_manifest["runs"] + if (row["task_id"], row["repetition"]) == key + ) + baseline_status = control_row["status"] + baseline_diagnostic = control_row.get("diagnostic") + diagnosis = build_diagnosis(baseline) if baseline is not None else None + finding_ids = ( + [ + finding["finding_id"] + for finding in diagnosis["findings"] + if finding["type"] == specification["focus"] + ] + if diagnosis and case["finding_ids"] + else [] + ) row = { "task_id": key[0], "repetition": key[1], - "baseline_status": case["capture_status"], + "baseline_status": baseline_status, "candidate_status": candidate_row["status"], "passed": False, "reasons": [], @@ -498,10 +591,15 @@ def finish(session_path, candidate_path, root, *, change): "replay": None, "intervention": None, } + if controls is not None: + row.update( + frozen_baseline_status=case["capture_status"], + control_ran_first=control_row.get("order") == 0, + ) if baseline is None or candidate is None: - row["reasons"].append("A planned baseline or candidate trace is unavailable") + row["reasons"].append(f"A planned {label.lower()} or candidate trace is unavailable") for side, status, diagnostic in ( - ("Baseline", case["capture_status"], case.get("capture_diagnostic")), + (label, baseline_status, baseline_diagnostic), ("Candidate", candidate_row["status"], candidate_row.get("diagnostic")), ): if status != "completed": @@ -512,7 +610,7 @@ def finish(session_path, candidate_path, root, *, change): quality = _quality(task, baseline, candidate) replay = build_replay_report(baseline, candidate, gates=gates, quality_report=quality) if ( - case["capture_status"] != "completed" + baseline_status != "completed" or candidate_row["status"] != "completed" or not all(_trace_usable(trace) for trace in (baseline, candidate)) ): @@ -523,13 +621,13 @@ def finish(session_path, candidate_path, root, *, change): for side in ("baseline", "candidate") ): row["reasons"].append( - "Baseline and candidate must both pass the declared quality checks" + f"{label} and candidate must both pass the declared quality checks" ) for result in quality["cases"]: for side in ("baseline", "candidate"): if not result[side]["passed"]: row["reasons"].append( - f"{side.title()} answer failed the declared {task.get('scorer', {}).get('type', 'exact_match')} criterion: {result[side]['detail']}" + f"{label if side == 'baseline' else 'Candidate'} answer failed the declared {task.get('scorer', {}).get('type', 'exact_match')} criterion: {result[side]['detail']}" ) if not replay["gates"]["passed"]: row["reasons"].append("Native replay gates failed") @@ -538,7 +636,7 @@ def finish(session_path, candidate_path, root, *, change): for gate in replay["gates"]["results"] if not gate["passed"] ) - if not case["diagnosis"].get("analysis_complete", True) or not candidate.report().get( + if not diagnosis.get("analysis_complete", True) or not candidate.report().get( "analysis_complete", True ): row["reasons"].append("Finding analysis is incomplete") @@ -567,14 +665,14 @@ def finish(session_path, candidate_path, root, *, change): cost_evaluable=replay["gates"]["cost_evaluable"], pricing_known=replay["gates"]["pricing_known"], ) - if case["finding_ids"]: + if finding_ids: record = build_intervention( baseline, candidate, - target_finding_ids=case["finding_ids"], + target_finding_ids=finding_ids, intervention_type=specification["focus"], configuration={"change": change}, - diagnosis=case["diagnosis"], + diagnosis=diagnosis, gates=gates, quality_report=quality, replay_report=replay, @@ -626,12 +724,19 @@ def finish(session_path, candidate_path, root, *, change): "candidate_capture_hash": fingerprint(candidate_manifest), "baseline_runner_hash": specification["baseline_capture"].get("runner_hash"), "candidate_runner_hash": candidate_manifest.get("runner_hash"), + "comparison_baseline": "frozen_baseline" if controls is None else "interleaved_control", + "control_capture_hash": None if controls is None else fingerprint(control_manifest), + "control_runner_hash": None if controls is None else control_manifest.get("runner_hash"), "planned_attempts": len(rows), "passed_attempts": sum(row["passed"] for row in rows), "passed": all(row["passed"] for row in rows), "synthetic": any( trace.metadata.get("synthetic") is True - for trace in [*baselines.values(), *candidates.values()] + for trace in [ + *baselines.values(), + *candidates.values(), + *(controls or {}).values(), + ] ), "cases": rows, "study": study, diff --git a/agentloop/improvement_cli.py b/agentloop/improvement_cli.py index 25845b6..f1f52db 100644 --- a/agentloop/improvement_cli.py +++ b/agentloop/improvement_cli.py @@ -49,6 +49,12 @@ def capture_command( show_runner_output: bool = typer.Option( False, help="Stream the runner's stdout and stderr for diagnosis." ), + control_arg: list[str] | None = typer.Option( + None, + "--control-arg", + help="One argument of the unchanged control runner; repeat for each argument. " + "Alternates control and candidate attempts so drift affects both.", + ), command: list[str] = typer.Argument( ..., help="Explicit runner argv after --. Receives a JSON request on stdin." ), @@ -61,19 +67,24 @@ def capture_command( command, timeout=timeout, show_output=show_runner_output, + control=control_arg or None, ) except (OSError, ValueError, TypeError, KeyError) as exc: raise typer.BadParameter(str(exc)) from exc - complete = sum(row["status"] == "completed" for row in manifest["runs"]) - console.print(f"Captured {complete} of {len(manifest['runs'])} planned attempts.", markup=False) + attempts = [("", row) for row in manifest["runs"]] + if control_arg: + control = improvement.load_json(out.resolve() / improvement.CONTROL_CAPTURE) + attempts += [("Control: ", row) for row in control["runs"]] + complete = sum(row["status"] == "completed" for _, row in attempts) + console.print(f"Captured {complete} of {len(attempts)} planned attempts.", markup=False) console.print(f"Capture manifest written to {out / 'capture.json'}", markup=False) - for attempt in manifest["runs"]: + for prefix, attempt in attempts: if attempt["status"] != "completed": console.print( - f"{attempt['task_id']} repetition {attempt['repetition']} — {attempt['status']}. {attempt.get('diagnostic') or 'Inspect the runner with --show-runner-output.'}", + f"{prefix}{attempt['task_id']} repetition {attempt['repetition']} — {attempt['status']}. {attempt.get('diagnostic') or 'Inspect the runner with --show-runner-output.'}", markup=False, ) - if complete != len(manifest["runs"]): + if complete != len(attempts): raise typer.Exit(1) diff --git a/agentloop/improvement_report.py b/agentloop/improvement_report.py index f3f8cfc..74e4da9 100644 --- a/agentloop/improvement_report.py +++ b/agentloop/improvement_report.py @@ -170,6 +170,18 @@ def _metric(summary, name): return f"{value:.3f}" if isinstance(value, float) else str(value) +def _basis(report): + if report.get("comparison_baseline") == "interleaved_control": + return ( + "Measurements compare the candidate with an unchanged control captured in " + "alternating order. Findings remain frozen from the original baseline." + ) + return ( + "Measurements compare the candidate with the frozen baseline captured before " + "the change. Run order is not counterbalanced." + ) + + def report_markdown(report): lines = [ f"# {markdown_heading(report['name'])}", @@ -184,6 +196,8 @@ def report_markdown(report): "", markdown_text(report["change"]), "", + _basis(report), + "", ] if report["synthetic"]: lines.extend( @@ -248,6 +262,7 @@ def export_report(root, report): sections = [ f"

{_text(report['name'])}

{report['passed_attempts']} of {report['planned_attempts']} planned attempts passed.

", f"

Applied change

{_text(report['change'])}

", + f"

{_text(_basis(report))}

", ] if report["synthetic"]: sections.append( diff --git a/docs/ADOPTION_GAPS.md b/docs/ADOPTION_GAPS.md index d5c2eb3..f6d0b0e 100644 --- a/docs/ADOPTION_GAPS.md +++ b/docs/ADOPTION_GAPS.md @@ -10,7 +10,7 @@ work prioritizes this path over adding another control or experiment system. | Representative agent work | Small Hermes repository questions, older model/tool studies, and a reproducible three-workload guided suite | Collect coding, retrieval and tool-heavy task sets from actual applications, with owners' independent acceptance criteria | | Recommendation precision | Native evidence/risk rankings, explicit assumptions, retained false positives and rejected changes | Review recommendations against independent labels; report unsupported recommendations and abstentions alongside accepted ones | | Quality coverage | Frozen task inputs and declared per-task checks; a wrong fast candidate cannot pass those checks | Expand adversarial and held-out tasks; measure quality dimensions beyond exact structured answers | -| Causal performance | Per-attempt measurements and disclosed fixed-order confounds | Counterbalance conditions, control cache/load effects, and increase independent task/host/model coverage | +| Causal performance | Per-attempt measurements, disclosed fixed-order confounds, and an optional interleaved control that alternates condition order | Use interleaved controls in real studies, control cache/load effects further, and increase independent task/host/model coverage | | Instrumentation overhead | Counterbalanced CPU controls separate recording, JSON export and native analysis at declared span volumes; model controls retain quality failures | Repeat controls around actual application execution, across hosts and realistic payload sizes | | Cost | Actual token provenance and qualified model-price estimates | Obtain authorized billing data; retain unknown local hardware/energy cost instead of inventing dollars | | Hermes compatibility | 0.21.6 contract coverage and real 0.22.7/0.23.1 conformance | Revalidate future hook changes against the pinned runtime scenarios before extending the supported range | diff --git a/docs/GUIDED_OPTIMIZATION.md b/docs/GUIDED_OPTIMIZATION.md index db88aa5..574996b 100644 --- a/docs/GUIDED_OPTIMIZATION.md +++ b/docs/GUIDED_OPTIMIZATION.md @@ -169,6 +169,32 @@ cost stays unavailable. A latency comparison can pass with an indeterminate, non-gating cost estimate. A required cost improvement fails when pricing or usable usage is missing. +## Compare against an interleaved control + +A plain candidate capture runs after the baseline, so cache warm-up, host load or +provider drift can look like an improvement. When you can keep the unchanged +agent runnable, for example in a separate `git worktree`, capture it as a control +alongside the candidate: + +```bash +git worktree add ../agent-before HEAD +agentloop improve capture --tasks tasks.json --out runs/paired --control-arg python --control-arg ../agent-before/your_agent.py -- python your_agent.py +agentloop improve finish --session runs/optimization/session.json --candidate runs/paired/capture.json --change "Describe the change applied to the agent" --out runs/results +``` + +Repeat `--control-arg` once for each argument of the control runner. Each +planned attempt runs the control and the candidate back to back. The control +runs first on even attempt indexes and second on odd ones. Use an even number +of attempts, through `repetitions`, so each condition runs first equally often. +The control capture is stored in `runs/paired/control/` and its manifest hash is +recorded in the candidate capture. + +`finish` then measures the candidate against the control instead of the frozen +baseline. Findings, the focus and the gates stay frozen from the original +baseline. The control must pass the same quality checks. Intervention records +link the control's matching findings. The report states which comparison basis +it used, and each case records whether the control ran first. + ## Failures, evidence and execution `capture` retains every planned attempt, including failed, timed-out, cancelled, @@ -194,8 +220,9 @@ default. Pass `--show-runner-output` to stream them when diagnosing a failing ru Raw task inputs, expected answers, traces, and outputs are retained in the local artifacts. Review them before sharing. HTML escapes supplied text and shows metrics and decisions. A fixed-order comparison describes the supplied tasks and -run conditions. Repeat across independent tasks and inspect variability before -generalizing a result. +run conditions. An interleaved control reduces order effects but does not remove +other differences between conditions. Repeat across independent tasks and +inspect variability before generalizing a result. ## Reproduce the Hermes example with local inference diff --git a/tests/test_improvement.py b/tests/test_improvement.py index f895da4..69d1f65 100644 --- a/tests/test_improvement.py +++ b/tests/test_improvement.py @@ -341,3 +341,103 @@ def test_rehashed_sessions_still_validate_the_full_attempt_inventory(tmp_path, w change="Invalid inventory", ) assert not (tmp_path / "results").exists() + + +def test_interleaved_control_alternates_order_and_becomes_the_comparison(tmp_path, workload): + tasks, worker = prepare(tmp_path, workload) + manifest = capture( + tasks, + tmp_path / "paired", + command(worker, "fast"), + control=command(worker, "baseline"), + ) + control = json.loads((tmp_path / "paired/control/capture.json").read_bytes()) + assert manifest["control"]["manifest_hash"] == fingerprint(control) + assert [row["order"] for row in control["runs"]] == [0, 1, 0, 1] + assert [row["order"] for row in manifest["runs"]] == [1, 0, 1, 0] + for index, (first, second) in enumerate(zip(control["runs"], manifest["runs"])): + ordered = (first, second) if index % 2 == 0 else (second, first) + assert ordered[0]["ended_at"] <= ordered[1]["started_at"] + report = finish( + tmp_path / "session/session.json", + tmp_path / "paired/capture.json", + tmp_path / "results", + change="Remove repeated work", + ) + assert report["passed"] is True + assert report["comparison_baseline"] == "interleaved_control" + assert report["control_capture_hash"] == fingerprint(control) + control_ids = { + json.loads((tmp_path / "paired/control" / row["trace"]).read_bytes())["run_id"] + for row in control["runs"] + } + for row in report["cases"]: + assert row["replay"]["baseline"]["run_id"] in control_ids + assert row["intervention"]["baseline_run_id"] in control_ids + assert row["frozen_baseline_status"] == "completed" + assert [row["control_ran_first"] for row in report["cases"]] == [True, False, True, False] + assert "alternating order" in (tmp_path / "results/report.md").read_text(encoding="utf-8") + + +def test_interleaved_control_must_also_pass_quality_checks(tmp_path, workload): + tasks, worker = prepare(tmp_path, workload) + capture( + tasks, + tmp_path / "paired", + command(worker, "fast"), + control=command(worker, "wrong"), + ) + report = finish( + tmp_path / "session/session.json", + tmp_path / "paired/capture.json", + tmp_path / "results", + change="Wrong control", + ) + assert report["passed"] is False + assert all( + any(reason.startswith("Control answer failed") for reason in row["reasons"]) + for row in report["cases"] + ) + + +def test_modified_interleaved_control_is_rejected(tmp_path, workload): + tasks, worker = prepare(tmp_path, workload) + capture( + tasks, + tmp_path / "paired", + command(worker, "fast"), + control=command(worker, "baseline"), + ) + path = tmp_path / "paired/control/capture.json" + control = json.loads(path.read_bytes()) + control["runner_hash"] = "changed" + path.write_text(json.dumps(control), encoding="utf-8") + with pytest.raises(ValueError, match="control capture changed"): + finish( + tmp_path / "session/session.json", + tmp_path / "paired/capture.json", + tmp_path / "results", + change="Tampered control", + ) + assert not (tmp_path / "results").exists() + + +def test_cli_accepts_repeated_control_arguments(tmp_path, workload): + tasks, worker = workload + result = CliRunner().invoke( + app, + [ + "improve", + "capture", + "--tasks", + str(tasks), + "--out", + str(tmp_path / "paired"), + *[item for arg in command(worker, "baseline") for item in ("--control-arg", arg)], + "--", + *command(worker, "fast"), + ], + ) + assert result.exit_code == 0, result.output + assert "Captured 8 of 8 planned attempts." in result.output + assert (tmp_path / "paired/control/capture.json").is_file()