diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c8c6c4..23e73a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ trace, study, receipt, API and database schema versions. ## [Unreleased] +### Added + +- Guided comparisons show trace latency and runner wall time in the CLI, JSON, + Markdown, and HTML. Per-attempt timing identifies the clock source. Paired + summaries show medians and coverage for completed, correct attempts. Latency + gates continue to use the trace measurement. + ## [0.2.3] - 2026-11-17 ### Added diff --git a/agentloop/improvement.py b/agentloop/improvement.py index 91361ec..ba65a67 100644 --- a/agentloop/improvement.py +++ b/agentloop/improvement.py @@ -12,6 +12,7 @@ from dataclasses import asdict from datetime import datetime, timezone from pathlib import Path +from time import perf_counter_ns from agentloop.experiment_artifacts import local_path, read_json, write_once from agentloop.experiment_types import fingerprint @@ -21,6 +22,7 @@ from agentloop.repetitions import add_repetition_findings from agentloop.replay import ReplayGates, build_replay_report from agentloop.replay import _summary as replay_summary +from agentloop.schema import TraceValidationError, validate_elapsed_ms from agentloop.tracer import AgentTrace VERSION = "1.0" @@ -234,12 +236,15 @@ def _attempt(root, index, row, task, command, timeout, show_output): "trace_path": str(path), } row["started_at"] = now() + started = perf_counter_ns() try: row.update(_invoke(command, request, timeout, show_output=show_output)) except KeyboardInterrupt: row["status"] = "cancelled" raise finally: + if row["status"] != "runner_error": + row["runner_elapsed_ms"] = (perf_counter_ns() - started) / 1_000_000 row["ended_at"] = now() if path.exists(): try: @@ -506,6 +511,65 @@ def _trace_usable(trace): return not any(event.parent_id is None and event.status == "error" for event in trace.events) +def _attempt_timing(row, trace): + """Describe the trace boundary and the capture process boundary independently.""" + timing = { + "trace_elapsed_ms": trace.elapsed_ms if trace is not None else None, + "runner_elapsed_ms": None, + "runner_clock": None, + } + if row.get("status") in {"runner_error", "not_started"}: + return timing + if "runner_elapsed_ms" in row: + try: + elapsed = validate_elapsed_ms(row["runner_elapsed_ms"]) + except TraceValidationError: + return timing + clock = "monotonic" + else: + # Capture 1.0 manifests predating runner timing retain UTC boundaries. + try: + started = datetime.fromisoformat(row.get("started_at")) + ended = datetime.fromisoformat(row.get("ended_at")) + if started.utcoffset() is None or ended.utcoffset() is None: + return timing + elapsed = (ended - started).total_seconds() * 1000 + except (TypeError, ValueError, OverflowError): + return timing + clock = "capture_timestamps" + if elapsed is not None and elapsed >= 0: + timing.update(runner_elapsed_ms=elapsed, runner_clock=clock) + return timing + + +def _median_duration(values): + if not values: + return None + ordered = sorted(values) + low, high = ordered[(len(ordered) - 1) // 2], ordered[len(ordered) // 2] + # Preserve the usual median and guard against overflow from adding two values. + middle = low + high + return middle / 2 if math.isfinite(middle) else low / 2 + high / 2 + + +def _timing_summary(rows): + eligible = [row["timing"] for row in rows if row["timing"]["eligible"]] + summary = {"eligible_pairs": len(eligible)} + for metric in ("trace_elapsed_ms", "runner_elapsed_ms"): + pairs = [ + (timing["baseline"][metric], timing["candidate"][metric]) + for timing in eligible + if all(timing[side][metric] is not None for side in ("baseline", "candidate")) + ] + summary[metric] = { + "paired_count": len(pairs), + "baseline_median_ms": _median_duration([before for before, _ in pairs]), + "candidate_median_ms": _median_duration([after for _, after in pairs]), + "median_paired_delta_ms": _median_duration([after - before for before, after in pairs]), + } + return summary + + def _sign_test_p_value(improved, regressed): """One-sided exact sign test: chance of at least this many improvements.""" trials = improved + regressed @@ -668,6 +732,15 @@ def finish(session_path, candidate_path, root, *, change): _completed(control_manifest, controls), ) gates = ReplayGates(**specification["gates"]) + comparison_manifest = ( + specification["baseline_capture"] if controls is None else control_manifest + ) + baseline_rows = { + (row["task_id"], row["repetition"]): row for row in comparison_manifest["runs"] + } + candidate_rows = { + (row["task_id"], row["repetition"]): row for row in candidate_manifest["runs"] + } root = Path(root).resolve() root.mkdir(parents=True, exist_ok=False) rows, study_paths = [], {"baseline": [], "candidate": []} @@ -675,9 +748,7 @@ def finish(session_path, candidate_path, root, *, change): key = (case["task_id"], case["repetition"]) task = next(task for task in workload["tasks"] if task["id"] == key[0]) candidate = candidates.get(key) - candidate_row = next( - row for row in candidate_manifest["runs"] if (row["task_id"], row["repetition"]) == key - ) + baseline_row, candidate_row = baseline_rows.get(key, {}), candidate_rows[key] if controls is None: label, baseline = "Baseline", baselines.get(key) baseline_status = case["capture_status"] @@ -687,13 +758,8 @@ def finish(session_path, candidate_path, root, *, change): # Findings stay frozen from the original baseline. Measurements compare # the candidate with the unchanged control run alongside it. label, baseline = "Control", controls.get(key) - control_row = next( - row - for row in control_manifest["runs"] - if (row["task_id"], row["repetition"]) == key - ) - baseline_status = control_row["status"] - baseline_diagnostic = control_row.get("diagnostic") + baseline_status = baseline_row["status"] + baseline_diagnostic = baseline_row.get("diagnostic") diagnosis = control_diagnoses.get(key) finding_ids = ( [ @@ -714,11 +780,16 @@ def finish(session_path, candidate_path, root, *, change): "quality": None, "replay": None, "intervention": None, + "timing": { + "baseline": _attempt_timing(baseline_row, baseline), + "candidate": _attempt_timing(candidate_row, candidate), + "eligible": False, + }, } if controls is not None: row.update( frozen_baseline_status=case["capture_status"], - control_ran_first=control_row.get("order") == 0, + control_ran_first=baseline_row.get("order") == 0, ) if baseline is None or candidate is None: row["reasons"].append(f"A planned {label.lower()} or candidate trace is unavailable") @@ -733,17 +804,18 @@ def finish(session_path, candidate_path, root, *, change): else: quality = _quality(task, baseline, candidate) replay = build_replay_report(baseline, candidate, gates=gates, quality_report=quality) - if ( - baseline_status != "completed" - or candidate_row["status"] != "completed" - or not all(_trace_usable(trace) for trace in (baseline, candidate)) - ): - row["reasons"].append("An attempt failed, was interrupted or did not complete") - if not quality["passed"] or any( - not result[side]["passed"] + completed = baseline_status == candidate_row["status"] == "completed" and all( + _trace_usable(trace) for trace in (baseline, candidate) + ) + quality_passed = quality["passed"] and all( + result[side]["passed"] for result in quality["cases"] for side in ("baseline", "candidate") - ): + ) + row["timing"]["eligible"] = completed and quality_passed + if not completed: + row["reasons"].append("An attempt failed, was interrupted or did not complete") + if not quality_passed: row["reasons"].append( f"{label} and candidate must both pass the declared quality checks" ) @@ -879,6 +951,7 @@ def finish(session_path, candidate_path, root, *, change): ] ), "cases": rows, + "timing_summary": _timing_summary(rows), "study": study, } from agentloop.improvement_report import export_report diff --git a/agentloop/improvement_cli.py b/agentloop/improvement_cli.py index 72ddf20..c2d6210 100644 --- a/agentloop/improvement_cli.py +++ b/agentloop/improvement_cli.py @@ -6,9 +6,11 @@ import typer from rich.console import Console +from rich.table import Table from agentloop import improvement from agentloop.improvement_init import initialize, project_tasks +from agentloop.improvement_report import comparison_label, decision_objective, timing_summary_rows improve_app = typer.Typer( help="Capture fixed tasks, review baseline findings, and verify an applied change." @@ -151,7 +153,7 @@ def finish_command( ) median = verdict["median_improvement_pct"] console.print( - f"Aggregate {verdict['objective']} decision " + f"Aggregate {decision_objective(verdict)} check " + ("passed" if verdict["passed"] else "failed") + f": median improvement {'unavailable' if median is None else f'{median:.2f}%'}, " f"{verdict['improved_pairs']} improved, {verdict['regressed_pairs']} regressed.", @@ -164,6 +166,16 @@ def finish_command( f"{report['passed_attempts']} of {report['planned_attempts']} planned comparisons passed.", markup=False, ) + timing = Table( + "Timing", + f"{comparison_label(report)} median", + "Candidate median", + "Measured pairs", + caption="Completed, correct pairs. Latency gates use trace latency.", + ) + for label, before, after, _, pairs in timing_summary_rows(report): + timing.add_row(label, before, after, pairs) + console.print(timing) console.print(f"Read {out / 'report.html'}", markup=False) for case in report["cases"]: if not case["passed"]: diff --git a/agentloop/improvement_report.py b/agentloop/improvement_report.py index 00557be..26a240c 100644 --- a/agentloop/improvement_report.py +++ b/agentloop/improvement_report.py @@ -13,6 +13,22 @@ markdown_text, ) +_TIMINGS = (("Trace latency", "trace_elapsed_ms"), ("Runner wall time", "runner_elapsed_ms")) +_MEASUREMENTS = ( + ("Input tokens", "input_tokens"), + ("Output tokens", "output_tokens"), + ("Model calls", "model_call_count"), + ("Tool operations", "tool_operations"), + ("Estimated model cost (USD)", "estimated_cost_usd"), + ("Quality score", "quality_score"), +) +_RUNNER_CLOCKS = {"monotonic": "Monotonic clock", "capture_timestamps": "Capture timestamps"} +_TIMING_SCOPE = ( + "Trace latency covers the instrumented execution. Runner wall time spans launch " + "through exit, including startup, trace export, and cleanup after interruption. " + "Latency gates use trace latency." +) + def plan_markdown(specification): lines = [ @@ -182,6 +198,72 @@ def _headline(report): return f"{report['passed_attempts']} of {report['planned_attempts']} planned attempts passed." +def comparison_label(report): + return "Control" if report.get("comparison_baseline") == "interleaved_control" else "Baseline" + + +def timing_summary_rows(report): + timing = report.get("timing_summary") + if timing is None: + return [] + return [ + [ + f"{label} (ms)", + _metric(timing[key], "baseline_median_ms"), + _metric(timing[key], "candidate_median_ms"), + _metric(timing[key], "median_paired_delta_ms"), + f"{timing[key]['paired_count']}/{report['planned_attempts']}", + ] + for label, key in _TIMINGS + ] + + +def _measurement_rows(row): + timing, replay = row.get("timing"), row["replay"] + sides = ("baseline", "candidate") + measurements = [] + if timing is not None: + measurements.extend( + [f"{label} (ms)", *(_metric(timing[side], key) for side in sides)] + for label, key in _TIMINGS + ) + measurements.append( + [ + "Runner clock", + *( + _RUNNER_CLOCKS.get(timing[side]["runner_clock"], "unavailable") + for side in sides + ), + ] + ) + elif replay: + measurements.append( + ["Trace latency (ms)", *(_metric(replay[side], "runtime_ms") for side in sides)] + ) + if replay: + measurements.extend( + [label, *(_metric(replay[side], key) for side in sides)] for label, key in _MEASUREMENTS + ) + return measurements + + +_TIMING_SUMMARY_SCOPE = ( + "Medians use pairs with completed traces, passing checks on both answers, and both " + "measurements for the given scope. The paired difference is candidate minus " + "comparison time for each pair, then summarized by the median." +) + + +def _timing_headers(comparison): + return [ + "Timing", + f"{comparison} median", + "Candidate median", + "Median paired difference", + "Pairs measured", + ] + + def _basis(report): if report.get("comparison_baseline") == "interleaved_control": return ( @@ -194,16 +276,21 @@ def _basis(report): ) +def decision_objective(verdict): + return "trace latency" if verdict["objective"] == "latency" else verdict["objective"] + + def _decision_lines(report): verdict = report.get("decision") or {"mode": "per_pair"} if verdict["mode"] != "aggregate": return ["Decision: every pair must pass correctness, completion and performance checks."] median = verdict["median_improvement_pct"] p_value = verdict["sign_test_p_value"] + objective = decision_objective(verdict) lines = [ "Decision: every pair must pass correctness and completion checks; " - f"{verdict['objective']} is judged across all pairs.", - f"Aggregate {verdict['objective']} decision " + f"{objective} is judged across all pairs.", + f"Aggregate {objective} check " + ("passed" if verdict["passed"] else "failed") + ": median paired improvement " + ("unavailable" if median is None else f"{median:.2f}%") @@ -224,6 +311,7 @@ def _decision_lines(report): def report_markdown(report): + comparison = comparison_label(report) lines = [ f"# {markdown_heading(report['name'])}", "", @@ -248,6 +336,23 @@ def report_markdown(report): "", ] ) + timing_rows = timing_summary_rows(report) + if timing_rows: + headers = _timing_headers(comparison) + lines.extend( + [ + "## Paired timing", + "", + _TIMING_SCOPE, + "", + _TIMING_SUMMARY_SCOPE, + "", + "| " + " | ".join(headers) + " |", + "| " + " | ".join(["---"] * len(headers)) + " |", + *("| " + " | ".join(values) + " |" for values in timing_rows), + "", + ] + ) for row in report["cases"]: lines.extend( [ @@ -258,21 +363,11 @@ def report_markdown(report): ] ) lines.extend(f"- {markdown_text(reason)}" for reason in row["reasons"]) + measurements = _measurement_rows(row) + if measurements: + lines.extend(["", f"| Measurement | {comparison} | Candidate |", "| --- | --- | --- |"]) + lines.extend("| " + " | ".join(values) + " |" for values in measurements) if row["replay"]: - lines.extend(["", "| Measurement | Baseline | Candidate |", "| --- | --- | --- |"]) - for label, metric in ( - ("Runtime (ms)", "runtime_ms"), - ("Input tokens", "input_tokens"), - ("Output tokens", "output_tokens"), - ("Model calls", "model_call_count"), - ("Tool operations", "tool_operations"), - ("Estimated model cost (USD)", "estimated_cost_usd"), - ("Quality score", "quality_score"), - ): - replay = row["replay"] - lines.append( - f"| {label} | {_metric(replay['baseline'], metric)} | {_metric(replay['candidate'], metric)} |" - ) lines.extend(["", "| Gate | Result | Detail |", "| --- | --- | --- |"]) for gate in row["replay"]["gates"]["results"]: status = ( @@ -301,6 +396,7 @@ def export_report(root, report): write_once(root, "report.json", report) markdown = report_markdown(report) (root / "report.md").write_text(markdown, encoding="utf-8") + comparison = comparison_label(report) sections = [ f"

{_text(report['name'])}

{_text(_headline(report))}

", f"

Applied change

{_text(report['change'])}

", @@ -311,36 +407,35 @@ def export_report(root, report): sections.append( '

This comparison includes synthetic evidence. Results apply to the supplied fixtures.

' ) - measurements = ( - ("Runtime (ms)", "runtime_ms"), - ("Input tokens", "input_tokens"), - ("Output tokens", "output_tokens"), - ("Model calls", "model_call_count"), - ("Tool operations", "tool_operations"), - ("Estimated model cost (USD)", "estimated_cost_usd"), - ("Quality score", "quality_score"), - ) + timing_rows = timing_summary_rows(report) + if timing_rows: + sections.extend( + [ + f"

Paired timing

{_text(_TIMING_SCOPE)}

{_text(_TIMING_SUMMARY_SCOPE)}

", + _table( + _timing_headers(comparison), + [[_text(cell) for cell in values] for values in timing_rows], + caption="Measured timing for completed, correct pairs", + ), + ] + ) for index, row in enumerate(report["cases"]): status = "Passed" if row["passed"] else "Rejected" sections.append( f"

{_text(row['task_id'])}, repetition {row['repetition']}

{status}

" ) sections.extend(f"

{_text(reason)}

" for reason in row["reasons"]) - if row["replay"]: - replay = row["replay"] - values = [ - [ - label, - _text(_metric(replay["baseline"], metric)), - _text(_metric(replay["candidate"], metric)), - ] - for label, metric in measurements - ] + measurements = _measurement_rows(row) + if measurements: sections.append( _table( - ["Measurement", "Baseline", "Candidate"], values, caption="Paired measurements" + ["Measurement", comparison, "Candidate"], + [[_text(cell) for cell in values] for values in measurements], + caption="Paired measurements", ) ) + if row["replay"]: + replay = row["replay"] checks = [ [ _text(gate["name"]), @@ -359,7 +454,7 @@ def export_report(root, report): + "" ) sections.append( - f'

Baseline trace · Candidate trace

' + f'

{comparison} trace · Candidate trace

' ) sections.append("
") sections.append( diff --git a/docs/GUIDED_OPTIMIZATION.md b/docs/GUIDED_OPTIMIZATION.md index bebac61..b7fcb5f 100644 --- a/docs/GUIDED_OPTIMIZATION.md +++ b/docs/GUIDED_OPTIMIZATION.md @@ -166,9 +166,23 @@ agentloop improve capture --tasks tasks.json --out runs/candidate -- python your agentloop improve finish --session runs/optimization/session.json --candidate runs/candidate/capture.json --change "Describe the change applied to the agent" --out runs/results ``` -Open `runs/results/report.html`. The report shows per-attempt quality, latency, +Open `runs/results/report.html`. The report shows per-attempt quality, timing, token provenance, model/tool counts, available model-cost estimates, and native -replay gate decisions. JSON and Markdown accompany it. Pairs with matching frozen +replay gate decisions. Builds after 0.2.3 show paired timing medians in the CLI, +HTML, and Markdown. +Trace latency covers the instrumented execution and drives latency gates. Runner +wall time covers process launch through exit, including startup, trace export, +and cleanup after interruption. It can help explain a small end-to-end gain when +the traced change is much faster. Runner wall time is descriptive. + +Timing medians use completed, correct pairs with available measurements on both +sides. Each metric lists how many pairs it could measure out of the full planned +set. Failed attempts retain any recorded timings in their individual rows. New +captures use a monotonic clock. Older captures with valid start and end timestamps +show the derived duration and its clock source. Missing timings show as +unavailable. + +JSON and Markdown accompany the HTML. Pairs with matching frozen findings also produce native intervention records. Available baseline and candidate traces produce a paired study. Study views retain the original trace fingerprints. diff --git a/scripts/smoke_guided_wheel.py b/scripts/smoke_guided_wheel.py index 2494237..6e32fc4 100644 --- a/scripts/smoke_guided_wheel.py +++ b/scripts/smoke_guided_wheel.py @@ -87,7 +87,7 @@ def cli(*args, expected=0): check=False, timeout=60, capture_output=True, - text=True, + encoding="utf-8", ) allowed = expected if isinstance(expected, tuple) else (expected,) if result.returncode not in allowed: @@ -147,6 +147,11 @@ def capture(mode, expected=0): raise RuntimeError("Installed guided outcome lost a planned attempt or quality failure") if not (root / ("report-" + mode) / "report.html").is_file(): raise RuntimeError("Installed guided HTML report missing") + timing = report["cases"][0]["timing"]["candidate"] + if timing["runner_clock"] != "monotonic" or timing["runner_elapsed_ms"] <= 0: + raise RuntimeError("Installed guided capture lost process timing") + if report["timing_summary"]["runner_elapsed_ms"]["paired_count"] != int(report["passed"]): + raise RuntimeError("Installed guided timing counted an incorrect or incomplete pair") cli( "improve", "finish", diff --git a/tests/test_improvement.py b/tests/test_improvement.py index 1341b2f..04b3529 100644 --- a/tests/test_improvement.py +++ b/tests/test_improvement.py @@ -83,6 +83,8 @@ def test_guided_workflow_uses_native_quality_replay_and_studies(tmp_path, worklo ) assert report["study"] is not None assert all(row["intervention"] for row in report["cases"]) + assert report["timing_summary"]["trace_elapsed_ms"]["paired_count"] == 4 + assert report["timing_summary"]["runner_elapsed_ms"]["paired_count"] == 4 assert (tmp_path / "results/report.html").is_file() assert (tmp_path / "session/plan.md").is_file() @@ -100,6 +102,16 @@ def test_faster_wrong_failed_or_missing_candidates_are_rejected(tmp_path, worklo assert report["passed"] is False assert report["planned_attempts"] == 4 assert report["passed_attempts"] == 0 + assert report["timing_summary"]["eligible_pairs"] == 0 + assert report["timing_summary"]["runner_elapsed_ms"]["paired_count"] == 0 + assert all(row["timing"]["candidate"]["runner_elapsed_ms"] > 0 for row in report["cases"]) + if mode == "missing": + assert all( + row["timing"]["candidate"]["trace_elapsed_ms"] is None for row in report["cases"] + ) + assert "| Trace latency (ms) |" in (tmp_path / "results/report.md").read_text( + encoding="utf-8" + ) assert all( row["intervention"] is None or row["intervention"]["gates_passed"] is False for row in report["cases"] @@ -235,6 +247,8 @@ def test_cli_runs_the_normal_entry_points_and_returns_quality_failure(tmp_path, ], ) assert result.exit_code == 1, result.output + assert "Trace latency (ms)" in result.output + assert "Runner wall time (ms)" in result.output assert (tmp_path / "results/report.json").is_file() @@ -278,6 +292,8 @@ def interrupt(*args, **kwargs): "not_started", "not_started", ] + assert manifest["runs"][0]["runner_elapsed_ms"] >= 0 + assert all("runner_elapsed_ms" not in row for row in manifest["runs"][1:]) def test_invalid_trace_keeps_the_runner_failure_status(tmp_path, workload): @@ -376,7 +392,60 @@ def test_interleaved_control_alternates_order_and_becomes_the_comparison(tmp_pat assert row["intervention"]["baseline_run_id"] in control_ids assert row["frozen_baseline_status"] == "completed" assert [row["control_ran_first"] for row in report["cases"]] == [True, False, True, False] - assert "alternating order" in (tmp_path / "results/report.md").read_text(encoding="utf-8") + for index, row in enumerate(report["cases"]): + timing = row["timing"] + assert ( + timing["baseline"]["runner_elapsed_ms"] == control["runs"][index]["runner_elapsed_ms"] + ) + assert timing["baseline"]["runner_clock"] == "monotonic" + assert ( + timing["candidate"]["runner_elapsed_ms"] == manifest["runs"][index]["runner_elapsed_ms"] + ) + markdown = (tmp_path / "results/report.md").read_text(encoding="utf-8") + assert "alternating order" in markdown + assert "| Measurement | Control | Candidate |" in markdown + page = (tmp_path / "results/report.html").read_text(encoding="utf-8") + assert "Control median" in page and "Control trace" in page + assert "Trace latency (ms)" in page and "Runner wall time (ms)" in page + + +def test_timing_summary_reads_legacy_receipts_and_keeps_runner_time_out_of_gates( + tmp_path, workload +): + tasks, worker = workload + baseline_path = tmp_path / "baseline/capture.json" + baseline = capture(tasks, baseline_path.parent, command(worker, "baseline")) + for row, end in zip(baseline["runs"], ("00.150", "00.350", "00.200", "00.099"), strict=True): + row.pop("runner_elapsed_ms") + row.update(started_at="2026-01-01T00:00:00.100Z", ended_at=f"2026-01-01T00:00:{end}+00:00") + baseline_path.write_text(json.dumps(baseline), encoding="utf-8") + session = tmp_path / "session" + start(tasks, baseline_path, session) + + candidate_path = tmp_path / "candidate/capture.json" + candidate = capture(tasks, candidate_path.parent, command(worker, "fast")) + for row, runner_ms in zip(candidate["runs"], (300, 350, True, 400), strict=True): + row["runner_elapsed_ms"] = runner_ms + candidate_path.write_text(json.dumps(candidate), encoding="utf-8") + + report = finish( + session / "session.json", + candidate_path, + tmp_path / "results", + change="Spend less time inside the trace", + ) + assert report["passed"] + assert report["timing_summary"]["eligible_pairs"] == 4 + assert report["timing_summary"]["trace_elapsed_ms"]["paired_count"] == 4 + assert report["timing_summary"]["runner_elapsed_ms"] == { + "paired_count": 2, + "baseline_median_ms": 150.0, + "candidate_median_ms": 325.0, + "median_paired_delta_ms": 175.0, + } + assert report["cases"][0]["timing"]["baseline"]["runner_clock"] == "capture_timestamps" + assert report["cases"][2]["timing"]["candidate"]["runner_clock"] is None + assert report["cases"][3]["timing"]["baseline"]["runner_elapsed_ms"] is None def test_interleaved_control_must_also_pass_quality_checks(tmp_path, workload): diff --git a/tests/test_improvement_init.py b/tests/test_improvement_init.py index f11b6fa..04fafae 100644 --- a/tests/test_improvement_init.py +++ b/tests/test_improvement_init.py @@ -18,7 +18,7 @@ def run_cli(project, *args, allowed=(0,)): [sys.executable, "-c", "from agentloop.entrypoint import app; app()", *map(str, args)], cwd=project, capture_output=True, - text=True, + encoding="utf-8", timeout=60, ) assert result.returncode in allowed, result.stdout + result.stderr