From c2cf9b4ff36906d9f65c0651773eb33819ce4154 Mon Sep 17 00:00:00 2001 From: Dipesh Babu Date: Tue, 17 Nov 2026 22:32:40 -0500 Subject: [PATCH 1/5] Report trace latency and runner wall time side by side --- CHANGELOG.md | 7 ++ agentloop/improvement.py | 113 ++++++++++++++++++---- agentloop/improvement_cli.py | 12 +++ agentloop/improvement_report.py | 160 +++++++++++++++++++++++++------- docs/GUIDED_OPTIMIZATION.md | 17 +++- scripts/smoke_guided_wheel.py | 5 + tests/test_improvement.py | 71 +++++++++++++- 7 files changed, 327 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c8c6c4..23e73a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ trace, study, receipt, API and database schema versions. ## [Unreleased] +### Added + +- Guided comparisons show trace latency and runner wall time in the CLI, JSON, + Markdown, and HTML. Per-attempt timing identifies the clock source. Paired + summaries show medians and coverage for completed, correct attempts. Latency + gates continue to use the trace measurement. + ## [0.2.3] - 2026-11-17 ### Added diff --git a/agentloop/improvement.py b/agentloop/improvement.py index 91361ec..ba65a67 100644 --- a/agentloop/improvement.py +++ b/agentloop/improvement.py @@ -12,6 +12,7 @@ from dataclasses import asdict from datetime import datetime, timezone from pathlib import Path +from time import perf_counter_ns from agentloop.experiment_artifacts import local_path, read_json, write_once from agentloop.experiment_types import fingerprint @@ -21,6 +22,7 @@ from agentloop.repetitions import add_repetition_findings from agentloop.replay import ReplayGates, build_replay_report from agentloop.replay import _summary as replay_summary +from agentloop.schema import TraceValidationError, validate_elapsed_ms from agentloop.tracer import AgentTrace VERSION = "1.0" @@ -234,12 +236,15 @@ def _attempt(root, index, row, task, command, timeout, show_output): "trace_path": str(path), } row["started_at"] = now() + started = perf_counter_ns() try: row.update(_invoke(command, request, timeout, show_output=show_output)) except KeyboardInterrupt: row["status"] = "cancelled" raise finally: + if row["status"] != "runner_error": + row["runner_elapsed_ms"] = (perf_counter_ns() - started) / 1_000_000 row["ended_at"] = now() if path.exists(): try: @@ -506,6 +511,65 @@ def _trace_usable(trace): return not any(event.parent_id is None and event.status == "error" for event in trace.events) +def _attempt_timing(row, trace): + """Describe the trace boundary and the capture process boundary independently.""" + timing = { + "trace_elapsed_ms": trace.elapsed_ms if trace is not None else None, + "runner_elapsed_ms": None, + "runner_clock": None, + } + if row.get("status") in {"runner_error", "not_started"}: + return timing + if "runner_elapsed_ms" in row: + try: + elapsed = validate_elapsed_ms(row["runner_elapsed_ms"]) + except TraceValidationError: + return timing + clock = "monotonic" + else: + # Capture 1.0 manifests predating runner timing retain UTC boundaries. + try: + started = datetime.fromisoformat(row.get("started_at")) + ended = datetime.fromisoformat(row.get("ended_at")) + if started.utcoffset() is None or ended.utcoffset() is None: + return timing + elapsed = (ended - started).total_seconds() * 1000 + except (TypeError, ValueError, OverflowError): + return timing + clock = "capture_timestamps" + if elapsed is not None and elapsed >= 0: + timing.update(runner_elapsed_ms=elapsed, runner_clock=clock) + return timing + + +def _median_duration(values): + if not values: + return None + ordered = sorted(values) + low, high = ordered[(len(ordered) - 1) // 2], ordered[len(ordered) // 2] + # Preserve the usual median and guard against overflow from adding two values. + middle = low + high + return middle / 2 if math.isfinite(middle) else low / 2 + high / 2 + + +def _timing_summary(rows): + eligible = [row["timing"] for row in rows if row["timing"]["eligible"]] + summary = {"eligible_pairs": len(eligible)} + for metric in ("trace_elapsed_ms", "runner_elapsed_ms"): + pairs = [ + (timing["baseline"][metric], timing["candidate"][metric]) + for timing in eligible + if all(timing[side][metric] is not None for side in ("baseline", "candidate")) + ] + summary[metric] = { + "paired_count": len(pairs), + "baseline_median_ms": _median_duration([before for before, _ in pairs]), + "candidate_median_ms": _median_duration([after for _, after in pairs]), + "median_paired_delta_ms": _median_duration([after - before for before, after in pairs]), + } + return summary + + def _sign_test_p_value(improved, regressed): """One-sided exact sign test: chance of at least this many improvements.""" trials = improved + regressed @@ -668,6 +732,15 @@ def finish(session_path, candidate_path, root, *, change): _completed(control_manifest, controls), ) gates = ReplayGates(**specification["gates"]) + comparison_manifest = ( + specification["baseline_capture"] if controls is None else control_manifest + ) + baseline_rows = { + (row["task_id"], row["repetition"]): row for row in comparison_manifest["runs"] + } + candidate_rows = { + (row["task_id"], row["repetition"]): row for row in candidate_manifest["runs"] + } root = Path(root).resolve() root.mkdir(parents=True, exist_ok=False) rows, study_paths = [], {"baseline": [], "candidate": []} @@ -675,9 +748,7 @@ def finish(session_path, candidate_path, root, *, change): key = (case["task_id"], case["repetition"]) task = next(task for task in workload["tasks"] if task["id"] == key[0]) candidate = candidates.get(key) - candidate_row = next( - row for row in candidate_manifest["runs"] if (row["task_id"], row["repetition"]) == key - ) + baseline_row, candidate_row = baseline_rows.get(key, {}), candidate_rows[key] if controls is None: label, baseline = "Baseline", baselines.get(key) baseline_status = case["capture_status"] @@ -687,13 +758,8 @@ def finish(session_path, candidate_path, root, *, change): # Findings stay frozen from the original baseline. Measurements compare # the candidate with the unchanged control run alongside it. label, baseline = "Control", controls.get(key) - control_row = next( - row - for row in control_manifest["runs"] - if (row["task_id"], row["repetition"]) == key - ) - baseline_status = control_row["status"] - baseline_diagnostic = control_row.get("diagnostic") + baseline_status = baseline_row["status"] + baseline_diagnostic = baseline_row.get("diagnostic") diagnosis = control_diagnoses.get(key) finding_ids = ( [ @@ -714,11 +780,16 @@ def finish(session_path, candidate_path, root, *, change): "quality": None, "replay": None, "intervention": None, + "timing": { + "baseline": _attempt_timing(baseline_row, baseline), + "candidate": _attempt_timing(candidate_row, candidate), + "eligible": False, + }, } if controls is not None: row.update( frozen_baseline_status=case["capture_status"], - control_ran_first=control_row.get("order") == 0, + control_ran_first=baseline_row.get("order") == 0, ) if baseline is None or candidate is None: row["reasons"].append(f"A planned {label.lower()} or candidate trace is unavailable") @@ -733,17 +804,18 @@ def finish(session_path, candidate_path, root, *, change): else: quality = _quality(task, baseline, candidate) replay = build_replay_report(baseline, candidate, gates=gates, quality_report=quality) - if ( - baseline_status != "completed" - or candidate_row["status"] != "completed" - or not all(_trace_usable(trace) for trace in (baseline, candidate)) - ): - row["reasons"].append("An attempt failed, was interrupted or did not complete") - if not quality["passed"] or any( - not result[side]["passed"] + completed = baseline_status == candidate_row["status"] == "completed" and all( + _trace_usable(trace) for trace in (baseline, candidate) + ) + quality_passed = quality["passed"] and all( + result[side]["passed"] for result in quality["cases"] for side in ("baseline", "candidate") - ): + ) + row["timing"]["eligible"] = completed and quality_passed + if not completed: + row["reasons"].append("An attempt failed, was interrupted or did not complete") + if not quality_passed: row["reasons"].append( f"{label} and candidate must both pass the declared quality checks" ) @@ -879,6 +951,7 @@ def finish(session_path, candidate_path, root, *, change): ] ), "cases": rows, + "timing_summary": _timing_summary(rows), "study": study, } from agentloop.improvement_report import export_report diff --git a/agentloop/improvement_cli.py b/agentloop/improvement_cli.py index 72ddf20..6c49b69 100644 --- a/agentloop/improvement_cli.py +++ b/agentloop/improvement_cli.py @@ -6,9 +6,11 @@ import typer from rich.console import Console +from rich.table import Table from agentloop import improvement from agentloop.improvement_init import initialize, project_tasks +from agentloop.improvement_report import comparison_label, timing_summary_rows improve_app = typer.Typer( help="Capture fixed tasks, review baseline findings, and verify an applied change." @@ -164,6 +166,16 @@ def finish_command( f"{report['passed_attempts']} of {report['planned_attempts']} planned comparisons passed.", markup=False, ) + timing = Table( + "Timing", + f"{comparison_label(report)} median", + "Candidate median", + "Measured pairs", + caption="Completed, correct pairs. Latency gates use trace latency.", + ) + for label, before, after, _, pairs in timing_summary_rows(report): + timing.add_row(label, before, after, pairs) + console.print(timing) console.print(f"Read {out / 'report.html'}", markup=False) for case in report["cases"]: if not case["passed"]: diff --git a/agentloop/improvement_report.py b/agentloop/improvement_report.py index 00557be..57a7bb5 100644 --- a/agentloop/improvement_report.py +++ b/agentloop/improvement_report.py @@ -13,6 +13,22 @@ markdown_text, ) +_TIMINGS = (("Trace latency", "trace_elapsed_ms"), ("Runner wall time", "runner_elapsed_ms")) +_MEASUREMENTS = ( + ("Input tokens", "input_tokens"), + ("Output tokens", "output_tokens"), + ("Model calls", "model_call_count"), + ("Tool operations", "tool_operations"), + ("Estimated model cost (USD)", "estimated_cost_usd"), + ("Quality score", "quality_score"), +) +_RUNNER_CLOCKS = {"monotonic": "Monotonic clock", "capture_timestamps": "Capture timestamps"} +_TIMING_SCOPE = ( + "Trace latency covers the instrumented execution. Runner wall time spans launch " + "through exit, including startup, trace export, and cleanup after interruption. " + "Latency gates use trace latency." +) + def plan_markdown(specification): lines = [ @@ -182,6 +198,72 @@ def _headline(report): return f"{report['passed_attempts']} of {report['planned_attempts']} planned attempts passed." +def comparison_label(report): + return "Control" if report.get("comparison_baseline") == "interleaved_control" else "Baseline" + + +def timing_summary_rows(report): + timing = report.get("timing_summary") + if timing is None: + return [] + return [ + [ + f"{label} (ms)", + _metric(timing[key], "baseline_median_ms"), + _metric(timing[key], "candidate_median_ms"), + _metric(timing[key], "median_paired_delta_ms"), + f"{timing[key]['paired_count']}/{report['planned_attempts']}", + ] + for label, key in _TIMINGS + ] + + +def _measurement_rows(row): + timing, replay = row.get("timing"), row["replay"] + sides = ("baseline", "candidate") + measurements = [] + if timing is not None: + measurements.extend( + [f"{label} (ms)", *(_metric(timing[side], key) for side in sides)] + for label, key in _TIMINGS + ) + measurements.append( + [ + "Runner clock", + *( + _RUNNER_CLOCKS.get(timing[side]["runner_clock"], "unavailable") + for side in sides + ), + ] + ) + elif replay: + measurements.append( + ["Trace latency (ms)", *(_metric(replay[side], "runtime_ms") for side in sides)] + ) + if replay: + measurements.extend( + [label, *(_metric(replay[side], key) for side in sides)] for label, key in _MEASUREMENTS + ) + return measurements + + +_TIMING_SUMMARY_SCOPE = ( + "Medians use pairs with completed traces, passing checks on both answers, and both " + "measurements for the given scope. The paired difference is candidate minus " + "comparison time for each pair, then summarized by the median." +) + + +def _timing_headers(comparison): + return [ + "Timing", + f"{comparison} median", + "Candidate median", + "Median paired difference", + "Pairs measured", + ] + + def _basis(report): if report.get("comparison_baseline") == "interleaved_control": return ( @@ -224,6 +306,7 @@ def _decision_lines(report): def report_markdown(report): + comparison = comparison_label(report) lines = [ f"# {markdown_heading(report['name'])}", "", @@ -248,6 +331,23 @@ def report_markdown(report): "", ] ) + timing_rows = timing_summary_rows(report) + if timing_rows: + headers = _timing_headers(comparison) + lines.extend( + [ + "## Paired timing", + "", + _TIMING_SCOPE, + "", + _TIMING_SUMMARY_SCOPE, + "", + "| " + " | ".join(headers) + " |", + "| " + " | ".join(["---"] * len(headers)) + " |", + *("| " + " | ".join(values) + " |" for values in timing_rows), + "", + ] + ) for row in report["cases"]: lines.extend( [ @@ -258,21 +358,11 @@ def report_markdown(report): ] ) lines.extend(f"- {markdown_text(reason)}" for reason in row["reasons"]) + measurements = _measurement_rows(row) + if measurements: + lines.extend(["", f"| Measurement | {comparison} | Candidate |", "| --- | --- | --- |"]) + lines.extend("| " + " | ".join(values) + " |" for values in measurements) if row["replay"]: - lines.extend(["", "| Measurement | Baseline | Candidate |", "| --- | --- | --- |"]) - for label, metric in ( - ("Runtime (ms)", "runtime_ms"), - ("Input tokens", "input_tokens"), - ("Output tokens", "output_tokens"), - ("Model calls", "model_call_count"), - ("Tool operations", "tool_operations"), - ("Estimated model cost (USD)", "estimated_cost_usd"), - ("Quality score", "quality_score"), - ): - replay = row["replay"] - lines.append( - f"| {label} | {_metric(replay['baseline'], metric)} | {_metric(replay['candidate'], metric)} |" - ) lines.extend(["", "| Gate | Result | Detail |", "| --- | --- | --- |"]) for gate in row["replay"]["gates"]["results"]: status = ( @@ -301,6 +391,7 @@ def export_report(root, report): write_once(root, "report.json", report) markdown = report_markdown(report) (root / "report.md").write_text(markdown, encoding="utf-8") + comparison = comparison_label(report) sections = [ f"

{_text(report['name'])}

{_text(_headline(report))}

", f"

Applied change

{_text(report['change'])}

", @@ -311,36 +402,35 @@ def export_report(root, report): sections.append( '

This comparison includes synthetic evidence. Results apply to the supplied fixtures.

' ) - measurements = ( - ("Runtime (ms)", "runtime_ms"), - ("Input tokens", "input_tokens"), - ("Output tokens", "output_tokens"), - ("Model calls", "model_call_count"), - ("Tool operations", "tool_operations"), - ("Estimated model cost (USD)", "estimated_cost_usd"), - ("Quality score", "quality_score"), - ) + timing_rows = timing_summary_rows(report) + if timing_rows: + sections.extend( + [ + f"

Paired timing

{_text(_TIMING_SCOPE)}

{_text(_TIMING_SUMMARY_SCOPE)}

", + _table( + _timing_headers(comparison), + [[_text(cell) for cell in values] for values in timing_rows], + caption="Measured timing for completed, correct pairs", + ), + ] + ) for index, row in enumerate(report["cases"]): status = "Passed" if row["passed"] else "Rejected" sections.append( f"

{_text(row['task_id'])}, repetition {row['repetition']}

{status}

" ) sections.extend(f"

{_text(reason)}

" for reason in row["reasons"]) - if row["replay"]: - replay = row["replay"] - values = [ - [ - label, - _text(_metric(replay["baseline"], metric)), - _text(_metric(replay["candidate"], metric)), - ] - for label, metric in measurements - ] + measurements = _measurement_rows(row) + if measurements: sections.append( _table( - ["Measurement", "Baseline", "Candidate"], values, caption="Paired measurements" + ["Measurement", comparison, "Candidate"], + [[_text(cell) for cell in values] for values in measurements], + caption="Paired measurements", ) ) + if row["replay"]: + replay = row["replay"] checks = [ [ _text(gate["name"]), @@ -359,7 +449,7 @@ def export_report(root, report): + "" ) sections.append( - f'

Baseline trace · Candidate trace

' + f'

{comparison} trace · Candidate trace

' ) sections.append("
") sections.append( diff --git a/docs/GUIDED_OPTIMIZATION.md b/docs/GUIDED_OPTIMIZATION.md index bebac61..efd96fa 100644 --- a/docs/GUIDED_OPTIMIZATION.md +++ b/docs/GUIDED_OPTIMIZATION.md @@ -166,9 +166,22 @@ agentloop improve capture --tasks tasks.json --out runs/candidate -- python your agentloop improve finish --session runs/optimization/session.json --candidate runs/candidate/capture.json --change "Describe the change applied to the agent" --out runs/results ``` -Open `runs/results/report.html`. The report shows per-attempt quality, latency, +Open `runs/results/report.html`. The report shows per-attempt quality, timing, token provenance, model/tool counts, available model-cost estimates, and native -replay gate decisions. JSON and Markdown accompany it. Pairs with matching frozen +replay gate decisions. The CLI, HTML, and Markdown show paired timing medians. +Trace latency covers the instrumented execution and drives latency gates. Runner +wall time covers process launch through exit, including startup, trace export, +and cleanup after interruption. It can help explain a small end-to-end gain when +the traced change is much faster. Runner wall time is descriptive. + +Timing medians use completed, correct pairs with available measurements on both +sides. Each metric lists how many pairs it could measure out of the full planned +set. Failed attempts retain any recorded timings in their individual rows. New +captures use a monotonic clock. Older captures with valid start and end timestamps +show the derived duration and its clock source. Missing timings show as +unavailable. + +JSON and Markdown accompany the HTML. Pairs with matching frozen findings also produce native intervention records. Available baseline and candidate traces produce a paired study. Study views retain the original trace fingerprints. diff --git a/scripts/smoke_guided_wheel.py b/scripts/smoke_guided_wheel.py index 2494237..5102d32 100644 --- a/scripts/smoke_guided_wheel.py +++ b/scripts/smoke_guided_wheel.py @@ -147,6 +147,11 @@ def capture(mode, expected=0): raise RuntimeError("Installed guided outcome lost a planned attempt or quality failure") if not (root / ("report-" + mode) / "report.html").is_file(): raise RuntimeError("Installed guided HTML report missing") + timing = report["cases"][0]["timing"]["candidate"] + if timing["runner_clock"] != "monotonic" or timing["runner_elapsed_ms"] <= 0: + raise RuntimeError("Installed guided capture lost process timing") + if report["timing_summary"]["runner_elapsed_ms"]["paired_count"] != int(report["passed"]): + raise RuntimeError("Installed guided timing counted an incorrect or incomplete pair") cli( "improve", "finish", diff --git a/tests/test_improvement.py b/tests/test_improvement.py index 1341b2f..04b3529 100644 --- a/tests/test_improvement.py +++ b/tests/test_improvement.py @@ -83,6 +83,8 @@ def test_guided_workflow_uses_native_quality_replay_and_studies(tmp_path, worklo ) assert report["study"] is not None assert all(row["intervention"] for row in report["cases"]) + assert report["timing_summary"]["trace_elapsed_ms"]["paired_count"] == 4 + assert report["timing_summary"]["runner_elapsed_ms"]["paired_count"] == 4 assert (tmp_path / "results/report.html").is_file() assert (tmp_path / "session/plan.md").is_file() @@ -100,6 +102,16 @@ def test_faster_wrong_failed_or_missing_candidates_are_rejected(tmp_path, worklo assert report["passed"] is False assert report["planned_attempts"] == 4 assert report["passed_attempts"] == 0 + assert report["timing_summary"]["eligible_pairs"] == 0 + assert report["timing_summary"]["runner_elapsed_ms"]["paired_count"] == 0 + assert all(row["timing"]["candidate"]["runner_elapsed_ms"] > 0 for row in report["cases"]) + if mode == "missing": + assert all( + row["timing"]["candidate"]["trace_elapsed_ms"] is None for row in report["cases"] + ) + assert "| Trace latency (ms) |" in (tmp_path / "results/report.md").read_text( + encoding="utf-8" + ) assert all( row["intervention"] is None or row["intervention"]["gates_passed"] is False for row in report["cases"] @@ -235,6 +247,8 @@ def test_cli_runs_the_normal_entry_points_and_returns_quality_failure(tmp_path, ], ) assert result.exit_code == 1, result.output + assert "Trace latency (ms)" in result.output + assert "Runner wall time (ms)" in result.output assert (tmp_path / "results/report.json").is_file() @@ -278,6 +292,8 @@ def interrupt(*args, **kwargs): "not_started", "not_started", ] + assert manifest["runs"][0]["runner_elapsed_ms"] >= 0 + assert all("runner_elapsed_ms" not in row for row in manifest["runs"][1:]) def test_invalid_trace_keeps_the_runner_failure_status(tmp_path, workload): @@ -376,7 +392,60 @@ def test_interleaved_control_alternates_order_and_becomes_the_comparison(tmp_pat assert row["intervention"]["baseline_run_id"] in control_ids assert row["frozen_baseline_status"] == "completed" assert [row["control_ran_first"] for row in report["cases"]] == [True, False, True, False] - assert "alternating order" in (tmp_path / "results/report.md").read_text(encoding="utf-8") + for index, row in enumerate(report["cases"]): + timing = row["timing"] + assert ( + timing["baseline"]["runner_elapsed_ms"] == control["runs"][index]["runner_elapsed_ms"] + ) + assert timing["baseline"]["runner_clock"] == "monotonic" + assert ( + timing["candidate"]["runner_elapsed_ms"] == manifest["runs"][index]["runner_elapsed_ms"] + ) + markdown = (tmp_path / "results/report.md").read_text(encoding="utf-8") + assert "alternating order" in markdown + assert "| Measurement | Control | Candidate |" in markdown + page = (tmp_path / "results/report.html").read_text(encoding="utf-8") + assert "Control median" in page and "Control trace" in page + assert "Trace latency (ms)" in page and "Runner wall time (ms)" in page + + +def test_timing_summary_reads_legacy_receipts_and_keeps_runner_time_out_of_gates( + tmp_path, workload +): + tasks, worker = workload + baseline_path = tmp_path / "baseline/capture.json" + baseline = capture(tasks, baseline_path.parent, command(worker, "baseline")) + for row, end in zip(baseline["runs"], ("00.150", "00.350", "00.200", "00.099"), strict=True): + row.pop("runner_elapsed_ms") + row.update(started_at="2026-01-01T00:00:00.100Z", ended_at=f"2026-01-01T00:00:{end}+00:00") + baseline_path.write_text(json.dumps(baseline), encoding="utf-8") + session = tmp_path / "session" + start(tasks, baseline_path, session) + + candidate_path = tmp_path / "candidate/capture.json" + candidate = capture(tasks, candidate_path.parent, command(worker, "fast")) + for row, runner_ms in zip(candidate["runs"], (300, 350, True, 400), strict=True): + row["runner_elapsed_ms"] = runner_ms + candidate_path.write_text(json.dumps(candidate), encoding="utf-8") + + report = finish( + session / "session.json", + candidate_path, + tmp_path / "results", + change="Spend less time inside the trace", + ) + assert report["passed"] + assert report["timing_summary"]["eligible_pairs"] == 4 + assert report["timing_summary"]["trace_elapsed_ms"]["paired_count"] == 4 + assert report["timing_summary"]["runner_elapsed_ms"] == { + "paired_count": 2, + "baseline_median_ms": 150.0, + "candidate_median_ms": 325.0, + "median_paired_delta_ms": 175.0, + } + assert report["cases"][0]["timing"]["baseline"]["runner_clock"] == "capture_timestamps" + assert report["cases"][2]["timing"]["candidate"]["runner_clock"] is None + assert report["cases"][3]["timing"]["baseline"]["runner_elapsed_ms"] is None def test_interleaved_control_must_also_pass_quality_checks(tmp_path, workload): From a39c725bd4c0df48bc2831ae3fd61e8e8ee298bd Mon Sep 17 00:00:00 2001 From: Dipesh Babu Date: Tue, 17 Nov 2026 22:34:19 -0500 Subject: [PATCH 2/5] Decode guided CLI smoke output as UTF-8 --- scripts/smoke_guided_wheel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/smoke_guided_wheel.py b/scripts/smoke_guided_wheel.py index 5102d32..6e32fc4 100644 --- a/scripts/smoke_guided_wheel.py +++ b/scripts/smoke_guided_wheel.py @@ -87,7 +87,7 @@ def cli(*args, expected=0): check=False, timeout=60, capture_output=True, - text=True, + encoding="utf-8", ) allowed = expected if isinstance(expected, tuple) else (expected,) if result.returncode not in allowed: From 64610a3c21dbe9a0903f652255d9ecf3e7a2cc37 Mon Sep 17 00:00:00 2001 From: Dipesh Babu Date: Tue, 17 Nov 2026 22:35:25 -0500 Subject: [PATCH 3/5] Name the trace-based check in aggregate timing reports --- agentloop/improvement_cli.py | 4 ++-- agentloop/improvement_report.py | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/agentloop/improvement_cli.py b/agentloop/improvement_cli.py index 6c49b69..c2d6210 100644 --- a/agentloop/improvement_cli.py +++ b/agentloop/improvement_cli.py @@ -10,7 +10,7 @@ from agentloop import improvement from agentloop.improvement_init import initialize, project_tasks -from agentloop.improvement_report import comparison_label, timing_summary_rows +from agentloop.improvement_report import comparison_label, decision_objective, timing_summary_rows improve_app = typer.Typer( help="Capture fixed tasks, review baseline findings, and verify an applied change." @@ -153,7 +153,7 @@ def finish_command( ) median = verdict["median_improvement_pct"] console.print( - f"Aggregate {verdict['objective']} decision " + f"Aggregate {decision_objective(verdict)} check " + ("passed" if verdict["passed"] else "failed") + f": median improvement {'unavailable' if median is None else f'{median:.2f}%'}, " f"{verdict['improved_pairs']} improved, {verdict['regressed_pairs']} regressed.", diff --git a/agentloop/improvement_report.py b/agentloop/improvement_report.py index 57a7bb5..26a240c 100644 --- a/agentloop/improvement_report.py +++ b/agentloop/improvement_report.py @@ -276,16 +276,21 @@ def _basis(report): ) +def decision_objective(verdict): + return "trace latency" if verdict["objective"] == "latency" else verdict["objective"] + + def _decision_lines(report): verdict = report.get("decision") or {"mode": "per_pair"} if verdict["mode"] != "aggregate": return ["Decision: every pair must pass correctness, completion and performance checks."] median = verdict["median_improvement_pct"] p_value = verdict["sign_test_p_value"] + objective = decision_objective(verdict) lines = [ "Decision: every pair must pass correctness and completion checks; " - f"{verdict['objective']} is judged across all pairs.", - f"Aggregate {verdict['objective']} decision " + f"{objective} is judged across all pairs.", + f"Aggregate {objective} check " + ("passed" if verdict["passed"] else "failed") + ": median paired improvement " + ("unavailable" if median is None else f"{median:.2f}%") From ca02dced5ad5022ccc1700936c6420df267496cc Mon Sep 17 00:00:00 2001 From: Dipesh Babu Date: Tue, 17 Nov 2026 22:35:58 -0500 Subject: [PATCH 4/5] Read UTF-8 output in the guided onboarding test --- tests/test_improvement_init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_improvement_init.py b/tests/test_improvement_init.py index f11b6fa..04fafae 100644 --- a/tests/test_improvement_init.py +++ b/tests/test_improvement_init.py @@ -18,7 +18,7 @@ def run_cli(project, *args, allowed=(0,)): [sys.executable, "-c", "from agentloop.entrypoint import app; app()", *map(str, args)], cwd=project, capture_output=True, - text=True, + encoding="utf-8", timeout=60, ) assert result.returncode in allowed, result.stdout + result.stderr From 4af75b0bcc2bc2a42ad10bc81592f14e67cfc02a Mon Sep 17 00:00:00 2001 From: Dipesh Babu Date: Tue, 17 Nov 2026 22:41:18 -0500 Subject: [PATCH 5/5] Document when paired runner timing is available --- docs/GUIDED_OPTIMIZATION.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/GUIDED_OPTIMIZATION.md b/docs/GUIDED_OPTIMIZATION.md index efd96fa..b7fcb5f 100644 --- a/docs/GUIDED_OPTIMIZATION.md +++ b/docs/GUIDED_OPTIMIZATION.md @@ -168,7 +168,8 @@ agentloop improve finish --session runs/optimization/session.json --candidate ru Open `runs/results/report.html`. The report shows per-attempt quality, timing, token provenance, model/tool counts, available model-cost estimates, and native -replay gate decisions. The CLI, HTML, and Markdown show paired timing medians. +replay gate decisions. Builds after 0.2.3 show paired timing medians in the CLI, +HTML, and Markdown. Trace latency covers the instrumented execution and drives latency gates. Runner wall time covers process launch through exit, including startup, trace export, and cleanup after interruption. It can help explain a small end-to-end gain when