diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8c8c6c4..23e73a4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,13 @@ trace, study, receipt, API and database schema versions.
## [Unreleased]
+### Added
+
+- Guided comparisons show trace latency and runner wall time in the CLI, JSON,
+ Markdown, and HTML. Per-attempt timing identifies the clock source. Paired
+ summaries show medians and coverage for completed, correct attempts. Latency
+ gates continue to use the trace measurement.
+
## [0.2.3] - 2026-11-17
### Added
diff --git a/agentloop/improvement.py b/agentloop/improvement.py
index 91361ec..ba65a67 100644
--- a/agentloop/improvement.py
+++ b/agentloop/improvement.py
@@ -12,6 +12,7 @@
from dataclasses import asdict
from datetime import datetime, timezone
from pathlib import Path
+from time import perf_counter_ns
from agentloop.experiment_artifacts import local_path, read_json, write_once
from agentloop.experiment_types import fingerprint
@@ -21,6 +22,7 @@
from agentloop.repetitions import add_repetition_findings
from agentloop.replay import ReplayGates, build_replay_report
from agentloop.replay import _summary as replay_summary
+from agentloop.schema import TraceValidationError, validate_elapsed_ms
from agentloop.tracer import AgentTrace
VERSION = "1.0"
@@ -234,12 +236,15 @@ def _attempt(root, index, row, task, command, timeout, show_output):
"trace_path": str(path),
}
row["started_at"] = now()
+ started = perf_counter_ns()
try:
row.update(_invoke(command, request, timeout, show_output=show_output))
except KeyboardInterrupt:
row["status"] = "cancelled"
raise
finally:
+ if row["status"] != "runner_error":
+ row["runner_elapsed_ms"] = (perf_counter_ns() - started) / 1_000_000
row["ended_at"] = now()
if path.exists():
try:
@@ -506,6 +511,65 @@ def _trace_usable(trace):
return not any(event.parent_id is None and event.status == "error" for event in trace.events)
+def _attempt_timing(row, trace):
+ """Describe the trace boundary and the capture process boundary independently."""
+ timing = {
+ "trace_elapsed_ms": trace.elapsed_ms if trace is not None else None,
+ "runner_elapsed_ms": None,
+ "runner_clock": None,
+ }
+ if row.get("status") in {"runner_error", "not_started"}:
+ return timing
+ if "runner_elapsed_ms" in row:
+ try:
+ elapsed = validate_elapsed_ms(row["runner_elapsed_ms"])
+ except TraceValidationError:
+ return timing
+ clock = "monotonic"
+ else:
+ # Capture 1.0 manifests predating runner timing retain UTC boundaries.
+ try:
+ started = datetime.fromisoformat(row.get("started_at"))
+ ended = datetime.fromisoformat(row.get("ended_at"))
+ if started.utcoffset() is None or ended.utcoffset() is None:
+ return timing
+ elapsed = (ended - started).total_seconds() * 1000
+ except (TypeError, ValueError, OverflowError):
+ return timing
+ clock = "capture_timestamps"
+ if elapsed is not None and elapsed >= 0:
+ timing.update(runner_elapsed_ms=elapsed, runner_clock=clock)
+ return timing
+
+
+def _median_duration(values):
+ if not values:
+ return None
+ ordered = sorted(values)
+ low, high = ordered[(len(ordered) - 1) // 2], ordered[len(ordered) // 2]
+ # Preserve the usual median and guard against overflow from adding two values.
+ middle = low + high
+ return middle / 2 if math.isfinite(middle) else low / 2 + high / 2
+
+
+def _timing_summary(rows):
+ eligible = [row["timing"] for row in rows if row["timing"]["eligible"]]
+ summary = {"eligible_pairs": len(eligible)}
+ for metric in ("trace_elapsed_ms", "runner_elapsed_ms"):
+ pairs = [
+ (timing["baseline"][metric], timing["candidate"][metric])
+ for timing in eligible
+ if all(timing[side][metric] is not None for side in ("baseline", "candidate"))
+ ]
+ summary[metric] = {
+ "paired_count": len(pairs),
+ "baseline_median_ms": _median_duration([before for before, _ in pairs]),
+ "candidate_median_ms": _median_duration([after for _, after in pairs]),
+ "median_paired_delta_ms": _median_duration([after - before for before, after in pairs]),
+ }
+ return summary
+
+
def _sign_test_p_value(improved, regressed):
"""One-sided exact sign test: chance of at least this many improvements."""
trials = improved + regressed
@@ -668,6 +732,15 @@ def finish(session_path, candidate_path, root, *, change):
_completed(control_manifest, controls),
)
gates = ReplayGates(**specification["gates"])
+ comparison_manifest = (
+ specification["baseline_capture"] if controls is None else control_manifest
+ )
+ baseline_rows = {
+ (row["task_id"], row["repetition"]): row for row in comparison_manifest["runs"]
+ }
+ candidate_rows = {
+ (row["task_id"], row["repetition"]): row for row in candidate_manifest["runs"]
+ }
root = Path(root).resolve()
root.mkdir(parents=True, exist_ok=False)
rows, study_paths = [], {"baseline": [], "candidate": []}
@@ -675,9 +748,7 @@ def finish(session_path, candidate_path, root, *, change):
key = (case["task_id"], case["repetition"])
task = next(task for task in workload["tasks"] if task["id"] == key[0])
candidate = candidates.get(key)
- candidate_row = next(
- row for row in candidate_manifest["runs"] if (row["task_id"], row["repetition"]) == key
- )
+ baseline_row, candidate_row = baseline_rows.get(key, {}), candidate_rows[key]
if controls is None:
label, baseline = "Baseline", baselines.get(key)
baseline_status = case["capture_status"]
@@ -687,13 +758,8 @@ def finish(session_path, candidate_path, root, *, change):
# Findings stay frozen from the original baseline. Measurements compare
# the candidate with the unchanged control run alongside it.
label, baseline = "Control", controls.get(key)
- control_row = next(
- row
- for row in control_manifest["runs"]
- if (row["task_id"], row["repetition"]) == key
- )
- baseline_status = control_row["status"]
- baseline_diagnostic = control_row.get("diagnostic")
+ baseline_status = baseline_row["status"]
+ baseline_diagnostic = baseline_row.get("diagnostic")
diagnosis = control_diagnoses.get(key)
finding_ids = (
[
@@ -714,11 +780,16 @@ def finish(session_path, candidate_path, root, *, change):
"quality": None,
"replay": None,
"intervention": None,
+ "timing": {
+ "baseline": _attempt_timing(baseline_row, baseline),
+ "candidate": _attempt_timing(candidate_row, candidate),
+ "eligible": False,
+ },
}
if controls is not None:
row.update(
frozen_baseline_status=case["capture_status"],
- control_ran_first=control_row.get("order") == 0,
+ control_ran_first=baseline_row.get("order") == 0,
)
if baseline is None or candidate is None:
row["reasons"].append(f"A planned {label.lower()} or candidate trace is unavailable")
@@ -733,17 +804,18 @@ def finish(session_path, candidate_path, root, *, change):
else:
quality = _quality(task, baseline, candidate)
replay = build_replay_report(baseline, candidate, gates=gates, quality_report=quality)
- if (
- baseline_status != "completed"
- or candidate_row["status"] != "completed"
- or not all(_trace_usable(trace) for trace in (baseline, candidate))
- ):
- row["reasons"].append("An attempt failed, was interrupted or did not complete")
- if not quality["passed"] or any(
- not result[side]["passed"]
+ completed = baseline_status == candidate_row["status"] == "completed" and all(
+ _trace_usable(trace) for trace in (baseline, candidate)
+ )
+ quality_passed = quality["passed"] and all(
+ result[side]["passed"]
for result in quality["cases"]
for side in ("baseline", "candidate")
- ):
+ )
+ row["timing"]["eligible"] = completed and quality_passed
+ if not completed:
+ row["reasons"].append("An attempt failed, was interrupted or did not complete")
+ if not quality_passed:
row["reasons"].append(
f"{label} and candidate must both pass the declared quality checks"
)
@@ -879,6 +951,7 @@ def finish(session_path, candidate_path, root, *, change):
]
),
"cases": rows,
+ "timing_summary": _timing_summary(rows),
"study": study,
}
from agentloop.improvement_report import export_report
diff --git a/agentloop/improvement_cli.py b/agentloop/improvement_cli.py
index 72ddf20..c2d6210 100644
--- a/agentloop/improvement_cli.py
+++ b/agentloop/improvement_cli.py
@@ -6,9 +6,11 @@
import typer
from rich.console import Console
+from rich.table import Table
from agentloop import improvement
from agentloop.improvement_init import initialize, project_tasks
+from agentloop.improvement_report import comparison_label, decision_objective, timing_summary_rows
improve_app = typer.Typer(
help="Capture fixed tasks, review baseline findings, and verify an applied change."
@@ -151,7 +153,7 @@ def finish_command(
)
median = verdict["median_improvement_pct"]
console.print(
- f"Aggregate {verdict['objective']} decision "
+ f"Aggregate {decision_objective(verdict)} check "
+ ("passed" if verdict["passed"] else "failed")
+ f": median improvement {'unavailable' if median is None else f'{median:.2f}%'}, "
f"{verdict['improved_pairs']} improved, {verdict['regressed_pairs']} regressed.",
@@ -164,6 +166,16 @@ def finish_command(
f"{report['passed_attempts']} of {report['planned_attempts']} planned comparisons passed.",
markup=False,
)
+ timing = Table(
+ "Timing",
+ f"{comparison_label(report)} median",
+ "Candidate median",
+ "Measured pairs",
+ caption="Completed, correct pairs. Latency gates use trace latency.",
+ )
+ for label, before, after, _, pairs in timing_summary_rows(report):
+ timing.add_row(label, before, after, pairs)
+ console.print(timing)
console.print(f"Read {out / 'report.html'}", markup=False)
for case in report["cases"]:
if not case["passed"]:
diff --git a/agentloop/improvement_report.py b/agentloop/improvement_report.py
index 00557be..26a240c 100644
--- a/agentloop/improvement_report.py
+++ b/agentloop/improvement_report.py
@@ -13,6 +13,22 @@
markdown_text,
)
+_TIMINGS = (("Trace latency", "trace_elapsed_ms"), ("Runner wall time", "runner_elapsed_ms"))
+_MEASUREMENTS = (
+ ("Input tokens", "input_tokens"),
+ ("Output tokens", "output_tokens"),
+ ("Model calls", "model_call_count"),
+ ("Tool operations", "tool_operations"),
+ ("Estimated model cost (USD)", "estimated_cost_usd"),
+ ("Quality score", "quality_score"),
+)
+_RUNNER_CLOCKS = {"monotonic": "Monotonic clock", "capture_timestamps": "Capture timestamps"}
+_TIMING_SCOPE = (
+ "Trace latency covers the instrumented execution. Runner wall time spans launch "
+ "through exit, including startup, trace export, and cleanup after interruption. "
+ "Latency gates use trace latency."
+)
+
def plan_markdown(specification):
lines = [
@@ -182,6 +198,72 @@ def _headline(report):
return f"{report['passed_attempts']} of {report['planned_attempts']} planned attempts passed."
+def comparison_label(report):
+ return "Control" if report.get("comparison_baseline") == "interleaved_control" else "Baseline"
+
+
+def timing_summary_rows(report):
+ timing = report.get("timing_summary")
+ if timing is None:
+ return []
+ return [
+ [
+ f"{label} (ms)",
+ _metric(timing[key], "baseline_median_ms"),
+ _metric(timing[key], "candidate_median_ms"),
+ _metric(timing[key], "median_paired_delta_ms"),
+ f"{timing[key]['paired_count']}/{report['planned_attempts']}",
+ ]
+ for label, key in _TIMINGS
+ ]
+
+
+def _measurement_rows(row):
+ timing, replay = row.get("timing"), row["replay"]
+ sides = ("baseline", "candidate")
+ measurements = []
+ if timing is not None:
+ measurements.extend(
+ [f"{label} (ms)", *(_metric(timing[side], key) for side in sides)]
+ for label, key in _TIMINGS
+ )
+ measurements.append(
+ [
+ "Runner clock",
+ *(
+ _RUNNER_CLOCKS.get(timing[side]["runner_clock"], "unavailable")
+ for side in sides
+ ),
+ ]
+ )
+ elif replay:
+ measurements.append(
+ ["Trace latency (ms)", *(_metric(replay[side], "runtime_ms") for side in sides)]
+ )
+ if replay:
+ measurements.extend(
+ [label, *(_metric(replay[side], key) for side in sides)] for label, key in _MEASUREMENTS
+ )
+ return measurements
+
+
+_TIMING_SUMMARY_SCOPE = (
+ "Medians use pairs with completed traces, passing checks on both answers, and both "
+ "measurements for the given scope. The paired difference is candidate minus "
+ "comparison time for each pair, then summarized by the median."
+)
+
+
+def _timing_headers(comparison):
+ return [
+ "Timing",
+ f"{comparison} median",
+ "Candidate median",
+ "Median paired difference",
+ "Pairs measured",
+ ]
+
+
def _basis(report):
if report.get("comparison_baseline") == "interleaved_control":
return (
@@ -194,16 +276,21 @@ def _basis(report):
)
+def decision_objective(verdict):
+ return "trace latency" if verdict["objective"] == "latency" else verdict["objective"]
+
+
def _decision_lines(report):
verdict = report.get("decision") or {"mode": "per_pair"}
if verdict["mode"] != "aggregate":
return ["Decision: every pair must pass correctness, completion and performance checks."]
median = verdict["median_improvement_pct"]
p_value = verdict["sign_test_p_value"]
+ objective = decision_objective(verdict)
lines = [
"Decision: every pair must pass correctness and completion checks; "
- f"{verdict['objective']} is judged across all pairs.",
- f"Aggregate {verdict['objective']} decision "
+ f"{objective} is judged across all pairs.",
+ f"Aggregate {objective} check "
+ ("passed" if verdict["passed"] else "failed")
+ ": median paired improvement "
+ ("unavailable" if median is None else f"{median:.2f}%")
@@ -224,6 +311,7 @@ def _decision_lines(report):
def report_markdown(report):
+ comparison = comparison_label(report)
lines = [
f"# {markdown_heading(report['name'])}",
"",
@@ -248,6 +336,23 @@ def report_markdown(report):
"",
]
)
+ timing_rows = timing_summary_rows(report)
+ if timing_rows:
+ headers = _timing_headers(comparison)
+ lines.extend(
+ [
+ "## Paired timing",
+ "",
+ _TIMING_SCOPE,
+ "",
+ _TIMING_SUMMARY_SCOPE,
+ "",
+ "| " + " | ".join(headers) + " |",
+ "| " + " | ".join(["---"] * len(headers)) + " |",
+ *("| " + " | ".join(values) + " |" for values in timing_rows),
+ "",
+ ]
+ )
for row in report["cases"]:
lines.extend(
[
@@ -258,21 +363,11 @@ def report_markdown(report):
]
)
lines.extend(f"- {markdown_text(reason)}" for reason in row["reasons"])
+ measurements = _measurement_rows(row)
+ if measurements:
+ lines.extend(["", f"| Measurement | {comparison} | Candidate |", "| --- | --- | --- |"])
+ lines.extend("| " + " | ".join(values) + " |" for values in measurements)
if row["replay"]:
- lines.extend(["", "| Measurement | Baseline | Candidate |", "| --- | --- | --- |"])
- for label, metric in (
- ("Runtime (ms)", "runtime_ms"),
- ("Input tokens", "input_tokens"),
- ("Output tokens", "output_tokens"),
- ("Model calls", "model_call_count"),
- ("Tool operations", "tool_operations"),
- ("Estimated model cost (USD)", "estimated_cost_usd"),
- ("Quality score", "quality_score"),
- ):
- replay = row["replay"]
- lines.append(
- f"| {label} | {_metric(replay['baseline'], metric)} | {_metric(replay['candidate'], metric)} |"
- )
lines.extend(["", "| Gate | Result | Detail |", "| --- | --- | --- |"])
for gate in row["replay"]["gates"]["results"]:
status = (
@@ -301,6 +396,7 @@ def export_report(root, report):
write_once(root, "report.json", report)
markdown = report_markdown(report)
(root / "report.md").write_text(markdown, encoding="utf-8")
+ comparison = comparison_label(report)
sections = [
f" {_text(_headline(report))}{_text(report['name'])}
{_text(report['change'])}
", @@ -311,36 +407,35 @@ def export_report(root, report): sections.append( 'This comparison includes synthetic evidence. Results apply to the supplied fixtures.
' ) - measurements = ( - ("Runtime (ms)", "runtime_ms"), - ("Input tokens", "input_tokens"), - ("Output tokens", "output_tokens"), - ("Model calls", "model_call_count"), - ("Tool operations", "tool_operations"), - ("Estimated model cost (USD)", "estimated_cost_usd"), - ("Quality score", "quality_score"), - ) + timing_rows = timing_summary_rows(report) + if timing_rows: + sections.extend( + [ + f"{_text(_TIMING_SCOPE)}
{_text(_TIMING_SUMMARY_SCOPE)}
", + _table( + _timing_headers(comparison), + [[_text(cell) for cell in values] for values in timing_rows], + caption="Measured timing for completed, correct pairs", + ), + ] + ) for index, row in enumerate(report["cases"]): status = "Passed" if row["passed"] else "Rejected" sections.append( f"{status}
" ) sections.extend(f"{_text(reason)}
" for reason in row["reasons"]) - if row["replay"]: - replay = row["replay"] - values = [ - [ - label, - _text(_metric(replay["baseline"], metric)), - _text(_metric(replay["candidate"], metric)), - ] - for label, metric in measurements - ] + measurements = _measurement_rows(row) + if measurements: sections.append( _table( - ["Measurement", "Baseline", "Candidate"], values, caption="Paired measurements" + ["Measurement", comparison, "Candidate"], + [[_text(cell) for cell in values] for values in measurements], + caption="Paired measurements", ) ) + if row["replay"]: + replay = row["replay"] checks = [ [ _text(gate["name"]), @@ -359,7 +454,7 @@ def export_report(root, report): + "" ) sections.append( - f'' + f'' ) sections.append("