Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ trace, study, receipt, API and database schema versions.
Markdown, and HTML. Per-attempt timing identifies the clock source. Paired
summaries show medians and coverage for completed, correct attempts. Latency
gates continue to use the trace measurement.
- Guided comparisons can require a trusted local Python verifier selected by
the developer. The frozen session binds the script and timeout, and both
sides must pass the verifier and their built-in quality checks.

## [0.2.3] - 2026-11-17

Expand Down
84 changes: 66 additions & 18 deletions agentloop/improvement.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from agentloop.experiment_artifacts import local_path, read_json, write_once
from agentloop.experiment_types import fingerprint
from agentloop.findings import build_diagnosis
from agentloop.improvement_verifier import DEFAULT_TIMEOUT, LocalVerifier, resume_verifier
from agentloop.interventions import build_intervention
from agentloop.quality import build_quality_report, validate_quality_fixtures
from agentloop.repetitions import add_repetition_findings
Expand Down Expand Up @@ -389,15 +390,25 @@ def _completed(manifest, traces):
return {key: trace for key, trace in traces.items() if key in done}


def _quality(task, baseline, candidate):
return build_quality_report(
[
def _quality(task, baseline, candidate, verification=None):
fixtures = [
{
"id": task["id"],
"expected": task["expected"],
"scorer": task.get("scorer", {"type": "exact_match"}),
}
]
if verification is not None:
fixtures.append(
{
"id": task["id"],
"expected": task["expected"],
"scorer": task.get("scorer", {"type": "exact_match"}),
"id": f"{task['id']}:local-verifier",
"expected": True,
"baseline_output": verification["baseline"]["passed"],
"candidate_output": verification["candidate"]["passed"],
}
],
)
return build_quality_report(
fixtures,
baseline_trace=baseline,
candidate_trace=candidate,
min_score=1.0,
Expand All @@ -413,9 +424,12 @@ def start(
objective="latency",
minimum_improvement=0,
decision="per_pair",
verifier=None,
verifier_timeout=DEFAULT_TIMEOUT,
):
workload = read_workload(workload_path)
manifest, traces = read_capture(baseline_path, workload)
verifier = LocalVerifier.read(verifier, verifier_timeout) if verifier is not None else None
require(objective in {"latency", "cost"}, "Objective must be latency or cost")
decision = decision.replace("-", "_") if isinstance(decision, str) else decision
require(decision in DECISIONS, "Decision must be per-pair or aggregate")
Expand Down Expand Up @@ -473,6 +487,13 @@ def start(
else [],
}
)
if verifier is not None:
cases[-1]["baseline_verifier"] = verifier.check(
task,
repetition,
trace,
usable=row["status"] == "completed" and trace is not None and _trace_usable(trace),
)
specification = {
"schema_version": VERSION,
"frozen_at": now(),
Expand All @@ -484,6 +505,8 @@ def start(
"cases": cases,
"baseline_capture": manifest,
}
if verifier is not None:
specification["verifier"] = verifier.declaration()
write_once(
root,
"session.json",
Expand Down Expand Up @@ -647,7 +670,7 @@ def _aggregate_decision(rows, objective, gates):
}


def finish(session_path, candidate_path, root, *, change):
def finish(session_path, candidate_path, root, *, change, verifier=None):
session_path = Path(session_path).resolve()
saved = load_json(session_path)
specification = saved["specification"]
Expand All @@ -657,6 +680,7 @@ def finish(session_path, candidate_path, root, *, change):
)
require(specification.get("schema_version") == VERSION, "Unsupported frozen session version")
workload = validate_workload(specification["workload"])
verifier = resume_verifier(specification.get("verifier"), verifier)
require(specification.get("objective") in {"latency", "cost"}, "Unsupported frozen objective")
# Sessions frozen before 0.2.3 have no decision field and keep per-pair judgment.
decision = specification.get("decision", "per_pair")
Expand Down Expand Up @@ -791,6 +815,28 @@ def finish(session_path, candidate_path, root, *, change):
frozen_baseline_status=case["capture_status"],
control_ran_first=baseline_row.get("order") == 0,
)
verification = None
if verifier is not None:
verification = {
side: verifier.check(
task,
key[1],
trace,
usable=(
trace is not None
and row[f"{side}_status"] == "completed"
and _trace_usable(trace)
),
)
for side, trace in (("baseline", baseline), ("candidate", candidate))
}
row["verifier"] = verification
for side, result in verification.items():
if not result["passed"]:
side_label = label if side == "baseline" else "Candidate"
row["reasons"].append(
f"{side_label} local verification failed: {result['detail']}"
)
if baseline is None or candidate is None:
row["reasons"].append(f"A planned {label.lower()} or candidate trace is unavailable")
for side, status, diagnostic in (
Expand All @@ -802,7 +848,7 @@ def finish(session_path, candidate_path, root, *, change):
f"{side} attempt is {status}. {diagnostic or 'Recapture this task with --show-runner-output to inspect the failure.'}"
)
else:
quality = _quality(task, baseline, candidate)
quality = _quality(task, baseline, candidate, verification)
replay = build_replay_report(baseline, candidate, gates=gates, quality_report=quality)
completed = baseline_status == candidate_row["status"] == "completed" and all(
_trace_usable(trace) for trace in (baseline, candidate)
Expand All @@ -819,12 +865,12 @@ def finish(session_path, candidate_path, root, *, change):
row["reasons"].append(
f"{label} and candidate must both pass the declared quality checks"
)
for result in quality["cases"]:
for side in ("baseline", "candidate"):
if not result[side]["passed"]:
row["reasons"].append(
f"{label if side == 'baseline' else 'Candidate'} answer failed the declared {task.get('scorer', {}).get('type', 'exact_match')} criterion: {result[side]['detail']}"
)
fixture_result = quality["cases"][0]
for side in ("baseline", "candidate"):
if not fixture_result[side]["passed"]:
row["reasons"].append(
f"{label if side == 'baseline' else 'Candidate'} answer failed the declared {task.get('scorer', {}).get('type', 'exact_match')} criterion: {fixture_result[side]['detail']}"
)
# Aggregate decisions judge speed and cost across all pairs after
# this loop; every other gate still applies to each pair.
judged = [
Expand Down Expand Up @@ -894,10 +940,10 @@ def finish(session_path, candidate_path, root, *, change):
source_trace_hash=fingerprint(trace.to_dict()),
)
if row["quality"]:
result = row["quality"]["cases"][0][side]
quality_cases = row["quality"]["cases"]
viewed["metadata"].update(
quality_score=result["score"],
success=result["passed"]
quality_score=row["quality"][f"{side}_score"],
success=all(result[side]["passed"] for result in quality_cases)
and _trace_usable(trace)
and row[f"{side}_status"] == "completed",
)
Expand Down Expand Up @@ -954,6 +1000,8 @@ def finish(session_path, candidate_path, root, *, change):
"timing_summary": _timing_summary(rows),
"study": study,
}
if verifier is not None:
report["verifier"] = verifier.declaration()
from agentloop.improvement_report import export_report

export_report(root, report)
Expand Down
21 changes: 20 additions & 1 deletion agentloop/improvement_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from agentloop import improvement
from agentloop.improvement_init import initialize, project_tasks
from agentloop.improvement_report import comparison_label, decision_objective, timing_summary_rows
from agentloop.improvement_verifier import DEFAULT_TIMEOUT

improve_app = typer.Typer(
help="Capture fixed tasks, review baseline findings, and verify an applied change."
Expand Down Expand Up @@ -108,6 +109,13 @@ def start_command(
help="per-pair requires every pair to improve; aggregate judges the median "
"paired improvement for noisy agents. Correctness is checked per pair in both.",
),
verifier: Path | None = typer.Option(None, help="Explicit trusted local Python verifier file."),
verifier_timeout: float = typer.Option(
DEFAULT_TIMEOUT,
min=0.01,
max=3600,
help="Seconds per verifier execution, frozen in the session.",
),
):
"""Freeze task checks and original findings before editing the agent."""
try:
Expand All @@ -119,6 +127,8 @@ def start_command(
objective=objective,
minimum_improvement=minimum_improvement,
decision=decision,
verifier=verifier,
verifier_timeout=verifier_timeout,
)
except (OSError, ValueError, TypeError, KeyError) as exc:
raise typer.BadParameter(str(exc)) from exc
Expand All @@ -130,6 +140,12 @@ def start_command(
markup=False,
)
console.print(f"Frozen session written to {out / 'session.json'}", markup=False)
if verifier is not None:
passed = sum(case["baseline_verifier"]["passed"] for case in specification["cases"])
total = len(specification["cases"])
console.print(
f"Local verifier passed on {passed} of {total} baseline attempts.", markup=False
)


@improve_app.command("finish")
Expand All @@ -138,10 +154,13 @@ def finish_command(
candidate: Path = typer.Option(..., help="Candidate capture.json recorded after start."),
change: str = typer.Option(..., help="Describe the change you applied."),
out: Path = typer.Option(..., help="Fresh report directory."),
verifier: Path | None = typer.Option(
None, help="Explicit trusted Python verifier matching the frozen session."
),
):
"""Compare every planned attempt and write JSON, Markdown and HTML results."""
try:
report = improvement.finish(session, candidate, out, change=change)
report = improvement.finish(session, candidate, out, change=change, verifier=verifier)
except (OSError, ValueError, TypeError, KeyError) as exc:
raise typer.BadParameter(str(exc)) from exc
verdict = report["decision"]
Expand Down
17 changes: 17 additions & 0 deletions agentloop/improvement_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,14 @@ def plan_markdown(specification):
else "The frozen decision requires every pair to improve the objective.",
"",
]
if specification.get("verifier"):
lines.extend([_verifier_scope(specification["verifier"]), ""])
for case in specification["cases"]:
lines.extend(
[f"## {markdown_heading(case['task_id'])}, repetition {case['repetition']}", ""]
)
if verification := case.get("baseline_verifier"):
lines.extend([f"Local verification: {markdown_text(verification['detail'])}.", ""])
diagnosis = case["diagnosis"]
quality = case.get("baseline_quality")
lines.extend(
Expand Down Expand Up @@ -244,6 +248,8 @@ def _measurement_rows(row):
measurements.extend(
[label, *(_metric(replay[side], key) for side in sides)] for label, key in _MEASUREMENTS
)
if verification := row.get("verifier"):
measurements.append(["Local verifier", *(verification[side]["detail"] for side in sides)])
return measurements


Expand Down Expand Up @@ -280,6 +286,13 @@ def decision_objective(verdict):
return "trace latency" if verdict["objective"] == "latency" else verdict["objective"]


def _verifier_scope(verifier):
return (
f"Local verifier SHA-256 {verifier['sha256']}. Each execution has a "
f"{verifier['timeout_s']:g}-second timeout. Both sides must pass."
)


def _decision_lines(report):
verdict = report.get("decision") or {"mode": "per_pair"}
if verdict["mode"] != "aggregate":
Expand Down Expand Up @@ -336,6 +349,8 @@ def report_markdown(report):
"",
]
)
if report.get("verifier"):
lines.extend([_verifier_scope(report["verifier"]), ""])
timing_rows = timing_summary_rows(report)
if timing_rows:
headers = _timing_headers(comparison)
Expand Down Expand Up @@ -407,6 +422,8 @@ def export_report(root, report):
sections.append(
'<p class="notice">This comparison includes synthetic evidence. Results apply to the supplied fixtures.</p>'
)
if report.get("verifier"):
sections.append(f"<p>{_text(_verifier_scope(report['verifier']))}</p>")
timing_rows = timing_summary_rows(report)
if timing_rows:
sections.extend(
Expand Down
95 changes: 95 additions & 0 deletions agentloop/improvement_verifier.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
"""Explicit local verifier snapshots for guided comparisons."""

from __future__ import annotations

import hashlib
import math
import sys
from dataclasses import dataclass, field
from pathlib import Path
from tempfile import TemporaryDirectory

from agentloop.quality import _MISSING, _trace_output

DEFAULT_TIMEOUT = 30.0
_MAX_SOURCE_BYTES = 1024 * 1024


@dataclass(frozen=True)
class LocalVerifier:
source: bytes = field(repr=False)
sha256: str
timeout_s: float

@classmethod
def read(cls, path, timeout_s):
if (
type(timeout_s) not in (int, float)
or not 0 < timeout_s <= 3600
or not math.isfinite(timeout_s)
):
raise ValueError("Verifier timeout must be positive and at most 3600 seconds")
path = Path(path)
if not path.is_file() or path.suffix.lower() != ".py":
raise ValueError("Select an existing Python verifier file with --verifier")
with path.open("rb") as handle:
source = handle.read(_MAX_SOURCE_BYTES + 1)
if not source or len(source) > _MAX_SOURCE_BYTES:
raise ValueError("Verifier source must be between 1 byte and 1 MiB")
return cls(source, hashlib.sha256(source).hexdigest(), float(timeout_s))

def declaration(self):
return {"sha256": self.sha256, "timeout_s": self.timeout_s}

def check(self, task, repetition, trace, *, usable):
if not usable:
return {
"status": "unavailable",
"passed": False,
"detail": "Attempt has no usable trace",
}
output = _trace_output(trace)
if output is _MISSING:
return {
"status": "unavailable",
"passed": False,
"detail": "Trace has no output for verification",
}
from agentloop.improvement import _invoke

request = {
"schema_version": "1.0",
"task_id": task["id"],
"repetition": repetition,
"input": task["input"],
"output": output,
}
with TemporaryDirectory(prefix="agentloop-verifier-") as directory:
# Execute the exact checked bytes even if the selected file changes.
snapshot = Path(directory) / "verifier.py"
snapshot.write_bytes(self.source)
result = _invoke([sys.executable, "-I", str(snapshot)], request, self.timeout_s)
state = result["status"]
detail = {
"completed": "Verifier exited successfully",
"failed": f"Verifier exited with code {result['exit_code']}",
"timed_out": f"Verifier exceeded its {self.timeout_s:g}-second timeout",
"cancelled": "Verifier was interrupted",
"runner_error": "Could not start the verifier",
}[state]
return {"status": state, "passed": state == "completed", "detail": detail}


def resume_verifier(declaration, path):
if declaration is None and path is None:
return None
if path is None:
raise ValueError("This session requires an explicit --verifier matching the frozen source")
if not isinstance(declaration, dict) or set(declaration) != {"sha256", "timeout_s"}:
raise ValueError("The session must freeze the local verifier in improve start")
verifier = LocalVerifier.read(path, declaration["timeout_s"])
if verifier.declaration() != declaration:
raise ValueError(
"Verifier source changed since improve start. Freeze a new session to change it"
)
return verifier
Loading
Loading