Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,13 @@ trace, study, receipt, API and database schema versions.

## [Unreleased]

### Fixed

- Repeated-tool findings now ask developers to investigate when only the tool
name repeats. Matching opaque input keys and declared read-only behavior can
support testing request-local reuse ahead of concurrency. Plans and estimates
identify which change the evidence supports.

### Added

- Guided comparisons show trace latency and runner wall time in the CLI, JSON,
Expand Down
14 changes: 13 additions & 1 deletion agentloop/estimates.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

from agentloop.costs import is_cost_evaluable
from agentloop.markdown import markdown_code_span, markdown_text
from agentloop.parallelism import REUSE_FORMULA


@dataclass(frozen=True)
Expand Down Expand Up @@ -92,8 +93,18 @@ def estimate_snapshot(candidate: Any, report: dict[str, Any], nodes: list[Any])
affected = set(candidate.affected_nodes or [])
durations = [node.duration_ms for node in nodes if node.node_id in affected]
cost_status = report.get("cost_status", "complete")
estimator_id = candidate.rule_id
extra_inputs = {}
if candidate.rule_id == "parallelize_tools":
retained = (candidate.observations or {}).get("retained_span_id")
if retained is not None:
estimator_id = "request_local_tool_reuse"
spec = Estimator(REUSE_FORMULA, ())
extra_inputs["retained_duration_ms"] = next(
node.duration_ms for node in nodes if node.node_id == retained
)
return {
"estimator_id": candidate.rule_id,
"estimator_id": estimator_id,
"estimator_version": spec.version,
"method": "heuristic",
"confidence": candidate.confidence,
Expand All @@ -110,6 +121,7 @@ def estimate_snapshot(candidate: Any, report: dict[str, Any], nodes: list[Any])
"repeated_context_ratio": report.get("repeated_context_ratio", 0.0),
"cost_status": cost_status,
"token_status": report.get("token_status", "unspecified"),
**extra_inputs,
},
"assumptions": list(dict.fromkeys([*(candidate.assumptions or []), *spec.assumptions])),
"unmodeled_metrics": (["latency"] if candidate.rule_id == "cache_context" else ["cost"]),
Expand Down
9 changes: 8 additions & 1 deletion agentloop/findings.py
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,13 @@ def _finding_from_card(
]
semantic = (card.get("estimate") or {}).get("estimator_id") == "semantic_leaf_removal"
observations = dict(card.get("observations", {}))
identity_title = title
if (
finding_type == "parallelize_tools"
and observations.get("recommended_action") == "inspect_repetition"
):
# The wording change preserves existing reviewed finding IDs.
identity_title = f"Parallelize repeated `{observations['tool_name']}` tool calls"
if semantic:
severity = "medium" if card.get("confidence") == "medium" else "low"
evidence.extend(
Expand All @@ -265,7 +272,7 @@ def _finding_from_card(
return OptimizationFinding(
finding_id=_stable_finding_id(
finding_type,
title,
identity_title,
affected_spans,
context={
"investigation_id": observations.get("investigation_id"),
Expand Down
3 changes: 2 additions & 1 deletion agentloop/graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -290,9 +290,10 @@ def dependency_summary(self):
and len(identities) == len(self.nodes),
}

def parallelizable_groups(self) -> list[dict[str, Any]]:
def parallelizable_groups(self, *, consider_reuse: bool = False) -> list[dict[str, Any]]:
return parallelization_candidates(
self.nodes,
consider_reuse=consider_reuse,
dependency_edges=(
(edge.source, edge.target)
for edge in self.edges
Expand Down
63 changes: 60 additions & 3 deletions agentloop/parallelism.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
from agentloop.workflow_types import stage_summary

PARALLELISM_NOTICE = (
"Repeated tool calls may be candidates for concurrent execution. "
"Verify output independence and shared-state safety before changing them."
"The tool name repeats in the same scope. Compare inputs and check dependencies "
"and shared state before changing the calls."
)
PARALLELISM_REWRITE = (
"After verifying output independence and shared-state safety, use asyncio.gather "
Expand All @@ -20,6 +20,16 @@
PARALLELISM_FORMULA = (
"sum(tool durations) - max(tool duration), assuming the calls can run concurrently"
)
REUSE_REWRITE = (
"Test reusing the first result within the same request. Confirm that the key covers "
"all effective inputs, the source stays stable, and task checks still pass."
)
REUSE_FORMULA = "sum_duration_ms - retained_duration_ms"
_REUSE_ASSUMPTIONS = (
"The declared key covers all effective inputs and the source stays stable within the request.",
"Reused results preserve required behavior and add negligible overhead.",
"The skipped calls are on the runtime-limiting path.",
)
_ASSUMPTIONS = (
"No call consumes another call's output.",
"Calls do not conflict through shared mutable state.",
Expand All @@ -28,7 +38,10 @@


def parallelization_candidates(
events: Iterable[Any], *, dependency_edges: Iterable[tuple[str, str]] = ()
events: Iterable[Any],
*,
dependency_edges: Iterable[tuple[str, str]] = (),
consider_reuse: bool = False,
) -> list[dict[str, Any]]:
"""Return hypotheses, never proof of independence or verified savings.

Expand All @@ -42,6 +55,7 @@ def parallelization_candidates(
invalid_dependencies: set[str] = set()
unsafe: set[str] = set()
declarations: set[str] = set()
reuse_keys: dict[str, str] = {}
for index, event in enumerate(events):
span_id = (
getattr(event, "event_id", None) or getattr(event, "node_id", None) or f"node_{index}"
Expand Down Expand Up @@ -70,6 +84,16 @@ def parallelization_candidates(
unsafe.add(span_id)
if operation_kind(event) == "tool":
groups.setdefault((event.name, getattr(event, "parent_id", None)), []).append(span_id)
reuse_key = metadata.get("tool_reuse_key")
if (
consider_reuse
and metadata.get("read_only") is True
and getattr(event, "status", None) == "ok"
and isinstance(reuse_key, str)
and 0 < len(reuse_key) <= 128
and reuse_key == reuse_key.strip()
):
reuse_keys[span_id] = reuse_key
for span_id, event in by_id.items():
parent_id = getattr(event, "parent_id", None)
if parent_id in by_id:
Expand Down Expand Up @@ -106,6 +130,35 @@ def parallelization_candidates(
sequential, parallel = sum(durations), max(durations)
if sequential <= parallel:
continue
keys = [reuse_keys.get(span_id) for span_id in span_ids]
if complete_timing and keys[0] is not None and all(key == keys[0] for key in keys):
retained = min(range(len(intervals)), key=intervals.__getitem__)
candidates.append(
{
"name": name,
"event_type": "tool_call",
"count": len(span_ids),
"node_ids": sorted(span_ids),
"sequential_time_ms": round(sequential, 3),
"estimated_savings_ms": round(sequential - durations[retained], 3),
"evidence_level": "declared",
"confidence": "medium",
"assumptions": list(_REUSE_ASSUMPTIONS),
"observations": {
"evidence_level": "observed",
"tool_call_count": len(span_ids),
"cumulative_span_time_ms": round(sequential, 3),
"timing": "non_overlapping",
"recommended_action": "test_request_local_reuse",
"same_input_key_declared": True,
"read_only_declared": True,
"retained_span_id": span_ids[retained],
},
"description": "The integration declares matching input keys and read-only behavior for these successful calls.",
"estimate_formula": REUSE_FORMULA,
}
)
continue
declared = members <= declarations
assumptions = list(_ASSUMPTIONS)
if not complete_timing:
Expand All @@ -127,6 +180,10 @@ def parallelization_candidates(
"tool_call_count": len(span_ids),
"cumulative_span_time_ms": round(sequential, 3),
"timing": "non_overlapping" if complete_timing else "unavailable",
"tool_name": name,
"recommended_action": "test_parallelization"
if declared
else "inspect_repetition",
},
"description": (
"The integration declares these calls safe to run concurrently; validate with replay."
Expand Down
18 changes: 16 additions & 2 deletions agentloop/patches.py
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,21 @@ def _plan_for_finding(
evidence_names = _evidence_names(finding)
files = _candidate_files(source_index, evidence_names)
framework = _select_framework(frameworks, files)
templates = _rewrite_templates(finding_type, framework)
action = (finding.get("observations") or {}).get("recommended_action")
notes = _notes(finding_type, files)
if finding_type == "parallelize_tools" and action in {
"inspect_repetition",
"test_request_local_reuse",
}:
templates = {
"risk": "medium",
"before_pattern": str(finding.get("metadata", {}).get("why", "Repeated tool calls.")),
"proposed_rewrite": finding["rewrite"]["hint"],
"suggested_diff": "Change one behavior after reviewing the call inputs, then capture it again and verify the result with replay.",
}
notes = _notes("", files)
else:
templates = _rewrite_templates(finding_type, framework)
return PatchPlan(
patch_id=f"patch_{index:03d}",
finding_id=str(finding["finding_id"]),
Expand All @@ -255,7 +269,7 @@ def _plan_for_finding(
finding.get("validation", {}).get("acceptance_criteria")
or "before/after replay passes configured quality and budget gates"
),
notes=_notes(finding_type, files),
notes=notes,
)


Expand Down
16 changes: 11 additions & 5 deletions agentloop/rules.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from agentloop.costs import is_cost_evaluable
from agentloop.estimates import ESTIMATORS, estimate_snapshot
from agentloop.graph import ExecutionGraph
from agentloop.parallelism import PARALLELISM_REWRITE
from agentloop.parallelism import PARALLELISM_NOTICE, PARALLELISM_REWRITE, REUSE_REWRITE
from agentloop.ranking import requirements
from agentloop.tokens import UNAVAILABLE

Expand Down Expand Up @@ -163,13 +163,19 @@ def run_rules(

def _parallelization_cards(graph: ExecutionGraph) -> list[FindingCandidate]:
cards = []
for group in graph.parallelizable_groups():
for group in graph.parallelizable_groups(consider_reuse=True):
action = group["observations"]["recommended_action"]
title, rewrite = {
"test_request_local_reuse": ("Test request-local reuse for", REUSE_REWRITE),
"test_parallelization": ("Parallelize repeated", PARALLELISM_REWRITE),
"inspect_repetition": ("Investigate repeated", PARALLELISM_NOTICE),
}[action]
cards.append(
FindingCandidate(
type=RecommendationType.PARALLELIZE_TOOLS,
title=f"Parallelize repeated `{group['name']}` tool calls",
title=f"{title} `{group['name']}` tool calls",
why=f"{group['count']} repeated calls. {group['description']}",
rewrite_hint=PARALLELISM_REWRITE,
rewrite_hint=rewrite,
confidence=group["confidence"],
estimated_latency_savings_ms=group["estimated_savings_ms"],
affected_nodes=group["node_ids"],
Expand Down Expand Up @@ -383,7 +389,7 @@ def _semantic_cards(context, family):


BUILTIN_RULES = (
FindingRule("parallelize_tools", "1.1", lambda ctx: _parallelization_cards(ctx.graph)),
FindingRule("parallelize_tools", "1.2", lambda ctx: _parallelization_cards(ctx.graph)),
FindingRule("cache_context", "1.1", lambda ctx: _context_cache_cards(ctx.report, ctx.graph)),
FindingRule("add_schema_validation", "1.2", lambda ctx: _retry_cards(ctx.graph)),
FindingRule("batch_model_calls", "1.2", lambda ctx: _batch_model_cards(ctx.graph)),
Expand Down
32 changes: 25 additions & 7 deletions docs/PARALLELIZATION.md
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
# Parallelization evidence

Three or more same-named tool calls in the same parent scope can suggest a
concurrency experiment. Names and ordering do not establish that their outputs
or shared state are independent. AgentLoop reports a candidate with explicit
assumptions and validates improvements through baseline/candidate replay.
Three or more same-named tool calls in the same parent scope are worth
investigating. The finding asks you to compare inputs and inspect dependencies
and shared state. When all calls declare concurrency safety, the plan suggests
testing concurrency. Matching read-only calls can instead qualify for a reuse
experiment.

## Evidence levels

Expand Down Expand Up @@ -42,6 +43,21 @@ malformed dependency lists, and references to missing dependency spans exclude t
affected group. Declarations cannot override these conflicts. Omit `parallel_safe`
when safety is unknown; a declaration on only some calls leaves the group inferred.

## Identical read-only calls

For an immutable request-scoped lookup, instrument each call with
`metadata.read_only: true` and the same opaque `metadata.tool_reuse_key`. Use a
nonempty string of at most 128 characters. The key must account for the effective
arguments, resource version, and execution context. Do not put raw arguments or
secrets in the key. Use distinct keys when the effective inputs differ.

The canonical finding suggests testing request-local reuse when the whole group
shares one key and all calls succeeded with complete, non-overlapping timing.
Existing dependency and safety exclusions still apply. The estimate subtracts
the first call's duration from total tool time. Cache lookup overhead, source
stability, and task correctness still need verification with replay. Tool keys
are omitted from the finding.

## Savings assumptions and limits

The estimated parallel time is the maximum selected call duration. Estimated
Expand All @@ -58,6 +74,8 @@ candidate and an additional assumption that the calls ran serially. The legacy
`sequential_time_ms` field is the sum of span durations, not a claim that serial
execution was measured. Estimated savings are hypotheses, not measured speedups.

Existing titles and finding IDs remain stable when the selected spans are unchanged.
Groups now excluded by dependency, scope, or timing evidence can become superseded
on explicit re-diagnosis under the existing finding lifecycle.
`parallelize_tools` remains the finding family. A clearer title for a name-only
investigation preserves the existing finding ID. A request-local reuse finding
has a distinct ID for the different proposed change. Dependency, scope, and
timing changes can supersede findings on an explicit re-diagnosis under the
existing lifecycle.
Loading
Loading