From dee34dcec068de78c7a4a51c66d30fbecb394242 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 13:00:59 -0700 Subject: [PATCH 01/22] test: cover core deck metrics and native child attribution --- tests/cli/test_sessions_deck_cmd.py | 37 ++ .../backends/test_child_task_transcripts.py | 70 +++ tests/execution/test_report_index.py | 59 +++ tests/execution/test_report_index_rows.py | 102 ++++ tests/execution/test_report_index_schema.py | 16 +- tests/execution/test_report_walk.py | 28 ++ tests/execution/test_session_parsing.py | 19 + tests/report/_fixtures.py | 47 ++ tests/report/test_deck_core_js.py | 146 +++++- tests/report/test_deck_html.py | 84 +++- tests/report/test_deck_payload.py | 437 +++++++++++++++++- tests/report/test_deck_views_js.py | 264 +++++++++++ 12 files changed, 1295 insertions(+), 14 deletions(-) create mode 100644 tests/report/test_deck_views_js.py diff --git a/tests/cli/test_sessions_deck_cmd.py b/tests/cli/test_sessions_deck_cmd.py index 174eeab213..c6a01cbb96 100644 --- a/tests/cli/test_sessions_deck_cmd.py +++ b/tests/cli/test_sessions_deck_cmd.py @@ -5,6 +5,8 @@ import json import re from pathlib import Path +from types import SimpleNamespace +from typing import Any import pytest @@ -14,6 +16,41 @@ pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium] +def test_sessions_deck_passes_all_fact_collections( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import autoskillit.execution as execution + import autoskillit.report as report + from autoskillit.cli.ops import _sessions + + log_root = tmp_path / "logs" + _configure_log_root(monkeypatch, log_root) + facts = SimpleNamespace( + sessions={"parent": {"key": "parent"}}, + requests={"request": {"key": "request"}}, + tools={"tool": {"key": "tool"}}, + subagents={"child": {"key": "child", "actor_level": "L0"}}, + ) + received: dict[str, Any] = {} + + def capture_deck(session_rows: Any, **kwargs: Any) -> str: + received["session_rows"] = list(session_rows) + received.update(kwargs) + return "" + + monkeypatch.setattr(_sessions, "_refresh_report_index", lambda *a, **kw: None) + monkeypatch.setattr(execution, "read_report_index", lambda *a: facts) + monkeypatch.setattr(report, "render_deck", capture_deck) + out = tmp_path / "deck.html" + + _sessions.sessions_deck(str(out)) + + assert received["session_rows"] == list(facts.sessions.values()) + for name in ("request", "tool", "subagent"): + assert list(received[f"{name}_rows"]) == list(getattr(facts, f"{name}s").values()) + assert out.read_text(encoding="utf-8") == "" + + def test_sessions_deck_refreshes_index_and_writes_session_data( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/execution/backends/test_child_task_transcripts.py b/tests/execution/backends/test_child_task_transcripts.py index 3f5b3dd391..dac8dbaaa8 100644 --- a/tests/execution/backends/test_child_task_transcripts.py +++ b/tests/execution/backends/test_child_task_transcripts.py @@ -12,6 +12,10 @@ from autoskillit.execution.backends import CompositeSessionLocator from autoskillit.execution.backends._claude.child_task import parse_claude_child_task from autoskillit.execution.backends._codex.child_task import parse_codex_child_task +from autoskillit.execution.backends._codex_parse import ( + extract_codex_child_turn_usage, + extract_codex_turn_usage, +) pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] @@ -58,6 +62,25 @@ def _write_codex(path: Path, child_id: str, events: list[dict[str, object]]) -> return _write_jsonl(path, [_codex_meta(child_id), *events]) +def _codex_usage_snapshot(cumulative_total: int, timestamp: str) -> dict[str, object]: + return { + "type": "event_msg", + "timestamp": timestamp, + "payload": { + "type": "token_count", + "info": { + "last_token_usage": { + "input_tokens": 10, + "cached_input_tokens": 2, + "output_tokens": 3, + "total_tokens": 13, + }, + "total_token_usage": {"total_tokens": cumulative_total}, + }, + }, + } + + @pytest.mark.parametrize( "prompt", [ @@ -491,3 +514,50 @@ def test_composite_locator_preserves_transcript_read_errors() -> None: with pytest.raises(OSError, match="unreadable"): locator.read_child_task("child") + + +def test_codex_child_usage_scans_whole_verified_rollout_and_deduplicates_snapshots( + tmp_path: Path, +) -> None: + child_thread_id = "child-thread" + path = _write_codex( + tmp_path / "child-rollout.jsonl", + child_thread_id, + [ + {"type": "turn_context", "payload": {"model": "codex-child-model"}}, + _codex_usage_snapshot(13, "2026-09-01T10:00:01Z"), + _codex_usage_snapshot(13, "2026-09-01T10:00:02Z"), + ], + ) + locator = SimpleNamespace(locate_session=lambda _thread_id: path) + + dedicated = extract_codex_child_turn_usage( + path, + child_thread_id, + provider_used="codex", + ) + interval = extract_codex_turn_usage( + locator, + child_thread_id, + "2026-09-01T10:00:00Z", + "2026-09-01T10:00:03Z", + provider_used="codex", + ) + + assert dedicated == interval + assert len(dedicated) == 1 + assert dedicated[0]["model"] == "codex-child-model" + assert dedicated[0]["input_tokens"] == 10 + + +def test_codex_child_usage_rejects_mismatched_rollout_identity(tmp_path: Path) -> None: + path = _write_codex(tmp_path / "child-rollout.jsonl", "actual-child", []) + + assert ( + extract_codex_child_turn_usage( + path, + "different-child", + provider_used="codex", + ) + == [] + ) diff --git a/tests/execution/test_report_index.py b/tests/execution/test_report_index.py index 53992ad2b9..78af5c3f40 100644 --- a/tests/execution/test_report_index.py +++ b/tests/execution/test_report_index.py @@ -396,6 +396,17 @@ def test_older_version_rows_parse_without_error(tmp_path: Path) -> None: } ) ) + (rows_path.parent / "state.json").write_text( + json.dumps( + { + "schema_version": 1, + "row_schema_version": REPORT_INDEX_SCHEMA_VERSION, + "walk": None, + "rows_bytes": rows_path.stat().st_size, + } + ), + encoding="utf-8", + ) indexed = read_report_index(rows_path.parent) @@ -410,6 +421,54 @@ def test_older_version_rows_parse_without_error(tmp_path: Path) -> None: assert len(indexed.sessions) == 2 +@pytest.mark.parametrize( + "row_schema_marker", + [None, 1, REPORT_INDEX_SCHEMA_VERSION + 1], + ids=["absent", "mismatched", "future"], +) +def test_incompatible_row_schema_is_rejected_and_rebuilt( + tmp_path: Path, + row_schema_marker: int | None, +) -> None: + assert not read_report_index(tmp_path / "absent-index").sessions + + root = tmp_path / "logs" + _write_jsonl(root / "sessions.jsonl", [_basic_session("current", "sid-current")]) + index_dir = tmp_path / "index" + update_report_index(root, index_dir) + + state_path = index_dir / "state.json" + state = json.loads(state_path.read_text(encoding="utf-8")) + if row_schema_marker is None: + state.pop("row_schema_version", None) + else: + state["row_schema_version"] = row_schema_marker + rows_path = index_dir / "rows.jsonl" + with rows_path.open("ab") as handle: + handle.write( + _json_line( + { + "schema_version": REPORT_INDEX_SCHEMA_VERSION, + "kind": "session", + "key": "stale-child-generation", + "session_id": "sid-stale", + } + ) + ) + state["rows_bytes"] = rows_path.stat().st_size + state_path.write_text(json.dumps(state), encoding="utf-8") + + with pytest.raises(ValueError, match="row schema"): + read_report_index(index_dir) + + update_report_index(root, index_dir) + + indexed = read_report_index(index_dir) + committed = json.loads(state_path.read_text(encoding="utf-8")) + assert set(indexed.sessions) == {"current"} + assert committed["row_schema_version"] == REPORT_INDEX_SCHEMA_VERSION + + def test_lost_state_keeps_rows_whose_sources_rotated_away(tmp_path: Path) -> None: root = tmp_path / "logs" _write_jsonl(root / "sessions.jsonl", [_basic_session("a", "sid-a")]) diff --git a/tests/execution/test_report_index_rows.py b/tests/execution/test_report_index_rows.py index ac09cb4005..de2737a37c 100644 --- a/tests/execution/test_report_index_rows.py +++ b/tests/execution/test_report_index_rows.py @@ -55,6 +55,7 @@ def _session_item( *, turns: tuple[tuple[str, ...], ...] = (), available: bool = True, + child_outcomes: tuple[dict[str, Any], ...] = (), ) -> WalkItem: return WalkItem( "session", @@ -67,6 +68,7 @@ def _session_item( {"turn_id": "private-turn-id", "timestamp": None, "tool_names": list(names)} for names in turns ], + "child_outcomes": child_outcomes, "transcripts_available": available, "transcript_unavailable_reasons": [], }, @@ -149,6 +151,106 @@ def test_session_row_carries_facets_pair_and_resolved_measures() -> None: assert "private-turn-id" not in serialized +def test_session_item_projects_verified_native_child_under_index_parent_key() -> None: + token_usage = { + "input_tokens": {"state": "measured", "value": 15}, + "output_tokens": {"state": "measured", "value": 4}, + "cache_read_tokens": {"state": "measured", "value": 3}, + "cache_write_tokens": {"state": "measured_zero", "value": 0}, + } + child = { + "child_id": "native-child-1", + "backend": "claude_code", + "parent_session_id": "native-parent-id", + "role": "audit-impl-slice-auditor", + "attribution_skill": "make-plan", + "effective_provider": "anthropic", + "effective_model": "claude-sonnet-child", + "evidence_source": "transcript_metadata", + "transcript_locator": "/private/logs/agent-native-child-1.jsonl", + "token_usage": token_usage, + "tool_counts": {"Bash": 1, "Read": 2}, + "transcript_state": "observed", + "usage_state": "observed", + } + managed_attempt = { + "child_id": "managed-attempt-1", + "launch_alias": "claude-managed-attempt", + "backend": "claude_code", + "parent_session_id": "native-parent-id", + "role": "audit-bugs", + "attribution_skill": "make-plan", + "evidence_source": "managed_attempt_reservation", + } + providerless_child = { + "child_id": "native-child-providerless", + "backend": "claude_code", + "parent_session_id": "native-parent-id", + "role": "audit-impl-slice-auditor", + "attribution_skill": "make-plan", + "effective_provider": "", + "effective_model": "", + "evidence_source": "transcript_metadata", + "transcript_locator": "/private/logs/agent-native-child-providerless.jsonl", + "token_usage": { + field: {"state": "unknown", "value": None} + for field in ( + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cache_write_tokens", + ) + }, + "tool_counts": {"Read": 1}, + "transcript_state": "observed", + "usage_state": "unknown", + } + parent = { + "session_id": "native-parent-id", + "timestamp": "2026-09-01T00:00:00Z", + "backend": "claude-code", + "provider_used": "openai", + "model_identifier": "parent-model", + "skill_command": "/autoskillit:make-plan", + "recipe_name": "research", + "step_name": "scope", + "session_type": "skill", + } + + rows = rows_for_walk_item( + _session_item( + "report-index-key", + parent, + child_outcomes=(child, managed_attempt, providerless_child), + ) + ) + + assert [row["kind"] for row in rows] == ["session", "subagent", "subagent"] + projected_by_id = {row["child_id"]: row for row in rows if row["kind"] == "subagent"} + projected = projected_by_id["native-child-1"] + assert projected["key"].startswith("report-index-key:") + assert projected["child_id"] == "native-child-1" + assert projected["parent_session_key"] == "report-index-key" + assert projected["native_parent_session_id"] == "native-parent-id" + assert projected["role"] == "audit-impl-slice-auditor" + assert projected["actor_level"] == "L0" + assert projected["skill"] == "make-plan" + assert projected["level"] == "skill" + assert projected["recipe"] == "research" + assert projected["step"] == "scope" + assert projected["provider"] == "anthropic" + assert projected["model"] == "claude-sonnet-child" + assert projected["time_ms"] == rows[0]["time_ms"] + assert projected["token_usage"] == token_usage + assert projected["tool_counts"] == {"Bash": 1, "Read": 2} + assert projected["transcript_state"] == projected["usage_state"] == "observed" + without_provider = projected_by_id["native-child-providerless"] + assert without_provider["provider"] == "unknown" + assert without_provider["model"] is None + assert without_provider["token_usage"]["input_tokens"]["state"] == "unknown" + assert without_provider["usage_state"] == "unknown" + + def test_session_row_legacy_zero_and_codex_sigil() -> None: row_data = { "schema_version": 13, diff --git a/tests/execution/test_report_index_schema.py b/tests/execution/test_report_index_schema.py index eebef314d4..bfaead3fae 100644 --- a/tests/execution/test_report_index_schema.py +++ b/tests/execution/test_report_index_schema.py @@ -15,7 +15,7 @@ pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] -_EXPECTED = (1, "8418c2eb676180f7865ce20982478b76f175e0d7d3527f5353f71e4a26117d06") +_EXPECTED = (2, "8418c2eb676180f7865ce20982478b76f175e0d7d3527f5353f71e4a26117d06") # Locked snapshot of per-kind field name orderings that produced ``_EXPECTED``. # Bump ``_EXPECTED[0]`` and refresh this dict in lockstep when row fields change; # ``test_report_index_schema_version_matches_field_digest`` is what fails first @@ -73,17 +73,31 @@ "tool_counts", ], "subagent": [ + "actor_level", "agent_type", + "child_id", "event_sequence", "final_model", "harness", "key", "kind", + "level", "model", "model_swapped", + "native_parent_session_id", + "parent_session_key", + "provider", + "recipe", + "role", "schema_version", "session_id", + "skill", + "step", "time_ms", + "tool_counts", + "token_usage", + "transcript_state", + "usage_state", ], "tool": [ "agent_name", diff --git a/tests/execution/test_report_walk.py b/tests/execution/test_report_walk.py index e7a823405f..1fd387c220 100644 --- a/tests/execution/test_report_walk.py +++ b/tests/execution/test_report_walk.py @@ -10,6 +10,7 @@ import pytest import zstandard +import autoskillit.execution.evidence.report_walk as report_walk from autoskillit.core import iter_merged_assistant_turns from autoskillit.execution.evidence.report_walk import ( SourceGapError, @@ -128,6 +129,33 @@ def test_first_pass_emits_joinable_records_and_tolerates_old_index_rows( assert read_tolerant_session_index_rows(root / "sessions.jsonl") == [current, old_schema] +def test_session_walk_reads_child_outcomes_by_native_parent_id( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = tmp_path / "logs" + child = {"child_id": "native-child", "role": "audit-bugs"} + calls: list[tuple[str, str, Path]] = [] + + def collect(*, backend: str, parent_session_id: str, log_root: Path): + calls.append((backend, parent_session_id, log_root)) + return (child,) + + monkeypatch.setattr(report_walk, "collect_child_outcomes", collect, raising=False) + _write_jsonl( + root / "sessions.jsonl", + [_session("report-index-key", "native-parent-id", backend="claude-code")], + ) + + (item,) = [candidate for candidate in iter_report_walk(root) if candidate.kind == "session"] + + assert calls == [("claude_code", "native-parent-id", root)] + assert item.source_id == "report-index-key" + assert item.session_id == "native-parent-id" + assert item.record is not None + assert item.record["child_outcomes"] == (child,) + assert item.watermark["projection"]["report-index-key"] + + @pytest.mark.parametrize("pause", ("before", "after")) def test_committed_watermark_resumes_without_missing_or_duplicate_records( tmp_path: Path, pause: str diff --git a/tests/execution/test_session_parsing.py b/tests/execution/test_session_parsing.py index 5c1a50f208..4077871f09 100644 --- a/tests/execution/test_session_parsing.py +++ b/tests/execution/test_session_parsing.py @@ -20,6 +20,7 @@ extract_token_usage, parse_session_result, ) +from autoskillit.execution.session.turn_usage import claude_inclusive_input_tokens from tests._helpers import UNKNOWN_MEASURE as _UNKNOWN from tests._helpers import _flush_structlog_proxy_caches as _flush_logger_proxy_caches from tests._helpers import observed_measure as _observed @@ -28,6 +29,24 @@ pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] +@pytest.mark.parametrize( + ("raw_input", "cache_read", "cache_write", "expected"), + [ + (10, 3, 2, 15), + (None, 3, 2, None), + (10, None, 2, None), + (10, 3, None, None), + ], +) +def test_claude_inclusive_input_requires_all_raw_components( + raw_input: int | None, + cache_read: int | None, + cache_write: int | None, + expected: int | None, +) -> None: + assert claude_inclusive_input_tokens(raw_input, cache_read, cache_write) == expected + + def _make_session_result( returncode: int = 0, stdout: str = "", diff --git a/tests/report/_fixtures.py b/tests/report/_fixtures.py index 8f4e304698..5fa08c4f6e 100644 --- a/tests/report/_fixtures.py +++ b/tests/report/_fixtures.py @@ -26,3 +26,50 @@ def session_row(key: str, **fields: Any) -> dict[str, Any]: ) row.update(fields) return row + + +def token_measure(value: int | None, *, state: str | None = None) -> dict[str, Any]: + if state is None: + state = "unknown" if value is None else ("measured_zero" if value == 0 else "measured") + return {"state": state, "value": value} + + +def subagent_row( + child_id: str, + *, + parent: dict[str, Any], + role: str, + skill: str, + provider: str, + input_tokens: int | None, + output_tokens: int | None, + cache_read_tokens: int | None, + cache_write_tokens: int | None, + tool_counts: dict[str, int], + transcript_state: str = "observed", + usage_state: str = "observed", +) -> dict[str, Any]: + """Build one canonical child fact linked to its parent cohort row.""" + return { + "child_id": child_id, + "role": role, + "parent_session_key": parent["key"], + "native_parent_session_id": parent["session_id"], + "skill": skill, + "provider": provider, + "model": "claude-sonnet", + "actor_level": "L0", + "time_ms": parent["time_ms"], + "level": parent["level"], + "recipe": parent["recipe"], + "step": parent["step"], + "token_usage": { + "input_tokens": token_measure(input_tokens), + "output_tokens": token_measure(output_tokens), + "cache_read_tokens": token_measure(cache_read_tokens), + "cache_write_tokens": token_measure(cache_write_tokens), + }, + "tool_counts": tool_counts, + "transcript_state": transcript_state, + "usage_state": usage_state, + } diff --git a/tests/report/test_deck_core_js.py b/tests/report/test_deck_core_js.py index ac9f8c6b98..8c7e33fd74 100644 --- a/tests/report/test_deck_core_js.py +++ b/tests/report/test_deck_core_js.py @@ -80,7 +80,7 @@ def test_href_for_keeps_cohort_params_and_applies_target_changes(deck_js: Any) - ) -def test_facet_selection_drops_struck_keys_and_preserves_one_live_key( +def test_facet_selection_drops_stale_keys_without_widening_scope( deck_js: Any, deck_model: dict[str, Any] ) -> None: chips = deck_model["chips"]["cohort"] @@ -97,9 +97,9 @@ def test_facet_selection_drops_struck_keys_and_preserves_one_live_key( assert mixed["dropped"] == ["L0"] assert mixed["widened"] is False - widened = deck_js.call("DeckCore.effectiveSelection", levels, ["L0"]) - assert widened["keys"] == ["unrecorded"] - assert widened["widened"] is True + stale = deck_js.call("DeckCore.effectiveSelection", levels, ["L0"]) + assert stale["keys"] == [] + assert stale["dropped"] == ["L0"] selected = deck_js.call("DeckCore.toggleSelection", harness, None, "codex") assert selected == ["claude-code"] @@ -109,17 +109,148 @@ def test_facet_selection_drops_struck_keys_and_preserves_one_live_key( assert deck_js.call("DeckCore.toggleSelection", harness, selected, "codex") is None -def test_window_selection_falls_back_to_all_for_absent_window( +def test_window_selection_does_not_fall_back_for_absent_window( deck_js: Any, deck_model: dict[str, Any] ) -> None: windows = deck_model["chips"]["cohort"]["window"] assert deck_js.call("DeckCore.windowSelection", windows, None)["chip"]["key"] == "all" absent = deck_js.call("DeckCore.windowSelection", windows, ["28d"]) - assert absent["chip"]["key"] == "all" + assert absent["chip"] is None assert absent["dropped"] == ["28d"] assert deck_js.call("DeckCore.windowSelection", windows, ["7d"])["chip"]["key"] == "7d" +def test_prepared_selection_uses_exact_scope_and_keeps_metric_cells_distinct( + deck_js: Any, +) -> None: + skill = { + "skill": "review", + "harness": "codex", + "provider": "openai", + "measures": {"input_tokens": {"state": "measured", "value": 17}}, + } + other_skill = {**skill, "harness": "claude-code", "provider": "anthropic"} + role = { + "role": "reviewer", + "provider": "openai", + "harnesses": [ + {"harness": "codex", "measures": {"input_tokens": {"value": 5}}}, + {"harness": "claude-code", "measures": {"input_tokens": {"value": 9}}}, + ], + } + chips = { + "window": [ + {"key": "7d", "match": "7d", "label": "7d", "state": "live", "days": 7}, + {"key": "all", "match": "all", "label": "all", "state": "live", "days": None}, + ], + "level": [ + {"key": "L1", "match": "skill", "label": "L1", "state": "live"}, + {"key": "L2", "match": "orchestrator", "label": "L2", "state": "live"}, + ], + "harness": [ + {"key": "codex", "match": "codex", "label": "codex", "state": "live"}, + { + "key": "claude-code", + "match": "claude-code", + "label": "claude-code", + "state": "live", + }, + ], + "provider": [ + {"key": "openai", "match": "openai", "label": "openai", "state": "live"}, + { + "key": "anthropic", + "match": "anthropic", + "label": "anthropic", + "state": "live", + }, + ], + } + relationship = { + "skill": "review", + "role": "reviewer", + "harness": "codex", + "provider": "openai", + } + definition = {"state": "available", "description": "Reads code"} + history = {"first_ms": 1, "last_ms": 2, "untimed": 0} + prepared = { + "skills": [ + {"window": "7d", "levels": ["skill"], "rows": [{**skill, "skill": "wrong-level"}]}, + { + "window": "all", + "levels": ["orchestrator", "skill"], + "rows": [{**skill, "skill": "wrong-window"}], + }, + { + "window": "7d", + "levels": ["orchestrator", "skill"], + "rows": [skill, other_skill], + }, + ], + "roles": [ + { + "window": "7d", + "levels": ["orchestrator", "skill"], + "rows": [role], + } + ], + "relationships": [relationship], + "definitions": {"reviewer": definition}, + "view_chips": {"efficiency": chips}, + "view_histories": {"efficiency": history}, + } + route = { + "params": { + "window": ["7d"], + "level": ["L1", "L2"], + "harness": ["codex"], + "provider": ["openai"], + } + } + + selected = deck_js.call("DeckCore.selectPrepared", prepared, "efficiency", chips, route) + + assert selected["skillRows"] == [skill] + assert selected["roleRows"] == [ + { + **role, + "harnesses": [role["harnesses"][0]], + } + ] + assert selected["relationships"] == [relationship] + assert selected["definitions"] == {"reviewer": definition} + assert selected["viewHistory"] == history + + +def test_prepared_selection_returns_no_metrics_for_stale_scope(deck_js: Any) -> None: + chips = { + "window": [{"key": "all", "match": "all", "label": "all", "state": "live", "days": None}], + "level": [{"key": "L1", "match": "skill", "label": "L1", "state": "live"}], + "harness": [{"key": "codex", "match": "codex", "label": "codex", "state": "live"}], + "provider": [{"key": "openai", "match": "openai", "label": "openai", "state": "live"}], + } + prepared = { + "skills": [{"window": "all", "levels": ["skill"], "rows": [{"skill": "review"}]}], + "roles": [{"window": "all", "levels": ["skill"], "rows": []}], + "relationships": [], + "definitions": {}, + "view_chips": {"skill": chips}, + "view_histories": {"skill": {"first_ms": 1, "last_ms": 1, "untimed": 0}}, + } + + selected = deck_js.call( + "DeckCore.selectPrepared", + prepared, + "skill", + chips, + {"params": {"window": ["deleted-window"], "level": ["L2"]}}, + ) + + assert selected["skillRows"] == [] + assert selected["roleRows"] == [] + + @pytest.mark.parametrize( ("params", "expected_keys", "untimed"), [ @@ -127,7 +258,8 @@ def test_window_selection_falls_back_to_all_for_absent_window( pytest.param({"harness": ["codex"]}, ["s4"], 0, id="codex"), pytest.param({"window": ["7d"]}, ["s1", "s2"], 1, id="last-week"), pytest.param({"harness": ["codex"], "window": ["7d"]}, [], 1, id="untimed-codex"), - pytest.param({"window": ["28d"]}, ["s1", "s2", "s3", "s4"], 0, id="absent-window"), + pytest.param({"window": ["28d"]}, [], 0, id="absent-window"), + pytest.param({"harness": ["retired"]}, [], 0, id="stale-harness"), ], ) def test_filter_rows_applies_facets_windows_and_untimed_count( diff --git a/tests/report/test_deck_html.py b/tests/report/test_deck_html.py index a9f7f10ff1..4a98f0546c 100644 --- a/tests/report/test_deck_html.py +++ b/tests/report/test_deck_html.py @@ -5,6 +5,7 @@ import json import re from html.parser import HTMLParser +from types import SimpleNamespace from typing import Any import pytest @@ -12,11 +13,12 @@ from autoskillit.core import pkg_root from autoskillit.report import render_deck from autoskillit.report.deck import _html as deck_html +from autoskillit.report.deck import _payload as deck_payload_module from autoskillit.report.deck import build_deck_payload from autoskillit.report.deck import render_deck as render_deck_from_deck from autoskillit.report.deck._html import render_deck_html from autoskillit.report.deck._registry import DECK_VIEWS -from tests.report._fixtures import DECK_GENERATED_AT +from tests.report._fixtures import DECK_GENERATED_AT, session_row, subagent_row, token_measure pytestmark = [pytest.mark.small] @@ -222,9 +224,85 @@ def test_render_deck_facades_match_html_wrapper( expected = render_deck_html(deck_payload) assert ( - render_deck(deck_rows, generated_at=DECK_GENERATED_AT, index_schema_version=1) == expected + render_deck( + deck_rows, + request_rows=(), + tool_rows=(), + subagent_rows=(), + generated_at=DECK_GENERATED_AT, + index_schema_version=1, + ) + == expected ) assert ( - render_deck_from_deck(deck_rows, generated_at=DECK_GENERATED_AT, index_schema_version=1) + render_deck_from_deck( + deck_rows, + request_rows=(), + tool_rows=(), + subagent_rows=(), + generated_at=DECK_GENERATED_AT, + index_schema_version=1, + ) == expected ) + + +def test_render_deck_forwards_child_facts_and_embeds_definitions_as_literal_json( + monkeypatch: pytest.MonkeyPatch, +) -> None: + role = "audit-impl-slice-auditor" + parent = session_row( + "parent", + session_id="native-parent", + time_ms=1_791_000_000_000, + skill="planner", + recipe="planner", + step="inspect", + level="skill", + input_tokens=token_measure(12), + output_tokens=token_measure(3), + ) + child = subagent_row( + "native-child", + parent=parent, + role=role, + skill="planner", + provider="anthropic", + input_tokens=20, + output_tokens=4, + cache_read_tokens=2, + cache_write_tokens=1, + tool_counts={"Bash": 3944, "Read": 6}, + ) + hostile_body = "Run git show. " + definition = SimpleNamespace( + name=role, + description="Inspect a committed slice.", + body=hostile_body, + tools=("Bash", "Read"), + model="sonnet", + reader_tools=(), + codex=SimpleNamespace(model="gpt-5.6-sol"), + ) + monkeypatch.setattr( + deck_payload_module, + "load_bundled_agent_definitions", + lambda: (definition,), + raising=False, + ) + html = deck_html.render_deck( + [parent], + request_rows=[{"request_id": "unlinked", "agent_name": role}], + tool_rows=[{"agent_name": role, "tool_name": "Bash", "count": 999_999}], + subagent_rows=[child], + generated_at=DECK_GENERATED_AT, + index_schema_version=1, + ) + collector = _collect(html) + data = next(source for attrs, source in collector.scripts if attrs.get("id") == "deck-data") + payload = json.loads(data) + + assert payload["prepared"]["definitions"][role]["body"] == hostile_body + assert any(row[0] == role for row in payload["tables"]["roles"]["rows"]) + assert "\\u003c/script\\u003e" in data + assert not any(tag == "img" for tag, _ in collector.elements) diff --git a/tests/report/test_deck_payload.py b/tests/report/test_deck_payload.py index 8ca06df34d..9cd7108a16 100644 --- a/tests/report/test_deck_payload.py +++ b/tests/report/test_deck_payload.py @@ -2,11 +2,20 @@ import json from datetime import datetime +from types import SimpleNamespace from typing import Any import pytest -from autoskillit.core import TokenMeasureState +from autoskillit.core import ( + MeasureRecord, + SourcePair, + TokenMeasure, + TokenMeasureState, + aggregate_measures, + measure_ratio, +) +from autoskillit.report.deck import _payload as deck_payload_module from autoskillit.report.deck import build_deck_payload from autoskillit.report.deck._payload import encode_table from autoskillit.report.deck._registry import ( @@ -16,7 +25,7 @@ SESSION_COLUMNS, SESSION_TABLE, ) -from tests.report._fixtures import DECK_GENERATED_AT +from tests.report._fixtures import DECK_GENERATED_AT, subagent_row, token_measure from tests.report._fixtures import session_row as _row pytestmark = [pytest.mark.small] @@ -24,13 +33,274 @@ def _payload(rows: list[dict[str, Any]]) -> dict[str, Any]: - return build_deck_payload(rows, generated_at=DECK_GENERATED_AT, index_schema_version=7) + return build_deck_payload( + rows, + request_rows=(), + tool_rows=(), + subagent_rows=(), + generated_at=DECK_GENERATED_AT, + index_schema_version=7, + ) def _chip(chips: dict[str, list[dict[str, Any]]], facet: str, key: str) -> dict[str, Any]: return next(chip for chip in chips[facet] if chip["key"] == key) +_TOKEN_FIELDS = ("input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens") +_AUDITOR = "audit-impl-slice-auditor" + + +def _session_fact( + key: str, + *, + days_ago: int, + skill: str, + level: str, + harness: str = "claude-code", + provider: str = "anthropic", + input_tokens: int | None = 10, + output_tokens: int | None = 2, + cache_read_tokens: int | None = 0, + cache_write_tokens: int | None = 0, +) -> dict[str, Any]: + return _row( + key, + session_id=f"native-{key}", + time_ms=GEN_MS - days_ago * 86_400_000, + harness=harness, + provider=provider, + skill=skill, + recipe=f"recipe-{skill}", + step="inspect", + level=level, + **{ + field: token_measure(value) + for field, value in zip( + _TOKEN_FIELDS, + (input_tokens, output_tokens, cache_read_tokens, cache_write_tokens), + strict=True, + ) + }, + ) + + +def _full_index_facts() -> tuple[ + list[dict[str, Any]], + list[dict[str, Any]], + list[dict[str, Any]], + list[dict[str, Any]], +]: + sessions = [ + _session_fact("skill", days_ago=1, skill="planner", level="skill", input_tokens=10), + _session_fact( + "orchestrator", days_ago=2, skill="planner", level="orchestrator", input_tokens=20 + ), + _session_fact("review", days_ago=3, skill="review", level="skill", input_tokens=5), + _session_fact( + "other-provider", + days_ago=4, + skill="planner", + level="skill", + provider="minimax", + input_tokens=7, + ), + _session_fact( + "codex", + days_ago=9, + skill="planner", + level="skill", + harness="codex", + provider="codex", + input_tokens=4, + ), + _session_fact("no-child", days_ago=0, skill="maintenance", level="skill", input_tokens=0), + ] + by_key = {row["key"]: row for row in sessions} + children = [ + subagent_row( + "child-a", + parent=by_key["skill"], + role=_AUDITOR, + skill="planner", + provider="anthropic", + input_tokens=100, + output_tokens=20, + cache_read_tokens=10, + cache_write_tokens=2, + tool_counts={"Bash": 3944, "Read": 6}, + ), + subagent_row( + "child-b", + parent=by_key["orchestrator"], + role=_AUDITOR, + skill="planner", + provider="anthropic", + input_tokens=0, + output_tokens=10, + cache_read_tokens=0, + cache_write_tokens=0, + tool_counts={"Bash": 1, "Read": 1}, + ), + subagent_row( + "child-c", + parent=by_key["review"], + role=_AUDITOR, + skill="review", + provider="anthropic", + input_tokens=5, + output_tokens=5, + cache_read_tokens=2, + cache_write_tokens=0, + tool_counts={"Read": 2}, + ), + subagent_row( + "child-d", + parent=by_key["other-provider"], + role=_AUDITOR, + skill="planner", + provider="minimax", + input_tokens=500, + output_tokens=25, + cache_read_tokens=20, + cache_write_tokens=5, + tool_counts={"Bash": 3}, + ), + subagent_row( + "child-e", + parent=by_key["codex"], + role="codex-reviewer", + skill="planner", + provider="codex", + input_tokens=9, + output_tokens=3, + cache_read_tokens=0, + cache_write_tokens=0, + tool_counts={"Read": 1}, + ), + subagent_row( + "child-f", + parent=by_key["review"], + role="unresolved-observer", + skill="review", + provider="anthropic", + input_tokens=None, + output_tokens=None, + cache_read_tokens=None, + cache_write_tokens=None, + tool_counts={}, + transcript_state="unknown", + usage_state="unknown", + ), + subagent_row( + "child-g", + parent=by_key["review"], + role="zero-denominator", + skill="review", + provider="anthropic", + input_tokens=2, + output_tokens=0, + cache_read_tokens=0, + cache_write_tokens=0, + tool_counts={"Read": 1}, + ), + ] + children[4]["token_usage"]["cache_read_tokens"] = token_measure(None, state="unavailable") + request_rows = [ + {"request_id": "unlinked-request", "agent_name": _AUDITOR, "input_tokens": 999_999} + ] + tool_rows = [{"agent_name": _AUDITOR, "tool_name": "Bash", "count": 999_999}] + return sessions, request_rows, tool_rows, children + + +def _source_records( + rows: list[dict[str, Any]], + *, + child: bool = False, + parents: dict[str, dict[str, Any]] | None = None, +) -> list[MeasureRecord]: + if child and parents is None: + raise ValueError("child records require their parent rows") + records = [] + for row in rows: + raw = row["token_usage"] if child else row + harness = parents[row["parent_session_key"]]["harness"] if child else row["harness"] + provider = row["provider"] + records.append( + MeasureRecord( + SourcePair(harness, provider), + {field: TokenMeasure.from_dict(raw[field]) for field in _TOKEN_FIELDS}, + ) + ) + return records + + +def _definition(name: str, body: str) -> SimpleNamespace: + return SimpleNamespace( + name=name, + description="Canonical reviewer role.", + body=body, + tools=("Bash", "Read"), + model="sonnet", + reader_tools=(), + codex=SimpleNamespace(model="gpt-5.6-sol"), + ) + + +def _metric_row( + blocks: list[dict[str, Any]], *, window: str, levels: set[str | None] +) -> dict[str, Any]: + block = next( + block for block in blocks if block["window"] == window and set(block["levels"]) == levels + ) + return block + + +def _assert_serialized_measure(actual: dict[str, Any], expected: Any) -> None: + assert actual["state"] == expected.value.state.value + assert actual["value"] == expected.value.value + assert actual["state_counts"] == { + state.value: count for state, count in expected.state_counts.items() + } + + +def _assert_serialized_ratio(actual: dict[str, Any], expected: Any, *, sample_unit: str) -> None: + assert actual["state"] == expected.state.value + assert actual["value"] == expected.value + assert actual["numerator"] == expected.numerator_field + assert actual["denominator"] == expected.denominator_field + assert actual["numerator_total"] == expected.numerator_total + assert actual["denominator_total"] == expected.denominator_total + assert actual["sample_unit"] == sample_unit + assert actual["sample_size"] == expected.sample_size + assert actual["excluded_runs"] == expected.excluded_runs + assert actual["unknown_runs"] == expected.unknown_runs + + +def _tool_records( + rows: list[dict[str, Any]], parents: dict[str, dict[str, Any]] +) -> list[MeasureRecord]: + tools = sorted({tool for row in rows for tool in row["tool_counts"]}) + base_records = _source_records(rows, child=True, parents=parents) + records = [] + for row, base in zip(rows, base_records, strict=True): + measures = dict(base.measures) + observed = row["transcript_state"] == "observed" + measures["tool_calls"] = ( + TokenMeasure.observed(sum(row["tool_counts"].values())) + if observed + else TokenMeasure.unknown() + ) + for tool in tools: + measures[f"tool:{tool}"] = ( + TokenMeasure.observed(row["tool_counts"].get(tool, 0)) + if observed + else TokenMeasure.unknown() + ) + records.append(MeasureRecord(base.pair, measures)) + return records + + def test_encode_table_aligns_rows_to_columns_and_fills_missing_keys() -> None: encoded = encode_table([{"key": "row-1", "time_ms": 42}], SESSION_COLUMNS) @@ -228,3 +498,164 @@ def test_empty_index_has_absent_declared_values_and_empty_observed_facets() -> N assert (chip["state"], chip["issue"]) == ("absent", 4622) assert all(chip["state"] == "absent" for chip in chips["window"][:-1]) assert (chips["window"][-1]["key"], chips["window"][-1]["state"]) == ("all", "live") + + +def test_prepared_metrics_keep_source_pairs_roles_and_shared_library_accounting( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sessions, requests, tools, children = _full_index_facts() + body = "Bash runs git show to inspect the committed slice." + loader_calls: list[None] = [] + monkeypatch.setattr( + deck_payload_module, + "load_bundled_agent_definitions", + lambda: loader_calls.append(None) or (_definition(_AUDITOR, body),), + raising=False, + ) + aggregate_calls: list[int] = [] + ratio_calls: list[int] = [] + original_aggregate = getattr(deck_payload_module, "aggregate_measures", aggregate_measures) + original_ratio = getattr(deck_payload_module, "measure_ratio", measure_ratio) + + def aggregate_spy(records: Any, fields: Any, **kwargs: Any) -> Any: + rows = list(records) + aggregate_calls.append(len(rows)) + return original_aggregate(rows, fields, **kwargs) + + def ratio_spy(records: Any, numerator: str, denominator: str, **kwargs: Any) -> Any: + rows = list(records) + ratio_calls.append(len(rows)) + return original_ratio(rows, numerator, denominator, **kwargs) + + monkeypatch.setattr(deck_payload_module, "aggregate_measures", aggregate_spy, raising=False) + monkeypatch.setattr(deck_payload_module, "measure_ratio", ratio_spy, raising=False) + payload = build_deck_payload( + sessions, + request_rows=requests, + tool_rows=tools, + subagent_rows=children, + generated_at=DECK_GENERATED_AT, + index_schema_version=7, + ) + + assert aggregate_calls and ratio_calls + assert len(loader_calls) == 1 + prepared = payload["prepared"] + levels = {"orchestrator", "skill"} + skill_block = _metric_row(prepared["skills"], window="all", levels=levels) + planner = next( + row + for row in skill_block["rows"] + if (row["skill"], row["harness"], row["provider"]) + == ("planner", "claude-code", "anthropic") + ) + planner_runs = [ + row + for row in sessions + if row["skill"] == "planner" + and row["harness"] == "claude-code" + and row["provider"] == "anthropic" + and row["level"] in levels + ] + planner_aggregate = aggregate_measures(_source_records(planner_runs), _TOKEN_FIELDS) + planner_ratio = measure_ratio(_source_records(planner_runs), "input_tokens", "output_tokens") + assert planner["measures"]["input_tokens"]["value"] == 30 + _assert_serialized_measure( + planner["measures"]["input_tokens"], planner_aggregate.fields["input_tokens"] + ) + _assert_serialized_ratio( + planner["ratios"]["input_output"], planner_ratio, sample_unit="skill-run" + ) + + role_block = _metric_row(prepared["roles"], window="all", levels=levels) + auditor = next( + row + for row in role_block["rows"] + if (row["role"], row["provider"]) == (_AUDITOR, "anthropic") + ) + claude_metrics = next(row for row in auditor["harnesses"] if row["harness"] == "claude-code") + auditor_runs = [ + row + for row in children + if row["role"] == _AUDITOR and row["provider"] == "anthropic" and row["level"] in levels + ] + parents = {row["key"]: row for row in sessions} + child_records = _source_records(auditor_runs, child=True, parents=parents) + child_aggregate = aggregate_measures(child_records, _TOKEN_FIELDS) + child_ratio = measure_ratio(child_records, "input_tokens", "output_tokens") + tool_records = _tool_records(auditor_runs, parents) + bash_ratio = measure_ratio(tool_records, "tool:Bash", "tool_calls") + assert claude_metrics["measures"]["input_tokens"]["value"] == 105 + assert claude_metrics["measures"]["input_tokens"]["value"] != 999_999 + _assert_serialized_measure( + claude_metrics["measures"]["input_tokens"], child_aggregate.fields["input_tokens"] + ) + _assert_serialized_ratio( + claude_metrics["ratios"]["input_output"], child_ratio, sample_unit="child-invocation" + ) + _assert_serialized_ratio( + claude_metrics["ratios"]["tool_mix"]["Bash"], + bash_ratio, + sample_unit="child-invocation", + ) + assert claude_metrics["ratios"]["input_output"]["sample_size"] == 3 + assert claude_metrics["ratios"]["tool_mix"]["Bash"]["review_eligible"] is True + assert claude_metrics["ratios"]["tool_mix"]["Bash"]["definition_roles"] == [_AUDITOR] + + zero_skill_block = _metric_row(prepared["skills"], window="all", levels={"skill"}) + maintenance = next(row for row in zero_skill_block["rows"] if row["skill"] == "maintenance") + zero_ratio = maintenance["ratios"]["input_output"] + assert (zero_ratio["state"], zero_ratio["value"], zero_ratio["sample_size"]) == ( + "measured_zero", + 0.0, + 1, + ) + + unresolved = next(row for row in role_block["rows"] if row["role"] == "unresolved-observer") + unresolved_ratio = unresolved["harnesses"][0]["ratios"]["input_output"] + assert unresolved_ratio["state"] == "unknown" + assert unresolved_ratio["review_eligible"] is False + assert unresolved_ratio["review_reason"] + assert prepared["definitions"][_AUDITOR]["state"] == "available" + assert prepared["definitions"][_AUDITOR]["body"] == body + assert prepared["definitions"]["unresolved-observer"]["state"] == "unavailable" + + zero_denominator = next(row for row in role_block["rows"] if row["role"] == "zero-denominator") + zero_denominator_ratio = zero_denominator["harnesses"][0]["ratios"]["input_output"] + assert (zero_denominator_ratio["state"], zero_denominator_ratio["sample_size"]) == ( + "not_applicable", + 1, + ) + codex = next(row for row in role_block["rows"] if row["role"] == "codex-reviewer") + codex_cache = codex["harnesses"][0]["ratios"]["cache_share"] + assert (codex_cache["state"], codex_cache["excluded_runs"]) == ("unavailable", 1) + + def table_rows(table: dict[str, Any]) -> list[dict[str, Any]]: + return [dict(zip(table["columns"], row, strict=True)) for row in table["rows"]] + + skill_identities = table_rows(payload["tables"]["skills"]) + role_identities = table_rows(payload["tables"]["roles"]) + assert {(row["skill"], row["harness"], row["provider"]) for row in skill_identities} >= { + ("planner", "claude-code", "anthropic"), + ("planner", "claude-code", "minimax"), + ("planner", "codex", "codex"), + } + assert {(row["role"], row["provider"]) for row in role_identities} >= { + (_AUDITOR, "anthropic"), + (_AUDITOR, "minimax"), + } + assert { + (edge["skill"], edge["role"], edge["harness"], edge["provider"]) + for edge in prepared["relationships"] + } >= { + ("planner", _AUDITOR, "claude-code", "anthropic"), + ("review", _AUDITOR, "claude-code", "anthropic"), + ("planner", _AUDITOR, "claude-code", "minimax"), + } + assert prepared["view_chips"] == payload["chips"] + assert prepared["view_histories"]["skill"]["last_ms"] == sessions[-1]["time_ms"] + assert ( + prepared["view_histories"]["role"]["last_ms"] + < prepared["view_histories"]["skill"]["last_ms"] + ) + json.dumps(payload, allow_nan=False) diff --git a/tests/report/test_deck_views_js.py b/tests/report/test_deck_views_js.py new file mode 100644 index 0000000000..f0198dec84 --- /dev/null +++ b/tests/report/test_deck_views_js.py @@ -0,0 +1,264 @@ +"""Browser-free renderer contracts for the observability deck's prepared views.""" + +from typing import Any + +import pytest +from py_mini_racer import MiniRacer + +pytestmark = [pytest.mark.small] + +VIEW_IDS = ("spend", "efficiency", "skill", "role") + +_RENDERER_HARNESS = r""" +globalThis.__deckViews = {}; +globalThis.DeckShell = { + registerView(id, render) { __deckViews[id] = render; } +}; +globalThis.location = {hash: "#/cohort"}; +globalThis.window = {location, addEventListener() {}}; + +function deckNode(tag, attrs = {}, children = null) { + const node = {tag, attrs: {...attrs}, children: [], events: {}}; + node.appendChild = child => { node.children.push(child); return child; }; + node.addEventListener = (name, callback) => { node.events[name] = callback; }; + if (Array.isArray(children)) node.children.push(...children); + else if (children !== null && children !== undefined) node.children.push(children); + return node; +} + +function deckText(node) { + if (node === null || node === undefined) return ""; + if (Array.isArray(node)) return node.map(deckText).join(" "); + if (typeof node !== "object") return String(node); + return [node.textContent || "", ...node.children.map(deckText)].join(" "); +} + +function deckLinks(node) { + if (node === null || node === undefined) return []; + if (Array.isArray(node)) return node.flatMap(deckLinks); + if (typeof node !== "object") return []; + return [...(node.tag === "a" ? [node.attrs.href] : []), ...node.children.flatMap(deckLinks)]; +} + +function deckFind(node, predicate) { + if (node === null || node === undefined) return null; + if (Array.isArray(node)) { + for (const child of node) { + const found = deckFind(child, predicate); + if (found) return found; + } + return null; + } + if (typeof node !== "object") return null; + if (predicate(node)) return node; + for (const child of node.children) { + const found = deckFind(child, predicate); + if (found) return found; + } + return null; +} + +function deckSnapshot(node) { + if (node === null || node === undefined || typeof node !== "object") return node; + return { + tag: node.tag, + attrs: node.attrs, + text: deckText(node), + links: deckLinks(node), + review: (() => { + const control = deckFind(node, child => + child.attrs && child.attrs["data-review-toggle"] !== undefined); + return control ? {attrs: control.attrs, text: deckText(control)} : null; + })() + }; +} + +function deckTable({columns, rows, defaultSort}) { + return deckNode("table", {"data-default-sort": defaultSort && defaultSort.key}, [ + deckNode("thead", {}, columns.map(column => deckNode("th", {}, column.label))), + deckNode("tbody", {}, rows.map(row => deckNode("tr", {}, columns.map(column => + deckNode("td", {}, column.cell ? column.cell(row) : row[column.key]))))) + ]); +} + +globalThis.DeckTest = { + render(id, input) { + const route = input.route; + const cohortKeys = ["harness", "provider", "level", "window"]; + const context = { + ...input, + model: {prepared: input.prepared}, + el: deckNode, + href: target => DeckCore.hrefFor(route, target, cohortKeys), + entityLink: (text, target) => deckNode("a", { + href: DeckCore.hrefFor(route, target, cohortKeys) + }, text), + sortableTable: deckTable, + availabilityCell: measure => deckNode("span", {"data-state": measure.state}, + measure.state === "measured" ? String(measure.value) : measure.state), + barChart: (items, options) => deckNode("svg", {"aria-label": options.label}, + JSON.stringify(items)) + }; + location.hash = DeckCore.encodeRoute(route, cohortKeys); + globalThis.__deckLastTree = __deckViews[id](context); + return deckSnapshot(__deckLastTree); + }, + clickReview() { + const control = deckFind(__deckLastTree, + child => child.attrs && child.attrs["data-review-toggle"] !== undefined); + if (!control || !control.events.click) throw new Error("review toggle has no click handler"); + control.events.click(); + return location.hash; + } +}; +""" + + +def _load_renderers(deck_asset: Any) -> MiniRacer: + context = MiniRacer() + context.eval(deck_asset("core.js")) + context.eval(_RENDERER_HARNESS) + for view_id in VIEW_IDS: + context.eval(deck_asset(f"views/{view_id}.js")) + return context + + +def _view_context(view_id: str, *, definitions_available: bool = True) -> dict[str, Any]: + definition_state = "available" if definitions_available else "unavailable" + roles = ("reader", "reviewer") + definitions = { + role: { + "state": definition_state, + "description": f"Definition for {role}: definition-first marker", + "body": f"{role} role instructions", + "tools": ["read_file"], + "model": "small-model", + "reader_tools": ["read_file"], + "codex_model": "small-model", + } + for role in roles + } + ratio = { + "state": "measured", + "value": 2.5, + "sample_unit": "child-invocation", + "sample_size": 2, + "review_eligible": True, + "review_reason": "Definitions are available for all contributors.", + "definition_roles": list(roles), + } + skill_row = { + "skill": "summarizer", + "harness": "codex", + "provider": "openai", + "measures": { + "input_tokens": {"state": "measured", "value": 101, "runs": 2}, + "output_tokens": {"state": "measured", "value": 40, "runs": 2}, + "cache_read_tokens": {"state": "measured", "value": 20, "runs": 2}, + "cache_write_tokens": {"state": "measured", "value": 3, "runs": 2}, + }, + "ratios": {"input_output": ratio, "cache_read_input": ratio}, + "recipe_steps": [], + } + role_row = { + "role": "reviewer", + "provider": "openai", + "harnesses": [ + { + "harness": "codex", + "measures": {"input_tokens": {"state": "measured", "value": 11, "runs": 2}}, + "ratios": { + "input_output": ratio, + "cache_read_input": ratio, + "tool_mix": {"read_file": ratio}, + }, + } + ], + } + prepared = { + "skills": [], + "roles": [], + "relationships": [ + {"skill": "summarizer", "role": role, "harness": "codex", "provider": "openai"} + for role in roles + ], + "definitions": definitions, + "view_chips": {}, + "view_histories": {view_id: {"first_ms": 1, "last_ms": 2, "untimed": 0}}, + } + return { + "prepared": prepared, + "metrics": [skill_row] if view_id == "skill" else [role_row], + "skillMetrics": [skill_row], + "roleMetrics": [role_row], + "relationships": prepared["relationships"], + "definitions": definitions, + "viewHistory": prepared["view_histories"][view_id], + "chips": {"window": [], "level": [], "harness": [], "provider": []}, + "selection": {}, + "route": { + "view": view_id, + "entity": "summarizer" + if view_id == "skill" + else "reviewer" + if view_id == "role" + else None, + "params": { + "window": ["all"], + "level": ["L1"], + "harness": ["codex"], + "provider": ["openai"], + }, + }, + } + + +def test_each_prepared_view_renders_its_population_and_cohort_links(deck_asset: Any) -> None: + context = _load_renderers(deck_asset) + try: + markers = {"spend": "101", "efficiency": "2.5", "skill": "summarizer", "role": "reviewer"} + for view_id, marker in markers.items(): + rendered = context.call("DeckTest.render", view_id, _view_context(view_id)) + assert marker in rendered["text"] + assert rendered["links"] + assert any( + "harness=codex" in href and "provider=openai" in href for href in rendered["links"] + ) + role_view = context.call("DeckTest.render", "role", _view_context("role")) + assert "L0" in role_view["text"] + assert "spawning skill level" in role_view["text"].lower() + finally: + context.close() + + +def test_review_toggle_requires_every_contributor_definition_and_resets_route( + deck_asset: Any, +) -> None: + context = _load_renderers(deck_asset) + try: + available = context.call("DeckTest.render", "role", _view_context("role")) + assert available["review"] is not None + assert "disabled" not in available["review"]["attrs"] + + unavailable = context.call( + "DeckTest.render", "role", _view_context("role", definitions_available=False) + ) + assert unavailable["review"] is not None + assert "disabled" in unavailable["review"]["attrs"] + assert "definition" in unavailable["text"].lower() + + context.call("DeckTest.render", "role", _view_context("role")) + enabled_hash = context.call("DeckTest.clickReview") + enabled_route = context.call("DeckCore.decodeRoute", enabled_hash, "cohort") + assert enabled_route["params"].get("review") + assert enabled_route["params"].get("harness") == ["codex"] + assert enabled_route["params"].get("provider") == ["openai"] + + context.call("DeckTest.render", "role", {**_view_context("role"), "route": enabled_route}) + reset_hash = context.call("DeckTest.clickReview") + reset_route = context.call("DeckCore.decodeRoute", reset_hash, "cohort") + assert "review" not in reset_route["params"] + assert reset_route["params"].get("harness") == ["codex"] + assert reset_route["params"].get("provider") == ["openai"] + finally: + context.close() From 33594ed73150e0c71c86a7300cd87d656adb9bbe Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 13:36:31 -0700 Subject: [PATCH 02/22] refactor: share native token parsing for child usage --- .../execution/backends/_codex_parse.py | 76 ++++++++++++++----- .../execution/session/_session_model.py | 7 +- .../execution/session/turn_usage.py | 11 +++ 3 files changed, 68 insertions(+), 26 deletions(-) diff --git a/src/autoskillit/execution/backends/_codex_parse.py b/src/autoskillit/execution/backends/_codex_parse.py index 4f03de1b80..42040f0824 100644 --- a/src/autoskillit/execution/backends/_codex_parse.py +++ b/src/autoskillit/execution/backends/_codex_parse.py @@ -229,13 +229,17 @@ def _codex_turn_usage_entry( info: Mapping[str, Any], last_usage: Mapping[str, Any], effective_model: str | None, - start: datetime, - end: datetime, + start: datetime | None, + end: datetime | None, provider_used: str, ) -> TurnTokenEntry | None: timestamp = first_nonempty_string(record.get("timestamp")) event_time = _utc_datetime(timestamp) - if event_time is None or event_time < start or event_time > end: + if ( + event_time is None + or (start is not None and event_time < start) + or (end is not None and event_time > end) + ): return None input_tokens = first_valid_token_count(last_usage, "input_tokens") @@ -304,30 +308,64 @@ def extract_codex_turn_usage( if path is None: return [] + return _extract_codex_turn_usage(path, start, end, provider_used) + + +def extract_codex_child_turn_usage( + rollout_path: Path, + child_thread_id: str, + *, + provider_used: str = "codex", +) -> list[TurnTokenEntry]: + """Extract snapshots from a dedicated Codex rollout after verifying its thread.""" + if not child_thread_id or _thread_id(rollout_path) != child_thread_id: + return [] + return _extract_codex_turn_usage(rollout_path, None, None, provider_used) + + +def _codex_usage_records(reader: BinaryIO) -> Iterator[Mapping[str, Any]]: + for raw_line in reader: + try: + record = json.loads(raw_line) + except (UnicodeDecodeError, json.JSONDecodeError): + continue + if not isinstance(record, Mapping) or not isinstance(record.get("payload"), Mapping): + continue + payload = record["payload"] + if record.get("type") == "event_msg" and payload.get("type") == "token_count": + info = payload.get("info") + if not isinstance(info, Mapping): + continue + if not isinstance(info.get("total_token_usage"), Mapping) or not isinstance( + info.get("last_token_usage"), Mapping + ): + continue + yield record + + +def _extract_codex_turn_usage( + path: Path, + start: datetime | None, + end: datetime | None, + provider_used: str, +) -> list[TurnTokenEntry]: + """Share rollout scanning and cumulative-snapshot handling across views.""" + rows: list[TurnTokenEntry] = [] current_model: str | None = None cumulative_high_water = 0 compacting = False try: with _logical_rollout_reader(path) as reader: - while raw_line := reader.readline(): - try: - record = json.loads(raw_line) - except (UnicodeDecodeError, json.JSONDecodeError): - continue - if not isinstance(record, Mapping): - continue - + for record in _codex_usage_records(reader): record_type = record.get("type") - payload = record.get("payload") + payload: Mapping[str, Any] = record["payload"] if record_type == "turn_context" and isinstance(payload, Mapping): current_model = first_nonempty_string(payload.get("model")) continue if record_type == "compacted": compacting = True continue - if not isinstance(payload, Mapping): - continue payload_type = payload.get("type") if record_type == "event_msg" and payload_type == "context_compacted": compacting = False @@ -335,13 +373,9 @@ def extract_codex_turn_usage( if record_type != "event_msg" or payload_type != "token_count": continue - info = payload.get("info") - if not isinstance(info, Mapping): - continue - total_usage = info.get("total_token_usage") - last_usage = info.get("last_token_usage") - if not isinstance(total_usage, Mapping) or not isinstance(last_usage, Mapping): - continue + info = payload["info"] + total_usage = info["total_token_usage"] + last_usage = info["last_token_usage"] cumulative_total = valid_token_count(total_usage.get("total_tokens")) if cumulative_total is None or cumulative_total <= cumulative_high_water: continue diff --git a/src/autoskillit/execution/session/_session_model.py b/src/autoskillit/execution/session/_session_model.py index 008711e3dd..82ca21dd6f 100644 --- a/src/autoskillit/execution/session/_session_model.py +++ b/src/autoskillit/execution/session/_session_model.py @@ -26,6 +26,7 @@ from autoskillit.execution.session.turn_usage import ( build_turn_token_entry, classify_token_measure, + claude_inclusive_input_tokens, first_nonempty_string, first_valid_token_count, merge_turn_usage, @@ -410,11 +411,7 @@ def _build_turn_usage_and_model_buckets( raw_input = row["input_tokens"] cache_read = row["cache_read_tokens"] cache_creation = row["cache_creation_tokens"] - inclusive_input = ( - raw_input + cache_read + cache_creation - if raw_input is not None and cache_read is not None and cache_creation is not None - else None - ) + inclusive_input = claude_inclusive_input_tokens(raw_input, cache_read, cache_creation) windows = model_windows.get(model, set()) if model is not None else set() context_window = next(iter(windows)) if len(windows) == 1 else None turn_usage.append( diff --git a/src/autoskillit/execution/session/turn_usage.py b/src/autoskillit/execution/session/turn_usage.py index 7883e174a1..82b3f9c9c2 100644 --- a/src/autoskillit/execution/session/turn_usage.py +++ b/src/autoskillit/execution/session/turn_usage.py @@ -230,6 +230,17 @@ def first_valid_token_count(usage: Mapping[str, Any], *fields: str) -> int | Non return None +def claude_inclusive_input_tokens( + raw_input: int | None, + cache_read: int | None, + cache_write: int | None, +) -> int | None: + """Return inclusive Claude input only when every raw component was observed.""" + if raw_input is None or cache_read is None or cache_write is None: + return None + return raw_input + cache_read + cache_write + + def valid_context_window(value: Any) -> int | None: """Return a positive provider-supplied context capacity.""" count = valid_token_count(value) From af88bc5f45cb2646fc5d6493c228ad8cf302e250 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 13:50:23 -0700 Subject: [PATCH 03/22] feat: index verified native child usage and refresh evidence --- .../execution/_report_index_rows.py | 164 +++++++- .../evidence/_native_child_projection.py | 385 ++++++++++++++++++ .../execution/evidence/report_walk.py | 106 ++++- src/autoskillit/execution/report_index.py | 55 ++- tests/execution/test_report_index.py | 124 +++++- tests/execution/test_report_index_rows.py | 36 +- tests/execution/test_report_index_schema.py | 2 +- tests/execution/test_report_walk.py | 129 +++++- 8 files changed, 961 insertions(+), 40 deletions(-) create mode 100644 src/autoskillit/execution/evidence/_native_child_projection.py diff --git a/src/autoskillit/execution/_report_index_rows.py b/src/autoskillit/execution/_report_index_rows.py index 45b467f630..fe9b80c7b0 100644 --- a/src/autoskillit/execution/_report_index_rows.py +++ b/src/autoskillit/execution/_report_index_rows.py @@ -34,7 +34,7 @@ from autoskillit.execution.evidence.report_walk import OTLP_WALK_KIND, SESSION_WALK_KIND, WalkItem from autoskillit.execution.session.turn_usage import classify_token_measure -REPORT_INDEX_SCHEMA_VERSION: Final[int] = 1 +REPORT_INDEX_SCHEMA_VERSION: Final[int] = 2 SESSION_KIND: Final[str] = "session" REQUEST_KIND: Final[str] = "request" TOOL_KIND: Final[str] = "tool" @@ -121,7 +121,7 @@ class ReportToolRow(ReportRowBase): class ReportSubagentRow(ReportRowBase): - """Subagent facts derived from a ``subagent_completed`` log record.""" + """Native child or OTLP completion facts, kept in one subagent fact family.""" harness: str agent_type: str | None @@ -129,6 +129,20 @@ class ReportSubagentRow(ReportRowBase): final_model: str | None model_swapped: bool | None event_sequence: int | None + child_id: str | None + native_parent_session_id: str | None + parent_session_key: str | None + role: str | None + actor_level: str | None + provider: str + skill: str | None + recipe: str | None + step: str | None + level: str | None + token_usage: dict[str, SerializedTokenMeasure] + tool_counts: dict[str, int] | None + transcript_state: str | None + usage_state: str | None REPORT_ROW_TYPES: Mapping[str, type] = MappingProxyType( @@ -176,6 +190,26 @@ def _count_map(value: object) -> dict[str, int] | None: return counts +def _token_usage_map(value: object) -> dict[str, SerializedTokenMeasure]: + mapping = value if isinstance(value, Mapping) else {} + measures: dict[str, SerializedTokenMeasure] = {} + for field in CANONICAL_ACCOUNTING_FIELDS: + raw = mapping.get(field) + try: + measures[field] = ( + TokenMeasure.from_dict(raw).to_dict() + if isinstance(raw, dict) + else ( + TokenMeasure.observed(raw).to_dict() + if isinstance(raw, int) and not isinstance(raw, bool) and raw >= 0 + else TokenMeasure.unknown().to_dict() + ) + ) + except ValueError: + measures[field] = TokenMeasure.unknown().to_dict() + return measures + + def _pair_text(value: object) -> str: return _text(value) or UNKNOWN_SOURCE @@ -257,6 +291,20 @@ def _raw_measure(value: object) -> object: "final_model": _text, "model_swapped": _flag, "event_sequence": _count, + "child_id": _text, + "native_parent_session_id": _text, + "parent_session_key": _text, + "role": _text, + "actor_level": _text, + "provider": _pair_text, + "skill": _text, + "recipe": _text, + "step": _text, + "level": _text, + "token_usage": _token_usage_map, + "tool_counts": _count_map, + "transcript_state": _text, + "usage_state": _text, }, } @@ -294,7 +342,8 @@ def rows_for_walk_item(item: WalkItem) -> list[dict[str, Any]]: if item.record is None or item.source_id is None: return [] if item.kind == SESSION_WALK_KIND: - return [_session_row(item.source_id, item.record)] + session = _session_row(item.source_id, item.record) + return [session, *_native_subagent_rows(item.source_id, item.record, session)] if item.kind == OTLP_WALK_KIND: return _otlp_rows(item.source_id, item.record) return [] @@ -375,6 +424,13 @@ def _skill_name(value: object) -> str | None: return extract_skill_name(command) +def _attribution_skill(value: object) -> str | None: + skill = _text(value) + if skill is None: + return None + return _skill_name(skill) if skill.startswith(("/", "$")) else skill + + def _tool_counts(record: dict[str, Any]) -> dict[str, int] | None: if record.get("transcripts_available") is not True: return None @@ -390,6 +446,108 @@ def _tool_counts(record: dict[str, Any]) -> dict[str, int] | None: return dict(sorted(counts.items())) +def _native_subagent_rows( + key: str, + record: dict[str, Any], + parent: Mapping[str, Any], +) -> list[dict[str, Any]]: + source_row = record.get("row") + outcomes = record.get("child_outcomes") + if not isinstance(source_row, dict) or not isinstance(outcomes, (tuple, list)): + return [] + native_parent_id = _text(source_row.get("session_id")) + parent_key = _text(source_row.get("dir_name")) or key + if native_parent_id is None: + return [] + + rows: list[dict[str, Any]] = [] + seen_keys: set[str] = set() + for child in outcomes: + if not isinstance(child, Mapping): + continue + child_id = _text(child.get("child_id")) + role = _text(child.get("role")) + backend = _text(child.get("backend")) + expected_source = { + "claude_code": "transcript_metadata", + "codex": "codex_rollout_metadata", + }.get(backend or "") + if ( + child_id is None + or role is None + or expected_source is None + or child.get("evidence_source") != expected_source + or child.get("parent_session_id") != native_parent_id + or child.get("parent_session_key") not in (None, parent_key) + ): + continue + child_key = f"{parent_key}:native:{backend}:{native_parent_id}:{child_id}" + if child_key in seen_keys: + continue + seen_keys.add(child_key) + + effective_provider = _text(child.get("effective_provider")) + if effective_provider is not None and effective_provider.casefold() == UNKNOWN_SOURCE: + effective_provider = None + provider = _pair_text(effective_provider) + token_usage = ( + _token_usage_map(child.get("token_usage")) + if effective_provider is not None + else _unknown_child_usage_map() + ) + if child.get("usage_state") == "unknown": + token_usage = _unknown_child_usage_map() + + parent_skill = parent["skill"] + attribution_skill = _attribution_skill(child.get("attribution_skill")) + skill = ( + parent_skill + if parent["level"] in ("skill", "orchestrator") + and parent_skill is not None + and attribution_skill == parent_skill + else None + ) + transcript_state = _text(child.get("transcript_state")) or "unknown" + tool_counts = ( + _count_map(child.get("tool_counts")) if transcript_state == "observed" else None + ) + rows.append( + { + "schema_version": REPORT_INDEX_SCHEMA_VERSION, + "kind": SUBAGENT_KIND, + "key": child_key, + "session_id": native_parent_id, + "time_ms": parent["time_ms"], + "harness": parent["harness"], + "agent_type": None, + "model": _text(child.get("effective_model")), + "final_model": None, + "model_swapped": None, + "event_sequence": None, + "child_id": child_id, + "native_parent_session_id": native_parent_id, + "parent_session_key": parent_key, + "role": role, + "actor_level": "L0", + "provider": provider, + "skill": skill, + "recipe": parent["recipe"], + "step": parent["step"], + "level": parent["level"], + "token_usage": token_usage, + "tool_counts": tool_counts, + "transcript_state": transcript_state, + "usage_state": _text(child.get("usage_state")) or "unknown", + } + ) + rows.sort(key=lambda row: row["key"]) + return rows + + +def _unknown_child_usage_map() -> dict[str, SerializedTokenMeasure]: + return {field: TokenMeasure.unknown().to_dict() for field in CANONICAL_ACCOUNTING_FIELDS} + + def _otlp_rows(source_id: str, record: dict[str, Any]) -> list[dict[str, Any]]: if record.get("signal") != "logs": return [] diff --git a/src/autoskillit/execution/evidence/_native_child_projection.py b/src/autoskillit/execution/evidence/_native_child_projection.py new file mode 100644 index 0000000000..17d8596158 --- /dev/null +++ b/src/autoskillit/execution/evidence/_native_child_projection.py @@ -0,0 +1,385 @@ +"""Read and attribute structurally verified native child evidence for reports.""" + +from __future__ import annotations + +import hashlib +import json +import math +from collections import Counter +from collections.abc import Mapping, Sequence +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import zstandard + +from autoskillit.core import ( + CANONICAL_ACCOUNTING_FIELDS, + SerializedTokenMeasure, + TokenMeasure, + iter_merged_assistant_turns, +) +from autoskillit.execution.backends._codex_execution_identity import ( + extract_codex_child_metadata, +) +from autoskillit.execution.backends._codex_parse import ( + _logical_rollout_reader, + extract_codex_child_turn_usage, +) +from autoskillit.execution.child_outcomes import ( + collect_child_outcomes, + enumerate_claude_subagent_transcripts, + normalize_backend_name, +) +from autoskillit.execution.session import extract_token_usage +from autoskillit.execution.session.turn_usage import classify_token_measure + + +def project_child_outcomes( + row: dict[str, Any], + *, + log_root: Path, + parent_rows: Mapping[tuple[str, str], Sequence[dict[str, Any]]], +) -> tuple[dict[str, Any], ...]: + parent_id = row.get("session_id") + backend = row.get("backend") + if not isinstance(parent_id, str) or not parent_id or not isinstance(backend, str): + return () + normalized_backend = normalize_backend_name(backend) + candidates = parent_rows.get((normalized_backend, parent_id), ()) + outcomes = collect_child_outcomes( + backend=normalized_backend, + parent_session_id=parent_id, + log_root=log_root, + ) + projected = [] + for outcome in outcomes: + child = _project_child_outcome( + row, + dict(outcome), + parent_id=parent_id, + backend=normalized_backend, + candidates=candidates, + ) + if child is not None: + projected.append(child) + return tuple(projected) + + +def _project_child_outcome( + row: dict[str, Any], + child: dict[str, Any], + *, + parent_id: str, + backend: str, + candidates: Sequence[dict[str, Any]], +) -> dict[str, Any] | None: + if not _is_native_child(child, backend, parent_id): + return None + path, identity_verified = _child_transcript_path(row, child, backend, parent_id) + if not identity_verified: + return None + text = _read_child_transcript(path, backend) if path is not None and path.is_file() else None + complete = _complete_jsonl(text) + child_timestamp = _first_transcript_timestamp(text) if text is not None else None + if not _child_belongs_to_parent(row, candidates, child_timestamp): + return None + + provider = child.get("effective_provider") + provider_used = ( + provider + if isinstance(provider, str) and provider and provider.casefold() != "unknown" + else "" + ) + tool_counts, token_usage, usage_state = _child_measurements( + backend, + child["child_id"], + path, + text, + complete, + provider_used, + ) + child.update( + { + "native_parent_session_id": parent_id, + "parent_session_key": row.get("dir_name"), + "transcript_state": "observed" if complete else "unknown", + "usage_state": usage_state, + "tool_counts": tool_counts, + "token_usage": token_usage, + "_child_timestamp": child_timestamp, + "_transcript_fingerprint": _transcript_fingerprint(path, text), + } + ) + return child + + +def _is_native_child(child: Mapping[str, Any], backend: str, parent_id: str) -> bool: + expected_source = { + "claude_code": "transcript_metadata", + "codex": "codex_rollout_metadata", + }.get(backend) + return ( + expected_source is not None + and child.get("backend") == backend + and child.get("parent_session_id") == parent_id + and isinstance(child.get("child_id"), str) + and bool(child.get("child_id")) + and isinstance(child.get("role"), str) + and bool(child.get("role")) + and child.get("evidence_source") == expected_source + ) + + +def _child_transcript_path( + row: dict[str, Any], child: Mapping[str, Any], backend: str, parent_id: str +) -> tuple[Path | None, bool]: + child_id = child.get("child_id") + if not isinstance(child_id, str) or not child_id: + return None, False + if backend == "claude_code": + return _claude_child_transcript_path(row, child_id, parent_id) + if backend == "codex": + return _codex_child_transcript_path(child, child_id, parent_id) + return None, False + + +def _claude_child_transcript_path( + row: dict[str, Any], child_id: str, parent_id: str +) -> tuple[Path | None, bool]: + parent_log = row.get("claude_code_log") + if not isinstance(parent_log, str) or not Path(parent_log).is_absolute(): + return None, True + if Path(parent_log).stem != parent_id: + return None, False + try: + paths = enumerate_claude_subagent_transcripts(Path(parent_log)) + except OSError: + return None, True + return next((path for path in paths if path.name == f"agent-{child_id}.jsonl"), None), True + + +def _codex_child_transcript_path( + child: Mapping[str, Any], child_id: str, parent_id: str +) -> tuple[Path | None, bool]: + locator = child.get("transcript_locator") + if not isinstance(locator, str) or not Path(locator).is_absolute(): + return None, True + path = Path(locator) + if not path.exists(): + return path, True + try: + extract_codex_child_metadata( + path, + expected_parent_id=parent_id, + expected_child_id=child_id, + ) + except (OSError, RuntimeError, ValueError, zstandard.ZstdError): + return None, False + return path, True + + +def _child_belongs_to_parent( + row: dict[str, Any], candidates: Sequence[dict[str, Any]], child_timestamp: float | None +) -> bool: + row_key = row.get("dir_name") + if not isinstance(row_key, str): + return False + if len(candidates) == 1: + return candidates[0].get("dir_name") == row_key + if child_timestamp is None: + return False + matches = [ + candidate for candidate in candidates if _invocation_contains(candidate, child_timestamp) + ] + return len(matches) == 1 and matches[0].get("dir_name") == row_key + + +def _invocation_contains(row: dict[str, Any], timestamp: float) -> bool: + start = _timestamp_seconds(row.get("timestamp")) + duration = row.get("duration_seconds") + if ( + start is None + or isinstance(duration, bool) + or not isinstance(duration, (int, float)) + or not math.isfinite(float(duration)) + or duration < 0 + ): + return False + return start <= timestamp <= start + float(duration) + + +def _timestamp_seconds(value: object) -> float | None: + if not isinstance(value, str) or not value: + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=UTC) + return parsed.timestamp() + + +def _first_transcript_timestamp(text: str) -> float | None: + for line in text.splitlines(): + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(record, Mapping): + timestamp = _timestamp_seconds(record.get("timestamp")) + if timestamp is not None: + return timestamp + return None + + +def _read_child_transcript(path: Path, backend: str) -> str | None: + try: + if backend == "codex": + with _logical_rollout_reader(path) as handle: + return handle.read().decode("utf-8") + return path.read_text(encoding="utf-8") + except (OSError, RuntimeError, UnicodeError, ValueError, zstandard.ZstdError): + return None + + +def _complete_jsonl(text: str | None) -> bool: + if not text or not text.endswith("\n"): + return False + for line in text.splitlines(): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError: + return False + if not isinstance(record, Mapping): + return False + return True + + +def _child_measurements( + backend: str, + child_id: str, + path: Path | None, + text: str | None, + complete: bool, + provider: str, +) -> tuple[dict[str, int] | None, dict[str, SerializedTokenMeasure], str]: + if not complete or text is None: + return None, _unknown_usage(), "unknown" + parser_backend = "claude" if backend == "claude_code" else backend + turns = list(iter_merged_assistant_turns(text, backend=parser_backend)) + tool_counts = dict(Counter(name for turn in turns for name in turn.tool_names)) + token_usage, usage_state = _child_token_usage(backend, child_id, path, text, provider) + return tool_counts, token_usage, usage_state + + +def _child_token_usage( + backend: str, child_id: str, path: Path | None, text: str, provider: str +) -> tuple[dict[str, SerializedTokenMeasure], str]: + if not provider: + return _unknown_usage(), "unknown" + if backend == "claude_code": + aggregate, _rows = extract_token_usage(text, provider_used=provider) + if aggregate is not None: + return ( + { + field: aggregate.get(field, TokenMeasure.unknown().to_dict()) + for field in CANONICAL_ACCOUNTING_FIELDS + }, + "observed", + ) + return _unknown_usage(), "unknown" + if backend == "codex" and path is not None: + rows = extract_codex_child_turn_usage(path, child_id, provider_used=provider) + if rows: + return _codex_usage_measures(rows, provider), "observed" + return _unknown_usage(), "unknown" + + +def _codex_usage_measures( + rows: Sequence[Mapping[str, Any]], provider: str +) -> dict[str, SerializedTokenMeasure]: + measures: dict[str, SerializedTokenMeasure] = {} + for field in CANONICAL_ACCOUNTING_FIELDS: + values = [row.get(field) for row in rows] + total = ( + sum( + value for value in values if isinstance(value, int) and not isinstance(value, bool) + ) + if values + and all( + isinstance(value, int) and not isinstance(value, bool) and value >= 0 + for value in values + ) + else None + ) + measures[field] = classify_token_measure("codex", provider, field, total).to_dict() + return measures + + +def _unknown_usage() -> dict[str, SerializedTokenMeasure]: + return {field: TokenMeasure.unknown().to_dict() for field in CANONICAL_ACCOUNTING_FIELDS} + + +def _transcript_fingerprint(path: Path | None, text: str | None) -> str | None: + if path is None: + return None + identity: list[int] | None = None + try: + stat = path.stat() + identity = [stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns] + except OSError: + pass + material = json.dumps( + { + "path": str(path), + "identity": identity, + "content": hashlib.sha256((text or "").encode("utf-8")).hexdigest(), + }, + sort_keys=True, + separators=(",", ":"), + ).encode() + return hashlib.sha256(material).hexdigest() + + +def child_evidence_fingerprint( + log_root: Path, + rows: Sequence[dict[str, Any]], + parent_rows: Mapping[tuple[str, str], Sequence[dict[str, Any]]], +) -> str: + """Digest owner-attributed child facts and the dedicated transcripts they use.""" + dependencies = [ + _child_dependency(row, child) + for row in rows + for child in project_child_outcomes(row, log_root=log_root, parent_rows=parent_rows) + ] + dependencies.sort(key=_canonical_json) + return hashlib.sha256(_canonical_json(dependencies).encode("utf-8")).hexdigest() + + +def _child_dependency(row: dict[str, Any], child: Mapping[str, Any]) -> dict[str, Any]: + return { + "parent_session_key": child.get("parent_session_key"), + "parent_timestamp": row.get("timestamp"), + "parent_duration_seconds": row.get("duration_seconds"), + "native_parent_session_id": child.get("native_parent_session_id"), + "child_id": child.get("child_id"), + "backend": child.get("backend"), + "role": child.get("role"), + "skill": child.get("attribution_skill"), + "provider": child.get("effective_provider"), + "model": child.get("effective_model"), + "evidence_source": child.get("evidence_source"), + "transcript_state": child.get("transcript_state"), + "usage_state": child.get("usage_state"), + "tool_counts": child.get("tool_counts"), + "token_usage": child.get("token_usage"), + "transcript_fingerprint": child.get("_transcript_fingerprint"), + } + + +def _canonical_json(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) diff --git a/src/autoskillit/execution/evidence/report_walk.py b/src/autoskillit/execution/evidence/report_walk.py index 56242f7695..126597b8b7 100644 --- a/src/autoskillit/execution/evidence/report_walk.py +++ b/src/autoskillit/execution/evidence/report_walk.py @@ -11,7 +11,7 @@ import hashlib import json import os -from collections.abc import Iterator +from collections.abc import Iterator, Mapping, Sequence from contextlib import ExitStack from dataclasses import dataclass from pathlib import Path @@ -19,7 +19,16 @@ from autoskillit.core import ArtifactLease, get_logger, iter_merged_assistant_turns from autoskillit.execution.backends._codex_parse import _logical_rollout_reader -from autoskillit.execution.child_outcomes import enumerate_claude_subagent_transcripts +from autoskillit.execution.child_outcomes import ( + enumerate_claude_subagent_transcripts, + normalize_backend_name, +) +from autoskillit.execution.evidence._native_child_projection import ( + child_evidence_fingerprint as _native_child_evidence_fingerprint, +) +from autoskillit.execution.evidence._native_child_projection import ( + project_child_outcomes, +) from autoskillit.execution.session_log.session_index import ( iter_tolerant_session_index_lines, read_tolerant_session_index_rows, @@ -269,11 +278,53 @@ def _transcript_turns(path: Path, backend: str) -> list[dict[str, Any]]: ] -def _session_record(row: dict[str, Any]) -> dict[str, Any]: +def _native_parent_rows( + rows: Sequence[dict[str, Any]], +) -> dict[tuple[str, str], list[dict[str, Any]]]: + by_key: dict[str, dict[str, Any]] = {} + for row in rows: + key = row.get("dir_name") + if isinstance(key, str) and key: + by_key[key] = row + parents: dict[tuple[str, str], list[dict[str, Any]]] = {} + for row in by_key.values(): + backend = row.get("backend") + session_id = row.get("session_id") + if not isinstance(backend, str) or not isinstance(session_id, str) or not session_id: + continue + identity = normalize_backend_name(backend), session_id + parents.setdefault(identity, []).append(row) + for candidates in parents.values(): + candidates.sort(key=lambda candidate: str(candidate.get("dir_name", ""))) + return parents + + +def _retained_session_rows(log_root: Path) -> list[dict[str, Any]]: + rows_by_key: dict[str, dict[str, Any]] = {} + for row in read_tolerant_session_index_rows(log_root / "sessions-archive.jsonl"): + key = row.get("dir_name") + if isinstance(key, str) and key: + rows_by_key[key] = row + with ArtifactLease.acquire_shared( + session_index_lock_path(log_root), timeout=_LEASE_TIMEOUT_SECONDS + ): + for row in read_tolerant_session_index_rows(log_root / "sessions.jsonl"): + key = row.get("dir_name") + if isinstance(key, str) and key: + rows_by_key[key] = row + return [rows_by_key[key] for key in sorted(rows_by_key)] + + +def _session_record( + row: dict[str, Any], + *, + log_root: Path, + parent_rows: Mapping[tuple[str, str], Sequence[dict[str, Any]]], +) -> dict[str, Any]: turns: list[dict[str, Any]] = [] unavailable_reasons: list[str] = [] has_transcript = False - for field, backend in (("claude_code_log", "claude"), ("codex_log", "codex")): + for field, transcript_backend in (("claude_code_log", "claude"), ("codex_log", "codex")): raw = row.get(field) if raw is None: if field in row: @@ -287,14 +338,16 @@ def _session_record(row: dict[str, Any]) -> dict[str, Any]: paths: tuple[Path, ...] try: paths = (path,) + ( - enumerate_claude_subagent_transcripts(path) if backend == "claude" else () + enumerate_claude_subagent_transcripts(path) + if transcript_backend == "claude" + else () ) except OSError as exc: unavailable_reasons.append(f"{field}:enumerate:{exc}") continue for transcript in paths: try: - turns.extend(_transcript_turns(transcript, backend)) + turns.extend(_transcript_turns(transcript, transcript_backend)) except (OSError, UnicodeError, ValueError) as exc: unavailable_reasons.append(f"{transcript}:{exc}") except RuntimeError as exc: @@ -303,16 +356,24 @@ def _session_record(row: dict[str, Any]) -> dict[str, Any]: # rather than masking the caller's intent. unavailable_reasons.append(f"{transcript}:runtime:{exc}") unavailable = not has_transcript or bool(unavailable_reasons) - return { + record: dict[str, Any] = { "row": row, "assistant_turn_count": None if unavailable else len(turns), "assistant_turns": turns, "transcripts_available": not unavailable, "transcript_unavailable_reasons": unavailable_reasons, } + record["child_outcomes"] = project_child_outcomes( + row, log_root=log_root, parent_rows=parent_rows + ) + return record -def _walk_archive(root: Path, state: dict[str, Any]) -> Iterator[WalkItem]: +def _walk_archive( + root: Path, + state: dict[str, Any], + parent_rows: Mapping[tuple[str, str], Sequence[dict[str, Any]]], +) -> Iterator[WalkItem]: # The session archive is an append-only retention file with no concurrent # writers in the live session workflow, so an ArtifactLease is not # required; the file handle is closed promptly and identity is captured @@ -350,12 +411,16 @@ def _walk_archive(root: Path, state: dict[str, Any]) -> Iterator[WalkItem]: SESSION_WALK_KIND, row["dir_name"], session_id, - _session_record(row), + _session_record(row, log_root=root, parent_rows=parent_rows), _copy(state), ) -def _walk_projection(root: Path, state: dict[str, Any]) -> Iterator[WalkItem]: +def _walk_projection( + root: Path, + state: dict[str, Any], + parent_rows: Mapping[tuple[str, str], Sequence[dict[str, Any]]], +) -> Iterator[WalkItem]: path = root / "sessions.jsonl" with ArtifactLease.acquire_shared( session_index_lock_path(root), timeout=_LEASE_TIMEOUT_SECONDS @@ -389,7 +454,13 @@ def _walk_projection(root: Path, state: dict[str, Any]) -> Iterator[WalkItem]: session_id = row.get("session_id") if session_id is not None and not isinstance(session_id, str): session_id = None - yield WalkItem(SESSION_WALK_KIND, name, session_id, _session_record(row), _copy(state)) + yield WalkItem( + SESSION_WALK_KIND, + name, + session_id, + _session_record(row, log_root=root, parent_rows=parent_rows), + _copy(state), + ) state["projection"] = current state["projection_identity"] = identity yield WalkItem(CHECKPOINT_WALK_KIND, None, None, None, _copy(state)) @@ -407,5 +478,14 @@ def iter_report_walk( should not assume one from ``_walk_otlp`` or ``_walk_archive``. """ state = _copy(watermark or {}) - for walk in (_walk_otlp, _walk_archive, _walk_projection): - yield from walk(log_root, state) + parent_rows = _native_parent_rows(_retained_session_rows(log_root)) + yield from _walk_otlp(log_root, state) + yield from _walk_archive(log_root, state, parent_rows) + yield from _walk_projection(log_root, state, parent_rows) + + +def child_evidence_fingerprint(log_root: Path) -> str: + """Return the current fingerprint of owner-attributed native child evidence.""" + rows = _retained_session_rows(log_root) + parent_rows = _native_parent_rows(rows) + return _native_child_evidence_fingerprint(log_root, rows, parent_rows) diff --git a/src/autoskillit/execution/report_index.py b/src/autoskillit/execution/report_index.py index fc02e6ea23..a260f08b13 100644 --- a/src/autoskillit/execution/report_index.py +++ b/src/autoskillit/execution/report_index.py @@ -7,9 +7,10 @@ the attributed harness/provider pair. For sources changed only in ways tracked by ``iter_report_walk``, an incremental -build and a rebuild have the same rows. This assumes transcripts are not modified -after their session row is walked; ``rebuild_report_index`` refreshes transcript -counts when that assumption no longer holds. +build and a rebuild have the same rows. Child snapshots and their dedicated +transcripts are fingerprinted on each update. The parent transcript is assumed not +to change after its session row is walked; ``rebuild_report_index`` refreshes its +derived counts when that assumption no longer holds. """ from __future__ import annotations @@ -52,6 +53,7 @@ VALID_SOURCE_KEYS, SourceGapError, WalkItem, + child_evidence_fingerprint, iter_report_walk, ) from autoskillit.execution.session_log.session_index import ( @@ -122,7 +124,20 @@ def _update(log_root: Path, index_dir: Path, *, rebuild: bool) -> ReportIndexUpd if rebuild: (index_dir / _ROWS_FILE).unlink(missing_ok=True) (index_dir / _STATE_FILE).unlink(missing_ok=True) - with _RowAppender.open(index_dir) as appender: + fingerprint = child_evidence_fingerprint(log_root) + state = read_versioned_json(index_dir / _STATE_FILE, _STATE_SCHEMA_VERSION) + row_schema_mismatch = ( + state.get("row_schema_version") != REPORT_INDEX_SCHEMA_VERSION + if state is not None + else (index_dir / _ROWS_FILE).exists() + ) + child_evidence_changed = ( + state is not None and state.get("child_evidence_fingerprint") != fingerprint + ) + if row_schema_mismatch or child_evidence_changed: + (index_dir / _ROWS_FILE).unlink(missing_ok=True) + (index_dir / _STATE_FILE).unlink(missing_ok=True) + with _RowAppender.open(index_dir, child_evidence_fingerprint=fingerprint) as appender: return _walk_into(log_root, appender) @@ -271,10 +286,12 @@ def __init__( index_dir: Path, handle: BinaryIO, watermark: dict[str, Any] | None, + child_evidence_fingerprint: str, ) -> None: self.index_dir = index_dir self.handle = handle self.watermark = watermark + self.child_evidence_fingerprint = child_evidence_fingerprint self._pending = bytearray() self._dirty = False self.items_walked = 0 @@ -282,7 +299,7 @@ def __init__( @classmethod @contextmanager - def open(cls, index_dir: Path) -> Iterator[_RowAppender]: + def open(cls, index_dir: Path, *, child_evidence_fingerprint: str) -> Iterator[_RowAppender]: rows_path = index_dir / _ROWS_FILE watermark, committed = _committed_state(index_dir, rows_path) fd = os.open( @@ -293,7 +310,7 @@ def open(cls, index_dir: Path) -> Iterator[_RowAppender]: with os.fdopen(fd, "r+b") as handle: handle.truncate(committed) handle.seek(committed) - yield cls(index_dir, handle, watermark) + yield cls(index_dir, handle, watermark, child_evidence_fingerprint) def add(self, item: WalkItem) -> None: for row in rows_for_walk_item(item): @@ -311,15 +328,20 @@ def add(self, item: WalkItem) -> None: if item.kind == CHECKPOINT_WALK_KIND or len(self._pending) >= _COMMIT_BYTES: self.commit() - def commit(self) -> None: - if not self._dirty: + def commit(self, *, force: bool = False) -> None: + if not self._dirty and not force: return self.handle.write(bytes(self._pending)) self.handle.flush() os.fsync(self.handle.fileno()) write_versioned_json( self.index_dir / _STATE_FILE, - {"walk": self.watermark, "rows_bytes": self.handle.tell()}, + { + "walk": self.watermark, + "rows_bytes": self.handle.tell(), + "row_schema_version": REPORT_INDEX_SCHEMA_VERSION, + "child_evidence_fingerprint": self.child_evidence_fingerprint, + }, schema_version=_STATE_SCHEMA_VERSION, ) self._pending.clear() @@ -352,7 +374,7 @@ def _walk_into(log_root: Path, appender: _RowAppender) -> ReportIndexUpdate: logger.warning("report_index_source_gap", source=gap.source, detail=str(gap)) appender.reset_source(gap.source) continue - appender.commit() + appender.commit(force=True) return ReportIndexUpdate(appender.items_walked, appender.rows_written, tuple(gaps)) @@ -449,8 +471,21 @@ def _sorted_rows(rows: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]: return dict(sorted(rows.items())) +def _require_row_schema(index_dir: Path) -> None: + rows_path = index_dir / _ROWS_FILE + state_path = index_dir / _STATE_FILE + if not rows_path.exists() and not state_path.exists(): + return + state = read_versioned_json(state_path, _STATE_SCHEMA_VERSION) + if state is None or state.get("row_schema_version") != REPORT_INDEX_SCHEMA_VERSION: + raise ValueError( + f"report index row schema mismatch: expected {REPORT_INDEX_SCHEMA_VERSION}" + ) + + def read_report_index(index_dir: Path) -> ReportIndex: """Read the latest valid fact for each key and resolve its session join.""" + _require_row_schema(index_dir) by_kind = _read_rows(index_dir / _ROWS_FILE) sessions = by_kind[SESSION_KIND] requests = by_kind[REQUEST_KIND] diff --git a/tests/execution/test_report_index.py b/tests/execution/test_report_index.py index 78af5c3f40..83d73802e4 100644 --- a/tests/execution/test_report_index.py +++ b/tests/execution/test_report_index.py @@ -17,6 +17,7 @@ update_report_index, ) from autoskillit.execution.evidence.report_walk import WalkItem +from autoskillit.hooks import _child_outcome_snapshot as child_snapshot from tests.execution._report_index_fixtures import ( CLAUDE_SCOPE, basic_session_row, @@ -69,6 +70,63 @@ def _basic_session(dir_name: str, session_id: str, **fields: Any) -> dict[str, A return basic_session_row(dir_name, session_id, **fields) +def _write_native_child(log_root: Path, parent_id: str, child_id: str, *, model: str) -> Path: + snapshot_path = child_snapshot.resolve_snapshot_path( + log_root, backend="claude_code", parent_session_id=parent_id + ) + child_snapshot.observe_child( + snapshot_path, + backend="claude_code", + parent_session_id=parent_id, + child_id=child_id, + ) + child_snapshot.record_terminal_evidence( + snapshot_path, + backend="claude_code", + parent_session_id=parent_id, + child_id=child_id, + evidence_key=f"{child_id}:metadata", + evidence={ + "role": "Explore", + "attribution_skill": "implement-worktree-no-merge", + "effective_provider": "openai", + "effective_model": model, + "evidence_source": "transcript_metadata", + }, + ) + transcript = log_root / parent_id / "subagents" / f"agent-{child_id}.jsonl" + transcript.parent.mkdir(parents=True, exist_ok=True) + transcript.write_text( + json.dumps( + { + "type": "assistant", + "agentId": child_id, + "attributionAgent": "Explore", + "attributionSkill": "implement-worktree-no-merge", + "message": {"id": f"message-{child_id}", "model": model}, + "sessionId": parent_id, + } + ) + + "\n", + encoding="utf-8", + ) + return snapshot_path + + +def _remove_native_child(snapshot_path: Path, child_id: str) -> None: + document = json.loads(snapshot_path.read_text(encoding="utf-8")) + document["children"].pop(child_id) + snapshot_path.write_text(json.dumps(document, sort_keys=True), encoding="utf-8") + + +def _indexed_native_children(index_dir: Path) -> dict[str, dict[str, Any]]: + return { + row["child_id"]: row + for row in read_report_index(index_dir).subagents.values() + if row["actor_level"] == "L0" and row["child_id"] is not None + } + + def test_update_then_read_joins_events_to_sessions(tmp_path: Path) -> None: root = tmp_path / "logs" session_dir = "session-a" @@ -301,6 +359,57 @@ def test_rebuild_from_scratch_equals_incremental_build(tmp_path: Path) -> None: assert read_report_index(inc) == read_report_index(full) +def test_child_only_changes_refresh_live_and_archive_rows_once(tmp_path: Path) -> None: + root = tmp_path / "logs" + index_dir = tmp_path / "index" + live_id, archive_id = "parent-live", "parent-archive" + live_transcript, archive_transcript = root / f"{live_id}.jsonl", root / f"{archive_id}.jsonl" + live_transcript.parent.mkdir(parents=True) + live_transcript.write_text("", encoding="utf-8") + archive_transcript.write_text("", encoding="utf-8") + _write_jsonl( + root / "sessions.jsonl", + [_basic_session("live", live_id, claude_code_log=str(live_transcript))], + ) + _write_jsonl( + root / "sessions-archive.jsonl", + [_basic_session("archive", archive_id, claude_code_log=str(archive_transcript))], + ) + + live_snapshot = _write_native_child(root, live_id, "child-live", model="model-v1") + _write_native_child(root, live_id, "child-removed", model="model-removed") + _write_native_child(root, archive_id, "child-archive", model="model-v1") + update_report_index(root, index_dir) + assert set(_indexed_native_children(index_dir)) == { + "child-live", + "child-removed", + "child-archive", + } + + _write_native_child(root, live_id, "child-live", model="model-v2") + _remove_native_child(live_snapshot, "child-removed") + (root / live_id / "subagents" / "agent-child-removed.jsonl").unlink() + _write_native_child(root, archive_id, "child-added", model="model-added") + + update_report_index(root, index_dir) + children = _indexed_native_children(index_dir) + assert set(children) == {"child-live", "child-archive", "child-added"} + assert children["child-live"]["model"] == "model-v2" + assert children["child-live"]["provider"] == "openai" + assert children["child-added"]["model"] == "model-added" + + rebuilt = tmp_path / "rebuilt" + rebuild_report_index(root, rebuilt) + expected = read_report_index(rebuilt) + assert read_report_index(index_dir) == expected + rows_path = index_dir / "rows.jsonl" + rows_size = rows_path.stat().st_size + repeated = update_report_index(root, index_dir) + assert repeated.rows_written == 0 + assert rows_path.stat().st_size == rows_size + assert read_report_index(index_dir) == expected + + def test_interrupted_update_resumes_to_the_same_index( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -422,13 +531,14 @@ def test_older_version_rows_parse_without_error(tmp_path: Path) -> None: @pytest.mark.parametrize( - "row_schema_marker", - [None, 1, REPORT_INDEX_SCHEMA_VERSION + 1], + ("row_schema_marker", "remove_rows"), + [(None, False), (1, False), (REPORT_INDEX_SCHEMA_VERSION + 1, True)], ids=["absent", "mismatched", "future"], ) def test_incompatible_row_schema_is_rejected_and_rebuilt( tmp_path: Path, row_schema_marker: int | None, + remove_rows: bool, ) -> None: assert not read_report_index(tmp_path / "absent-index").sessions @@ -457,6 +567,8 @@ def test_incompatible_row_schema_is_rejected_and_rebuilt( ) state["rows_bytes"] = rows_path.stat().st_size state_path.write_text(json.dumps(state), encoding="utf-8") + if remove_rows: + rows_path.unlink() with pytest.raises(ValueError, match="row schema"): read_report_index(index_dir) @@ -466,10 +578,11 @@ def test_incompatible_row_schema_is_rejected_and_rebuilt( indexed = read_report_index(index_dir) committed = json.loads(state_path.read_text(encoding="utf-8")) assert set(indexed.sessions) == {"current"} + assert committed["schema_version"] == report_index._STATE_SCHEMA_VERSION assert committed["row_schema_version"] == REPORT_INDEX_SCHEMA_VERSION -def test_lost_state_keeps_rows_whose_sources_rotated_away(tmp_path: Path) -> None: +def test_lost_state_rebuilds_rows_from_retained_sources(tmp_path: Path) -> None: root = tmp_path / "logs" _write_jsonl(root / "sessions.jsonl", [_basic_session("a", "sid-a")]) (root / "otlp.jsonl").write_bytes( @@ -481,11 +594,14 @@ def test_lost_state_keeps_rows_whose_sources_rotated_away(tmp_path: Path) -> Non (root / "otlp.jsonl").unlink() (index_dir / "state.json").unlink() + with pytest.raises(ValueError, match="row schema"): + read_report_index(index_dir) + update_report_index(root, index_dir) indexed = read_report_index(index_dir) assert indexed.sessions == sessions_before - assert {row["request_id"] for row in indexed.requests.values()} == {"a"} + assert indexed.requests == {} def test_archive_gap_reindexes_archive_then_projection(tmp_path: Path) -> None: diff --git a/tests/execution/test_report_index_rows.py b/tests/execution/test_report_index_rows.py index de2737a37c..26848e1042 100644 --- a/tests/execution/test_report_index_rows.py +++ b/tests/execution/test_report_index_rows.py @@ -151,7 +151,10 @@ def test_session_row_carries_facets_pair_and_resolved_measures() -> None: assert "private-turn-id" not in serialized -def test_session_item_projects_verified_native_child_under_index_parent_key() -> None: +@pytest.mark.parametrize("session_type", ["skill", "orchestrator"]) +def test_session_item_projects_verified_native_child_under_index_parent_key( + session_type: str, +) -> None: token_usage = { "input_tokens": {"state": "measured", "value": 15}, "output_tokens": {"state": "measured", "value": 4}, @@ -205,6 +208,20 @@ def test_session_item_projects_verified_native_child_under_index_parent_key() -> "transcript_state": "observed", "usage_state": "unknown", } + mismatched_skill_child = { + "child_id": "native-child-mismatched-skill", + "backend": "claude_code", + "parent_session_id": "native-parent-id", + "role": "audit-impl-slice-auditor", + "attribution_skill": "other-skill", + "effective_provider": "anthropic", + "effective_model": "claude-sonnet-child", + "evidence_source": "transcript_metadata", + "token_usage": token_usage, + "tool_counts": {}, + "transcript_state": "observed", + "usage_state": "observed", + } parent = { "session_id": "native-parent-id", "timestamp": "2026-09-01T00:00:00Z", @@ -214,18 +231,23 @@ def test_session_item_projects_verified_native_child_under_index_parent_key() -> "skill_command": "/autoskillit:make-plan", "recipe_name": "research", "step_name": "scope", - "session_type": "skill", + "session_type": session_type, } rows = rows_for_walk_item( _session_item( "report-index-key", parent, - child_outcomes=(child, managed_attempt, providerless_child), + child_outcomes=(child, managed_attempt, providerless_child, mismatched_skill_child), ) ) - assert [row["kind"] for row in rows] == ["session", "subagent", "subagent"] + assert [row["kind"] for row in rows] == [ + "session", + "subagent", + "subagent", + "subagent", + ] projected_by_id = {row["child_id"]: row for row in rows if row["kind"] == "subagent"} projected = projected_by_id["native-child-1"] assert projected["key"].startswith("report-index-key:") @@ -235,7 +257,7 @@ def test_session_item_projects_verified_native_child_under_index_parent_key() -> assert projected["role"] == "audit-impl-slice-auditor" assert projected["actor_level"] == "L0" assert projected["skill"] == "make-plan" - assert projected["level"] == "skill" + assert projected["level"] == session_type assert projected["recipe"] == "research" assert projected["step"] == "scope" assert projected["provider"] == "anthropic" @@ -249,6 +271,10 @@ def test_session_item_projects_verified_native_child_under_index_parent_key() -> assert without_provider["model"] is None assert without_provider["token_usage"]["input_tokens"]["state"] == "unknown" assert without_provider["usage_state"] == "unknown" + mismatched = projected_by_id["native-child-mismatched-skill"] + assert mismatched["skill"] is None + assert mismatched["role"] == "audit-impl-slice-auditor" + assert mismatched["level"] == session_type def test_session_row_legacy_zero_and_codex_sigil() -> None: diff --git a/tests/execution/test_report_index_schema.py b/tests/execution/test_report_index_schema.py index bfaead3fae..74f771d174 100644 --- a/tests/execution/test_report_index_schema.py +++ b/tests/execution/test_report_index_schema.py @@ -15,7 +15,7 @@ pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] -_EXPECTED = (2, "8418c2eb676180f7865ce20982478b76f175e0d7d3527f5353f71e4a26117d06") +_EXPECTED = (2, "e574d4e83971be001c1c92d29e807a421edbf705acbe5e72a03e7e513d43c4ba") # Locked snapshot of per-kind field name orderings that produced ``_EXPECTED``. # Bump ``_EXPECTED[0]`` and refresh this dict in lockstep when row fields change; # ``test_report_index_schema_version_matches_field_digest`` is what fails first diff --git a/tests/execution/test_report_walk.py b/tests/execution/test_report_walk.py index 1fd387c220..66ea09958f 100644 --- a/tests/execution/test_report_walk.py +++ b/tests/execution/test_report_walk.py @@ -10,7 +10,7 @@ import pytest import zstandard -import autoskillit.execution.evidence.report_walk as report_walk +import autoskillit.execution.evidence._native_child_projection as native_child_projection from autoskillit.core import iter_merged_assistant_turns from autoskillit.execution.evidence.report_walk import ( SourceGapError, @@ -133,14 +133,23 @@ def test_session_walk_reads_child_outcomes_by_native_parent_id( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: root = tmp_path / "logs" - child = {"child_id": "native-child", "role": "audit-bugs"} + child = { + "child_id": "native-child", + "backend": "claude_code", + "parent_session_id": "native-parent-id", + "role": "audit-bugs", + "attribution_skill": "make-plan", + "effective_provider": "", + "effective_model": "", + "evidence_source": "transcript_metadata", + } calls: list[tuple[str, str, Path]] = [] def collect(*, backend: str, parent_session_id: str, log_root: Path): calls.append((backend, parent_session_id, log_root)) return (child,) - monkeypatch.setattr(report_walk, "collect_child_outcomes", collect, raising=False) + monkeypatch.setattr(native_child_projection, "collect_child_outcomes", collect) _write_jsonl( root / "sessions.jsonl", [_session("report-index-key", "native-parent-id", backend="claude-code")], @@ -152,10 +161,122 @@ def collect(*, backend: str, parent_session_id: str, log_root: Path): assert item.source_id == "report-index-key" assert item.session_id == "native-parent-id" assert item.record is not None - assert item.record["child_outcomes"] == (child,) + (projected_child,) = item.record["child_outcomes"] + assert projected_child["child_id"] == "native-child" + assert projected_child["native_parent_session_id"] == "native-parent-id" + assert projected_child["parent_session_key"] == "report-index-key" + assert projected_child["transcript_state"] == "unknown" + assert projected_child["usage_state"] == "unknown" assert item.watermark["projection"]["report-index-key"] +def test_resumed_native_parent_owns_only_timestamp_matched_child( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = tmp_path / "logs" + parent_log = tmp_path / "project" / "native-parent-id.jsonl" + parent_log.parent.mkdir(parents=True) + parent_log.write_text("{}\n", encoding="utf-8") + child_dir = parent_log.parent / "native-parent-id" / "subagents" + _write_jsonl( + child_dir / "agent-child-in-second-resume.jsonl", + [ + { + "type": "assistant", + "timestamp": "2026-10-05T10:00:25Z", + "message": {"id": "m1", "content": []}, + } + ], + ) + _write_jsonl( + child_dir / "agent-child-without-time.jsonl", + [{"type": "assistant", "message": {"id": "m2", "content": []}}], + ) + (child_dir / "agent-incomplete-child.jsonl").write_text( + json.dumps( + { + "type": "assistant", + "timestamp": "2026-10-05T10:00:26Z", + "message": {"id": "m3", "content": []}, + } + ), + encoding="utf-8", + ) + children = ( + { + "child_id": "child-in-second-resume", + "backend": "claude_code", + "parent_session_id": "native-parent-id", + "role": "audit-bugs", + "attribution_skill": "make-plan", + "effective_provider": "anthropic", + "effective_model": "claude-child", + "evidence_source": "transcript_metadata", + }, + { + "child_id": "child-without-time", + "backend": "claude_code", + "parent_session_id": "native-parent-id", + "role": "audit-bugs", + "attribution_skill": "make-plan", + "effective_provider": "anthropic", + "effective_model": "claude-child", + "evidence_source": "transcript_metadata", + }, + { + "child_id": "incomplete-child", + "backend": "claude_code", + "parent_session_id": "native-parent-id", + "role": "audit-bugs", + "attribution_skill": "make-plan", + "effective_provider": "anthropic", + "effective_model": "claude-child", + "evidence_source": "transcript_metadata", + }, + ) + monkeypatch.setattr( + native_child_projection, + "collect_child_outcomes", + lambda **_kwargs: children, + ) + _write_jsonl( + root / "sessions.jsonl", + [ + _session( + "resume-a", + "native-parent-id", + backend="claude-code", + claude_code_log=str(parent_log), + timestamp="2026-10-05T10:00:00Z", + duration_seconds=10, + ), + _session( + "resume-b", + "native-parent-id", + backend="claude-code", + claude_code_log=str(parent_log), + timestamp="2026-10-05T10:00:20Z", + duration_seconds=10, + ), + ], + ) + + sessions = { + item.source_id: item.record for item in iter_report_walk(root) if item.kind == "session" + } + + assert sessions["resume-a"]["child_outcomes"] == () + owned_children = {child["child_id"]: child for child in sessions["resume-b"]["child_outcomes"]} + tool_free = owned_children["child-in-second-resume"] + assert tool_free["tool_counts"] == {} + assert tool_free["transcript_state"] == "observed" + assert tool_free["usage_state"] == "unknown" + incomplete = owned_children["incomplete-child"] + assert incomplete["tool_counts"] is None + assert incomplete["transcript_state"] == "unknown" + assert incomplete["usage_state"] == "unknown" + + @pytest.mark.parametrize("pause", ("before", "after")) def test_committed_watermark_resumes_without_missing_or_duplicate_records( tmp_path: Path, pause: str From 91a808b84626b657a4e3ee0ab1623a416e989cb5 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 14:14:42 -0700 Subject: [PATCH 04/22] feat: prepare core deck measures and role relationships --- src/autoskillit/cli/ops/_sessions.py | 3 + src/autoskillit/report/deck/_html.py | 15 +- src/autoskillit/report/deck/_payload.py | 580 +++++++++++++++++++++++- tests/report/test_deck_payload.py | 47 +- 4 files changed, 627 insertions(+), 18 deletions(-) diff --git a/src/autoskillit/cli/ops/_sessions.py b/src/autoskillit/cli/ops/_sessions.py index ce3d17f0dd..5a0fcc4655 100644 --- a/src/autoskillit/cli/ops/_sessions.py +++ b/src/autoskillit/cli/ops/_sessions.py @@ -79,6 +79,9 @@ def sessions_deck(output: str) -> None: report = read_report_index(index_dir) html = render_deck( report.sessions.values(), + request_rows=report.requests.values(), + tool_rows=report.tools.values(), + subagent_rows=report.subagents.values(), generated_at=datetime.now(UTC), index_schema_version=REPORT_INDEX_SCHEMA_VERSION, ) diff --git a/src/autoskillit/report/deck/_html.py b/src/autoskillit/report/deck/_html.py index bdcbbf2a68..0da3411f9a 100644 --- a/src/autoskillit/report/deck/_html.py +++ b/src/autoskillit/report/deck/_html.py @@ -67,10 +67,21 @@ def render_deck_html(payload: Mapping[str, Any]) -> str: def render_deck( - session_rows: Iterable[Mapping[str, Any]], *, generated_at: datetime, index_schema_version: int + session_rows: Iterable[Mapping[str, Any]], + *, + request_rows: Iterable[Mapping[str, Any]] = (), + tool_rows: Iterable[Mapping[str, Any]] = (), + subagent_rows: Iterable[Mapping[str, Any]] = (), + generated_at: datetime, + index_schema_version: int, ) -> str: return render_deck_html( build_deck_payload( - session_rows, generated_at=generated_at, index_schema_version=index_schema_version + session_rows, + request_rows=request_rows, + tool_rows=tool_rows, + subagent_rows=subagent_rows, + generated_at=generated_at, + index_schema_version=index_schema_version, ) ) diff --git a/src/autoskillit/report/deck/_payload.py b/src/autoskillit/report/deck/_payload.py index 8374f0ea00..6d7e963924 100644 --- a/src/autoskillit/report/deck/_payload.py +++ b/src/autoskillit/report/deck/_payload.py @@ -3,9 +3,18 @@ from collections import Counter from collections.abc import Iterable, Mapping, Sequence from datetime import UTC, datetime, timedelta +from itertools import combinations from typing import Any -from autoskillit.core import TokenMeasureState +from autoskillit.core import ( + MeasureRecord, + SourcePair, + TokenMeasure, + TokenMeasureState, + aggregate_measures, + load_bundled_agent_definitions, + measure_ratio, +) from ._registry import ( AVAILABILITY_VOCABULARY, @@ -24,6 +33,8 @@ ) DAY_MS = 86_400_000 +TOKEN_FIELDS = ("input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens") +SKILL_LEVELS = ("skill", "orchestrator", None) def encode_table(rows: Sequence[Mapping[str, Any]], columns: Sequence[str]) -> dict[str, Any]: @@ -104,6 +115,7 @@ def _window_chips( *, generated_at_ms: int, history: Mapping[str, Any], + population_label: str | None = None, ) -> list[dict[str, Any]]: chips = [] for window in WINDOWS: @@ -113,7 +125,11 @@ def _window_chips( state, count = ChipState.LIVE, len(rows) elif history["first_ms"] is None: state = ChipState.ABSENT - reason = ReasonDef("no session row in this index carries a timestamp") + reason = ReasonDef( + f"no {population_label} row in this view carries a timestamp" + if population_label + else "no session row in this index carries a timestamp" + ) else: first = history["first_ms"] retained = (generated_at_ms - first) // DAY_MS @@ -132,7 +148,11 @@ def _window_chips( ) state = ChipState.LIVE if count else ChipState.ABSENT if not count: - reason = ReasonDef(f"no rows in the last {window.label}") + reason = ReasonDef( + f"no {population_label} rows in the last {window.label}" + if population_label + else f"no rows in the last {window.label}" + ) chips.append(_chip(window.key, window.label, state, count, reason, days=window.days)) return chips @@ -143,15 +163,550 @@ def resolve_view_chips( *, generated_at_ms: int, history: Mapping[str, Any], + population_label: str | None = None, ) -> dict[str, list[dict[str, Any]]]: return {f.facet_id: _value_chips(view, f, rows) for f in FACETS} | { - WINDOW_FACET_ID: _window_chips(rows, generated_at_ms=generated_at_ms, history=history) + WINDOW_FACET_ID: _window_chips( + rows, + generated_at_ms=generated_at_ms, + history=history, + population_label=population_label, + ) + } + + +def _canonical_children( + sessions: Sequence[Mapping[str, Any]], + subagents: Iterable[Mapping[str, Any]], +) -> list[dict[str, Any]]: + parents = {row.get("key"): row for row in sessions if isinstance(row.get("key"), str)} + children = [] + for raw in subagents: + row = dict(raw) + parent_key = row.get("parent_session_key") + parent = parents.get(parent_key) + role = row.get("role") + if ( + row.get("actor_level") != "L0" + or not isinstance(row.get("child_id"), str) + or not row["child_id"] + or not isinstance(role, str) + or not role + or parent is None + or not isinstance(row.get("native_parent_session_id"), str) + or row["native_parent_session_id"] != parent.get("session_id") + ): + continue + children.append( + { + **row, + "harness": parent.get("harness") or "unknown", + "provider": row.get("provider") or "unknown", + "time_ms": parent.get("time_ms"), + "level": parent.get("level"), + "recipe": parent.get("recipe"), + "step": parent.get("step"), + "skill": row.get("skill") if isinstance(row.get("skill"), str) else None, + "_parent_provider": parent.get("provider") or "unknown", + } + ) + return children + + +def _window_level_blocks( + rows: Sequence[Mapping[str, Any]], *, generated_at_ms: int +) -> list[tuple[str, list[str | None], list[Mapping[str, Any]]]]: + blocks: list[tuple[str, list[str | None], list[Mapping[str, Any]]]] = [] + for window in WINDOWS: + in_window = [ + row + for row in rows + if window.days is None + or ( + row.get("time_ms") is not None + and row["time_ms"] >= generated_at_ms - window.days * DAY_MS + ) + ] + levels = sorted( + {row.get("level") for row in in_window}, + key=lambda level: (level is None, level or ""), + ) + for size in range(1, len(levels) + 1): + for selected in combinations(levels, size): + selected_set = set(selected) + members = [row for row in in_window if row.get("level") in selected_set] + if members: + blocks.append((window.key, list(selected), members)) + return blocks + + +def _measure_records( + rows: Sequence[Mapping[str, Any]], *, tool_names: Sequence[str] = () +) -> list[MeasureRecord]: + records = [] + for row in rows: + harness = row.get("harness") + provider = row.get("provider") + pair = SourcePair( + harness if isinstance(harness, str) and harness else "unknown", + provider if isinstance(provider, str) and provider else "unknown", + ) + child = row.get("actor_level") == "L0" + raw = row.get("token_usage") if child else row + raw = raw if isinstance(raw, Mapping) else {} + usage_known = not child or row.get("usage_state") == "observed" + measures = { + field: TokenMeasure.measure_from_raw(raw.get(field)) + if usage_known + else TokenMeasure.unknown() + for field in TOKEN_FIELDS + } + if child: + counts = row.get("tool_counts") + transcript_known = row.get("transcript_state") == "observed" and isinstance( + counts, Mapping + ) + counts = counts if isinstance(counts, Mapping) else {} + measures["tool_calls"] = ( + TokenMeasure.observed(sum(counts.values())) + if transcript_known + else TokenMeasure.unknown() + ) + for tool in tool_names: + measures[f"tool:{tool}"] = ( + TokenMeasure.observed(counts.get(tool, 0)) + if transcript_known + else TokenMeasure.unknown() + ) + records.append(MeasureRecord(pair, measures)) + return records + + +def _measure_payload(records: Sequence[MeasureRecord], fields: Sequence[str]) -> dict[str, Any]: + result = aggregate_measures(records, fields) + return { + field: { + "state": aggregate.value.state.value, + "value": aggregate.value.value, + "state_counts": { + state.value: count for state, count in aggregate.state_counts.items() + }, + } + for field, aggregate in result.fields.items() + } + + +def _ratio_payload( + records: Sequence[MeasureRecord], + numerator: str, + denominator: str, + *, + sample_unit: str, + definition_roles: Sequence[str], + definitions: Mapping[str, Mapping[str, Any]], +) -> dict[str, Any]: + result = measure_ratio(records, numerator, denominator) + roles = sorted(set(definition_roles)) + missing = [role for role in roles if definitions.get(role, {}).get("state") != "available"] + eligible = ( + result.state in (TokenMeasureState.MEASURED, TokenMeasureState.MEASURED_ZERO) + and result.value is not None + and result.sample_size > 0 + and bool(roles) + and not missing + ) + reasons = [] + if result.state is TokenMeasureState.UNKNOWN: + reasons.append("the ratio has unknown contributing measurements") + elif result.state is TokenMeasureState.UNAVAILABLE: + reasons.append("the ratio is unavailable for this source") + elif result.state is TokenMeasureState.NOT_APPLICABLE: + reasons.append("the ratio has no applicable positive denominator") + if not roles: + reasons.append("no contributor role definitions are linked") + if missing: + reasons.append("missing definitions: " + ", ".join(missing)) + return { + "state": result.state.value, + "value": result.value, + "numerator": result.numerator_field, + "denominator": result.denominator_field, + "numerator_total": result.numerator_total, + "denominator_total": result.denominator_total, + "sample_unit": sample_unit, + "sample_size": result.sample_size, + "excluded_runs": result.excluded_runs, + "unknown_runs": result.unknown_runs, + "definition_roles": roles, + "review_eligible": eligible, + "review_reason": None if eligible else "; ".join(reasons), } +def _metric_payload( + rows: Sequence[Mapping[str, Any]], + *, + sample_unit: str, + definition_roles: Sequence[str], + definitions: Mapping[str, Mapping[str, Any]], +) -> dict[str, Any]: + tool_names = sorted( + { + tool + for row in rows + if row.get("transcript_state") == "observed" + and isinstance(row.get("tool_counts"), Mapping) + for tool in row["tool_counts"] + if isinstance(tool, str) + } + ) + records = _measure_records(rows, tool_names=tool_names) + fields: tuple[str, ...] = TOKEN_FIELDS + has_tool_metrics = sample_unit == "child-invocation" + if has_tool_metrics: + fields += ("tool_calls",) + tuple(f"tool:{tool}" for tool in tool_names) + measures = _measure_payload(records, fields) + ratios = { + "input_output": _ratio_payload( + records, + "input_tokens", + "output_tokens", + sample_unit=sample_unit, + definition_roles=definition_roles, + definitions=definitions, + ), + "cache_share": _ratio_payload( + records, + "cache_read_tokens", + "input_tokens", + sample_unit=sample_unit, + definition_roles=definition_roles, + definitions=definitions, + ), + } + if has_tool_metrics: + ratios["tool_mix"] = { + tool: _ratio_payload( + records, + f"tool:{tool}", + "tool_calls", + sample_unit=sample_unit, + definition_roles=definition_roles, + definitions=definitions, + ) + for tool in tool_names + } + return {"measures": measures, "ratios": ratios} + + +def _skill_blocks( + sessions: Sequence[Mapping[str, Any]], + children: Sequence[Mapping[str, Any]], + *, + generated_at_ms: int, + definitions: Mapping[str, Mapping[str, Any]], +) -> list[dict[str, Any]]: + skill_rows: list[Mapping[str, Any]] = [ + row + for row in sessions + if row.get("level") in SKILL_LEVELS and isinstance(row.get("skill"), str) + ] + child_blocks = { + (window, tuple(levels)): rows + for window, levels, rows in _window_level_blocks(children, generated_at_ms=generated_at_ms) + } + blocks = [] + for window, levels, selected in _window_level_blocks( + skill_rows, generated_at_ms=generated_at_ms + ): + groups: dict[tuple[str, str, str], list[Mapping[str, Any]]] = {} + for row in selected: + identity = ( + row["skill"], + row.get("harness") or "unknown", + row.get("provider") or "unknown", + ) + groups.setdefault(identity, []).append(row) + rows_out = [] + selected_children = child_blocks.get((window, tuple(levels)), []) + for skill, harness, provider in sorted(groups): + runs = groups[(skill, harness, provider)] + related = [ + row + for row in selected_children + if row.get("skill") == skill + and row.get("harness") == harness + and row.get("_parent_provider") == provider + ] + related_roles = sorted({row["role"] for row in related}) + metric = _metric_payload( + runs, + sample_unit="skill-run", + definition_roles=related_roles, + definitions=definitions, + ) + step_groups: dict[tuple[str | None, str | None], list[Mapping[str, Any]]] = {} + for run in runs: + recipe = run.get("recipe") if isinstance(run.get("recipe"), str) else None + step = run.get("step") if isinstance(run.get("step"), str) else None + step_groups.setdefault((recipe, step), []).append(run) + recipe_steps = [] + for recipe, step in sorted( + step_groups, + key=lambda value: ( + value[0] is None, + value[0] or "", + value[1] is None, + value[1] or "", + ), + ): + step_runs = step_groups[(recipe, step)] + step_children = [ + row + for row in related + if row.get("recipe") == recipe and row.get("step") == step + ] + step_metric = _metric_payload( + step_runs, + sample_unit="skill-run", + definition_roles=sorted({row["role"] for row in step_children}), + definitions=definitions, + ) + recipe_steps.append( + { + "recipe": recipe, + "step": step, + **step_metric, + "exact_retransmission": TokenMeasure.unavailable().to_dict(), + } + ) + role_groups: dict[tuple[str, str, str], list[Mapping[str, Any]]] = {} + for child in related: + key = (child["role"], child["provider"], child["harness"]) + role_groups.setdefault(key, []).append(child) + child_roles = [] + for role, role_provider, role_harness in sorted(role_groups): + role_runs = role_groups[(role, role_provider, role_harness)] + role_metric = _metric_payload( + role_runs, + sample_unit="child-invocation", + definition_roles=[role], + definitions=definitions, + ) + child_roles.append( + { + "role": role, + "provider": role_provider, + "harness": role_harness, + "models": sorted( + { + row["model"] + for row in role_runs + if isinstance(row.get("model"), str) + } + ), + **role_metric, + } + ) + rows_out.append( + { + "skill": skill, + "harness": harness, + "provider": provider, + **metric, + "exact_retransmission": TokenMeasure.unavailable().to_dict(), + "recipe_steps": recipe_steps, + "child_roles": child_roles, + } + ) + blocks.append({"window": window, "levels": levels, "rows": rows_out}) + return blocks + + +def _role_blocks( + children: Sequence[Mapping[str, Any]], + *, + generated_at_ms: int, + definitions: Mapping[str, Mapping[str, Any]], +) -> list[dict[str, Any]]: + role_rows: list[Mapping[str, Any]] = [ + row for row in children if row.get("level") in SKILL_LEVELS + ] + blocks = [] + for window, levels, selected in _window_level_blocks( + role_rows, generated_at_ms=generated_at_ms + ): + groups: dict[tuple[str, str], list[Mapping[str, Any]]] = {} + for row in selected: + groups.setdefault((row["role"], row["provider"]), []).append(row) + rows_out = [] + for role, provider in sorted(groups): + harness_groups: dict[str, list[Mapping[str, Any]]] = {} + for row in groups[(role, provider)]: + harness_groups.setdefault(row["harness"], []).append(row) + harnesses = [] + for harness in sorted(harness_groups): + invocations = harness_groups[harness] + metric = _metric_payload( + invocations, + sample_unit="child-invocation", + definition_roles=[role], + definitions=definitions, + ) + harnesses.append( + { + "harness": harness, + "models": sorted( + { + row["model"] + for row in invocations + if isinstance(row.get("model"), str) + } + ), + **metric, + } + ) + rows_out.append({"role": role, "provider": provider, "harnesses": harnesses}) + blocks.append({"window": window, "levels": levels, "rows": rows_out}) + return blocks + + +def _prepare_deck_data( + sessions: Sequence[Mapping[str, Any]], + subagents: Iterable[Mapping[str, Any]], + *, + generated_at_ms: int, +) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]: + children = _canonical_children(sessions, subagents) + skill_rows: list[Mapping[str, Any]] = [ + row + for row in sessions + if row.get("level") in SKILL_LEVELS and isinstance(row.get("skill"), str) + ] + role_rows: list[Mapping[str, Any]] = [ + row for row in children if row.get("level") in SKILL_LEVELS + ] + bundled = {definition.name: definition for definition in load_bundled_agent_definitions()} + roles = sorted({row["role"] for row in role_rows}) + definitions: dict[str, dict[str, Any]] = {} + for role in roles: + definition = bundled.get(role) + definitions[role] = ( + { + "state": "available", + "description": definition.description, + "body": definition.body, + "tools": list(definition.tools), + "model": definition.model, + "reader_tools": list(definition.reader_tools), + "codex_model": getattr(getattr(definition, "codex", None), "model", None), + } + if definition is not None + else { + "state": "unavailable", + "description": None, + "body": None, + "tools": [], + "model": None, + "reader_tools": [], + "codex_model": None, + } + ) + skill_identities = sorted( + { + ( + row["skill"], + row.get("harness") or "unknown", + row.get("provider") or "unknown", + ) + for row in skill_rows + } + ) + role_identities = sorted({(row["role"], row["provider"]) for row in role_rows}) + relationships = [ + {"skill": skill, "role": role, "harness": harness, "provider": provider} + for skill, role, harness, provider in sorted( + { + (row["skill"], row["role"], row["harness"], row["provider"]) + for row in role_rows + if isinstance(row.get("skill"), str) and row["skill"] + } + ) + ] + view_rows: dict[str, list[Mapping[str, Any]]] = { + "cohort": list(sessions), + "spend": skill_rows, + "efficiency": [*skill_rows, *role_rows], + "skill": skill_rows, + "role": role_rows, + } + population_labels = { + "spend": "skill", + "efficiency": "contributor", + "skill": "skill", + "role": "child", + } + view_chips: dict[str, dict[str, list[dict[str, Any]]]] = {} + view_histories: dict[str, dict[str, Any]] = {} + for view in DECK_VIEWS: + if view.planned_issue is not None: + continue + rows = view_rows.get(view.view_id, list(sessions)) + history = _history(rows) + view_chips[view.view_id] = resolve_view_chips( + view, + rows, + generated_at_ms=generated_at_ms, + history=history, + population_label=population_labels.get(view.view_id), + ) + view_histories[view.view_id] = history + return ( + { + "skills": _skill_blocks( + skill_rows, + role_rows, + generated_at_ms=generated_at_ms, + definitions=definitions, + ), + "roles": _role_blocks( + role_rows, generated_at_ms=generated_at_ms, definitions=definitions + ), + "relationships": relationships, + "definitions": definitions, + "view_chips": view_chips, + "view_histories": view_histories, + }, + { + "skills": encode_table( + [ + {"skill": skill, "harness": harness, "provider": provider} + for skill, harness, provider in skill_identities + ], + ("skill", "harness", "provider"), + ), + "roles": encode_table( + [{"role": role, "provider": provider} for role, provider in role_identities], + ("role", "provider"), + ), + }, + ) + + def build_deck_payload( - session_rows: Iterable[Mapping[str, Any]], *, generated_at: datetime, index_schema_version: int + session_rows: Iterable[Mapping[str, Any]], + *, + request_rows: Iterable[Mapping[str, Any]] = (), + tool_rows: Iterable[Mapping[str, Any]] = (), + subagent_rows: Iterable[Mapping[str, Any]] = (), + generated_at: datetime, + index_schema_version: int, ) -> dict[str, Any]: + """Encode the session cohort and prepare attributed skill and child-role metrics. + + Request and tool rows remain available at this boundary, but contribute to no metric + without a verified child identity. Native child rows already carry folded usage. + """ if generated_at.utcoffset() is None: raise ValueError("generated_at must be timezone-aware") epoch = datetime(1970, 1, 1, tzinfo=UTC) @@ -161,12 +716,16 @@ def build_deck_payload( key=lambda r: (r.get("time_ms") is None, r.get("time_ms") or 0, r.get("key") or ""), ) history = _history(rows) + prepared, identity_tables = _prepare_deck_data( + rows, subagent_rows, generated_at_ms=generated_at_ms + ) + tables = {SESSION_TABLE: encode_table(rows, SESSION_COLUMNS), **identity_tables} return { "generated_at_ms": generated_at_ms, "index_schema_version": index_schema_version, "landing": LANDING_VIEW, "history": history, - "tables": {SESSION_TABLE: encode_table(rows, SESSION_COLUMNS)}, + "tables": tables, "facets": [ {"id": f.facet_id, "label": f.label, "column": f.column, "kind": "values"} for f in FACETS @@ -183,13 +742,8 @@ def build_deck_payload( } for v in DECK_VIEWS ], - "chips": { - v.view_id: resolve_view_chips( - v, rows, generated_at_ms=generated_at_ms, history=history - ) - for v in DECK_VIEWS - if v.planned_issue is None - }, + "chips": prepared["view_chips"], + "prepared": prepared, "availability": [ { "state": s.value, diff --git a/tests/report/test_deck_payload.py b/tests/report/test_deck_payload.py index 9cd7108a16..b0b32b79c2 100644 --- a/tests/report/test_deck_payload.py +++ b/tests/report/test_deck_payload.py @@ -221,10 +221,11 @@ def _source_records( ) -> list[MeasureRecord]: if child and parents is None: raise ValueError("child records require their parent rows") + parent_rows = parents or {} records = [] for row in rows: raw = row["token_usage"] if child else row - harness = parents[row["parent_session_key"]]["harness"] if child else row["harness"] + harness = parent_rows[row["parent_session_key"]]["harness"] if child else row["harness"] provider = row["provider"] records.append( MeasureRecord( @@ -506,10 +507,15 @@ def test_prepared_metrics_keep_source_pairs_roles_and_shared_library_accounting( sessions, requests, tools, children = _full_index_facts() body = "Bash runs git show to inspect the committed slice." loader_calls: list[None] = [] + + def definition_loader() -> tuple[SimpleNamespace, ...]: + loader_calls.append(None) + return (_definition(_AUDITOR, body),) + monkeypatch.setattr( deck_payload_module, "load_bundled_agent_definitions", - lambda: loader_calls.append(None) or (_definition(_AUDITOR, body),), + definition_loader, raising=False, ) aggregate_calls: list[int] = [] @@ -541,7 +547,7 @@ def ratio_spy(records: Any, numerator: str, denominator: str, **kwargs: Any) -> assert aggregate_calls and ratio_calls assert len(loader_calls) == 1 prepared = payload["prepared"] - levels = {"orchestrator", "skill"} + levels: set[str | None] = {"orchestrator", "skill"} skill_block = _metric_row(prepared["skills"], window="all", levels=levels) planner = next( row @@ -560,6 +566,11 @@ def ratio_spy(records: Any, numerator: str, denominator: str, **kwargs: Any) -> planner_aggregate = aggregate_measures(_source_records(planner_runs), _TOKEN_FIELDS) planner_ratio = measure_ratio(_source_records(planner_runs), "input_tokens", "output_tokens") assert planner["measures"]["input_tokens"]["value"] == 30 + assert planner["exact_retransmission"] == {"state": "unavailable", "value": None} + assert planner["recipe_steps"][0]["exact_retransmission"] == { + "state": "unavailable", + "value": None, + } _assert_serialized_measure( planner["measures"]["input_tokens"], planner_aggregate.fields["input_tokens"] ) @@ -659,3 +670,33 @@ def table_rows(table: dict[str, Any]) -> list[dict[str, Any]]: < prepared["view_histories"]["skill"]["last_ms"] ) json.dumps(payload, allow_nan=False) + + +def test_observed_zero_ratio_can_be_flagged_with_complete_definitions( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sessions, _, _, children = _full_index_facts() + for child in children: + if child.get("role") == _AUDITOR: + child["token_usage"]["cache_read_tokens"] = token_measure(0) + monkeypatch.setattr( + deck_payload_module, + "load_bundled_agent_definitions", + lambda: (_definition(_AUDITOR, "Inspect the slice with git show."),), + ) + payload = build_deck_payload( + sessions, + subagent_rows=children, + generated_at=DECK_GENERATED_AT, + index_schema_version=7, + ) + block = _metric_row(payload["prepared"]["roles"], window="all", levels={"skill"}) + auditor = next( + row for row in block["rows"] if row["role"] == _AUDITOR and row["provider"] == "anthropic" + ) + signal = auditor["harnesses"][0]["ratios"]["cache_share"] + assert (signal["state"], signal["value"]) == ("measured_zero", 0.0) + assert signal["sample_size"] > 0 + assert signal["definition_roles"] == [_AUDITOR] + assert signal["review_eligible"] is True + assert signal["review_reason"] is None From 4e3dbb57f126e5f43c9db70157c8ab8557bb07c4 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 14:37:08 -0700 Subject: [PATCH 05/22] feat: select prepared deck cohorts and preserve empty details --- src/autoskillit/assets/deck/core.js | 86 +++++++++++++++++++++++----- src/autoskillit/assets/deck/shell.js | 40 ++++++++++--- tests/report/test_deck_core_js.py | 35 ++++++++--- 3 files changed, 132 insertions(+), 29 deletions(-) diff --git a/src/autoskillit/assets/deck/core.js b/src/autoskillit/assets/deck/core.js index 6640fa1b9f..2e2184ed99 100644 --- a/src/autoskillit/assets/deck/core.js +++ b/src/autoskillit/assets/deck/core.js @@ -64,7 +64,7 @@ globalThis.DeckCore = (() => { if (selected == null) return {keys: live, dropped: [], widened: false}; const keys = live.filter(k => selected.includes(k)); const dropped = selected.filter(k => !live.includes(k)); - return {keys: keys.length ? keys : live, dropped, widened: keys.length === 0}; + return {keys, dropped, widened: false}; } function toggleSelection(chips, selected, key) { @@ -77,9 +77,66 @@ globalThis.DeckCore = (() => { function windowSelection(chips, selected) { const want = selected?.[0]; - const chip = chips.find(c => c.state === "live" && c.key === want) || - chips.find(c => c.key === "all"); - return {chip, dropped: want && chip.key !== want ? [want] : []}; + const chip = want == null ? chips.find(c => c.state === "live" && c.key === "all") : + chips.find(c => c.state === "live" && c.key === want); + return {chip: chip ?? null, dropped: want != null && !chip ? [want] : []}; + } + + function sameValues(left, right) { + return left.length === right.length && left.every(value => right.includes(value)); + } + + function selectPrepared(prepared, viewId, chips, route) { + prepared = prepared ?? {}; + chips = chips ?? {}; + const params = route.params ?? {}; + const window = windowSelection(chips.window ?? [], params.window ?? null); + const level = effectiveSelection(chips.level ?? [], params.level ?? null); + const harness = effectiveSelection(chips.harness ?? [], params.harness ?? null); + const provider = effectiveSelection(chips.provider ?? [], params.provider ?? null); + const harnesses = new Set((chips.harness ?? []) + .filter(chip => harness.keys.includes(chip.key)).map(chip => chip.match)); + const providers = new Set((chips.provider ?? []) + .filter(chip => provider.keys.includes(chip.key)).map(chip => chip.match)); + + function selectedRows(blocks) { + if (!window.chip) return []; + const inWindow = blocks.filter(block => block.window === window.chip.key); + const domain = [...new Set(inWindow.flatMap(block => block.levels))]; + const requested = params.level == null ? domain : level.keys + .map(key => (chips.level ?? []).find(chip => chip.key === key)?.match) + .filter(value => domain.includes(value)); + if (!requested.length) return []; + const block = inWindow.find(item => sameValues(item.levels, requested)); + return block?.rows ?? []; + } + + let skillRows = viewId === "role" ? [] : selectedRows(prepared.skills ?? []).filter(row => + harnesses.has(row.harness) && providers.has(row.provider)); + let roleRows = viewId === "skill" ? [] : selectedRows(prepared.roles ?? []).filter(row => + providers.has(row.provider)).map(row => ({...row, + harnesses: (row.harnesses ?? []).filter(cell => harnesses.has(cell.harness)) + })).filter(row => row.harnesses.length > 0); + if (viewId === "skill" && route.entity != null) { + skillRows = skillRows.filter(row => row.skill === route.entity); + } + if (viewId === "role" && route.entity != null) { + roleRows = roleRows.filter(row => row.role === route.entity); + } + const relationships = (prepared.relationships ?? []).filter(edge => { + if (viewId === "skill" && route.entity && edge.skill !== route.entity) return false; + if (viewId === "role" && route.entity && edge.role !== route.entity) return false; + return harnesses.has(edge.harness) && providers.has(edge.provider); + }); + + return { + skillRows, + roleRows, + relationships, + definitions: prepared.definitions ?? {}, + viewHistory: prepared.view_histories?.[viewId] ?? null, + selection: {window, level, harness, provider} + }; } function filterRows(model, rows, chips, route) { @@ -91,7 +148,9 @@ globalThis.DeckCore = (() => { } let untimed = 0; const window = windowSelection(chips.window, route.params.window ?? null).chip; - if (window.days != null) { + if (!window) { + filtered = []; + } else if (window.days != null) { filtered = filtered.filter(row => { if (row.time_ms == null) { untimed += 1; return false; } return row.time_ms >= model.generated_at_ms - window.days * DAY_MS; @@ -115,15 +174,16 @@ globalThis.DeckCore = (() => { const selection = effectiveSelection(values, route.params[facet.id] ?? null); const live = values.filter(c => c.state === "live"); const labels = values.filter(c => selection.keys.includes(c.key)).map(c => c.label); - parts.push(facet.label + " " + (!live.length ? "none recorded" : labels.join(" + ") + - (selection.keys.length === live.length ? " (all)" : ""))); + const selected = !live.length ? "none recorded" : !selection.keys.length ? "none selected" : + labels.join(" + ") + (selection.keys.length === live.length ? " (all)" : ""); + parts.push(facet.label + " " + selected); notes.push(...selection.dropped.map(k => droppedNote(values, k))); - if (selection.widened) notes.push("no selected " + facet.label + - " is selectable here — showing every selectable " + facet.label); } const selection = windowSelection(chips.window, route.params.window ?? null); const window = selection.chip; - if (window.days != null) { + if (!window) { + parts.push("window unavailable"); + } else if (window.days != null) { parts.push("window last " + window.label + " (" + formatDate(model.generated_at_ms - window.days * DAY_MS) + " → " + formatDate(model.generated_at_ms) + ")"); @@ -199,7 +259,7 @@ globalThis.DeckCore = (() => { } return Object.freeze({decodeTable, encodeRoute, decodeRoute, hrefFor, effectiveSelection, - toggleSelection, windowSelection, filterRows, populationSentence, summarizePairs, sortRows, - parseSort, chipPresentation, availabilityPresentation, barLayout, formatCount, formatDate, - CHIP_STATES, DAY_MS}); + toggleSelection, windowSelection, selectPrepared, filterRows, populationSentence, + summarizePairs, sortRows, parseSort, chipPresentation, availabilityPresentation, + barLayout, formatCount, formatDate, CHIP_STATES, DAY_MS}); })(); diff --git a/src/autoskillit/assets/deck/shell.js b/src/autoskillit/assets/deck/shell.js index a42d05b890..4d2dc838c9 100644 --- a/src/autoskillit/assets/deck/shell.js +++ b/src/autoskillit/assets/deck/shell.js @@ -155,7 +155,7 @@ globalThis.DeckShell = (() => { } } - function renderCohort(chips, result, sentence) { + function renderCohort(chips, result, sentence, facets = model.facets) { const cohort = document.getElementById("deck-cohort"); cohort.replaceChildren(); cohort.appendChild(el("div", {class: "cohort-sentence"}, [ @@ -164,10 +164,11 @@ globalThis.DeckShell = (() => { cohort.appendChild(el("ul", {class: "cohort-notes"}, sentence.notes.map(note => el("li", {}, note)))); - for (const facet of model.facets) { + for (const facet of facets) { const facetChips = facet.kind === "window" ? chips.window : chips[facet.id]; - const selectedKeys = facet.kind === "window" ? - [DeckCore.windowSelection(chips.window, route.params.window ?? null).chip.key] : + const window = facet.kind === "window" ? + DeckCore.windowSelection(chips.window, route.params.window ?? null).chip : null; + const selectedKeys = facet.kind === "window" ? (window ? [window.key] : []) : DeckCore.effectiveSelection(chips[facet.id], route.params[facet.id] ?? null).keys; const group = el("div", {class: "cohort-group"}, [el("span", {}, facet.label)]); const unavailable = []; @@ -236,10 +237,21 @@ globalThis.DeckShell = (() => { notice = "No view named " + route.view; } - const table = model.tables.sessions; - const result = DeckCore.filterRows(model, table, chips, route); - const sentence = DeckCore.populationSentence(model, chips, route, result); - renderCohort(chips, result, sentence); + const selected = renderer && view.id !== model.landing ? + DeckCore.selectPrepared(model.prepared, view.id, chips, route) : null; + const primary = view?.id === "role" ? selected?.roleRows : selected?.skillRows; + const result = selected ? {rows: primary, untimed: 0} : + DeckCore.filterRows(model, model.tables.sessions, chips, route); + const populationModel = selected ? {...model, history: selected.viewHistory, + facets: model.facets.map(facet => view.id === "role" && facet.id === "level" ? + {...facet, label: "Spawning skill level"} : facet)} : model; + const sentence = DeckCore.populationSentence(populationModel, chips, route, result); + if (selected) { + const noun = view.id === "role" ? "role/provider row" : "skill cell"; + sentence.headline = DeckCore.formatCount(result.rows.length) + " " + noun + + (result.rows.length === 1 ? "" : "s"); + } + renderCohort(chips, result, sentence, populationModel.facets); const content = document.getElementById("deck-view"); content.replaceChildren(); @@ -247,6 +259,18 @@ globalThis.DeckShell = (() => { const ctx = {model, route, rows: result.rows, chips, href: target => DeckCore.hrefFor(route, target, cohortKeys), el, entityLink, sortableTable, availabilityCell, barChart}; + if (view.id !== model.landing) { + ctx.prepared = model.prepared; + ctx.metrics = view.id === "role" ? selected.roleRows : selected.skillRows; + ctx.relationships = selected.relationships; + ctx.definitions = selected.definitions; + ctx.viewHistory = selected.viewHistory; + ctx.selection = selected.selection; + if (view.id === "spend" || view.id === "efficiency") { + ctx.skillMetrics = selected.skillRows; + ctx.roleMetrics = selected.roleRows; + } + } content.appendChild(renderer(ctx)); } else { const noticeCard = el("section", {class: "card"}, [el("p", {}, notice)]); diff --git a/tests/report/test_deck_core_js.py b/tests/report/test_deck_core_js.py index 8c7e33fd74..e10968b01e 100644 --- a/tests/report/test_deck_core_js.py +++ b/tests/report/test_deck_core_js.py @@ -222,6 +222,23 @@ def test_prepared_selection_uses_exact_scope_and_keeps_metric_cells_distinct( assert selected["definitions"] == {"reviewer": definition} assert selected["viewHistory"] == history + route["params"]["window"] = ["missing-window"] + empty = deck_js.call("DeckCore.selectPrepared", prepared, "efficiency", chips, route) + assert empty["skillRows"] == empty["roleRows"] == [] + assert empty["relationships"] == [relationship] + assert empty["definitions"] == {"reviewer": definition} + + route["params"]["window"] = ["7d"] + prepared["skills"] = [{"window": "7d", "levels": ["skill"], "rows": [skill]}] + selected = deck_js.call("DeckCore.selectPrepared", prepared, "skill", chips, route) + assert selected["skillRows"] == [skill] + route["params"]["level"] = ["L2"] + selected = deck_js.call("DeckCore.selectPrepared", prepared, "skill", chips, route) + assert selected["skillRows"] == [] + route["params"]["level"] = [] + selected = deck_js.call("DeckCore.selectPrepared", prepared, "skill", chips, route) + assert selected["skillRows"] == [skill] + def test_prepared_selection_returns_no_metrics_for_stale_scope(deck_js: Any) -> None: chips = { @@ -302,29 +319,31 @@ def test_filter_rows_applies_facets_windows_and_untimed_count( ), pytest.param( _route(level=["L0"]), - "4 runs", - ALL_POPULATION_DETAIL, + "0 runs", + ALL_POPULATION_DETAIL.replace("level unrecorded (all)", "level none selected"), [ "L0 is not selectable on this view — L0 leaf agents write no session row; " "they exist only as subagent transcripts", - "no selected level is selectable here — showing every selectable level", ], id="struck-level", ), pytest.param( _route(harness=["nope"]), - "4 runs", - ALL_POPULATION_DETAIL, + "0 runs", + ALL_POPULATION_DETAIL.replace( + "harness claude-code + codex (all)", "harness none selected" + ), [ "nope does not appear in this index", - "no selected harness is selectable here — showing every selectable harness", ], id="unknown-harness", ), pytest.param( _route(window=["28d"]), - "4 runs", - ALL_POPULATION_DETAIL, + "0 runs", + ALL_POPULATION_DETAIL.replace( + "window all history (2026-09-25 → 2026-10-04)", "window unavailable" + ), [ "28 days is not selectable on this view — index history begins 2026-09-25 — " "9 days retained (#4621)" From 57ede0526d185affdfa65930ae7500a56e8e5fa7 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 14:53:27 -0700 Subject: [PATCH 06/22] fix: canonicalize native Claude input before indexing --- .../evidence/_native_child_projection.py | 23 ++++-- tests/execution/test_report_index.py | 75 +++++++++++++++++++ 2 files changed, 93 insertions(+), 5 deletions(-) diff --git a/src/autoskillit/execution/evidence/_native_child_projection.py b/src/autoskillit/execution/evidence/_native_child_projection.py index 17d8596158..32ef513efe 100644 --- a/src/autoskillit/execution/evidence/_native_child_projection.py +++ b/src/autoskillit/execution/evidence/_native_child_projection.py @@ -32,7 +32,10 @@ normalize_backend_name, ) from autoskillit.execution.session import extract_token_usage -from autoskillit.execution.session.turn_usage import classify_token_measure +from autoskillit.execution.session.turn_usage import ( + classify_token_measure, + claude_inclusive_input_tokens, +) def project_child_outcomes( @@ -284,11 +287,21 @@ def _child_token_usage( if backend == "claude_code": aggregate, _rows = extract_token_usage(text, provider_used=provider) if aggregate is not None: + usage = { + field: aggregate.get(field, TokenMeasure.unknown().to_dict()) + for field in CANONICAL_ACCOUNTING_FIELDS + } + inclusive_input = claude_inclusive_input_tokens( + *( + TokenMeasure.from_dict(usage[field]).value + for field in ("input_tokens", "cache_read_tokens", "cache_write_tokens") + ) + ) + usage["input_tokens"] = classify_token_measure( + backend, provider, "input_tokens", inclusive_input + ).to_dict() return ( - { - field: aggregate.get(field, TokenMeasure.unknown().to_dict()) - for field in CANONICAL_ACCOUNTING_FIELDS - }, + usage, "observed", ) return _unknown_usage(), "unknown" diff --git a/tests/execution/test_report_index.py b/tests/execution/test_report_index.py index 83d73802e4..4b277de14a 100644 --- a/tests/execution/test_report_index.py +++ b/tests/execution/test_report_index.py @@ -127,6 +127,81 @@ def _indexed_native_children(index_dir: Path) -> dict[str, dict[str, Any]]: } +@pytest.mark.parametrize("cache_write", [0, None]) +def test_native_claude_input_is_inclusive_only_with_complete_components( + tmp_path: Path, cache_write: int | None +) -> None: + root = tmp_path / "logs" + parent_id, child_id = "native-parent", "native-child" + root.mkdir() + parent_log = root / f"{parent_id}.jsonl" + parent_log.write_text("", encoding="utf-8") + _write_jsonl( + root / "sessions.jsonl", + [ + _basic_session( + "parent-key", + parent_id, + claude_code_log=str(parent_log), + session_type="skill", + skill_command="/autoskillit:implement-worktree-no-merge", + ) + ], + ) + snapshot = _write_native_child(root, parent_id, child_id, model="child-model") + child_snapshot.record_terminal_evidence( + snapshot, + backend="claude_code", + parent_session_id=parent_id, + child_id=child_id, + evidence_key=f"{child_id}:metadata", + evidence={ + "role": "Explore", + "attribution_skill": "implement-worktree-no-merge", + "effective_provider": "anthropic", + "effective_model": "child-model", + "evidence_source": "transcript_metadata", + }, + ) + usage = {"input_tokens": 12, "output_tokens": 4, "cache_read_input_tokens": 3} + if cache_write is not None: + usage["cache_creation_input_tokens"] = cache_write + line = ( + json.dumps( + { + "type": "assistant", + "agentId": child_id, + "sessionId": parent_id, + "requestId": "request-1", + "message": { + "id": "message-1", + "model": "child-model", + "content": [], + "usage": usage, + }, + } + ) + + "\n" + ) + (root / parent_id / "subagents" / f"agent-{child_id}.jsonl").write_text(line * 2) + index_dir = tmp_path / "index" + + update_report_index(root, index_dir) + + child = _indexed_native_children(index_dir)[child_id] + expected = ( + {"state": "measured", "value": 15} + if cache_write is not None + else { + "state": "unknown", + "value": None, + } + ) + assert child["token_usage"]["input_tokens"] == expected + assert child["token_usage"]["output_tokens"] == {"state": "measured", "value": 4} + assert child["token_usage"]["cache_read_tokens"] == {"state": "measured", "value": 3} + + def test_update_then_read_joins_events_to_sessions(tmp_path: Path) -> None: root = tmp_path / "logs" session_dir = "session-a" From 6bdb2fe46821c65a5b3f2cbc4e7a44dbe98c6c60 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 15:15:41 -0700 Subject: [PATCH 07/22] fix: preserve Codex compaction and fold child request snapshots --- .../execution/backends/_codex_parse.py | 15 ++++--- .../evidence/_native_child_projection.py | 5 ++- .../backends/test_child_task_transcripts.py | 42 +++++++++++++++++++ 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/src/autoskillit/execution/backends/_codex_parse.py b/src/autoskillit/execution/backends/_codex_parse.py index 42040f0824..a3a0e78ef0 100644 --- a/src/autoskillit/execution/backends/_codex_parse.py +++ b/src/autoskillit/execution/backends/_codex_parse.py @@ -329,7 +329,12 @@ def _codex_usage_records(reader: BinaryIO) -> Iterator[Mapping[str, Any]]: record = json.loads(raw_line) except (UnicodeDecodeError, json.JSONDecodeError): continue - if not isinstance(record, Mapping) or not isinstance(record.get("payload"), Mapping): + if not isinstance(record, Mapping): + continue + if record.get("type") == "compacted": + yield record + continue + if not isinstance(record.get("payload"), Mapping): continue payload = record["payload"] if record.get("type") == "event_msg" and payload.get("type") == "token_count": @@ -349,7 +354,7 @@ def _extract_codex_turn_usage( end: datetime | None, provider_used: str, ) -> list[TurnTokenEntry]: - """Share rollout scanning and cumulative-snapshot handling across views.""" + """Read advancing request snapshots within optional rollout time bounds.""" rows: list[TurnTokenEntry] = [] current_model: str | None = None @@ -359,13 +364,13 @@ def _extract_codex_turn_usage( with _logical_rollout_reader(path) as reader: for record in _codex_usage_records(reader): record_type = record.get("type") + if record_type == "compacted": + compacting = True + continue payload: Mapping[str, Any] = record["payload"] if record_type == "turn_context" and isinstance(payload, Mapping): current_model = first_nonempty_string(payload.get("model")) continue - if record_type == "compacted": - compacting = True - continue payload_type = payload.get("type") if record_type == "event_msg" and payload_type == "context_compacted": compacting = False diff --git a/src/autoskillit/execution/evidence/_native_child_projection.py b/src/autoskillit/execution/evidence/_native_child_projection.py index 32ef513efe..01ca365fa6 100644 --- a/src/autoskillit/execution/evidence/_native_child_projection.py +++ b/src/autoskillit/execution/evidence/_native_child_projection.py @@ -17,6 +17,7 @@ CANONICAL_ACCOUNTING_FIELDS, SerializedTokenMeasure, TokenMeasure, + TurnTokenEntry, iter_merged_assistant_turns, ) from autoskillit.execution.backends._codex_execution_identity import ( @@ -35,6 +36,7 @@ from autoskillit.execution.session.turn_usage import ( classify_token_measure, claude_inclusive_input_tokens, + merge_turn_usage, ) @@ -313,8 +315,9 @@ def _child_token_usage( def _codex_usage_measures( - rows: Sequence[Mapping[str, Any]], provider: str + rows: Sequence[TurnTokenEntry], provider: str ) -> dict[str, SerializedTokenMeasure]: + rows = merge_turn_usage(rows) measures: dict[str, SerializedTokenMeasure] = {} for field in CANONICAL_ACCOUNTING_FIELDS: values = [row.get(field) for row in rows] diff --git a/tests/execution/backends/test_child_task_transcripts.py b/tests/execution/backends/test_child_task_transcripts.py index dac8dbaaa8..4cd47533fe 100644 --- a/tests/execution/backends/test_child_task_transcripts.py +++ b/tests/execution/backends/test_child_task_transcripts.py @@ -550,6 +550,48 @@ def test_codex_child_usage_scans_whole_verified_rollout_and_deduplicates_snapsho assert dedicated[0]["input_tokens"] == 10 +def test_codex_compaction_exclusion_is_preserved_for_both_extractors(tmp_path: Path) -> None: + child_id = "child-thread" + path = _write_codex( + tmp_path / "child.jsonl", + child_id, + [ + _codex_usage_snapshot(13, "2026-09-01T10:00:01Z"), + {"type": "compacted"}, + _codex_usage_snapshot(26, "2026-09-01T10:00:02Z"), + _codex_event("event_msg", {"type": "context_compacted"}), + _codex_usage_snapshot(39, "2026-09-01T10:00:03Z"), + ], + ) + locator = SimpleNamespace(locate_session=lambda _thread_id: path) + dedicated = extract_codex_child_turn_usage(path, child_id, provider_used="codex") + interval = extract_codex_turn_usage( + locator, child_id, "2026-09-01T10:00:00Z", "2026-09-01T10:00:04Z", provider_used="codex" + ) + assert dedicated == interval + assert len(dedicated) == 2 + + +def test_native_codex_usage_folds_advancing_snapshots_of_one_request(tmp_path: Path) -> None: + from autoskillit.execution.evidence._native_child_projection import _child_token_usage + + snapshots = [ + _codex_usage_snapshot(13, "2026-09-01T10:00:01Z"), + _codex_usage_snapshot(26, "2026-09-01T10:00:02Z"), + ] + for snapshot in snapshots: + payload = snapshot["payload"] + assert isinstance(payload, dict) + payload["request_id"] = "same-request" + path = _write_codex(tmp_path / "child.jsonl", "child-thread", snapshots) + usage, state = _child_token_usage( + "codex", "child-thread", path, path.read_text(encoding="utf-8"), "codex" + ) + assert state == "observed" + assert usage["input_tokens"] == {"state": "measured", "value": 10} + assert usage["output_tokens"] == {"state": "measured", "value": 3} + + def test_codex_child_usage_rejects_mismatched_rollout_identity(tmp_path: Path) -> None: path = _write_codex(tmp_path / "child-rollout.jsonl", "actual-child", []) From d3469982d1004078f1812ccb47699d27f9879c17 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 15:41:35 -0700 Subject: [PATCH 08/22] feat: render spend efficiency skill and role journeys --- src/autoskillit/assets/deck/core.js | 46 ++++- src/autoskillit/assets/deck/deck.css | 22 ++- .../assets/deck/views/efficiency.js | 153 ++++++++++++++++ src/autoskillit/assets/deck/views/role.js | 138 +++++++++++++++ src/autoskillit/assets/deck/views/skill.js | 164 ++++++++++++++++++ src/autoskillit/assets/deck/views/spend.js | 79 +++++++++ src/autoskillit/report/deck/_registry.py | 8 +- tests/report/test_deck_core_js.py | 92 ++++++++++ tests/report/test_deck_views_js.py | 44 ++++- 9 files changed, 731 insertions(+), 15 deletions(-) create mode 100644 src/autoskillit/assets/deck/views/efficiency.js create mode 100644 src/autoskillit/assets/deck/views/role.js create mode 100644 src/autoskillit/assets/deck/views/skill.js create mode 100644 src/autoskillit/assets/deck/views/spend.js diff --git a/src/autoskillit/assets/deck/core.js b/src/autoskillit/assets/deck/core.js index 2e2184ed99..5417642840 100644 --- a/src/autoskillit/assets/deck/core.js +++ b/src/autoskillit/assets/deck/core.js @@ -5,6 +5,43 @@ globalThis.DeckCore = (() => { const enc = encodeURIComponent; const formatCount = n => new Intl.NumberFormat("en-US").format(n); const formatDate = ms => new Date(ms).toISOString().slice(0, 10); + + function formatRatio(ratio, percent = false) { + if (!ratio || (ratio.state !== "measured" && ratio.state !== "measured_zero")) return null; + if (ratio.state === "measured_zero") return percent ? "0%" : "0"; + if (typeof ratio.value !== "number" || !Number.isFinite(ratio.value)) return null; + const value = Math.round(ratio.value * (percent ? 100 : 1) * 100) / 100; + return String(value) + (percent ? "%" : ""); + } + + function ratioSample(ratio) { + if (ratio?.sample_size == null) return "sample size unavailable"; + const unit = ratio.sample_unit === "skill-run" ? "skill run" : + ratio.sample_unit === "child-invocation" ? "child invocation" : ratio.sample_unit; + return formatCount(ratio.sample_size) + " " + unit + (ratio.sample_size === 1 ? "" : "s"); + } + + function reviewEligibility(ratio, definitions) { + const roles = [...new Set(ratio?.definition_roles ?? [])]; + if (!ratio || (ratio.state !== "measured" && ratio.state !== "measured_zero")) { + return {eligible: false, + reason: ratio?.review_reason || "No measured review signal is available.", roles}; + } + if (ratio.review_eligible !== true || !(ratio.sample_size > 0)) { + return {eligible: false, + reason: ratio.review_reason || "This signal is not eligible for review.", roles}; + } + if (!roles.length) { + return {eligible: false, + reason: "No contributor definitions are linked to this signal.", roles}; + } + const unavailable = roles.filter(role => definitions?.[role]?.state !== "available"); + if (unavailable.length) { + return {eligible: false, + reason: "Contributor definitions unavailable: " + unavailable.join(", ") + ".", roles}; + } + return {eligible: true, reason: null, roles}; + } const decodeTable = ({columns, rows}) => rows.map(r => Object.fromEntries(columns.map((c, i) => [c, r[i]]))); @@ -124,9 +161,9 @@ globalThis.DeckCore = (() => { roleRows = roleRows.filter(row => row.role === route.entity); } const relationships = (prepared.relationships ?? []).filter(edge => { - if (viewId === "skill" && route.entity && edge.skill !== route.entity) return false; - if (viewId === "role" && route.entity && edge.role !== route.entity) return false; - return harnesses.has(edge.harness) && providers.has(edge.provider); + if (viewId === "skill" && route.entity != null) return edge.skill === route.entity; + if (viewId === "role" && route.entity != null) return edge.role === route.entity; + return true; }); return { @@ -261,5 +298,6 @@ globalThis.DeckCore = (() => { return Object.freeze({decodeTable, encodeRoute, decodeRoute, hrefFor, effectiveSelection, toggleSelection, windowSelection, selectPrepared, filterRows, populationSentence, summarizePairs, sortRows, parseSort, chipPresentation, availabilityPresentation, - barLayout, formatCount, formatDate, CHIP_STATES, DAY_MS}); + barLayout, formatCount, formatRatio, ratioSample, reviewEligibility, formatDate, + CHIP_STATES, DAY_MS}); })(); diff --git a/src/autoskillit/assets/deck/deck.css b/src/autoskillit/assets/deck/deck.css index c1c25d4d72..be96c6f5e0 100644 --- a/src/autoskillit/assets/deck/deck.css +++ b/src/autoskillit/assets/deck/deck.css @@ -82,12 +82,31 @@ svg { display: block; max-width: 100%; height: auto; } .hatch-line { stroke: var(--hatch); stroke-width: 2; } .bar-label, .bar-value { fill: var(--ink-2); font: 11px var(--mono); } .defs { position: absolute; } +.view-stack { display: grid; gap: 18px; } +.view-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 18px; } +.view-lede, .view-note { color: var(--ink-2); line-height: 1.55; } +.view-empty { color: var(--ink-3); font-style: italic; } +.view-links, .view-pills { display: flex; flex-wrap: wrap; gap: 8px; } +.view-pills span, .view-sample { color: var(--ink-3); font: 11px var(--mono); } +.view-measure { white-space: nowrap; } +.view-definition { border-top: 1px solid var(--rule); padding: 12px 0; } +.view-definition h3 { font-size: 13px; margin: 0 0 6px; } +.view-definition pre { white-space: pre-wrap; overflow-wrap: anywhere; color: var(--ink-2); } +.view-review { border: 1px solid var(--rule); border-radius: 6px; padding: 12px; } +.view-review button { border: 1px solid var(--rule-2); border-radius: 5px; padding: 7px 10px; + color: var(--ink); background: var(--surface-2); cursor: pointer; } +.view-review button:focus-visible, .view-links a:focus-visible { outline: 2px solid var(--s1); + outline-offset: 2px; } +.view-review button:disabled { cursor: not-allowed; opacity: .6; } +.view-review__reason { color: var(--warn); font-size: 12px; } +.view-actor { color: var(--ink-3); font: 11px var(--mono); } .foot { color: var(--ink-3); font: 10px/1.5 var(--mono); padding: 0 28px 24px; } @media screen and (max-width: 860px) { .shell { grid-template-columns: minmax(0,1fr); } .rail { border-right: 0; border-bottom: 1px solid var(--rule); } .cohort { position: static; } .cohort, #deck-view { padding: 18px; } + .view-grid { grid-template-columns: minmax(0,1fr); } } @media print { @page { margin: 12mm; } @@ -95,6 +114,7 @@ svg { display: block; max-width: 100%; height: auto; } .rail { display: none; } .shell { display: block; } .cohort { position: static; } - .card, table, svg { break-inside: avoid; } + .card, table, svg, .view-definition, .view-review { break-inside: avoid; } + .view-review button { display: none; } #deck-view { padding: 18px 0; } } diff --git a/src/autoskillit/assets/deck/views/efficiency.js b/src/autoskillit/assets/deck/views/efficiency.js new file mode 100644 index 0000000000..2420bb673e --- /dev/null +++ b/src/autoskillit/assets/deck/views/efficiency.js @@ -0,0 +1,153 @@ +DeckShell.registerView("efficiency", ctx => { + const skillRows = ctx.skillMetrics ?? []; + const roleRows = (ctx.roleMetrics ?? []).flatMap(row => (row.harnesses ?? []).map(cell => ({ + role: row.role, + provider: row.provider, + harness: cell.harness, + models: cell.models ?? [], + measures: cell.measures, + ratios: cell.ratios + }))); + + function ratioSignals(ratios = {}) { + const values = []; + if (ratios.input_output) values.push({label: "Input exposure per output", ratio: ratios.input_output}); + if (ratios.cache_share) values.push({label: "Cache-read share", ratio: ratios.cache_share, + percent: true}); + for (const [tool, ratio] of Object.entries(ratios.tool_mix ?? {})) { + values.push({label: "Tool mix · " + tool, ratio, percent: true}); + } + return values; + } + + function ratioCell(ratio, percent = false) { + const text = DeckCore.formatRatio(ratio, percent); + const value = text == null ? ctx.availabilityCell(ratio ?? {state: "unavailable"}) : + ctx.el("span", {class: "view-measure"}, text); + return ctx.el("div", {class: "view-measure"}, [value, + ctx.el("small", {class: "view-sample"}, DeckCore.ratioSample(ratio))]); + } + + function toolMix(ratios = {}) { + const entries = Object.entries(ratios.tool_mix ?? {}); + return entries.length ? ctx.el("div", {class: "view-pills"}, entries.map(([tool, ratio]) => { + const value = DeckCore.formatRatio(ratio, true); + return ctx.el("span", {}, [ + ctx.el("span", {}, tool), + ctx.el("span", {}, " · "), + value == null ? ctx.availabilityCell(ratio) : ctx.el("span", {}, value), + ctx.el("small", {class: "view-sample"}, DeckCore.ratioSample(ratio)) + ]); + })) : + ctx.el("span", {class: "view-empty"}, "No observed tool calls"); + } + + function reviewable(ratios = {}) { + return ratioSignals(ratios).some(signal => + DeckCore.reviewEligibility(signal.ratio, ctx.definitions).eligible); + } + + const definitions = ctx.definitions ?? {}; + const contributorRoles = new Set((ctx.relationships ?? []).map(edge => edge.role)); + for (const row of skillRows) { + for (const signal of ratioSignals(row.ratios)) { + for (const role of signal.ratio.definition_roles ?? []) contributorRoles.add(role); + } + } + for (const row of roleRows) { + for (const signal of ratioSignals(row.ratios)) { + for (const role of signal.ratio.definition_roles ?? []) contributorRoles.add(role); + } + } + const definitionCards = [...contributorRoles].sort().map(role => { + const definition = definitions[role]; + const available = definition?.state === "available"; + return ctx.el("article", {class: "view-definition"}, [ + ctx.el("h3", {}, ctx.entityLink(role, {view: "role", entity: role})), + available ? ctx.el("p", {}, definition.description ?? "Role definition loaded.") : + ctx.el("p", {class: "view-review__reason"}, "Definition unavailable for " + role + "."), + available ? ctx.el("p", {}, "Declared tools: " + + ((definition.tools ?? []).join(", ") || "none recorded")) : null, + available ? ctx.el("pre", {}, definition.body ?? "") : null + ]); + }); + const signals = [...skillRows, ...roleRows].flatMap(row => ratioSignals(row.ratios)); + const eligibility = signals.map(signal => + DeckCore.reviewEligibility(signal.ratio, definitions)); + const reviewEnabled = eligibility.some(status => status.eligible); + const disabledReason = eligibility.find(status => !status.eligible)?.reason ?? + "No prepared measured review signals are available in this cohort."; + const reviewMode = ctx.route.params.review?.[0] === "eligible"; + const toggle = ctx.el("button", { + type: "button", + "data-review-toggle": "true", + "aria-label": "Filter to reviewable signals", + "aria-pressed": String(reviewMode && reviewEnabled), + disabled: !reviewEnabled + }, reviewMode ? "Show all signals" : "Show reviewable signals"); + if (reviewEnabled) { + toggle.addEventListener("click", () => { + const params = {...ctx.route.params, review: reviewMode ? [] : ["eligible"]}; + location.hash = ctx.href({view: "efficiency", entity: ctx.route.entity, params}); + }); + } + const reviewControl = ctx.el("section", {class: "view-review"}, [ + toggle, + reviewEnabled ? ctx.el("p", {class: "view-note"}, reviewMode ? + "Showing prepared review-eligible signals." : + "Review eligibility reflects prepared evidence and linked role definitions.") : + ctx.el("p", {class: "view-review__reason"}, disabledReason) + ]); + + const shownSkills = reviewMode && reviewEnabled ? skillRows.filter(row => reviewable(row.ratios)) : + skillRows; + const shownRoles = reviewMode && reviewEnabled ? roleRows.filter(row => reviewable(row.ratios)) : + roleRows; + const skillTable = shownSkills.length ? ctx.sortableTable({ + columns: [ + {key: "skill", label: "Skill", cell: row => ctx.entityLink(row.skill, { + view: "skill", entity: row.skill + })}, + {key: "harness", label: "Harness"}, + {key: "provider", label: "Provider"}, + {key: "input_output", label: "Input / output", cell: row => + ratioCell(row.ratios?.input_output)}, + {key: "cache_share", label: "Cache-read share", cell: row => + ratioCell(row.ratios?.cache_share, true)}, + {key: "tool_mix", label: "Observed tool mix", cell: row => toolMix(row.ratios)} + ], + rows: shownSkills, + defaultSort: {key: "skill", dir: "asc"} + }) : ctx.el("p", {class: "view-empty"}, "No skill-run ratios are available in this cohort."); + const roleTable = shownRoles.length ? ctx.sortableTable({ + columns: [ + {key: "role", label: "Role", cell: row => ctx.entityLink(row.role, { + view: "role", entity: row.role + })}, + {key: "harness", label: "Harness"}, + {key: "provider", label: "Provider"}, + {key: "input_output", label: "Input / output", cell: row => + ratioCell(row.ratios?.input_output)}, + {key: "cache_share", label: "Cache-read share", cell: row => + ratioCell(row.ratios?.cache_share, true)}, + {key: "tool_mix", label: "Observed tool mix", cell: row => toolMix(row.ratios)} + ], + rows: shownRoles, + defaultSort: {key: "role", dir: "asc"} + }) : ctx.el("p", {class: "view-empty"}, "No child-invocation ratios are available in this cohort."); + + return ctx.el("div", {class: "view-stack"}, [ + ctx.el("p", {class: "view-lede"}, + "Prepared ratios compare eligible input, output, cache-read, and observed tool totals. " + + "They are evidence signals, not quality diagnoses or automatic model recommendations."), + ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, "Contributor definitions"), + ...(definitionCards.length ? definitionCards : [ + ctx.el("p", {class: "view-empty"}, "No contributor definitions are linked in this cohort.") + ]), + reviewControl + ]), + ctx.el("section", {class: "card"}, [ctx.el("h2", {}, "Skill-run efficiency"), skillTable]), + ctx.el("section", {class: "card"}, [ctx.el("h2", {}, "Child-invocation efficiency"), roleTable]) + ]); +}); diff --git a/src/autoskillit/assets/deck/views/role.js b/src/autoskillit/assets/deck/views/role.js new file mode 100644 index 0000000000..fcc4e60dbe --- /dev/null +++ b/src/autoskillit/assets/deck/views/role.js @@ -0,0 +1,138 @@ +DeckShell.registerView("role", ctx => { + const roleName = ctx.route.entity; + const identities = ctx.model.tables?.roles ?? []; + const definitions = ctx.definitions ?? {}; + const rows = ctx.metrics ?? []; + const roleNames = new Set([ + ...identities.map(row => row.role), + ...Object.keys(definitions), + ...(roleName == null ? [] : [roleName]) + ]); + const roleRows = roleName == null ? [...roleNames].sort().map(role => ({ + role, + definition: definitions[role], + hasMetrics: rows.some(row => row.role === role) + })) : rows.filter(row => row.role === roleName); + + function measureCell(measure) { + return ctx.availabilityCell(measure ?? {state: "unavailable"}); + } + + function ratioCell(ratio, percent = false) { + const value = DeckCore.formatRatio(ratio, percent); + return ctx.el("div", {class: "view-measure"}, [ + value == null ? measureCell(ratio) : ctx.el("span", {}, value), + ctx.el("small", {class: "view-sample"}, DeckCore.ratioSample(ratio)) + ]); + } + + function toolMix(ratios = {}) { + const tools = Object.entries(ratios.tool_mix ?? {}); + return tools.length ? ctx.el("div", {class: "view-pills"}, tools.map(([tool, ratio]) => { + const value = DeckCore.formatRatio(ratio, true); + return ctx.el("span", {}, [ + ctx.el("span", {}, tool), + ctx.el("span", {}, " · "), + value == null ? ctx.availabilityCell(ratio) : ctx.el("span", {}, value), + ctx.el("small", {class: "view-sample"}, DeckCore.ratioSample(ratio)) + ]); + })) : + ctx.el("span", {class: "view-empty"}, "No observed tool calls"); + } + + const spawningLevels = (ctx.selection?.level?.keys ?? []).map(key => + (ctx.chips?.level ?? []).find(chip => chip.key === key)?.label ?? key); + const edges = ctx.relationships ?? []; + const spawningTable = edges.length ? ctx.sortableTable({ + columns: [ + {key: "skill", label: "Spawning skill", cell: edge => ctx.entityLink(edge.skill, { + view: "skill", entity: edge.skill + })}, + {key: "harness", label: "Harness"}, + {key: "provider", label: "Child provider"} + ], + rows: edges, + defaultSort: {key: "skill", dir: "asc"} + }) : ctx.el("p", {class: "view-empty"}, "No verified spawning-skill links are recorded."); + + const harnessRows = roleRows.flatMap(row => (row.harnesses ?? []).map(cell => ({ + role: row.role, + provider: row.provider, + harness: cell.harness, + models: cell.models ?? [], + measures: cell.measures, + ratios: cell.ratios + }))); + const usageTable = harnessRows.length ? ctx.sortableTable({ + columns: [ + {key: "harness", label: "Harness"}, + {key: "provider", label: "Provider"}, + {key: "models", label: "Observed models", cell: row => + (row.models ?? []).join(", ") || "No model recorded"}, + {key: "input_tokens", label: "Input tokens", numeric: true, + cell: row => measureCell(row.measures?.input_tokens)}, + {key: "output_tokens", label: "Output tokens", numeric: true, + cell: row => measureCell(row.measures?.output_tokens)}, + {key: "cache_read_tokens", label: "Cache-read tokens", numeric: true, + cell: row => measureCell(row.measures?.cache_read_tokens)}, + {key: "cache_write_tokens", label: "Cache-write tokens", numeric: true, + cell: row => measureCell(row.measures?.cache_write_tokens)}, + {key: "input_output", label: "Input / output", cell: row => + ratioCell(row.ratios?.input_output)}, + {key: "cache_share", label: "Cache-read share", cell: row => + ratioCell(row.ratios?.cache_share, true)}, + {key: "tool_mix", label: "Observed tool mix", cell: row => toolMix(row.ratios)} + ], + rows: harnessRows, + defaultSort: {key: "harness", dir: "asc"} + }) : ctx.el("p", {class: "view-empty"}, "No child-invocation metrics match these facets."); + + const definition = roleName == null ? null : definitions[roleName]; + const available = definition?.state === "available"; + const definitionCard = roleName == null ? null : ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, "Role definition · " + roleName), + available ? ctx.el("p", {}, definition.description ?? "Role definition loaded.") : + ctx.el("p", {class: "view-review__reason"}, "Definition unavailable for " + roleName + "."), + available ? ctx.el("p", {}, "Claude Code declared tools: " + + ((definition.tools ?? []).join(", ") || "none recorded")) : null, + available ? ctx.el("p", {}, "Claude Code declared model: " + + (definition.model ?? "not declared")) : null, + available ? ctx.el("p", {}, "Codex read-only tools: " + + ((definition.reader_tools ?? []).join(", ") || "none declared")) : null, + available ? ctx.el("p", {}, "Codex model: " + + (definition.codex_model ?? "not declared")) : null, + available ? ctx.el("pre", {}, definition.body ?? "") : null + ]); + const indexTable = roleName == null ? ctx.sortableTable({ + columns: [ + {key: "role", label: "Role", cell: row => ctx.entityLink(row.role, { + view: "role", entity: row.role + })}, + {key: "state", label: "Definition", cell: row => row.definition?.state ?? "unavailable"}, + {key: "hasMetrics", label: "Selected metrics", cell: row => + row.hasMetrics ? "available" : "none in this cohort"} + ], + rows: roleRows, + defaultSort: {key: "role", dir: "asc"} + }) : null; + + return ctx.el("div", {class: "view-stack"}, [ + ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, roleName ?? "Known specialized roles"), + ctx.el("p", {class: "view-actor"}, roleName == null ? + "Native specialized child actor · L0" : "Actor level: L0 · Spawning skill level: " + + (spawningLevels.join(" + ") || "all observed levels")), + ctx.el("p", {class: "view-lede"}, + "Observed child-invocation usage stays separate from its spawning skill's session spend."), + indexTable + ]), + definitionCard, + ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, "Observed role usage"), + ctx.el("p", {class: "view-note"}, + "Observed models and tool mix come from child evidence. Declared tools and models above come from the role definition."), + usageTable + ]), + ctx.el("section", {class: "card"}, [ctx.el("h2", {}, "Spawning skills"), spawningTable]) + ]); +}); diff --git a/src/autoskillit/assets/deck/views/skill.js b/src/autoskillit/assets/deck/views/skill.js new file mode 100644 index 0000000000..c6fb91304f --- /dev/null +++ b/src/autoskillit/assets/deck/views/skill.js @@ -0,0 +1,164 @@ +DeckShell.registerView("skill", ctx => { + const rows = ctx.metrics ?? []; + const routeSkill = ctx.route.entity; + const relationships = ctx.relationships ?? []; + const selected = routeSkill == null ? [] : rows.filter(row => row.skill === routeSkill); + const skillNames = [...new Set(selected.map(row => row.skill))]; + if (routeSkill != null && !skillNames.includes(routeSkill)) skillNames.unshift(routeSkill); + + function measureCell(measure) { + return ctx.availabilityCell(measure ?? {state: "unavailable"}); + } + + function ratioCell(ratio, percent = false) { + const value = DeckCore.formatRatio(ratio, percent); + return ctx.el("div", {class: "view-measure"}, [ + value == null ? measureCell(ratio) : ctx.el("span", {}, value), + ctx.el("small", {class: "view-sample"}, DeckCore.ratioSample(ratio)) + ]); + } + + function definitionCard(role) { + const definition = ctx.definitions?.[role]; + return ctx.el("article", {class: "view-definition"}, [ + ctx.el("h3", {}, ctx.entityLink(role, {view: "role", entity: role})), + definition?.state === "available" ? + ctx.el("p", {}, definition.description ?? "Role definition loaded.") : + ctx.el("p", {class: "view-review__reason"}, "Definition unavailable for " + role + "."), + definition?.state === "available" ? ctx.el("p", {}, "Declared tools: " + + ((definition.tools ?? []).join(", ") || "none recorded")) : null, + definition?.state === "available" ? ctx.el("pre", {}, definition.body ?? "") : null + ]); + } + + const roleNames = new Set(relationships.map(edge => edge.role)); + for (const row of selected) { + for (const child of row.child_roles ?? []) roleNames.add(child.role); + for (const role of row.ratios?.input_output?.definition_roles ?? []) roleNames.add(role); + for (const role of row.ratios?.cache_share?.definition_roles ?? []) roleNames.add(role); + } + const definitions = [...roleNames].sort().map(definitionCard); + + const skillTable = rows.length ? ctx.sortableTable({ + columns: [ + {key: "skill", label: "Skill", cell: row => ctx.entityLink(row.skill, { + view: "skill", entity: row.skill + })}, + {key: "harness", label: "Harness"}, + {key: "provider", label: "Provider"}, + {key: "input_tokens", label: "Input tokens", numeric: true, + cell: row => measureCell(row.measures?.input_tokens)}, + {key: "output_tokens", label: "Output tokens", numeric: true, + cell: row => measureCell(row.measures?.output_tokens)}, + {key: "cache_read_tokens", label: "Cache-read tokens", numeric: true, + cell: row => measureCell(row.measures?.cache_read_tokens)}, + {key: "cache_write_tokens", label: "Cache-write tokens", numeric: true, + cell: row => measureCell(row.measures?.cache_write_tokens)}, + {key: "input_output", label: "Input / output", cell: row => + ratioCell(row.ratios?.input_output)}, + {key: "cache_share", label: "Cache-read share", cell: row => + ratioCell(row.ratios?.cache_share, true)}, + {key: "exact_retransmission", label: "Exact prompt retransmission", cell: row => + measureCell(row.exact_retransmission)} + ], + rows, + defaultSort: {key: "skill", dir: "asc"} + }) : ctx.el("p", {class: "view-empty"}, "No skill-run metrics match these facets."); + + if (routeSkill == null) { + return ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, "Skills in this cohort"), + ctx.el("p", {class: "view-lede"}, + "Select a skill to inspect its recipe-step and child-role attribution."), + skillTable + ]); + } + + const steps = selected.flatMap(row => (row.recipe_steps ?? []).map(step => ({ + skill: row.skill, + recipe: step.recipe, + step: step.step, + measures: step.measures, + ratios: step.ratios, + exact_retransmission: step.exact_retransmission + }))); + const stepTable = steps.length ? ctx.sortableTable({ + columns: [ + {key: "recipe", label: "Recipe"}, + {key: "step", label: "Step"}, + {key: "input_tokens", label: "Input tokens", cell: row => + measureCell(row.measures?.input_tokens)}, + {key: "output_tokens", label: "Output tokens", cell: row => + measureCell(row.measures?.output_tokens)}, + {key: "cache_read_tokens", label: "Cache-read tokens", cell: row => + measureCell(row.measures?.cache_read_tokens)}, + {key: "exact_retransmission", label: "Exact prompt retransmission", cell: row => + measureCell(row.exact_retransmission)} + ], + rows: steps, + defaultSort: {key: "recipe", dir: "asc"} + }) : ctx.el("p", {class: "view-empty"}, "No recipe-step attribution is available for this skill."); + + const childRows = selected.flatMap(row => (row.child_roles ?? []).map(child => ({ + skill: row.skill, + ...child + }))); + const childTable = childRows.length ? ctx.sortableTable({ + columns: [ + {key: "role", label: "Child role", cell: row => ctx.entityLink(row.role, { + view: "role", entity: row.role + })}, + {key: "harness", label: "Harness"}, + {key: "provider", label: "Provider"}, + {key: "input_tokens", label: "Child input tokens", cell: row => + measureCell(row.measures?.input_tokens)}, + {key: "output_tokens", label: "Child output tokens", cell: row => + measureCell(row.measures?.output_tokens)}, + {key: "input_output", label: "Input / output", cell: row => + ratioCell(row.ratios?.input_output)}, + {key: "cache_share", label: "Cache-read share", cell: row => + ratioCell(row.ratios?.cache_share, true)} + ], + rows: childRows, + defaultSort: {key: "role", dir: "asc"} + }) : ctx.el("p", {class: "view-empty"}, "No verified child-role attribution is linked to this skill."); + + const heading = routeSkill ?? "Skills in this cohort"; + const related = relationships.length ? ctx.sortableTable({ + columns: [ + {key: "role", label: "Spawned role", cell: edge => ctx.entityLink(edge.role, { + view: "role", entity: edge.role + })}, + {key: "harness", label: "Harness"}, + {key: "provider", label: "Child provider"} + ], + rows: relationships, + defaultSort: {key: "role", dir: "asc"} + }) : ctx.el("p", {class: "view-empty"}, "No verified child relationships are recorded."); + + return ctx.el("div", {class: "view-stack"}, [ + ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, heading), + routeSkill == null ? ctx.el("p", {class: "view-lede"}, + "Select a skill to inspect its recipe-step and child-role attribution.") : + ctx.el("p", {class: "view-lede"}, + "Input exposure and cache-read share are retransmission proxies. Exact repeated-prompt tokens are not reported when absent; cache-read is already included in input."), + skillTable + ]), + ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, "Contributor definitions"), + ...(definitions.length ? definitions : [ + ctx.el("p", {class: "view-empty"}, "No child-role definitions are linked to this skill.") + ]) + ]), + ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, "Recipe-step attribution"), + ctx.el("p", {class: "view-note"}, + "Step measures remain separate from child-role usage and from the skill-run total."), + stepTable + ]), + ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, "Child-role attribution"), childTable, related + ]) + ]); +}); diff --git a/src/autoskillit/assets/deck/views/spend.js b/src/autoskillit/assets/deck/views/spend.js new file mode 100644 index 0000000000..4a85e241ef --- /dev/null +++ b/src/autoskillit/assets/deck/views/spend.js @@ -0,0 +1,79 @@ +DeckShell.registerView("spend", ctx => { + const skillRows = ctx.skillMetrics ?? ctx.metrics ?? []; + const roleRows = ctx.roleMetrics ?? []; + const amount = measure => measure && + (measure.state === "measured" || measure.state === "measured_zero") ? measure.value : null; + const skillChart = skillRows.map(row => ({ + label: row.skill + " · " + row.harness + " · " + row.provider, + value: amount(row.measures?.input_tokens), + href: ctx.href({view: "skill", entity: row.skill}) + })); + const roleCells = roleRows.flatMap(row => (row.harnesses ?? []).map(cell => ({ + role: row.role, + provider: row.provider, + harness: cell.harness, + measures: cell.measures + }))); + const roleChart = roleCells.map(row => ({ + label: row.role + " · " + row.harness + " · " + row.provider, + value: amount(row.measures?.input_tokens), + href: ctx.href({view: "role", entity: row.role}) + })); + const measureCell = measure => ctx.availabilityCell(measure ?? {state: "unavailable"}); + const skillTable = skillRows.length ? ctx.sortableTable({ + columns: [ + {key: "skill", label: "Skill", cell: row => ctx.entityLink(row.skill, { + view: "skill", entity: row.skill + })}, + {key: "harness", label: "Harness"}, + {key: "provider", label: "Provider"}, + {key: "input_tokens", label: "Input tokens", numeric: true, + cell: row => measureCell(row.measures?.input_tokens)}, + {key: "output_tokens", label: "Output tokens", numeric: true, + cell: row => measureCell(row.measures?.output_tokens)}, + {key: "cache_read_tokens", label: "Cache-read tokens", numeric: true, + cell: row => measureCell(row.measures?.cache_read_tokens)}, + {key: "cache_write_tokens", label: "Cache-write tokens", numeric: true, + cell: row => measureCell(row.measures?.cache_write_tokens)} + ], + rows: skillRows, + defaultSort: {key: "skill", dir: "asc"} + }) : ctx.el("p", {class: "view-empty"}, "No skill-run spend is available in this cohort."); + const roleTable = roleCells.length ? ctx.sortableTable({ + columns: [ + {key: "role", label: "Role", cell: row => ctx.entityLink(row.role, { + view: "role", entity: row.role + })}, + {key: "harness", label: "Harness"}, + {key: "provider", label: "Provider"}, + {key: "input_tokens", label: "Input tokens", numeric: true, + cell: row => measureCell(row.measures?.input_tokens)}, + {key: "output_tokens", label: "Output tokens", numeric: true, + cell: row => measureCell(row.measures?.output_tokens)}, + {key: "cache_read_tokens", label: "Cache-read tokens", numeric: true, + cell: row => measureCell(row.measures?.cache_read_tokens)}, + {key: "cache_write_tokens", label: "Cache-write tokens", numeric: true, + cell: row => measureCell(row.measures?.cache_write_tokens)} + ], + rows: roleCells, + defaultSort: {key: "role", dir: "asc"} + }) : ctx.el("p", {class: "view-empty"}, "No verified child-role spend is available in this cohort."); + + return ctx.el("div", {class: "view-stack"}, [ + ctx.el("p", {class: "view-lede"}, + "Input, output, cache-read, and cache-write tokens remain separate inclusive measures. " + + "Cache-read is already included in input. Child-role usage is shown as its own population."), + ctx.el("div", {class: "view-grid"}, [ + ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, "Skill-run spend"), + ctx.barChart(skillChart, {label: "Input tokens by skill run identity"}), + skillTable + ]), + ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, "Child-role spend"), + ctx.barChart(roleChart, {label: "Input tokens by child role identity"}), + roleTable + ]) + ]) + ]); +}); diff --git a/src/autoskillit/report/deck/_registry.py b/src/autoskillit/report/deck/_registry.py index c057f6dbf8..019c862694 100644 --- a/src/autoskillit/report/deck/_registry.py +++ b/src/autoskillit/report/deck/_registry.py @@ -152,8 +152,8 @@ class DeckViewDef(NamedTuple): "Where it is worth spending optimization time at all", "What do you need to know?", None, + "views/spend.js", None, - 4651, ), DeckViewDef( "efficiency", @@ -161,8 +161,8 @@ class DeckViewDef(NamedTuple): "Which skill or role is wasteful, as opposed to merely big", "What do you need to know?", None, + "views/efficiency.js", None, - 4651, ), DeckViewDef( "skill", @@ -170,8 +170,8 @@ class DeckViewDef(NamedTuple): "Which step, which subagent, how much is re-transmitted prompt", "What do you need to know?", None, + "views/skill.js", None, - 4651, ), DeckViewDef( "role", @@ -179,8 +179,8 @@ class DeckViewDef(NamedTuple): "Whether to restrict a role to read-only on a smaller model", "What do you need to know?", None, + "views/role.js", None, - 4651, ), DeckViewDef( "context", diff --git a/tests/report/test_deck_core_js.py b/tests/report/test_deck_core_js.py index e10968b01e..51324fbe4f 100644 --- a/tests/report/test_deck_core_js.py +++ b/tests/report/test_deck_core_js.py @@ -268,6 +268,98 @@ def test_prepared_selection_returns_no_metrics_for_stale_scope(deck_js: Any) -> assert selected["roleRows"] == [] +@pytest.mark.parametrize( + ("levels", "expected"), + [ + pytest.param(None, ["short-window-skill"], id="window-local-default"), + pytest.param(["L1", "L2"], ["short-window-skill"], id="mixed-levels-intersect"), + pytest.param(["L2"], [], id="unobserved-window-level-is-empty"), + ], +) +def test_prepared_selection_uses_the_selected_windows_level_domain( + deck_js: Any, + levels: list[str] | None, + expected: list[str], +) -> None: + chips = { + "window": [ + {"key": "7d", "match": "7d", "label": "7d", "state": "live", "days": 7}, + {"key": "all", "match": "all", "label": "all", "state": "live", "days": None}, + ], + "level": [ + {"key": "L1", "match": "skill", "label": "L1", "state": "live"}, + {"key": "L2", "match": "orchestrator", "label": "L2", "state": "live"}, + ], + "harness": [{"key": "codex", "match": "codex", "label": "codex", "state": "live"}], + "provider": [{"key": "openai", "match": "openai", "label": "openai", "state": "live"}], + } + short_row = {"skill": "short-window-skill", "harness": "codex", "provider": "openai"} + all_row = {"skill": "older-orchestrator", "harness": "codex", "provider": "openai"} + prepared = { + "skills": [ + {"window": "7d", "levels": ["skill"], "rows": [short_row]}, + {"window": "all", "levels": ["orchestrator", "skill"], "rows": [all_row]}, + ], + "roles": [], + "relationships": [], + "definitions": {}, + } + params = {"window": ["7d"], "harness": ["codex"], "provider": ["openai"]} + if levels is not None: + params["level"] = levels + + selected = deck_js.call( + "DeckCore.selectPrepared", prepared, "skill", chips, {"params": params} + ) + + assert [row["skill"] for row in selected["skillRows"]] == expected + + +def test_prepared_selection_keeps_skill_edges_with_unknown_child_provider( + deck_js: Any, +) -> None: + chips = { + "window": [{"key": "all", "match": "all", "label": "all", "state": "live", "days": None}], + "level": [{"key": "L1", "match": "skill", "label": "L1", "state": "live"}], + "harness": [{"key": "codex", "match": "codex", "label": "codex", "state": "live"}], + "provider": [{"key": "openai", "match": "openai", "label": "openai", "state": "live"}], + } + skill = { + "skill": "parent-skill", + "harness": "codex", + "provider": "openai", + "measures": {"input_tokens": {"state": "measured", "value": 42}}, + } + edge = { + "skill": "parent-skill", + "role": "native-reader", + "harness": "codex", + "provider": "unknown", + } + prepared = { + "skills": [{"window": "all", "levels": ["skill"], "rows": [skill]}], + "roles": [], + "relationships": [edge, {**edge, "skill": "another-skill"}], + "definitions": {"native-reader": {"state": "available"}}, + "view_histories": {"skill": {"first_ms": 1, "last_ms": 1, "untimed": 0}}, + } + route = { + "view": "skill", + "entity": "parent-skill", + "params": { + "window": ["all"], + "level": ["L1"], + "harness": ["codex"], + "provider": ["openai"], + }, + } + + selected = deck_js.call("DeckCore.selectPrepared", prepared, "skill", chips, route) + + assert selected["skillRows"] == [skill] + assert selected["relationships"] == [edge] + + @pytest.mark.parametrize( ("params", "expected_keys", "untimed"), [ diff --git a/tests/report/test_deck_views_js.py b/tests/report/test_deck_views_js.py index f0198dec84..84089d5584 100644 --- a/tests/report/test_deck_views_js.py +++ b/tests/report/test_deck_views_js.py @@ -147,6 +147,7 @@ def _view_context(view_id: str, *, definitions_available: bool = True) -> dict[s "review_reason": "Definitions are available for all contributors.", "definition_roles": list(roles), } + skill_ratio = {**ratio, "sample_unit": "skill-run"} skill_row = { "skill": "summarizer", "harness": "codex", @@ -157,7 +158,7 @@ def _view_context(view_id: str, *, definitions_available: bool = True) -> dict[s "cache_read_tokens": {"state": "measured", "value": 20, "runs": 2}, "cache_write_tokens": {"state": "measured", "value": 3, "runs": 2}, }, - "ratios": {"input_output": ratio, "cache_read_input": ratio}, + "ratios": {"input_output": skill_ratio, "cache_share": skill_ratio}, "recipe_steps": [], } role_row = { @@ -166,10 +167,11 @@ def _view_context(view_id: str, *, definitions_available: bool = True) -> dict[s "harnesses": [ { "harness": "codex", + "models": ["observed-model"], "measures": {"input_tokens": {"state": "measured", "value": 11, "runs": 2}}, "ratios": { "input_output": ratio, - "cache_read_input": ratio, + "cache_share": ratio, "tool_mix": {"read_file": ratio}, }, } @@ -227,6 +229,9 @@ def test_each_prepared_view_renders_its_population_and_cohort_links(deck_asset: role_view = context.call("DeckTest.render", "role", _view_context("role")) assert "L0" in role_view["text"] assert "spawning skill level" in role_view["text"].lower() + assert "observed-model" in role_view["text"] + assert "Claude Code declared tools" in role_view["text"] + assert "Codex read-only tools" in role_view["text"] finally: context.close() @@ -236,25 +241,34 @@ def test_review_toggle_requires_every_contributor_definition_and_resets_route( ) -> None: context = _load_renderers(deck_asset) try: - available = context.call("DeckTest.render", "role", _view_context("role")) + available = context.call("DeckTest.render", "efficiency", _view_context("efficiency")) assert available["review"] is not None assert "disabled" not in available["review"]["attrs"] + assert available["text"].index("definition-first marker") < available["text"].index( + "Show reviewable signals" + ) unavailable = context.call( - "DeckTest.render", "role", _view_context("role", definitions_available=False) + "DeckTest.render", + "efficiency", + _view_context("efficiency", definitions_available=False), ) assert unavailable["review"] is not None assert "disabled" in unavailable["review"]["attrs"] assert "definition" in unavailable["text"].lower() - context.call("DeckTest.render", "role", _view_context("role")) + context.call("DeckTest.render", "efficiency", _view_context("efficiency")) enabled_hash = context.call("DeckTest.clickReview") enabled_route = context.call("DeckCore.decodeRoute", enabled_hash, "cohort") assert enabled_route["params"].get("review") assert enabled_route["params"].get("harness") == ["codex"] assert enabled_route["params"].get("provider") == ["openai"] - context.call("DeckTest.render", "role", {**_view_context("role"), "route": enabled_route}) + context.call( + "DeckTest.render", + "efficiency", + {**_view_context("efficiency"), "route": enabled_route}, + ) reset_hash = context.call("DeckTest.clickReview") reset_route = context.call("DeckCore.decodeRoute", reset_hash, "cohort") assert "review" not in reset_route["params"] @@ -262,3 +276,21 @@ def test_review_toggle_requires_every_contributor_definition_and_resets_route( assert reset_route["params"].get("provider") == ["openai"] finally: context.close() + + +def test_role_definition_and_spawning_links_survive_empty_metric_scope(deck_asset: Any) -> None: + context = _load_renderers(deck_asset) + try: + view = _view_context("role") + view["route"]["entity"] = "reviewer" + view["metrics"] = [] + view["roleMetrics"] = [] + + rendered = context.call("DeckTest.render", "role", view) + + assert "reviewer" in rendered["text"] + assert "reviewer role instructions" in rendered["text"] + assert "No child-invocation metrics match these facets." in rendered["text"] + assert any("#/skill/summarizer" in href for href in rendered["links"]) + finally: + context.close() From 490d648c8d30c9726d7fde9b7526f854513f16e3 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 15:49:42 -0700 Subject: [PATCH 09/22] docs: describe native role attribution and core deck contracts --- docs/developer/diagnostics.md | 82 +++++++++++++++++++++++++++++------ 1 file changed, 69 insertions(+), 13 deletions(-) diff --git a/docs/developer/diagnostics.md b/docs/developer/diagnostics.md index 5aaa80267f..b87bcc720b 100644 --- a/docs/developer/diagnostics.md +++ b/docs/developer/diagnostics.md @@ -303,19 +303,38 @@ its rotated `otlp.jsonl.1` generation. It stores report facts separately from the retained `sessions.jsonl` projection. -Every v1 row carries `schema_version`, `kind`, `key`, `session_id`, and +Every v2 row carries `schema_version`, `kind`, `key`, `session_id`, and `time_ms`. `time_ms` is epoch milliseconds: session rows use the source session timestamp; OTLP rows use `timeUnixNano`, falling back to `observedTimeUnixNano`. Rows are append-only upserts keyed by `(kind, key)`; when a key is written again, readers keep its last row. -| Kind | Key | Additional v1 fields | +| Kind | Key | Additional fields | |---|---|---| | `session` | The session directory name (`dir_name`) | `harness`, `provider`, `model`, `skill`, `recipe`, `step`, `level`, `kitchen_id`, `order_id`, `dispatch_id`, `campaign_id`, `caller_session_id`, `parent_session_id`, `success`, `subtype`, `adjudication_reason`, `adjudication_subtype`, `duration_seconds`, `input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_write_tokens`, `assistant_turn_count`, `tool_counts` | | `request` | `{session_id}:{request_id}` | `harness`, `request_id`, `agent_name`, `query_source`, `model`, `event_sequence`, the four token fields, `cost_usd`, `duration_ms` | | `tool` | `{session_id}:{tool_use_id}`, or `{source_id}#{ordinal}` when there is no tool-use ID | `harness`, `agent_name`, `tool_name`, `tool_use_id`, `error_type`, `success`, `duration_ms`, `tool_input_size_bytes`, `tool_result_size_bytes`, `event_sequence` | | `subagent` | `{source_id}#{ordinal}` | `harness`, `agent_type`, `model`, `final_model`, `model_swapped`, `event_sequence` | +Native child `subagent` rows use +`{parent_session_key}:native:{backend}:{native_parent_session_id}:{child_id}`. +They additionally carry `child_id`, `native_parent_session_id`, `parent_session_key`, +canonical `role`, `actor_level` (`L0`), child `provider` and `model`, attribution `skill`, +spawning `recipe`, `step`, `level` and `time_ms`, canonical `token_usage`, `tool_counts`, +`transcript_state`, and `usage_state`. OTLP completion events remain separate from these +invocation observations. A managed attempt or launch alias alone cannot establish a +native actor. Resumed parents sharing a native session ID require unique invocation +ownership; ambiguous children remain unattributed. + +Native child tokens and tools come from that child's verified dedicated transcript, +with native message/request snapshots deduplicated. Tokens require the child's own +effective provider; missing provider/model identities never inherit parent values. +Claude's raw-exclusive input becomes inclusive through the shared conversion of input, +cache-read and cache-write counts, only when every component is observed. Codex input +and serialized canonical input already include cache and are not expanded again. +Raw OTLP request counters do not establish this convention through their transport or +field names, and uncorrelated request/tool facts do not become role usage. + Session token fields are serialized structured measures. Request rows persist raw token-counter observations as `int | None`; the reader resolves them to structured measures. Session `provider` is persisted verbatim from @@ -327,7 +346,10 @@ measures remain `unavailable` or `unknown` according to that pair; they are never treated as zero. Session rows do not copy `cwd`, transcript paths, or turn IDs. -The reader tolerates older or partial rows. Blank, malformed, non-object, +An existing generation must carry the current `row_schema_version` in its committed +state before read admission. An absent index remains empty; absent, old, or future +row-schema markers require an update/rebuild. Within an admitted generation, the row +decoder tolerates partial rows. Blank, malformed, non-object, unknown-kind, and empty-key rows are skipped; unknown fields are ignored, and missing or wrongly typed fields are treated as absent. Invalid entries in a tool-count map are dropped. The reader adds `session_key` to event rows in @@ -343,10 +365,18 @@ is attributed only when its session ID has exactly one attempt; otherwise The writer holds an exclusive lease for the report index. Each commit appends and fsyncs `rows.jsonl` before atomically replacing the versioned `state.json` -with the committed walk watermark and byte offset. On the next open, bytes past -that offset are truncated. If state is missing or invalid, recovery preserves -complete row lines, drops an incomplete final line, and walks sources from the -beginning; last-row-wins upserts repair rows re-derived by that walk. +with the committed walk watermark, byte offset, `row_schema_version`, and +`child_evidence_fingerprint`. On the next open, bytes past that offset are truncated. +The row-schema marker is an application-controlled compatibility boundary, distinct +from the state envelope version. Missing/incompatible row-schema evidence resets the +generation before append admission. Recoverable byte/watermark corruption within a +known current schema retains the existing complete-line recovery behavior. + +The child-evidence fingerprint covers canonical native child facts and dedicated +transcript dependencies, including changes, additions and removals. Drift selects a +full replay under the existing exclusive index lease before unchanged live or consumed +archive parents can be skipped. Markers commit with the successful row/watermark state; +this reset does not preserve rows whose original sources are no longer retained. An incremental walk whose archive or OTLP cursor is no longer retained resets that source cursor and re-walks the retained data. An archive reset also resets @@ -359,9 +389,8 @@ Rebuilding reads only the sources still retained on disk. Rows from OTLP data that has rotated out survive through incremental updates, but a rebuild cannot re-derive those older facts. A rebuild equals an incremental index when source changes are limited to those tracked by the walk: appended OTLP, OTLP rotation, -and changed, added, or evicted session rows. This assumes transcripts have not -changed since their session row was walked. `--rebuild` refreshes -transcript-derived counts. +and changed, added, or evicted session rows, plus native child evidence changes. +Parent transcript changes still require `--rebuild` when their session row is unchanged. Codex OTLP token events are not projected: they do not carry a stable request identity for deduplication. Codex token measures enter the report index through @@ -371,25 +400,52 @@ the session rows. `autoskillit sessions deck ` incrementally refreshes the report index and writes one self-contained HTML deck. It can be opened from `file://` without a server or network -connection. The deck embeds the index's session rows and filters them in the browser. +connection. It receives all four index fact collections as mapping iterables; the +report layer imports core types and helpers rather than the execution index type. +The cohort view retains its session table. Spend, efficiency, skill and role views +select metric blocks prepared by the shared aggregation library, without browser +aggregation of measures. ADR-0017 records why charts use first-party SVG. ### Payload contract The payload keys are `generated_at_ms`, `index_schema_version`, `landing`, and `history`; -`tables.sessions{columns,rows}`; `facets[{id,label,column,kind}]`; +column tables `tables.sessions`, `tables.skills`, and `tables.roles`; `facets[{id,label,column,kind}]`; `views[{id,question,decision,group,status,issue}]`; `chips{view:{facet:[{key,label,state,count,reason,issue,match|days}]}}`; and `availability[{state,label,description}]`. `match` is used for value facets and `days` for windows. `views[].issue` tracks the issue for building a planned view; a chip's `issue` identifies why that choice is unavailable. These fields have separate meanings. +`prepared.skills` and `prepared.roles` contain fixed-window, exact stored-level subset +blocks; `prepared.relationships`, `definitions`, `view_chips` and `view_histories` carry +the independent relationship graph, bundled agent definitions and population metadata. +Skill cells are `(skill, harness, provider)` and represent recorded L1/L2 skill runs. +Role rows are `(role, provider)` with separate harness-scoped metrics for terminal L0 +child invocations. Providers and token source pairs are not pooled. Role usage is an +attribution population and is not added to or subtracted from session spend. + +Token measures retain their availability states. Input/output exposure, cache-read/input +share and tool/total-tool ratios are eligible sums with explicit sample sizes, units, +excluded and unknown counts. Cache annotations overlap inclusive input; cache evidence +is a reuse proxy, while unobserved exact prompt retransmission remains not reported. +Role level selections mean the spawning skill level, while the actor remains L0. +Identity details, definitions and known cross-references remain available in empty +usage cohorts. A stale explicit window or facet selection yields no measurements. + +Canonical role routes provide offline definition links. The artifact preserves Claude +tools/model separately from Codex reader tools/model and renders original definition +body text literally. Manual ratio/tool-mix review controls require observed eligible +signals and all contributor definition links; missing evidence has an inline reason. +Flags reset on rerender and do not diagnose a configuration or recommend a downgrade. + ### Chip states Each cohort choice is `live`, `struck`, or `absent`. Live choices filter the rows. Struck choices remain visible with a reason and cannot be selected. Absent choices have no matching rows. L0 is struck because L0 leaf agents write no session row and exist -only as subagent transcripts. On a legacy index, unavailable orchestration levels cite +only as subagent transcripts in the cohort's session population. Role views use the +spawning skill's level facet. On a legacy source, unavailable orchestration levels cite [#4622](https://github.com/TalonT-Org/AutoSkillit/issues/4622); windows extending before the retained history cite [#4621](https://github.com/TalonT-Org/AutoSkillit/issues/4621). From 517457d12a70c46fbe9920a13ac9557a89450352 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 16:56:41 -0700 Subject: [PATCH 10/22] fix: align core view contracts and add per-signal review flags --- src/autoskillit/assets/deck/core.js | 40 ++++++- src/autoskillit/assets/deck/deck.css | 13 +- src/autoskillit/assets/deck/shell.js | 2 +- .../assets/deck/views/efficiency.js | 54 ++------- src/autoskillit/assets/deck/views/role.js | 6 +- src/autoskillit/assets/deck/views/skill.js | 3 +- .../execution/_report_index_rows.py | 14 +++ .../evidence/_native_child_projection.py | 16 ++- src/autoskillit/report/deck/_registry.py | 8 +- tests/_test_filter.py | 4 +- tests/arch/test_hook_flock_nonblocking.py | 5 + tests/contracts/test_package_gateways.py | 1 + tests/execution/test_report_index.py | 2 +- tests/execution/test_report_walk.py | 9 ++ tests/report/_fixtures.py | 2 +- tests/report/test_deck_html.py | 4 +- tests/report/test_deck_payload.py | 8 +- tests/report/test_deck_registry.py | 16 +++ tests/report/test_deck_views_js.py | 112 ++++++++++++------ 19 files changed, 210 insertions(+), 109 deletions(-) diff --git a/src/autoskillit/assets/deck/core.js b/src/autoskillit/assets/deck/core.js index 5417642840..f2930f7cb2 100644 --- a/src/autoskillit/assets/deck/core.js +++ b/src/autoskillit/assets/deck/core.js @@ -42,6 +42,44 @@ globalThis.DeckCore = (() => { } return {eligible: true, reason: null, roles}; } + + function reviewSignal(ctx, ratio) { + const eligibility = reviewEligibility(ratio, ctx.definitions); + const links = eligibility.roles.map(role => ctx.entityLink(role, { + view: "role", entity: role + })); + const definitions = links.length ? ctx.el("div", { + class: "view-review__definitions", + "aria-label": "Contributor definitions" + }, links) : null; + const marker = ctx.el("span", { + class: "view-review__marker", + "aria-live": "polite", + hidden: true + }, "Flagged for review"); + const button = ctx.el("button", { + type: "button", + class: "view-review__flag", + "data-review-flag": "true", + "aria-label": "Flag this signal for review", + "aria-pressed": "false", + disabled: !eligibility.eligible + }, "Flag for review"); + let flagged = false; + if (eligibility.eligible) { + button.addEventListener("click", () => { + flagged = !flagged; + button.setAttribute("aria-pressed", String(flagged)); + marker.hidden = !flagged; + }); + } + return ctx.el("div", {class: "view-review-signal", "data-review-signal": "true"}, [ + definitions, + button, + marker, + eligibility.reason ? ctx.el("p", {class: "view-review__reason"}, eligibility.reason) : null + ]); + } const decodeTable = ({columns, rows}) => rows.map(r => Object.fromEntries(columns.map((c, i) => [c, r[i]]))); @@ -298,6 +336,6 @@ globalThis.DeckCore = (() => { return Object.freeze({decodeTable, encodeRoute, decodeRoute, hrefFor, effectiveSelection, toggleSelection, windowSelection, selectPrepared, filterRows, populationSentence, summarizePairs, sortRows, parseSort, chipPresentation, availabilityPresentation, - barLayout, formatCount, formatRatio, ratioSample, reviewEligibility, formatDate, + barLayout, formatCount, formatRatio, ratioSample, reviewEligibility, reviewSignal, formatDate, CHIP_STATES, DAY_MS}); })(); diff --git a/src/autoskillit/assets/deck/deck.css b/src/autoskillit/assets/deck/deck.css index be96c6f5e0..33b6ebb74e 100644 --- a/src/autoskillit/assets/deck/deck.css +++ b/src/autoskillit/assets/deck/deck.css @@ -92,12 +92,17 @@ svg { display: block; max-width: 100%; height: auto; } .view-definition { border-top: 1px solid var(--rule); padding: 12px 0; } .view-definition h3 { font-size: 13px; margin: 0 0 6px; } .view-definition pre { white-space: pre-wrap; overflow-wrap: anywhere; color: var(--ink-2); } +.view-review-signal { display: grid; justify-items: start; gap: 4px; margin-top: 5px; } +.view-review__definitions { display: flex; flex-wrap: wrap; gap: 6px; font-size: 11px; } +.view-review__marker { color: var(--warn); font-size: 11px; font-weight: 600; } .view-review { border: 1px solid var(--rule); border-radius: 6px; padding: 12px; } -.view-review button { border: 1px solid var(--rule-2); border-radius: 5px; padding: 7px 10px; +.view-review button, .view-review-signal button { border: 1px solid var(--rule-2); + border-radius: 5px; padding: 5px 8px; color: var(--ink); background: var(--surface-2); cursor: pointer; } -.view-review button:focus-visible, .view-links a:focus-visible { outline: 2px solid var(--s1); +.view-review button:focus-visible, .view-review-signal button:focus-visible, +.view-links a:focus-visible { outline: 2px solid var(--s1); outline-offset: 2px; } -.view-review button:disabled { cursor: not-allowed; opacity: .6; } +.view-review button:disabled, .view-review-signal button:disabled { cursor: not-allowed; opacity: .6; } .view-review__reason { color: var(--warn); font-size: 12px; } .view-actor { color: var(--ink-3); font: 11px var(--mono); } .foot { color: var(--ink-3); font: 10px/1.5 var(--mono); padding: 0 28px 24px; } @@ -115,6 +120,6 @@ svg { display: block; max-width: 100%; height: auto; } .shell { display: block; } .cohort { position: static; } .card, table, svg, .view-definition, .view-review { break-inside: avoid; } - .view-review button { display: none; } + .view-review button, .view-review-signal button { display: none; } #deck-view { padding: 18px 0; } } diff --git a/src/autoskillit/assets/deck/shell.js b/src/autoskillit/assets/deck/shell.js index 4d2dc838c9..e0fd0014ab 100644 --- a/src/autoskillit/assets/deck/shell.js +++ b/src/autoskillit/assets/deck/shell.js @@ -288,7 +288,7 @@ globalThis.DeckShell = (() => { DeckCore.formatDate(first) + " → " + DeckCore.formatDate(last); const generated = new Date(model.generated_at_ms).toISOString().slice(0, 16); document.getElementById("deck-foot").textContent = "report index v" + - model.index_schema_version + " · " + DeckCore.formatCount(table.length) + + model.index_schema_version + " · " + DeckCore.formatCount(model.tables.sessions.length) + " session rows · history " + history + " · generated " + generated + " UTC"; } diff --git a/src/autoskillit/assets/deck/views/efficiency.js b/src/autoskillit/assets/deck/views/efficiency.js index 2420bb673e..c6cffd5dfe 100644 --- a/src/autoskillit/assets/deck/views/efficiency.js +++ b/src/autoskillit/assets/deck/views/efficiency.js @@ -25,7 +25,8 @@ DeckShell.registerView("efficiency", ctx => { const value = text == null ? ctx.availabilityCell(ratio ?? {state: "unavailable"}) : ctx.el("span", {class: "view-measure"}, text); return ctx.el("div", {class: "view-measure"}, [value, - ctx.el("small", {class: "view-sample"}, DeckCore.ratioSample(ratio))]); + ctx.el("small", {class: "view-sample"}, DeckCore.ratioSample(ratio)), + DeckCore.reviewSignal(ctx, ratio)]); } function toolMix(ratios = {}) { @@ -36,17 +37,13 @@ DeckShell.registerView("efficiency", ctx => { ctx.el("span", {}, tool), ctx.el("span", {}, " · "), value == null ? ctx.availabilityCell(ratio) : ctx.el("span", {}, value), - ctx.el("small", {class: "view-sample"}, DeckCore.ratioSample(ratio)) + ctx.el("small", {class: "view-sample"}, DeckCore.ratioSample(ratio)), + DeckCore.reviewSignal(ctx, ratio) ]); })) : ctx.el("span", {class: "view-empty"}, "No observed tool calls"); } - function reviewable(ratios = {}) { - return ratioSignals(ratios).some(signal => - DeckCore.reviewEligibility(signal.ratio, ctx.definitions).eligible); - } - const definitions = ctx.definitions ?? {}; const contributorRoles = new Set((ctx.relationships ?? []).map(edge => edge.role)); for (const row of skillRows) { @@ -71,39 +68,7 @@ DeckShell.registerView("efficiency", ctx => { available ? ctx.el("pre", {}, definition.body ?? "") : null ]); }); - const signals = [...skillRows, ...roleRows].flatMap(row => ratioSignals(row.ratios)); - const eligibility = signals.map(signal => - DeckCore.reviewEligibility(signal.ratio, definitions)); - const reviewEnabled = eligibility.some(status => status.eligible); - const disabledReason = eligibility.find(status => !status.eligible)?.reason ?? - "No prepared measured review signals are available in this cohort."; - const reviewMode = ctx.route.params.review?.[0] === "eligible"; - const toggle = ctx.el("button", { - type: "button", - "data-review-toggle": "true", - "aria-label": "Filter to reviewable signals", - "aria-pressed": String(reviewMode && reviewEnabled), - disabled: !reviewEnabled - }, reviewMode ? "Show all signals" : "Show reviewable signals"); - if (reviewEnabled) { - toggle.addEventListener("click", () => { - const params = {...ctx.route.params, review: reviewMode ? [] : ["eligible"]}; - location.hash = ctx.href({view: "efficiency", entity: ctx.route.entity, params}); - }); - } - const reviewControl = ctx.el("section", {class: "view-review"}, [ - toggle, - reviewEnabled ? ctx.el("p", {class: "view-note"}, reviewMode ? - "Showing prepared review-eligible signals." : - "Review eligibility reflects prepared evidence and linked role definitions.") : - ctx.el("p", {class: "view-review__reason"}, disabledReason) - ]); - - const shownSkills = reviewMode && reviewEnabled ? skillRows.filter(row => reviewable(row.ratios)) : - skillRows; - const shownRoles = reviewMode && reviewEnabled ? roleRows.filter(row => reviewable(row.ratios)) : - roleRows; - const skillTable = shownSkills.length ? ctx.sortableTable({ + const skillTable = skillRows.length ? ctx.sortableTable({ columns: [ {key: "skill", label: "Skill", cell: row => ctx.entityLink(row.skill, { view: "skill", entity: row.skill @@ -116,10 +81,10 @@ DeckShell.registerView("efficiency", ctx => { ratioCell(row.ratios?.cache_share, true)}, {key: "tool_mix", label: "Observed tool mix", cell: row => toolMix(row.ratios)} ], - rows: shownSkills, + rows: skillRows, defaultSort: {key: "skill", dir: "asc"} }) : ctx.el("p", {class: "view-empty"}, "No skill-run ratios are available in this cohort."); - const roleTable = shownRoles.length ? ctx.sortableTable({ + const roleTable = roleRows.length ? ctx.sortableTable({ columns: [ {key: "role", label: "Role", cell: row => ctx.entityLink(row.role, { view: "role", entity: row.role @@ -132,7 +97,7 @@ DeckShell.registerView("efficiency", ctx => { ratioCell(row.ratios?.cache_share, true)}, {key: "tool_mix", label: "Observed tool mix", cell: row => toolMix(row.ratios)} ], - rows: shownRoles, + rows: roleRows, defaultSort: {key: "role", dir: "asc"} }) : ctx.el("p", {class: "view-empty"}, "No child-invocation ratios are available in this cohort."); @@ -144,8 +109,7 @@ DeckShell.registerView("efficiency", ctx => { ctx.el("h2", {}, "Contributor definitions"), ...(definitionCards.length ? definitionCards : [ ctx.el("p", {class: "view-empty"}, "No contributor definitions are linked in this cohort.") - ]), - reviewControl + ]) ]), ctx.el("section", {class: "card"}, [ctx.el("h2", {}, "Skill-run efficiency"), skillTable]), ctx.el("section", {class: "card"}, [ctx.el("h2", {}, "Child-invocation efficiency"), roleTable]) diff --git a/src/autoskillit/assets/deck/views/role.js b/src/autoskillit/assets/deck/views/role.js index fcc4e60dbe..f373c1b6d4 100644 --- a/src/autoskillit/assets/deck/views/role.js +++ b/src/autoskillit/assets/deck/views/role.js @@ -22,7 +22,8 @@ DeckShell.registerView("role", ctx => { const value = DeckCore.formatRatio(ratio, percent); return ctx.el("div", {class: "view-measure"}, [ value == null ? measureCell(ratio) : ctx.el("span", {}, value), - ctx.el("small", {class: "view-sample"}, DeckCore.ratioSample(ratio)) + ctx.el("small", {class: "view-sample"}, DeckCore.ratioSample(ratio)), + DeckCore.reviewSignal(ctx, ratio) ]); } @@ -34,7 +35,8 @@ DeckShell.registerView("role", ctx => { ctx.el("span", {}, tool), ctx.el("span", {}, " · "), value == null ? ctx.availabilityCell(ratio) : ctx.el("span", {}, value), - ctx.el("small", {class: "view-sample"}, DeckCore.ratioSample(ratio)) + ctx.el("small", {class: "view-sample"}, DeckCore.ratioSample(ratio)), + DeckCore.reviewSignal(ctx, ratio) ]); })) : ctx.el("span", {class: "view-empty"}, "No observed tool calls"); diff --git a/src/autoskillit/assets/deck/views/skill.js b/src/autoskillit/assets/deck/views/skill.js index c6fb91304f..e0421bb2d0 100644 --- a/src/autoskillit/assets/deck/views/skill.js +++ b/src/autoskillit/assets/deck/views/skill.js @@ -14,7 +14,8 @@ DeckShell.registerView("skill", ctx => { const value = DeckCore.formatRatio(ratio, percent); return ctx.el("div", {class: "view-measure"}, [ value == null ? measureCell(ratio) : ctx.el("span", {}, value), - ctx.el("small", {class: "view-sample"}, DeckCore.ratioSample(ratio)) + ctx.el("small", {class: "view-sample"}, DeckCore.ratioSample(ratio)), + DeckCore.reviewSignal(ctx, ratio) ]); } diff --git a/src/autoskillit/execution/_report_index_rows.py b/src/autoskillit/execution/_report_index_rows.py index fe9b80c7b0..eb5eb6ebd7 100644 --- a/src/autoskillit/execution/_report_index_rows.py +++ b/src/autoskillit/execution/_report_index_rows.py @@ -633,6 +633,20 @@ def _subagent_fields( "final_model": unique_string_attribute(attributes, "final_model"), "model_swapped": unique_flag_attribute(attributes, "model_swapped"), "event_sequence": unique_count_attribute(attributes, "event.sequence"), + "child_id": None, + "native_parent_session_id": None, + "parent_session_key": None, + "role": None, + "actor_level": None, + "provider": UNKNOWN_SOURCE, + "skill": None, + "recipe": None, + "step": None, + "level": None, + "token_usage": _unknown_child_usage_map(), + "tool_counts": None, + "transcript_state": None, + "usage_state": None, } diff --git a/src/autoskillit/execution/evidence/_native_child_projection.py b/src/autoskillit/execution/evidence/_native_child_projection.py index 01ca365fa6..80b95192ca 100644 --- a/src/autoskillit/execution/evidence/_native_child_projection.py +++ b/src/autoskillit/execution/evidence/_native_child_projection.py @@ -14,6 +14,8 @@ import zstandard from autoskillit.core import ( + AGENT_BACKEND_CLAUDE_CODE, + AGENT_BACKEND_CODEX, CANONICAL_ACCOUNTING_FIELDS, SerializedTokenMeasure, TokenMeasure, @@ -142,9 +144,9 @@ def _child_transcript_path( child_id = child.get("child_id") if not isinstance(child_id, str) or not child_id: return None, False - if backend == "claude_code": + if backend == normalize_backend_name(AGENT_BACKEND_CLAUDE_CODE): return _claude_child_transcript_path(row, child_id, parent_id) - if backend == "codex": + if backend == normalize_backend_name(AGENT_BACKEND_CODEX): return _codex_child_transcript_path(child, child_id, parent_id) return None, False @@ -241,7 +243,7 @@ def _first_transcript_timestamp(text: str) -> float | None: def _read_child_transcript(path: Path, backend: str) -> str | None: try: - if backend == "codex": + if backend == normalize_backend_name(AGENT_BACKEND_CODEX): with _logical_rollout_reader(path) as handle: return handle.read().decode("utf-8") return path.read_text(encoding="utf-8") @@ -274,7 +276,9 @@ def _child_measurements( ) -> tuple[dict[str, int] | None, dict[str, SerializedTokenMeasure], str]: if not complete or text is None: return None, _unknown_usage(), "unknown" - parser_backend = "claude" if backend == "claude_code" else backend + parser_backend = ( + "claude" if backend == normalize_backend_name(AGENT_BACKEND_CLAUDE_CODE) else backend + ) turns = list(iter_merged_assistant_turns(text, backend=parser_backend)) tool_counts = dict(Counter(name for turn in turns for name in turn.tool_names)) token_usage, usage_state = _child_token_usage(backend, child_id, path, text, provider) @@ -286,7 +290,7 @@ def _child_token_usage( ) -> tuple[dict[str, SerializedTokenMeasure], str]: if not provider: return _unknown_usage(), "unknown" - if backend == "claude_code": + if backend == normalize_backend_name(AGENT_BACKEND_CLAUDE_CODE): aggregate, _rows = extract_token_usage(text, provider_used=provider) if aggregate is not None: usage = { @@ -307,7 +311,7 @@ def _child_token_usage( "observed", ) return _unknown_usage(), "unknown" - if backend == "codex" and path is not None: + if backend == normalize_backend_name(AGENT_BACKEND_CODEX) and path is not None: rows = extract_codex_child_turn_usage(path, child_id, provider_used=provider) if rows: return _codex_usage_measures(rows, provider), "observed" diff --git a/src/autoskillit/report/deck/_registry.py b/src/autoskillit/report/deck/_registry.py index 019c862694..13d2cc8076 100644 --- a/src/autoskillit/report/deck/_registry.py +++ b/src/autoskillit/report/deck/_registry.py @@ -151,7 +151,7 @@ class DeckViewDef(NamedTuple): "Where do the tokens go?", "Where it is worth spending optimization time at all", "What do you need to know?", - None, + "skills", "views/spend.js", None, ), @@ -160,7 +160,7 @@ class DeckViewDef(NamedTuple): "What's inefficient?", "Which skill or role is wasteful, as opposed to merely big", "What do you need to know?", - None, + "skills", "views/efficiency.js", None, ), @@ -169,7 +169,7 @@ class DeckViewDef(NamedTuple): "What is this skill spending on?", "Which step, which subagent, how much is re-transmitted prompt", "What do you need to know?", - None, + "skills", "views/skill.js", None, ), @@ -178,7 +178,7 @@ class DeckViewDef(NamedTuple): "Can this agent be cheaper?", "Whether to restrict a role to read-only on a smaller model", "What do you need to know?", - None, + "roles", "views/role.js", None, ), diff --git a/tests/_test_filter.py b/tests/_test_filter.py index 32bbe61997..b4f16e98b0 100644 --- a/tests/_test_filter.py +++ b/tests/_test_filter.py @@ -606,7 +606,7 @@ class SourceMapRejection(enum.StrEnum): "_type_figure_spec": frozenset({"core", "report"}), "_type_session_shape": frozenset({"core", "cli", "fleet", "pipeline", "server"}), "_type_capture": frozenset({"core", "fleet", "recipe", "cli"}), - "_type_token": frozenset({"cli", "core", "execution", "fleet", "pipeline"}), + "_type_token": frozenset({"cli", "core", "execution", "fleet", "pipeline", "report"}), "_type_constants_env": frozenset( { "cli", @@ -1507,6 +1507,8 @@ class SourceMapRejection(enum.StrEnum): # file-level: exercises the hook-authored child-outcome snapshot module # directly (issue #4623) via autoskillit.hooks's public re-export. "execution/test_child_outcomes.py", + # file-level: report-index refresh tests seed native child snapshots through hooks. + "execution/test_report_index.py", # hook_registry imports from autoskillit.hooks at module scope # (HOOK_REGISTRY, generate_hooks_json) for the registry init wiring. "hook_registry", diff --git a/tests/arch/test_hook_flock_nonblocking.py b/tests/arch/test_hook_flock_nonblocking.py index 7dea6a01e1..8af3b237fe 100644 --- a/tests/arch/test_hook_flock_nonblocking.py +++ b/tests/arch/test_hook_flock_nonblocking.py @@ -103,6 +103,11 @@ "_open_otlp_handles", "ArtifactLease.acquire_shared", ), + ( + "execution/evidence/report_walk.py", + "_retained_session_rows", + "ArtifactLease.acquire_shared", + ), ( "execution/evidence/report_walk.py", "_walk_projection", diff --git a/tests/contracts/test_package_gateways.py b/tests/contracts/test_package_gateways.py index ae21e2d528..a75911004e 100644 --- a/tests/contracts/test_package_gateways.py +++ b/tests/contracts/test_package_gateways.py @@ -506,6 +506,7 @@ def test_measure_aggregation_ships_as_projected_hook_asset() -> None: "execution_evidence": frozenset( { "anomaly_detection", + "_native_child_projection", "linux_tracing", "otlp_sink", "report_walk", diff --git a/tests/execution/test_report_index.py b/tests/execution/test_report_index.py index 4b277de14a..0333f87a54 100644 --- a/tests/execution/test_report_index.py +++ b/tests/execution/test_report_index.py @@ -8,6 +8,7 @@ import pytest +import autoskillit.execution.child_outcomes as child_snapshot from autoskillit.core import ArtifactLease, ArtifactLeaseContention from autoskillit.execution import ( REPORT_INDEX_SCHEMA_VERSION, @@ -17,7 +18,6 @@ update_report_index, ) from autoskillit.execution.evidence.report_walk import WalkItem -from autoskillit.hooks import _child_outcome_snapshot as child_snapshot from tests.execution._report_index_fixtures import ( CLAUDE_SCOPE, basic_session_row, diff --git a/tests/execution/test_report_walk.py b/tests/execution/test_report_walk.py index 66ea09958f..51ee45b5af 100644 --- a/tests/execution/test_report_walk.py +++ b/tests/execution/test_report_walk.py @@ -47,12 +47,21 @@ def _session( *, claude_code_log: object = ..., codex_log: object = ..., + backend: str | None = None, + timestamp: str | None = None, + duration_seconds: float | None = None, ) -> dict[str, Any]: row: dict[str, Any] = {"dir_name": dir_name, "session_id": session_id} if claude_code_log is not ...: row["claude_code_log"] = claude_code_log if codex_log is not ...: row["codex_log"] = codex_log + if backend is not None: + row["backend"] = backend + if timestamp is not None: + row["timestamp"] = timestamp + if duration_seconds is not None: + row["duration_seconds"] = duration_seconds return row diff --git a/tests/report/_fixtures.py b/tests/report/_fixtures.py index 5fa08c4f6e..08bae77ac8 100644 --- a/tests/report/_fixtures.py +++ b/tests/report/_fixtures.py @@ -57,7 +57,7 @@ def subagent_row( "native_parent_session_id": parent["session_id"], "skill": skill, "provider": provider, - "model": "claude-sonnet", + "model": "test-child-model", "actor_level": "L0", "time_ms": parent["time_ms"], "level": parent["level"], diff --git a/tests/report/test_deck_html.py b/tests/report/test_deck_html.py index 4a98f0546c..f239a77194 100644 --- a/tests/report/test_deck_html.py +++ b/tests/report/test_deck_html.py @@ -280,9 +280,9 @@ def test_render_deck_forwards_child_facts_and_embeds_definitions_as_literal_json description="Inspect a committed slice.", body=hostile_body, tools=("Bash", "Read"), - model="sonnet", + model="test-claude-model", reader_tools=(), - codex=SimpleNamespace(model="gpt-5.6-sol"), + codex=SimpleNamespace(model="test-codex-model"), ) monkeypatch.setattr( deck_payload_module, diff --git a/tests/report/test_deck_payload.py b/tests/report/test_deck_payload.py index b0b32b79c2..f3bcf4cfe0 100644 --- a/tests/report/test_deck_payload.py +++ b/tests/report/test_deck_payload.py @@ -242,9 +242,9 @@ def _definition(name: str, body: str) -> SimpleNamespace: description="Canonical reviewer role.", body=body, tools=("Bash", "Read"), - model="sonnet", + model="test-claude-model", reader_tools=(), - codex=SimpleNamespace(model="gpt-5.6-sol"), + codex=SimpleNamespace(model="test-codex-model"), ) @@ -547,6 +547,10 @@ def ratio_spy(records: Any, numerator: str, denominator: str, **kwargs: Any) -> assert aggregate_calls and ratio_calls assert len(loader_calls) == 1 prepared = payload["prepared"] + built_views = {view.view_id for view in DECK_VIEWS if view.planned_issue is None} + assert set(prepared["view_chips"]) == built_views + assert set(prepared["view_histories"]) == built_views + assert set(payload["tables"]) == {SESSION_TABLE, "skills", "roles"} levels: set[str | None] = {"orchestrator", "skill"} skill_block = _metric_row(prepared["skills"], window="all", levels=levels) planner = next( diff --git a/tests/report/test_deck_registry.py b/tests/report/test_deck_registry.py index 7c33a70e6a..7c208808ae 100644 --- a/tests/report/test_deck_registry.py +++ b/tests/report/test_deck_registry.py @@ -6,6 +6,7 @@ from autoskillit.execution._report_index_rows import ReportSessionRow from autoskillit.report.deck._registry import ( AVAILABILITY_VOCABULARY, + DECK_VIEWS, FACETS, SESSION_COLUMNS, SESSION_TABLE, @@ -93,6 +94,21 @@ def test_session_columns_follow_report_session_row_schema() -> None: assert len(SESSION_COLUMNS) == len(set(SESSION_COLUMNS)) +def test_reserved_measure_views_have_primary_tables_and_registered_scripts() -> None: + views = {view.view_id: view for view in DECK_VIEWS} + expected = { + "spend": ("skills", "views/spend.js"), + "efficiency": ("skills", "views/efficiency.js"), + "skill": ("skills", "views/skill.js"), + "role": ("roles", "views/role.js"), + } + + assert { + view_id: (views[view_id].table, views[view_id].script) for view_id in expected + } == expected + assert all(views[view_id].planned_issue is None for view_id in expected) + + def test_availability_vocabulary_covers_every_token_measure_state() -> None: assert set(AVAILABILITY_VOCABULARY) == set(TokenMeasureState) diff --git a/tests/report/test_deck_views_js.py b/tests/report/test_deck_views_js.py index 84089d5584..1ffd893bc5 100644 --- a/tests/report/test_deck_views_js.py +++ b/tests/report/test_deck_views_js.py @@ -18,9 +18,11 @@ globalThis.window = {location, addEventListener() {}}; function deckNode(tag, attrs = {}, children = null) { - const node = {tag, attrs: {...attrs}, children: [], events: {}}; + const node = {tag, attrs: {...attrs}, children: [], events: {}, hidden: attrs.hidden === true, + textContent: ""}; node.appendChild = child => { node.children.push(child); return child; }; node.addEventListener = (name, callback) => { node.events[name] = callback; }; + node.setAttribute = (name, value) => { node.attrs[name] = String(value); }; if (Array.isArray(children)) node.children.push(...children); else if (children !== null && children !== undefined) node.children.push(children); return node; @@ -58,6 +60,14 @@ return null; } +function deckFindAll(node, predicate) { + if (node === null || node === undefined) return []; + if (Array.isArray(node)) return node.flatMap(child => deckFindAll(child, predicate)); + if (typeof node !== "object") return []; + return [...(predicate(node) ? [node] : []), ...node.children.flatMap(child => + deckFindAll(child, predicate))]; +} + function deckSnapshot(node) { if (node === null || node === undefined || typeof node !== "object") return node; return { @@ -65,11 +75,22 @@ attrs: node.attrs, text: deckText(node), links: deckLinks(node), - review: (() => { - const control = deckFind(node, child => - child.attrs && child.attrs["data-review-toggle"] !== undefined); - return control ? {attrs: control.attrs, text: deckText(control)} : null; - })() + reviews: deckFindAll(node, child => + child.attrs && child.attrs["data-review-signal"] !== undefined).map(signal => { + const control = deckFind(signal, child => + child.attrs && child.attrs["data-review-flag"] !== undefined); + const marker = deckFind(signal, child => + child.attrs && child.attrs.class === "view-review__marker"); + const definitions = deckFind(signal, child => + child.attrs && child.attrs.class === "view-review__definitions"); + return { + attrs: control.attrs, + markerHidden: marker.hidden, + markerText: deckText(marker), + linksBeforeFlag: definitions !== null && signal.children.indexOf(definitions) < + signal.children.indexOf(control) + }; + }) }; } @@ -103,13 +124,17 @@ globalThis.__deckLastTree = __deckViews[id](context); return deckSnapshot(__deckLastTree); }, - clickReview() { - const control = deckFind(__deckLastTree, - child => child.attrs && child.attrs["data-review-toggle"] !== undefined); - if (!control || !control.events.click) throw new Error("review toggle has no click handler"); + clickReview(index = 0) { + const signals = deckFindAll(__deckLastTree, + child => child.attrs && child.attrs["data-review-signal"] !== undefined); + const control = deckFind(signals[index], + child => child.attrs && child.attrs["data-review-flag"] !== undefined); + if (!control || !control.events.click) throw new Error("review flag has no click handler"); control.events.click(); - return location.hash; - } + return deckSnapshot(__deckLastTree).reviews[index]; + }, + snapshot() { return deckSnapshot(__deckLastTree); }, + currentHash() { return location.hash; } }; """ @@ -124,11 +149,10 @@ def _load_renderers(deck_asset: Any) -> MiniRacer: def _view_context(view_id: str, *, definitions_available: bool = True) -> dict[str, Any]: - definition_state = "available" if definitions_available else "unavailable" roles = ("reader", "reviewer") definitions = { role: { - "state": definition_state, + "state": "available" if definitions_available or role == "reader" else "unavailable", "description": f"Definition for {role}: definition-first marker", "body": f"{role} role instructions", "tools": ["read_file"], @@ -226,54 +250,66 @@ def test_each_prepared_view_renders_its_population_and_cohort_links(deck_asset: assert any( "harness=codex" in href and "provider=openai" in href for href in rendered["links"] ) + if view_id in {"efficiency", "skill", "role"}: + assert rendered["reviews"] + assert all(control["linksBeforeFlag"] for control in rendered["reviews"]) role_view = context.call("DeckTest.render", "role", _view_context("role")) assert "L0" in role_view["text"] assert "spawning skill level" in role_view["text"].lower() assert "observed-model" in role_view["text"] assert "Claude Code declared tools" in role_view["text"] assert "Codex read-only tools" in role_view["text"] + efficiency_view = context.call( + "DeckTest.render", "efficiency", _view_context("efficiency") + ) + assert "2 skill runs" in efficiency_view["text"] + assert "2 child invocations" in efficiency_view["text"] finally: context.close() -def test_review_toggle_requires_every_contributor_definition_and_resets_route( +def test_manual_review_flags_require_definitions_and_reset_on_rerender( deck_asset: Any, ) -> None: context = _load_renderers(deck_asset) try: available = context.call("DeckTest.render", "efficiency", _view_context("efficiency")) - assert available["review"] is not None - assert "disabled" not in available["review"]["attrs"] + assert len(available["reviews"]) >= 3 + assert all( + control["attrs"].get("disabled") is not True for control in available["reviews"] + ) + assert all( + control["attrs"].get("aria-pressed") == "false" for control in available["reviews"] + ) + assert all(control["markerHidden"] for control in available["reviews"]) + assert all(control["linksBeforeFlag"] for control in available["reviews"]) assert available["text"].index("definition-first marker") < available["text"].index( - "Show reviewable signals" + "Flag for review" ) + initial_hash = context.call("DeckTest.currentHash") + + flagged = context.call("DeckTest.clickReview", 0) + after_one_flag = context.call("DeckTest.snapshot")["reviews"] + assert flagged["attrs"]["aria-pressed"] == "true" + assert flagged["markerHidden"] is False + assert flagged["markerText"] == "Flagged for review" + assert after_one_flag[1]["attrs"]["aria-pressed"] == "false" + assert context.call("DeckTest.currentHash") == initial_hash unavailable = context.call( "DeckTest.render", "efficiency", _view_context("efficiency", definitions_available=False), ) - assert unavailable["review"] is not None - assert "disabled" in unavailable["review"]["attrs"] - assert "definition" in unavailable["text"].lower() - - context.call("DeckTest.render", "efficiency", _view_context("efficiency")) - enabled_hash = context.call("DeckTest.clickReview") - enabled_route = context.call("DeckCore.decodeRoute", enabled_hash, "cohort") - assert enabled_route["params"].get("review") - assert enabled_route["params"].get("harness") == ["codex"] - assert enabled_route["params"].get("provider") == ["openai"] + assert unavailable["reviews"] + assert all("disabled" in control["attrs"] for control in unavailable["reviews"]) + assert all(control["linksBeforeFlag"] for control in unavailable["reviews"]) + assert "Contributor definitions unavailable" in unavailable["text"] - context.call( - "DeckTest.render", - "efficiency", - {**_view_context("efficiency"), "route": enabled_route}, - ) - reset_hash = context.call("DeckTest.clickReview") - reset_route = context.call("DeckCore.decodeRoute", reset_hash, "cohort") - assert "review" not in reset_route["params"] - assert reset_route["params"].get("harness") == ["codex"] - assert reset_route["params"].get("provider") == ["openai"] + reset = context.call("DeckTest.render", "efficiency", _view_context("efficiency")) + assert all(control["attrs"]["aria-pressed"] == "false" for control in reset["reviews"]) + assert all(control["markerHidden"] for control in reset["reviews"]) + assert "review" not in _view_context("efficiency")["route"]["params"] finally: context.close() From 22c32d878a763403cbda9b76bc4e7d60ecc9e597 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 16:59:21 -0700 Subject: [PATCH 11/22] fix: wrap embedded role definitions for screen and print --- src/autoskillit/assets/deck/views/role.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autoskillit/assets/deck/views/role.js b/src/autoskillit/assets/deck/views/role.js index f373c1b6d4..557a26349f 100644 --- a/src/autoskillit/assets/deck/views/role.js +++ b/src/autoskillit/assets/deck/views/role.js @@ -91,7 +91,7 @@ DeckShell.registerView("role", ctx => { const definition = roleName == null ? null : definitions[roleName]; const available = definition?.state === "available"; - const definitionCard = roleName == null ? null : ctx.el("section", {class: "card"}, [ + const definitionCard = roleName == null ? null : ctx.el("section", {class: "card view-definition"}, [ ctx.el("h2", {}, "Role definition · " + roleName), available ? ctx.el("p", {}, definition.description ?? "Role definition loaded.") : ctx.el("p", {class: "view-review__reason"}, "Definition unavailable for " + roleName + "."), From 10b2076aa4c063ab72df99cc355ec6504cd9854a Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 17:09:15 -0700 Subject: [PATCH 12/22] fix: fold Codex request snapshots and canonicalize prepared level selection --- src/autoskillit/assets/deck/core.js | 9 +++++--- .../evidence/_native_child_projection.py | 23 +++++++++++++++++++ tests/_test_filter.py | 2 -- tests/infra/test_plugin_source_ratchets.py | 8 +++---- tests/report/test_deck_views_js.py | 2 +- tests/test_test_filter_core_cascade.py | 2 +- 6 files changed, 35 insertions(+), 11 deletions(-) diff --git a/src/autoskillit/assets/deck/core.js b/src/autoskillit/assets/deck/core.js index f2930f7cb2..229032aff3 100644 --- a/src/autoskillit/assets/deck/core.js +++ b/src/autoskillit/assets/deck/core.js @@ -136,7 +136,9 @@ globalThis.DeckCore = (() => { function effectiveSelection(chips, selected) { const live = chips.filter(c => c.state === "live").map(c => c.key); - if (selected == null) return {keys: live, dropped: [], widened: false}; + if (selected == null || selected.length === 0) { + return {keys: live, dropped: [], widened: false}; + } const keys = live.filter(k => selected.includes(k)); const dropped = selected.filter(k => !live.includes(k)); return {keys, dropped, widened: false}; @@ -178,9 +180,10 @@ globalThis.DeckCore = (() => { if (!window.chip) return []; const inWindow = blocks.filter(block => block.window === window.chip.key); const domain = [...new Set(inWindow.flatMap(block => block.levels))]; - const requested = params.level == null ? domain : level.keys + const selectedLevels = new Set(level.keys .map(key => (chips.level ?? []).find(chip => chip.key === key)?.match) - .filter(value => domain.includes(value)); + ); + const requested = domain.filter(value => selectedLevels.has(value)); if (!requested.length) return []; const block = inWindow.find(item => sameValues(item.levels, requested)); return block?.rows ?? []; diff --git a/src/autoskillit/execution/evidence/_native_child_projection.py b/src/autoskillit/execution/evidence/_native_child_projection.py index 80b95192ca..af20c85fd3 100644 --- a/src/autoskillit/execution/evidence/_native_child_projection.py +++ b/src/autoskillit/execution/evidence/_native_child_projection.py @@ -322,6 +322,29 @@ def _codex_usage_measures( rows: Sequence[TurnTokenEntry], provider: str ) -> dict[str, SerializedTokenMeasure]: rows = merge_turn_usage(rows) + request_positions: dict[tuple[str, str, str], int] = {} + folded_rows: list[TurnTokenEntry] = [] + for row in rows: + request_id = row["request_id"] + identity = (row["backend"], row["provider_used"], request_id or "") + if not request_id or identity not in request_positions: + folded_rows.append(row.copy()) + if request_id: + request_positions[identity] = len(folded_rows) - 1 + continue + + current = folded_rows[request_positions[identity]] + for field in ( + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cache_creation_tokens", + "context_window_tokens", + ): + if row[field] is not None: + current[field] = row[field] + + rows = folded_rows measures: dict[str, SerializedTokenMeasure] = {} for field in CANONICAL_ACCOUNTING_FIELDS: values = [row.get(field) for row in rows] diff --git a/tests/_test_filter.py b/tests/_test_filter.py index b4f16e98b0..f54327234e 100644 --- a/tests/_test_filter.py +++ b/tests/_test_filter.py @@ -1507,8 +1507,6 @@ class SourceMapRejection(enum.StrEnum): # file-level: exercises the hook-authored child-outcome snapshot module # directly (issue #4623) via autoskillit.hooks's public re-export. "execution/test_child_outcomes.py", - # file-level: report-index refresh tests seed native child snapshots through hooks. - "execution/test_report_index.py", # hook_registry imports from autoskillit.hooks at module scope # (HOOK_REGISTRY, generate_hooks_json) for the registry init wiring. "hook_registry", diff --git a/tests/infra/test_plugin_source_ratchets.py b/tests/infra/test_plugin_source_ratchets.py index 1c6e1c5b7c..cf82e52067 100644 --- a/tests/infra/test_plugin_source_ratchets.py +++ b/tests/infra/test_plugin_source_ratchets.py @@ -51,12 +51,12 @@ PLUGIN_MUTATION_ALLOWLIST: dict[tuple[str, str, str], tuple[int, str]] = { ("execution/report_index.py", "_update", "(index_dir / _ROWS_FILE).unlink"): ( - 1, - "Rebuild removes derived rows while holding the report-index writer lease.", + 2, + "Rebuilds and incompatible generations remove derived rows under the writer lease.", ), ("execution/report_index.py", "_update", "(index_dir / _STATE_FILE).unlink"): ( - 1, - "Rebuild removes the derived watermark while holding the report-index writer lease.", + 2, + "Rebuilds and incompatible generations remove the watermark under the writer lease.", ), ( "server/tools/tools_kitchen/_close_kitchen.py", diff --git a/tests/report/test_deck_views_js.py b/tests/report/test_deck_views_js.py index 1ffd893bc5..70bd32ea77 100644 --- a/tests/report/test_deck_views_js.py +++ b/tests/report/test_deck_views_js.py @@ -292,7 +292,7 @@ def test_manual_review_flags_require_definitions_and_reset_on_rerender( after_one_flag = context.call("DeckTest.snapshot")["reviews"] assert flagged["attrs"]["aria-pressed"] == "true" assert flagged["markerHidden"] is False - assert flagged["markerText"] == "Flagged for review" + assert flagged["markerText"].strip() == "Flagged for review" assert after_one_flag[1]["attrs"]["aria-pressed"] == "false" assert context.call("DeckTest.currentHash") == initial_hash diff --git a/tests/test_test_filter_core_cascade.py b/tests/test_test_filter_core_cascade.py index 9714f4e5ba..1fe52a7c01 100644 --- a/tests/test_test_filter_core_cascade.py +++ b/tests/test_test_filter_core_cascade.py @@ -349,7 +349,7 @@ def test_type_capture_cascade(self) -> None: def test_type_token_cascade(self) -> None: assert MODULE_CASCADE_CORE["_type_token"] == frozenset( - {"cli", "core", "execution", "fleet", "pipeline"} + {"cli", "core", "execution", "fleet", "pipeline", "report"} ) def test_type_protocols_backend_cascade(self) -> None: From 3c1f7d65b8bd34e6116ac173fed0f681e9174089 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 5 Oct 2026 21:08:09 -0700 Subject: [PATCH 13/22] test: verify core deck view scripts in the built wheel --- tests/report/test_deck_html.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/report/test_deck_html.py b/tests/report/test_deck_html.py index f239a77194..8f25fd71a5 100644 --- a/tests/report/test_deck_html.py +++ b/tests/report/test_deck_html.py @@ -4,9 +4,12 @@ import hashlib import json import re +import subprocess from html.parser import HTMLParser +from pathlib import Path from types import SimpleNamespace from typing import Any +from zipfile import ZipFile import pytest @@ -20,7 +23,7 @@ from autoskillit.report.deck._registry import DECK_VIEWS from tests.report._fixtures import DECK_GENERATED_AT, session_row, subagent_row, token_measure -pytestmark = [pytest.mark.small] +pytestmark = [pytest.mark.medium] class _HTMLCollector(HTMLParser): @@ -206,6 +209,28 @@ def test_deck_asset_directory_contains_only_registered_shell_assets() -> None: assert actual == {"deck.html", "deck.css", "core.js", "shell.js"} | view_assets +def test_built_wheel_contains_the_four_new_view_scripts(tmp_path: Path) -> None: + # REQ-108/REQ-158: isolate the plan's wheel-check directory per test worker. + wheel_out = tmp_path / "wheel-check" + result = subprocess.run( + ["uv", "build", "--wheel", "--out-dir", str(wheel_out), "--offline"], + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + text=True, + timeout=120, + ) + assert result.returncode == 0, (result.stdout + result.stderr)[-4000:] + wheels = list(wheel_out.glob("*.whl")) + assert len(wheels) == 1, f"Expected one built wheel, found {wheels}" + with ZipFile(wheels[0]) as wheel: + required = { + f"autoskillit/assets/deck/views/{view}.js" + for view in ("spend", "efficiency", "skill", "role") + } + missing = required - set(wheel.namelist()) + assert not missing, f"Built wheel is missing view scripts: {sorted(missing)}" + + def test_embedded_cohort_chips_cover_all_states(deck_payload: dict[str, Any]) -> None: collector = _collect(render_deck_html(deck_payload)) data = next(source for attrs, source in collector.scripts if attrs.get("id") == "deck-data") From dd521c5766518594210873e0aaf80d3913bac706 Mon Sep 17 00:00:00 2001 From: Trecek Date: Tue, 6 Oct 2026 08:10:54 -0700 Subject: [PATCH 14/22] fix(review): share tool mix rendering across deck views --- src/autoskillit/assets/deck/core.js | 16 +++++++++++++++- .../assets/deck/views/efficiency.js | 19 ++----------------- src/autoskillit/assets/deck/views/role.js | 17 +---------------- 3 files changed, 18 insertions(+), 34 deletions(-) diff --git a/src/autoskillit/assets/deck/core.js b/src/autoskillit/assets/deck/core.js index 229032aff3..17910ea0bc 100644 --- a/src/autoskillit/assets/deck/core.js +++ b/src/autoskillit/assets/deck/core.js @@ -80,6 +80,20 @@ globalThis.DeckCore = (() => { eligibility.reason ? ctx.el("p", {class: "view-review__reason"}, eligibility.reason) : null ]); } + function toolMix(ctx, ratios = {}) { + const tools = Object.entries(ratios.tool_mix ?? {}); + return tools.length ? ctx.el("div", {class: "view-pills"}, tools.map(([tool, ratio]) => { + const value = formatRatio(ratio, true); + return ctx.el("span", {}, [ + ctx.el("span", {}, tool), + ctx.el("span", {}, " · "), + value == null ? ctx.availabilityCell(ratio) : ctx.el("span", {}, value), + ctx.el("small", {class: "view-sample"}, ratioSample(ratio)), + reviewSignal(ctx, ratio) + ]); + })) : ctx.el("span", {class: "view-empty"}, "No observed tool calls"); + } + const decodeTable = ({columns, rows}) => rows.map(r => Object.fromEntries(columns.map((c, i) => [c, r[i]]))); @@ -339,6 +353,6 @@ globalThis.DeckCore = (() => { return Object.freeze({decodeTable, encodeRoute, decodeRoute, hrefFor, effectiveSelection, toggleSelection, windowSelection, selectPrepared, filterRows, populationSentence, summarizePairs, sortRows, parseSort, chipPresentation, availabilityPresentation, - barLayout, formatCount, formatRatio, ratioSample, reviewEligibility, reviewSignal, formatDate, + barLayout, formatCount, formatRatio, ratioSample, reviewEligibility, reviewSignal, toolMix, formatDate, CHIP_STATES, DAY_MS}); })(); diff --git a/src/autoskillit/assets/deck/views/efficiency.js b/src/autoskillit/assets/deck/views/efficiency.js index c6cffd5dfe..7e28bd1cf5 100644 --- a/src/autoskillit/assets/deck/views/efficiency.js +++ b/src/autoskillit/assets/deck/views/efficiency.js @@ -29,21 +29,6 @@ DeckShell.registerView("efficiency", ctx => { DeckCore.reviewSignal(ctx, ratio)]); } - function toolMix(ratios = {}) { - const entries = Object.entries(ratios.tool_mix ?? {}); - return entries.length ? ctx.el("div", {class: "view-pills"}, entries.map(([tool, ratio]) => { - const value = DeckCore.formatRatio(ratio, true); - return ctx.el("span", {}, [ - ctx.el("span", {}, tool), - ctx.el("span", {}, " · "), - value == null ? ctx.availabilityCell(ratio) : ctx.el("span", {}, value), - ctx.el("small", {class: "view-sample"}, DeckCore.ratioSample(ratio)), - DeckCore.reviewSignal(ctx, ratio) - ]); - })) : - ctx.el("span", {class: "view-empty"}, "No observed tool calls"); - } - const definitions = ctx.definitions ?? {}; const contributorRoles = new Set((ctx.relationships ?? []).map(edge => edge.role)); for (const row of skillRows) { @@ -79,7 +64,7 @@ DeckShell.registerView("efficiency", ctx => { ratioCell(row.ratios?.input_output)}, {key: "cache_share", label: "Cache-read share", cell: row => ratioCell(row.ratios?.cache_share, true)}, - {key: "tool_mix", label: "Observed tool mix", cell: row => toolMix(row.ratios)} + {key: "tool_mix", label: "Observed tool mix", cell: row => DeckCore.toolMix(ctx, row.ratios)} ], rows: skillRows, defaultSort: {key: "skill", dir: "asc"} @@ -95,7 +80,7 @@ DeckShell.registerView("efficiency", ctx => { ratioCell(row.ratios?.input_output)}, {key: "cache_share", label: "Cache-read share", cell: row => ratioCell(row.ratios?.cache_share, true)}, - {key: "tool_mix", label: "Observed tool mix", cell: row => toolMix(row.ratios)} + {key: "tool_mix", label: "Observed tool mix", cell: row => DeckCore.toolMix(ctx, row.ratios)} ], rows: roleRows, defaultSort: {key: "role", dir: "asc"} diff --git a/src/autoskillit/assets/deck/views/role.js b/src/autoskillit/assets/deck/views/role.js index 557a26349f..78d3a9584c 100644 --- a/src/autoskillit/assets/deck/views/role.js +++ b/src/autoskillit/assets/deck/views/role.js @@ -27,21 +27,6 @@ DeckShell.registerView("role", ctx => { ]); } - function toolMix(ratios = {}) { - const tools = Object.entries(ratios.tool_mix ?? {}); - return tools.length ? ctx.el("div", {class: "view-pills"}, tools.map(([tool, ratio]) => { - const value = DeckCore.formatRatio(ratio, true); - return ctx.el("span", {}, [ - ctx.el("span", {}, tool), - ctx.el("span", {}, " · "), - value == null ? ctx.availabilityCell(ratio) : ctx.el("span", {}, value), - ctx.el("small", {class: "view-sample"}, DeckCore.ratioSample(ratio)), - DeckCore.reviewSignal(ctx, ratio) - ]); - })) : - ctx.el("span", {class: "view-empty"}, "No observed tool calls"); - } - const spawningLevels = (ctx.selection?.level?.keys ?? []).map(key => (ctx.chips?.level ?? []).find(chip => chip.key === key)?.label ?? key); const edges = ctx.relationships ?? []; @@ -83,7 +68,7 @@ DeckShell.registerView("role", ctx => { ratioCell(row.ratios?.input_output)}, {key: "cache_share", label: "Cache-read share", cell: row => ratioCell(row.ratios?.cache_share, true)}, - {key: "tool_mix", label: "Observed tool mix", cell: row => toolMix(row.ratios)} + {key: "tool_mix", label: "Observed tool mix", cell: row => DeckCore.toolMix(ctx, row.ratios)} ], rows: harnessRows, defaultSort: {key: "harness", dir: "asc"} From 3f281bda69d818cb4d65c942d87058a0edb82f09 Mon Sep 17 00:00:00 2001 From: Trecek Date: Tue, 6 Oct 2026 08:12:05 -0700 Subject: [PATCH 15/22] fix(review): share ratio cells and contributor definition cards --- src/autoskillit/assets/deck/core.js | 29 +++++++++- .../assets/deck/views/efficiency.js | 32 ++-------- src/autoskillit/assets/deck/views/role.js | 25 ++------ src/autoskillit/assets/deck/views/skill.js | 58 +++++-------------- 4 files changed, 56 insertions(+), 88 deletions(-) diff --git a/src/autoskillit/assets/deck/core.js b/src/autoskillit/assets/deck/core.js index 17910ea0bc..5666ebe9fd 100644 --- a/src/autoskillit/assets/deck/core.js +++ b/src/autoskillit/assets/deck/core.js @@ -80,6 +80,32 @@ globalThis.DeckCore = (() => { eligibility.reason ? ctx.el("p", {class: "view-review__reason"}, eligibility.reason) : null ]); } + function measureCell(ctx, measure) { + return ctx.availabilityCell(measure ?? {state: "unavailable"}); + } + + function ratioCell(ctx, ratio, percent = false) { + const value = formatRatio(ratio, percent); + return ctx.el("div", {class: "view-measure"}, [ + value == null ? measureCell(ctx, ratio) : ctx.el("span", {}, value), + ctx.el("small", {class: "view-sample"}, ratioSample(ratio)), + reviewSignal(ctx, ratio) + ]); + } + + function definitionCard(ctx, role) { + const definition = ctx.definitions?.[role]; + const available = definition?.state === "available"; + return ctx.el("article", {class: "view-definition"}, [ + ctx.el("h3", {}, ctx.entityLink(role, {view: "role", entity: role})), + available ? ctx.el("p", {}, definition.description ?? "Role definition loaded.") : + ctx.el("p", {class: "view-review__reason"}, "Definition unavailable for " + role + "."), + available ? ctx.el("p", {}, "Declared tools: " + + ((definition.tools ?? []).join(", ") || "none recorded")) : null, + available ? ctx.el("pre", {}, definition.body ?? "") : null + ]); + } + function toolMix(ctx, ratios = {}) { const tools = Object.entries(ratios.tool_mix ?? {}); return tools.length ? ctx.el("div", {class: "view-pills"}, tools.map(([tool, ratio]) => { @@ -353,6 +379,7 @@ globalThis.DeckCore = (() => { return Object.freeze({decodeTable, encodeRoute, decodeRoute, hrefFor, effectiveSelection, toggleSelection, windowSelection, selectPrepared, filterRows, populationSentence, summarizePairs, sortRows, parseSort, chipPresentation, availabilityPresentation, - barLayout, formatCount, formatRatio, ratioSample, reviewEligibility, reviewSignal, toolMix, formatDate, + barLayout, formatCount, formatRatio, ratioSample, reviewEligibility, reviewSignal, + measureCell, ratioCell, definitionCard, toolMix, formatDate, CHIP_STATES, DAY_MS}); })(); diff --git a/src/autoskillit/assets/deck/views/efficiency.js b/src/autoskillit/assets/deck/views/efficiency.js index 7e28bd1cf5..7dad0ff776 100644 --- a/src/autoskillit/assets/deck/views/efficiency.js +++ b/src/autoskillit/assets/deck/views/efficiency.js @@ -20,16 +20,6 @@ DeckShell.registerView("efficiency", ctx => { return values; } - function ratioCell(ratio, percent = false) { - const text = DeckCore.formatRatio(ratio, percent); - const value = text == null ? ctx.availabilityCell(ratio ?? {state: "unavailable"}) : - ctx.el("span", {class: "view-measure"}, text); - return ctx.el("div", {class: "view-measure"}, [value, - ctx.el("small", {class: "view-sample"}, DeckCore.ratioSample(ratio)), - DeckCore.reviewSignal(ctx, ratio)]); - } - - const definitions = ctx.definitions ?? {}; const contributorRoles = new Set((ctx.relationships ?? []).map(edge => edge.role)); for (const row of skillRows) { for (const signal of ratioSignals(row.ratios)) { @@ -41,18 +31,8 @@ DeckShell.registerView("efficiency", ctx => { for (const role of signal.ratio.definition_roles ?? []) contributorRoles.add(role); } } - const definitionCards = [...contributorRoles].sort().map(role => { - const definition = definitions[role]; - const available = definition?.state === "available"; - return ctx.el("article", {class: "view-definition"}, [ - ctx.el("h3", {}, ctx.entityLink(role, {view: "role", entity: role})), - available ? ctx.el("p", {}, definition.description ?? "Role definition loaded.") : - ctx.el("p", {class: "view-review__reason"}, "Definition unavailable for " + role + "."), - available ? ctx.el("p", {}, "Declared tools: " + - ((definition.tools ?? []).join(", ") || "none recorded")) : null, - available ? ctx.el("pre", {}, definition.body ?? "") : null - ]); - }); + const definitionCards = [...contributorRoles].sort().map(role => + DeckCore.definitionCard(ctx, role)); const skillTable = skillRows.length ? ctx.sortableTable({ columns: [ {key: "skill", label: "Skill", cell: row => ctx.entityLink(row.skill, { @@ -61,9 +41,9 @@ DeckShell.registerView("efficiency", ctx => { {key: "harness", label: "Harness"}, {key: "provider", label: "Provider"}, {key: "input_output", label: "Input / output", cell: row => - ratioCell(row.ratios?.input_output)}, + DeckCore.ratioCell(ctx, row.ratios?.input_output)}, {key: "cache_share", label: "Cache-read share", cell: row => - ratioCell(row.ratios?.cache_share, true)}, + DeckCore.ratioCell(ctx, row.ratios?.cache_share, true)}, {key: "tool_mix", label: "Observed tool mix", cell: row => DeckCore.toolMix(ctx, row.ratios)} ], rows: skillRows, @@ -77,9 +57,9 @@ DeckShell.registerView("efficiency", ctx => { {key: "harness", label: "Harness"}, {key: "provider", label: "Provider"}, {key: "input_output", label: "Input / output", cell: row => - ratioCell(row.ratios?.input_output)}, + DeckCore.ratioCell(ctx, row.ratios?.input_output)}, {key: "cache_share", label: "Cache-read share", cell: row => - ratioCell(row.ratios?.cache_share, true)}, + DeckCore.ratioCell(ctx, row.ratios?.cache_share, true)}, {key: "tool_mix", label: "Observed tool mix", cell: row => DeckCore.toolMix(ctx, row.ratios)} ], rows: roleRows, diff --git a/src/autoskillit/assets/deck/views/role.js b/src/autoskillit/assets/deck/views/role.js index 78d3a9584c..50200f99ec 100644 --- a/src/autoskillit/assets/deck/views/role.js +++ b/src/autoskillit/assets/deck/views/role.js @@ -14,19 +14,6 @@ DeckShell.registerView("role", ctx => { hasMetrics: rows.some(row => row.role === role) })) : rows.filter(row => row.role === roleName); - function measureCell(measure) { - return ctx.availabilityCell(measure ?? {state: "unavailable"}); - } - - function ratioCell(ratio, percent = false) { - const value = DeckCore.formatRatio(ratio, percent); - return ctx.el("div", {class: "view-measure"}, [ - value == null ? measureCell(ratio) : ctx.el("span", {}, value), - ctx.el("small", {class: "view-sample"}, DeckCore.ratioSample(ratio)), - DeckCore.reviewSignal(ctx, ratio) - ]); - } - const spawningLevels = (ctx.selection?.level?.keys ?? []).map(key => (ctx.chips?.level ?? []).find(chip => chip.key === key)?.label ?? key); const edges = ctx.relationships ?? []; @@ -57,17 +44,17 @@ DeckShell.registerView("role", ctx => { {key: "models", label: "Observed models", cell: row => (row.models ?? []).join(", ") || "No model recorded"}, {key: "input_tokens", label: "Input tokens", numeric: true, - cell: row => measureCell(row.measures?.input_tokens)}, + cell: row => DeckCore.measureCell(ctx, row.measures?.input_tokens)}, {key: "output_tokens", label: "Output tokens", numeric: true, - cell: row => measureCell(row.measures?.output_tokens)}, + cell: row => DeckCore.measureCell(ctx, row.measures?.output_tokens)}, {key: "cache_read_tokens", label: "Cache-read tokens", numeric: true, - cell: row => measureCell(row.measures?.cache_read_tokens)}, + cell: row => DeckCore.measureCell(ctx, row.measures?.cache_read_tokens)}, {key: "cache_write_tokens", label: "Cache-write tokens", numeric: true, - cell: row => measureCell(row.measures?.cache_write_tokens)}, + cell: row => DeckCore.measureCell(ctx, row.measures?.cache_write_tokens)}, {key: "input_output", label: "Input / output", cell: row => - ratioCell(row.ratios?.input_output)}, + DeckCore.ratioCell(ctx, row.ratios?.input_output)}, {key: "cache_share", label: "Cache-read share", cell: row => - ratioCell(row.ratios?.cache_share, true)}, + DeckCore.ratioCell(ctx, row.ratios?.cache_share, true)}, {key: "tool_mix", label: "Observed tool mix", cell: row => DeckCore.toolMix(ctx, row.ratios)} ], rows: harnessRows, diff --git a/src/autoskillit/assets/deck/views/skill.js b/src/autoskillit/assets/deck/views/skill.js index e0421bb2d0..09e809c0f5 100644 --- a/src/autoskillit/assets/deck/views/skill.js +++ b/src/autoskillit/assets/deck/views/skill.js @@ -6,39 +6,13 @@ DeckShell.registerView("skill", ctx => { const skillNames = [...new Set(selected.map(row => row.skill))]; if (routeSkill != null && !skillNames.includes(routeSkill)) skillNames.unshift(routeSkill); - function measureCell(measure) { - return ctx.availabilityCell(measure ?? {state: "unavailable"}); - } - - function ratioCell(ratio, percent = false) { - const value = DeckCore.formatRatio(ratio, percent); - return ctx.el("div", {class: "view-measure"}, [ - value == null ? measureCell(ratio) : ctx.el("span", {}, value), - ctx.el("small", {class: "view-sample"}, DeckCore.ratioSample(ratio)), - DeckCore.reviewSignal(ctx, ratio) - ]); - } - - function definitionCard(role) { - const definition = ctx.definitions?.[role]; - return ctx.el("article", {class: "view-definition"}, [ - ctx.el("h3", {}, ctx.entityLink(role, {view: "role", entity: role})), - definition?.state === "available" ? - ctx.el("p", {}, definition.description ?? "Role definition loaded.") : - ctx.el("p", {class: "view-review__reason"}, "Definition unavailable for " + role + "."), - definition?.state === "available" ? ctx.el("p", {}, "Declared tools: " + - ((definition.tools ?? []).join(", ") || "none recorded")) : null, - definition?.state === "available" ? ctx.el("pre", {}, definition.body ?? "") : null - ]); - } - const roleNames = new Set(relationships.map(edge => edge.role)); for (const row of selected) { for (const child of row.child_roles ?? []) roleNames.add(child.role); for (const role of row.ratios?.input_output?.definition_roles ?? []) roleNames.add(role); for (const role of row.ratios?.cache_share?.definition_roles ?? []) roleNames.add(role); } - const definitions = [...roleNames].sort().map(definitionCard); + const definitions = [...roleNames].sort().map(role => DeckCore.definitionCard(ctx, role)); const skillTable = rows.length ? ctx.sortableTable({ columns: [ @@ -48,19 +22,19 @@ DeckShell.registerView("skill", ctx => { {key: "harness", label: "Harness"}, {key: "provider", label: "Provider"}, {key: "input_tokens", label: "Input tokens", numeric: true, - cell: row => measureCell(row.measures?.input_tokens)}, + cell: row => DeckCore.measureCell(ctx, row.measures?.input_tokens)}, {key: "output_tokens", label: "Output tokens", numeric: true, - cell: row => measureCell(row.measures?.output_tokens)}, + cell: row => DeckCore.measureCell(ctx, row.measures?.output_tokens)}, {key: "cache_read_tokens", label: "Cache-read tokens", numeric: true, - cell: row => measureCell(row.measures?.cache_read_tokens)}, + cell: row => DeckCore.measureCell(ctx, row.measures?.cache_read_tokens)}, {key: "cache_write_tokens", label: "Cache-write tokens", numeric: true, - cell: row => measureCell(row.measures?.cache_write_tokens)}, + cell: row => DeckCore.measureCell(ctx, row.measures?.cache_write_tokens)}, {key: "input_output", label: "Input / output", cell: row => - ratioCell(row.ratios?.input_output)}, + DeckCore.ratioCell(ctx, row.ratios?.input_output)}, {key: "cache_share", label: "Cache-read share", cell: row => - ratioCell(row.ratios?.cache_share, true)}, + DeckCore.ratioCell(ctx, row.ratios?.cache_share, true)}, {key: "exact_retransmission", label: "Exact prompt retransmission", cell: row => - measureCell(row.exact_retransmission)} + DeckCore.measureCell(ctx, row.exact_retransmission)} ], rows, defaultSort: {key: "skill", dir: "asc"} @@ -88,13 +62,13 @@ DeckShell.registerView("skill", ctx => { {key: "recipe", label: "Recipe"}, {key: "step", label: "Step"}, {key: "input_tokens", label: "Input tokens", cell: row => - measureCell(row.measures?.input_tokens)}, + DeckCore.measureCell(ctx, row.measures?.input_tokens)}, {key: "output_tokens", label: "Output tokens", cell: row => - measureCell(row.measures?.output_tokens)}, + DeckCore.measureCell(ctx, row.measures?.output_tokens)}, {key: "cache_read_tokens", label: "Cache-read tokens", cell: row => - measureCell(row.measures?.cache_read_tokens)}, + DeckCore.measureCell(ctx, row.measures?.cache_read_tokens)}, {key: "exact_retransmission", label: "Exact prompt retransmission", cell: row => - measureCell(row.exact_retransmission)} + DeckCore.measureCell(ctx, row.exact_retransmission)} ], rows: steps, defaultSort: {key: "recipe", dir: "asc"} @@ -112,13 +86,13 @@ DeckShell.registerView("skill", ctx => { {key: "harness", label: "Harness"}, {key: "provider", label: "Provider"}, {key: "input_tokens", label: "Child input tokens", cell: row => - measureCell(row.measures?.input_tokens)}, + DeckCore.measureCell(ctx, row.measures?.input_tokens)}, {key: "output_tokens", label: "Child output tokens", cell: row => - measureCell(row.measures?.output_tokens)}, + DeckCore.measureCell(ctx, row.measures?.output_tokens)}, {key: "input_output", label: "Input / output", cell: row => - ratioCell(row.ratios?.input_output)}, + DeckCore.ratioCell(ctx, row.ratios?.input_output)}, {key: "cache_share", label: "Cache-read share", cell: row => - ratioCell(row.ratios?.cache_share, true)} + DeckCore.ratioCell(ctx, row.ratios?.cache_share, true)} ], rows: childRows, defaultSort: {key: "role", dir: "asc"} From bd1babb0a12a25d8e1f05cb754eb79bcd3be707e Mon Sep 17 00:00:00 2001 From: Trecek Date: Tue, 6 Oct 2026 08:12:44 -0700 Subject: [PATCH 16/22] fix(review): share role harness projection across deck views --- src/autoskillit/assets/deck/core.js | 13 ++++++++++++- src/autoskillit/assets/deck/views/efficiency.js | 9 +-------- src/autoskillit/assets/deck/views/role.js | 9 +-------- src/autoskillit/assets/deck/views/spend.js | 7 +------ 4 files changed, 15 insertions(+), 23 deletions(-) diff --git a/src/autoskillit/assets/deck/core.js b/src/autoskillit/assets/deck/core.js index 5666ebe9fd..cbcdf8b49d 100644 --- a/src/autoskillit/assets/deck/core.js +++ b/src/autoskillit/assets/deck/core.js @@ -120,6 +120,17 @@ globalThis.DeckCore = (() => { })) : ctx.el("span", {class: "view-empty"}, "No observed tool calls"); } + function roleHarnessRows(rows) { + return rows.flatMap(row => (row.harnesses ?? []).map(cell => ({ + role: row.role, + provider: row.provider, + harness: cell.harness, + models: cell.models ?? [], + measures: cell.measures, + ratios: cell.ratios + }))); + } + const decodeTable = ({columns, rows}) => rows.map(r => Object.fromEntries(columns.map((c, i) => [c, r[i]]))); @@ -380,6 +391,6 @@ globalThis.DeckCore = (() => { toggleSelection, windowSelection, selectPrepared, filterRows, populationSentence, summarizePairs, sortRows, parseSort, chipPresentation, availabilityPresentation, barLayout, formatCount, formatRatio, ratioSample, reviewEligibility, reviewSignal, - measureCell, ratioCell, definitionCard, toolMix, formatDate, + measureCell, ratioCell, definitionCard, toolMix, roleHarnessRows, formatDate, CHIP_STATES, DAY_MS}); })(); diff --git a/src/autoskillit/assets/deck/views/efficiency.js b/src/autoskillit/assets/deck/views/efficiency.js index 7dad0ff776..3f2970ca02 100644 --- a/src/autoskillit/assets/deck/views/efficiency.js +++ b/src/autoskillit/assets/deck/views/efficiency.js @@ -1,13 +1,6 @@ DeckShell.registerView("efficiency", ctx => { const skillRows = ctx.skillMetrics ?? []; - const roleRows = (ctx.roleMetrics ?? []).flatMap(row => (row.harnesses ?? []).map(cell => ({ - role: row.role, - provider: row.provider, - harness: cell.harness, - models: cell.models ?? [], - measures: cell.measures, - ratios: cell.ratios - }))); + const roleRows = DeckCore.roleHarnessRows(ctx.roleMetrics ?? []); function ratioSignals(ratios = {}) { const values = []; diff --git a/src/autoskillit/assets/deck/views/role.js b/src/autoskillit/assets/deck/views/role.js index 50200f99ec..553a635158 100644 --- a/src/autoskillit/assets/deck/views/role.js +++ b/src/autoskillit/assets/deck/views/role.js @@ -29,14 +29,7 @@ DeckShell.registerView("role", ctx => { defaultSort: {key: "skill", dir: "asc"} }) : ctx.el("p", {class: "view-empty"}, "No verified spawning-skill links are recorded."); - const harnessRows = roleRows.flatMap(row => (row.harnesses ?? []).map(cell => ({ - role: row.role, - provider: row.provider, - harness: cell.harness, - models: cell.models ?? [], - measures: cell.measures, - ratios: cell.ratios - }))); + const harnessRows = DeckCore.roleHarnessRows(roleRows); const usageTable = harnessRows.length ? ctx.sortableTable({ columns: [ {key: "harness", label: "Harness"}, diff --git a/src/autoskillit/assets/deck/views/spend.js b/src/autoskillit/assets/deck/views/spend.js index 4a85e241ef..ce4c5a0dce 100644 --- a/src/autoskillit/assets/deck/views/spend.js +++ b/src/autoskillit/assets/deck/views/spend.js @@ -8,12 +8,7 @@ DeckShell.registerView("spend", ctx => { value: amount(row.measures?.input_tokens), href: ctx.href({view: "skill", entity: row.skill}) })); - const roleCells = roleRows.flatMap(row => (row.harnesses ?? []).map(cell => ({ - role: row.role, - provider: row.provider, - harness: cell.harness, - measures: cell.measures - }))); + const roleCells = DeckCore.roleHarnessRows(roleRows); const roleChart = roleCells.map(row => ({ label: row.role + " · " + row.harness + " · " + row.provider, value: amount(row.measures?.input_tokens), From 4efb1da216d618877603e5382afe2b1a676536fc Mon Sep 17 00:00:00 2001 From: Trecek Date: Tue, 6 Oct 2026 08:15:31 -0700 Subject: [PATCH 17/22] fix(review): include found row schema in mismatch diagnostics --- src/autoskillit/execution/report_index.py | 6 ++++-- tests/execution/test_report_index.py | 6 +++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/autoskillit/execution/report_index.py b/src/autoskillit/execution/report_index.py index a260f08b13..0b875f820a 100644 --- a/src/autoskillit/execution/report_index.py +++ b/src/autoskillit/execution/report_index.py @@ -477,9 +477,11 @@ def _require_row_schema(index_dir: Path) -> None: if not rows_path.exists() and not state_path.exists(): return state = read_versioned_json(state_path, _STATE_SCHEMA_VERSION) - if state is None or state.get("row_schema_version") != REPORT_INDEX_SCHEMA_VERSION: + found = state.get("row_schema_version") if state is not None else None + if found != REPORT_INDEX_SCHEMA_VERSION: raise ValueError( - f"report index row schema mismatch: expected {REPORT_INDEX_SCHEMA_VERSION}" + f"report index row schema mismatch: expected {REPORT_INDEX_SCHEMA_VERSION}, " + f"found {found!r}" ) diff --git a/tests/execution/test_report_index.py b/tests/execution/test_report_index.py index 0333f87a54..5f6359ad18 100644 --- a/tests/execution/test_report_index.py +++ b/tests/execution/test_report_index.py @@ -645,8 +645,12 @@ def test_incompatible_row_schema_is_rejected_and_rebuilt( if remove_rows: rows_path.unlink() - with pytest.raises(ValueError, match="row schema"): + with pytest.raises(ValueError, match="row schema") as exc_info: read_report_index(index_dir) + assert str(exc_info.value) == ( + f"report index row schema mismatch: expected {REPORT_INDEX_SCHEMA_VERSION}, " + f"found {row_schema_marker!r}" + ) update_report_index(root, index_dir) From b2df6a151c76b537da32f009b885511454ffba79 Mon Sep 17 00:00:00 2001 From: Trecek Date: Tue, 6 Oct 2026 08:16:30 -0700 Subject: [PATCH 18/22] fix(review): centralize paired report index cleanup --- src/autoskillit/execution/report_index.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/autoskillit/execution/report_index.py b/src/autoskillit/execution/report_index.py index 0b875f820a..a3bfd207c0 100644 --- a/src/autoskillit/execution/report_index.py +++ b/src/autoskillit/execution/report_index.py @@ -118,12 +118,16 @@ def rebuild_report_index(log_root: Path, index_dir: Path) -> ReportIndexUpdate: return _update(log_root, index_dir, rebuild=True) +def _clear_index_files(index_dir: Path) -> None: + (index_dir / _ROWS_FILE).unlink(missing_ok=True) + (index_dir / _STATE_FILE).unlink(missing_ok=True) + + def _update(log_root: Path, index_dir: Path, *, rebuild: bool) -> ReportIndexUpdate: index_dir.mkdir(mode=0o700, parents=True, exist_ok=True) with ArtifactLease.acquire_exclusive(index_dir / _LOCK_FILE, timeout=_LEASE_TIMEOUT_SECONDS): if rebuild: - (index_dir / _ROWS_FILE).unlink(missing_ok=True) - (index_dir / _STATE_FILE).unlink(missing_ok=True) + _clear_index_files(index_dir) fingerprint = child_evidence_fingerprint(log_root) state = read_versioned_json(index_dir / _STATE_FILE, _STATE_SCHEMA_VERSION) row_schema_mismatch = ( @@ -135,8 +139,7 @@ def _update(log_root: Path, index_dir: Path, *, rebuild: bool) -> ReportIndexUpd state is not None and state.get("child_evidence_fingerprint") != fingerprint ) if row_schema_mismatch or child_evidence_changed: - (index_dir / _ROWS_FILE).unlink(missing_ok=True) - (index_dir / _STATE_FILE).unlink(missing_ok=True) + _clear_index_files(index_dir) with _RowAppender.open(index_dir, child_evidence_fingerprint=fingerprint) as appender: return _walk_into(log_root, appender) From e050c872cec65e6bac3eebea5f20427bcb0010e4 Mon Sep 17 00:00:00 2001 From: Trecek Date: Tue, 6 Oct 2026 08:17:08 -0700 Subject: [PATCH 19/22] fix(review): remove redundant token sentinel assertion --- tests/report/test_deck_payload.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/report/test_deck_payload.py b/tests/report/test_deck_payload.py index f3bcf4cfe0..0551a58392 100644 --- a/tests/report/test_deck_payload.py +++ b/tests/report/test_deck_payload.py @@ -601,7 +601,6 @@ def ratio_spy(records: Any, numerator: str, denominator: str, **kwargs: Any) -> tool_records = _tool_records(auditor_runs, parents) bash_ratio = measure_ratio(tool_records, "tool:Bash", "tool_calls") assert claude_metrics["measures"]["input_tokens"]["value"] == 105 - assert claude_metrics["measures"]["input_tokens"]["value"] != 999_999 _assert_serialized_measure( claude_metrics["measures"]["input_tokens"], child_aggregate.fields["input_tokens"] ) From 57794ad48f339e4d125f2bbba7937e292da7ad93 Mon Sep 17 00:00:00 2001 From: Trecek Date: Tue, 6 Oct 2026 08:18:08 -0700 Subject: [PATCH 20/22] fix(review): split prepared payload scenarios with shared fixture --- tests/report/test_deck_payload.py | 48 +++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/tests/report/test_deck_payload.py b/tests/report/test_deck_payload.py index 0551a58392..06d1f0970b 100644 --- a/tests/report/test_deck_payload.py +++ b/tests/report/test_deck_payload.py @@ -501,9 +501,10 @@ def test_empty_index_has_absent_declared_values_and_empty_observed_facets() -> N assert (chips["window"][-1]["key"], chips["window"][-1]["state"]) == ("all", "live") -def test_prepared_metrics_keep_source_pairs_roles_and_shared_library_accounting( +@pytest.fixture +def prepared_metrics( monkeypatch: pytest.MonkeyPatch, -) -> None: +) -> SimpleNamespace: sessions, requests, tools, children = _full_index_facts() body = "Bash runs git show to inspect the committed slice." loader_calls: list[None] = [] @@ -544,13 +545,26 @@ def ratio_spy(records: Any, numerator: str, denominator: str, **kwargs: Any) -> index_schema_version=7, ) - assert aggregate_calls and ratio_calls - assert len(loader_calls) == 1 + return SimpleNamespace( + payload=payload, + sessions=sessions, + children=children, + definition_body=body, + aggregate_calls=aggregate_calls, + ratio_calls=ratio_calls, + loader_calls=loader_calls, + ) + + +def test_prepared_skill_and_child_metrics_use_shared_accounting( + prepared_metrics: SimpleNamespace, +) -> None: + payload = prepared_metrics.payload + sessions = prepared_metrics.sessions + children = prepared_metrics.children + assert prepared_metrics.aggregate_calls and prepared_metrics.ratio_calls + assert len(prepared_metrics.loader_calls) == 1 prepared = payload["prepared"] - built_views = {view.view_id for view in DECK_VIEWS if view.planned_issue is None} - assert set(prepared["view_chips"]) == built_views - assert set(prepared["view_histories"]) == built_views - assert set(payload["tables"]) == {SESSION_TABLE, "skills", "roles"} levels: set[str | None] = {"orchestrator", "skill"} skill_block = _metric_row(prepared["skills"], window="all", levels=levels) planner = next( @@ -616,6 +630,12 @@ def ratio_spy(records: Any, numerator: str, denominator: str, **kwargs: Any) -> assert claude_metrics["ratios"]["tool_mix"]["Bash"]["review_eligible"] is True assert claude_metrics["ratios"]["tool_mix"]["Bash"]["definition_roles"] == [_AUDITOR] + +def test_prepared_ratio_states_and_definition_availability( + prepared_metrics: SimpleNamespace, +) -> None: + prepared = prepared_metrics.payload["prepared"] + role_block = _metric_row(prepared["roles"], window="all", levels={"orchestrator", "skill"}) zero_skill_block = _metric_row(prepared["skills"], window="all", levels={"skill"}) maintenance = next(row for row in zero_skill_block["rows"] if row["skill"] == "maintenance") zero_ratio = maintenance["ratios"]["input_output"] @@ -631,7 +651,7 @@ def ratio_spy(records: Any, numerator: str, denominator: str, **kwargs: Any) -> assert unresolved_ratio["review_eligible"] is False assert unresolved_ratio["review_reason"] assert prepared["definitions"][_AUDITOR]["state"] == "available" - assert prepared["definitions"][_AUDITOR]["body"] == body + assert prepared["definitions"][_AUDITOR]["body"] == prepared_metrics.definition_body assert prepared["definitions"]["unresolved-observer"]["state"] == "unavailable" zero_denominator = next(row for row in role_block["rows"] if row["role"] == "zero-denominator") @@ -644,6 +664,16 @@ def ratio_spy(records: Any, numerator: str, denominator: str, **kwargs: Any) -> codex_cache = codex["harnesses"][0]["ratios"]["cache_share"] assert (codex_cache["state"], codex_cache["excluded_runs"]) == ("unavailable", 1) + +def test_prepared_identity_tables_and_view_projections(prepared_metrics: SimpleNamespace) -> None: + payload = prepared_metrics.payload + sessions = prepared_metrics.sessions + prepared = payload["prepared"] + built_views = {view.view_id for view in DECK_VIEWS if view.planned_issue is None} + assert set(prepared["view_chips"]) == built_views + assert set(prepared["view_histories"]) == built_views + assert set(payload["tables"]) == {SESSION_TABLE, "skills", "roles"} + def table_rows(table: dict[str, Any]) -> list[dict[str, Any]]: return [dict(zip(table["columns"], row, strict=True)) for row in table["rows"]] From 04c411b50532a57903920ce8035ec83d2d5f6efb Mon Sep 17 00:00:00 2001 From: Trecek Date: Tue, 6 Oct 2026 08:18:38 -0700 Subject: [PATCH 21/22] fix(review): remove tautological route fixture assertion --- tests/report/test_deck_views_js.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/report/test_deck_views_js.py b/tests/report/test_deck_views_js.py index 70bd32ea77..e6da2c5274 100644 --- a/tests/report/test_deck_views_js.py +++ b/tests/report/test_deck_views_js.py @@ -309,7 +309,6 @@ def test_manual_review_flags_require_definitions_and_reset_on_rerender( reset = context.call("DeckTest.render", "efficiency", _view_context("efficiency")) assert all(control["attrs"]["aria-pressed"] == "false" for control in reset["reviews"]) assert all(control["markerHidden"] for control in reset["reviews"]) - assert "review" not in _view_context("efficiency")["route"]["params"] finally: context.close() From 25972d4dcb70043d90ea6d5a896c23332574064e Mon Sep 17 00:00:00 2001 From: Trecek Date: Tue, 6 Oct 2026 08:26:46 -0700 Subject: [PATCH 22/22] fix(review): update mutation inventory for shared index cleanup --- tests/infra/test_plugin_source_ratchets.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/infra/test_plugin_source_ratchets.py b/tests/infra/test_plugin_source_ratchets.py index cf82e52067..92124befef 100644 --- a/tests/infra/test_plugin_source_ratchets.py +++ b/tests/infra/test_plugin_source_ratchets.py @@ -50,12 +50,12 @@ } PLUGIN_MUTATION_ALLOWLIST: dict[tuple[str, str, str], tuple[int, str]] = { - ("execution/report_index.py", "_update", "(index_dir / _ROWS_FILE).unlink"): ( - 2, + ("execution/report_index.py", "_clear_index_files", "(index_dir / _ROWS_FILE).unlink"): ( + 1, "Rebuilds and incompatible generations remove derived rows under the writer lease.", ), - ("execution/report_index.py", "_update", "(index_dir / _STATE_FILE).unlink"): ( - 2, + ("execution/report_index.py", "_clear_index_files", "(index_dir / _STATE_FILE).unlink"): ( + 1, "Rebuilds and incompatible generations remove the watermark under the writer lease.", ), (