diff --git a/docs/developer/diagnostics.md b/docs/developer/diagnostics.md index 015117784e..8168f94bd0 100644 --- a/docs/developer/diagnostics.md +++ b/docs/developer/diagnostics.md @@ -303,19 +303,38 @@ its rotated `otlp.jsonl.1` generation. It stores report facts separately from the retained `sessions.jsonl` projection. -Every v1 row carries `schema_version`, `kind`, `key`, `session_id`, and +Every v2 row carries `schema_version`, `kind`, `key`, `session_id`, and `time_ms`. `time_ms` is epoch milliseconds: session rows use the source session timestamp; OTLP rows use `timeUnixNano`, falling back to `observedTimeUnixNano`. Rows are append-only upserts keyed by `(kind, key)`; when a key is written again, readers keep its last row. -| Kind | Key | Additional v1 fields | +| Kind | Key | Additional fields | |---|---|---| | `session` | The session directory name (`dir_name`) | `harness`, `provider`, `model`, `skill`, `recipe`, `step`, `level`, `kitchen_id`, `order_id`, `dispatch_id`, `campaign_id`, `caller_session_id`, `parent_session_id`, `success`, `subtype`, `adjudication_reason`, `adjudication_subtype`, `duration_seconds`, `input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_write_tokens`, `assistant_turn_count`, `tool_counts` | | `request` | `{session_id}:{request_id}` | `harness`, `request_id`, `agent_name`, `query_source`, `model`, `event_sequence`, the four token fields, `cost_usd`, `duration_ms` | | `tool` | `{session_id}:{tool_use_id}`, or `{source_id}#{ordinal}` when there is no tool-use ID | `harness`, `agent_name`, `tool_name`, `tool_use_id`, `error_type`, `success`, `duration_ms`, `tool_input_size_bytes`, `tool_result_size_bytes`, `event_sequence` | | `subagent` | `{source_id}#{ordinal}` | `harness`, `agent_type`, `model`, `final_model`, `model_swapped`, `event_sequence` | +Native child `subagent` rows use +`{parent_session_key}:native:{backend}:{native_parent_session_id}:{child_id}`. +They additionally carry `child_id`, `native_parent_session_id`, `parent_session_key`, +canonical `role`, `actor_level` (`L0`), child `provider` and `model`, attribution `skill`, +spawning `recipe`, `step`, `level` and `time_ms`, canonical `token_usage`, `tool_counts`, +`transcript_state`, and `usage_state`. OTLP completion events remain separate from these +invocation observations. A managed attempt or launch alias alone cannot establish a +native actor. Resumed parents sharing a native session ID require unique invocation +ownership; ambiguous children remain unattributed. + +Native child tokens and tools come from that child's verified dedicated transcript, +with native message/request snapshots deduplicated. Tokens require the child's own +effective provider; missing provider/model identities never inherit parent values. +Claude's raw-exclusive input becomes inclusive through the shared conversion of input, +cache-read and cache-write counts, only when every component is observed. Codex input +and serialized canonical input already include cache and are not expanded again. +Raw OTLP request counters do not establish this convention through their transport or +field names, and uncorrelated request/tool facts do not become role usage. + Session token fields are serialized structured measures. Request rows persist raw token-counter observations as `int | None`; the reader resolves them to structured measures. Session `provider` is persisted verbatim from @@ -327,7 +346,10 @@ measures remain `unavailable` or `unknown` according to that pair; they are never treated as zero. Session rows do not copy `cwd`, transcript paths, or turn IDs. -The reader tolerates older or partial rows. Blank, malformed, non-object, +An existing generation must carry the current `row_schema_version` in its committed +state before read admission. An absent index remains empty; absent, old, or future +row-schema markers require an update/rebuild. Within an admitted generation, the row +decoder tolerates partial rows. Blank, malformed, non-object, unknown-kind, and empty-key rows are skipped; unknown fields are ignored, and missing or wrongly typed fields are treated as absent. Invalid entries in a tool-count map are dropped. The reader adds `session_key` to event rows in @@ -343,10 +365,18 @@ is attributed only when its session ID has exactly one attempt; otherwise The writer holds an exclusive lease for the report index. Each commit appends and fsyncs `rows.jsonl` before atomically replacing the versioned `state.json` -with the committed walk watermark and byte offset. On the next open, bytes past -that offset are truncated. If state is missing or invalid, recovery preserves -complete row lines, drops an incomplete final line, and walks sources from the -beginning; last-row-wins upserts repair rows re-derived by that walk. +with the committed walk watermark, byte offset, `row_schema_version`, and +`child_evidence_fingerprint`. On the next open, bytes past that offset are truncated. +The row-schema marker is an application-controlled compatibility boundary, distinct +from the state envelope version. Missing/incompatible row-schema evidence resets the +generation before append admission. Recoverable byte/watermark corruption within a +known current schema retains the existing complete-line recovery behavior. + +The child-evidence fingerprint covers canonical native child facts and dedicated +transcript dependencies, including changes, additions and removals. Drift selects a +full replay under the existing exclusive index lease before unchanged live or consumed +archive parents can be skipped. Markers commit with the successful row/watermark state; +this reset does not preserve rows whose original sources are no longer retained. An incremental walk whose archive or OTLP cursor is no longer retained resets that source cursor and re-walks the retained data. An archive reset also resets @@ -359,9 +389,8 @@ Rebuilding reads only the sources still retained on disk. Rows from OTLP data that has rotated out survive through incremental updates, but a rebuild cannot re-derive those older facts. A rebuild equals an incremental index when source changes are limited to those tracked by the walk: appended OTLP, OTLP rotation, -and changed, added, or evicted session rows. This assumes transcripts have not -changed since their session row was walked. `--rebuild` refreshes -transcript-derived counts. +and changed, added, or evicted session rows, plus native child evidence changes. +Parent transcript changes still require `--rebuild` when their session row is unchanged. Codex OTLP token events are not projected: they do not carry a stable request identity for deduplication. Codex token measures enter the report index through @@ -371,25 +400,52 @@ the session rows. `autoskillit sessions deck ` incrementally refreshes the report index and writes one self-contained HTML deck. It can be opened from `file://` without a server or network -connection. The deck embeds the index's session rows and filters them in the browser. +connection. It receives all four index fact collections as mapping iterables; the +report layer imports core types and helpers rather than the execution index type. +The cohort view retains its session table. Spend, efficiency, skill and role views +select metric blocks prepared by the shared aggregation library, without browser +aggregation of measures. ADR-0017 records why charts use first-party SVG. ### Payload contract The payload keys are `generated_at_ms`, `index_schema_version`, `landing`, and `history`; -`tables.sessions{columns,rows}`; `facets[{id,label,column,kind}]`; +column tables `tables.sessions`, `tables.skills`, and `tables.roles`; `facets[{id,label,column,kind}]`; `views[{id,question,decision,group,status,issue}]`; `chips{view:{facet:[{key,label,state,count,reason,issue,match|days}]}}`; and `availability[{state,label,description}]`. `match` is used for value facets and `days` for windows. `views[].issue` tracks the issue for building a planned view; a chip's `issue` identifies why that choice is unavailable. These fields have separate meanings. +`prepared.skills` and `prepared.roles` contain fixed-window, exact stored-level subset +blocks; `prepared.relationships`, `definitions`, `view_chips` and `view_histories` carry +the independent relationship graph, bundled agent definitions and population metadata. +Skill cells are `(skill, harness, provider)` and represent recorded L1/L2 skill runs. +Role rows are `(role, provider)` with separate harness-scoped metrics for terminal L0 +child invocations. Providers and token source pairs are not pooled. Role usage is an +attribution population and is not added to or subtracted from session spend. + +Token measures retain their availability states. Input/output exposure, cache-read/input +share and tool/total-tool ratios are eligible sums with explicit sample sizes, units, +excluded and unknown counts. Cache annotations overlap inclusive input; cache evidence +is a reuse proxy, while unobserved exact prompt retransmission remains not reported. +Role level selections mean the spawning skill level, while the actor remains L0. +Identity details, definitions and known cross-references remain available in empty +usage cohorts. A stale explicit window or facet selection yields no measurements. + +Canonical role routes provide offline definition links. The artifact preserves Claude +tools/model separately from Codex reader tools/model and renders original definition +body text literally. Manual ratio/tool-mix review controls require observed eligible +signals and all contributor definition links; missing evidence has an inline reason. +Flags reset on rerender and do not diagnose a configuration or recommend a downgrade. + ### Chip states Each cohort choice is `live`, `struck`, or `absent`. Live choices filter the rows. Struck choices remain visible with a reason and cannot be selected. Absent choices have no matching rows. L0 is struck because L0 leaf agents write no session row and exist -only as subagent transcripts. On a legacy index, unavailable orchestration levels cite +only as subagent transcripts in the cohort's session population. Role views use the +spawning skill's level facet. On a legacy source, unavailable orchestration levels cite [#4622](https://github.com/TalonT-Org/AutoSkillit/issues/4622); windows extending before the retained history cite [#4621](https://github.com/TalonT-Org/AutoSkillit/issues/4621). diff --git a/src/autoskillit/assets/deck/core.js b/src/autoskillit/assets/deck/core.js index 6640fa1b9f..cbcdf8b49d 100644 --- a/src/autoskillit/assets/deck/core.js +++ b/src/autoskillit/assets/deck/core.js @@ -5,6 +5,132 @@ globalThis.DeckCore = (() => { const enc = encodeURIComponent; const formatCount = n => new Intl.NumberFormat("en-US").format(n); const formatDate = ms => new Date(ms).toISOString().slice(0, 10); + + function formatRatio(ratio, percent = false) { + if (!ratio || (ratio.state !== "measured" && ratio.state !== "measured_zero")) return null; + if (ratio.state === "measured_zero") return percent ? "0%" : "0"; + if (typeof ratio.value !== "number" || !Number.isFinite(ratio.value)) return null; + const value = Math.round(ratio.value * (percent ? 100 : 1) * 100) / 100; + return String(value) + (percent ? "%" : ""); + } + + function ratioSample(ratio) { + if (ratio?.sample_size == null) return "sample size unavailable"; + const unit = ratio.sample_unit === "skill-run" ? "skill run" : + ratio.sample_unit === "child-invocation" ? "child invocation" : ratio.sample_unit; + return formatCount(ratio.sample_size) + " " + unit + (ratio.sample_size === 1 ? "" : "s"); + } + + function reviewEligibility(ratio, definitions) { + const roles = [...new Set(ratio?.definition_roles ?? [])]; + if (!ratio || (ratio.state !== "measured" && ratio.state !== "measured_zero")) { + return {eligible: false, + reason: ratio?.review_reason || "No measured review signal is available.", roles}; + } + if (ratio.review_eligible !== true || !(ratio.sample_size > 0)) { + return {eligible: false, + reason: ratio.review_reason || "This signal is not eligible for review.", roles}; + } + if (!roles.length) { + return {eligible: false, + reason: "No contributor definitions are linked to this signal.", roles}; + } + const unavailable = roles.filter(role => definitions?.[role]?.state !== "available"); + if (unavailable.length) { + return {eligible: false, + reason: "Contributor definitions unavailable: " + unavailable.join(", ") + ".", roles}; + } + return {eligible: true, reason: null, roles}; + } + + function reviewSignal(ctx, ratio) { + const eligibility = reviewEligibility(ratio, ctx.definitions); + const links = eligibility.roles.map(role => ctx.entityLink(role, { + view: "role", entity: role + })); + const definitions = links.length ? ctx.el("div", { + class: "view-review__definitions", + "aria-label": "Contributor definitions" + }, links) : null; + const marker = ctx.el("span", { + class: "view-review__marker", + "aria-live": "polite", + hidden: true + }, "Flagged for review"); + const button = ctx.el("button", { + type: "button", + class: "view-review__flag", + "data-review-flag": "true", + "aria-label": "Flag this signal for review", + "aria-pressed": "false", + disabled: !eligibility.eligible + }, "Flag for review"); + let flagged = false; + if (eligibility.eligible) { + button.addEventListener("click", () => { + flagged = !flagged; + button.setAttribute("aria-pressed", String(flagged)); + marker.hidden = !flagged; + }); + } + return ctx.el("div", {class: "view-review-signal", "data-review-signal": "true"}, [ + definitions, + button, + marker, + eligibility.reason ? ctx.el("p", {class: "view-review__reason"}, eligibility.reason) : null + ]); + } + function measureCell(ctx, measure) { + return ctx.availabilityCell(measure ?? {state: "unavailable"}); + } + + function ratioCell(ctx, ratio, percent = false) { + const value = formatRatio(ratio, percent); + return ctx.el("div", {class: "view-measure"}, [ + value == null ? measureCell(ctx, ratio) : ctx.el("span", {}, value), + ctx.el("small", {class: "view-sample"}, ratioSample(ratio)), + reviewSignal(ctx, ratio) + ]); + } + + function definitionCard(ctx, role) { + const definition = ctx.definitions?.[role]; + const available = definition?.state === "available"; + return ctx.el("article", {class: "view-definition"}, [ + ctx.el("h3", {}, ctx.entityLink(role, {view: "role", entity: role})), + available ? ctx.el("p", {}, definition.description ?? "Role definition loaded.") : + ctx.el("p", {class: "view-review__reason"}, "Definition unavailable for " + role + "."), + available ? ctx.el("p", {}, "Declared tools: " + + ((definition.tools ?? []).join(", ") || "none recorded")) : null, + available ? ctx.el("pre", {}, definition.body ?? "") : null + ]); + } + + function toolMix(ctx, ratios = {}) { + const tools = Object.entries(ratios.tool_mix ?? {}); + return tools.length ? ctx.el("div", {class: "view-pills"}, tools.map(([tool, ratio]) => { + const value = formatRatio(ratio, true); + return ctx.el("span", {}, [ + ctx.el("span", {}, tool), + ctx.el("span", {}, " · "), + value == null ? ctx.availabilityCell(ratio) : ctx.el("span", {}, value), + ctx.el("small", {class: "view-sample"}, ratioSample(ratio)), + reviewSignal(ctx, ratio) + ]); + })) : ctx.el("span", {class: "view-empty"}, "No observed tool calls"); + } + + function roleHarnessRows(rows) { + return rows.flatMap(row => (row.harnesses ?? []).map(cell => ({ + role: row.role, + provider: row.provider, + harness: cell.harness, + models: cell.models ?? [], + measures: cell.measures, + ratios: cell.ratios + }))); + } + const decodeTable = ({columns, rows}) => rows.map(r => Object.fromEntries(columns.map((c, i) => [c, r[i]]))); @@ -61,10 +187,12 @@ globalThis.DeckCore = (() => { function effectiveSelection(chips, selected) { const live = chips.filter(c => c.state === "live").map(c => c.key); - if (selected == null) return {keys: live, dropped: [], widened: false}; + if (selected == null || selected.length === 0) { + return {keys: live, dropped: [], widened: false}; + } const keys = live.filter(k => selected.includes(k)); const dropped = selected.filter(k => !live.includes(k)); - return {keys: keys.length ? keys : live, dropped, widened: keys.length === 0}; + return {keys, dropped, widened: false}; } function toggleSelection(chips, selected, key) { @@ -77,9 +205,67 @@ globalThis.DeckCore = (() => { function windowSelection(chips, selected) { const want = selected?.[0]; - const chip = chips.find(c => c.state === "live" && c.key === want) || - chips.find(c => c.key === "all"); - return {chip, dropped: want && chip.key !== want ? [want] : []}; + const chip = want == null ? chips.find(c => c.state === "live" && c.key === "all") : + chips.find(c => c.state === "live" && c.key === want); + return {chip: chip ?? null, dropped: want != null && !chip ? [want] : []}; + } + + function sameValues(left, right) { + return left.length === right.length && left.every(value => right.includes(value)); + } + + function selectPrepared(prepared, viewId, chips, route) { + prepared = prepared ?? {}; + chips = chips ?? {}; + const params = route.params ?? {}; + const window = windowSelection(chips.window ?? [], params.window ?? null); + const level = effectiveSelection(chips.level ?? [], params.level ?? null); + const harness = effectiveSelection(chips.harness ?? [], params.harness ?? null); + const provider = effectiveSelection(chips.provider ?? [], params.provider ?? null); + const harnesses = new Set((chips.harness ?? []) + .filter(chip => harness.keys.includes(chip.key)).map(chip => chip.match)); + const providers = new Set((chips.provider ?? []) + .filter(chip => provider.keys.includes(chip.key)).map(chip => chip.match)); + + function selectedRows(blocks) { + if (!window.chip) return []; + const inWindow = blocks.filter(block => block.window === window.chip.key); + const domain = [...new Set(inWindow.flatMap(block => block.levels))]; + const selectedLevels = new Set(level.keys + .map(key => (chips.level ?? []).find(chip => chip.key === key)?.match) + ); + const requested = domain.filter(value => selectedLevels.has(value)); + if (!requested.length) return []; + const block = inWindow.find(item => sameValues(item.levels, requested)); + return block?.rows ?? []; + } + + let skillRows = viewId === "role" ? [] : selectedRows(prepared.skills ?? []).filter(row => + harnesses.has(row.harness) && providers.has(row.provider)); + let roleRows = viewId === "skill" ? [] : selectedRows(prepared.roles ?? []).filter(row => + providers.has(row.provider)).map(row => ({...row, + harnesses: (row.harnesses ?? []).filter(cell => harnesses.has(cell.harness)) + })).filter(row => row.harnesses.length > 0); + if (viewId === "skill" && route.entity != null) { + skillRows = skillRows.filter(row => row.skill === route.entity); + } + if (viewId === "role" && route.entity != null) { + roleRows = roleRows.filter(row => row.role === route.entity); + } + const relationships = (prepared.relationships ?? []).filter(edge => { + if (viewId === "skill" && route.entity != null) return edge.skill === route.entity; + if (viewId === "role" && route.entity != null) return edge.role === route.entity; + return true; + }); + + return { + skillRows, + roleRows, + relationships, + definitions: prepared.definitions ?? {}, + viewHistory: prepared.view_histories?.[viewId] ?? null, + selection: {window, level, harness, provider} + }; } function filterRows(model, rows, chips, route) { @@ -91,7 +277,9 @@ globalThis.DeckCore = (() => { } let untimed = 0; const window = windowSelection(chips.window, route.params.window ?? null).chip; - if (window.days != null) { + if (!window) { + filtered = []; + } else if (window.days != null) { filtered = filtered.filter(row => { if (row.time_ms == null) { untimed += 1; return false; } return row.time_ms >= model.generated_at_ms - window.days * DAY_MS; @@ -115,15 +303,16 @@ globalThis.DeckCore = (() => { const selection = effectiveSelection(values, route.params[facet.id] ?? null); const live = values.filter(c => c.state === "live"); const labels = values.filter(c => selection.keys.includes(c.key)).map(c => c.label); - parts.push(facet.label + " " + (!live.length ? "none recorded" : labels.join(" + ") + - (selection.keys.length === live.length ? " (all)" : ""))); + const selected = !live.length ? "none recorded" : !selection.keys.length ? "none selected" : + labels.join(" + ") + (selection.keys.length === live.length ? " (all)" : ""); + parts.push(facet.label + " " + selected); notes.push(...selection.dropped.map(k => droppedNote(values, k))); - if (selection.widened) notes.push("no selected " + facet.label + - " is selectable here — showing every selectable " + facet.label); } const selection = windowSelection(chips.window, route.params.window ?? null); const window = selection.chip; - if (window.days != null) { + if (!window) { + parts.push("window unavailable"); + } else if (window.days != null) { parts.push("window last " + window.label + " (" + formatDate(model.generated_at_ms - window.days * DAY_MS) + " → " + formatDate(model.generated_at_ms) + ")"); @@ -199,7 +388,9 @@ globalThis.DeckCore = (() => { } return Object.freeze({decodeTable, encodeRoute, decodeRoute, hrefFor, effectiveSelection, - toggleSelection, windowSelection, filterRows, populationSentence, summarizePairs, sortRows, - parseSort, chipPresentation, availabilityPresentation, barLayout, formatCount, formatDate, + toggleSelection, windowSelection, selectPrepared, filterRows, populationSentence, + summarizePairs, sortRows, parseSort, chipPresentation, availabilityPresentation, + barLayout, formatCount, formatRatio, ratioSample, reviewEligibility, reviewSignal, + measureCell, ratioCell, definitionCard, toolMix, roleHarnessRows, formatDate, CHIP_STATES, DAY_MS}); })(); diff --git a/src/autoskillit/assets/deck/deck.css b/src/autoskillit/assets/deck/deck.css index c1c25d4d72..33b6ebb74e 100644 --- a/src/autoskillit/assets/deck/deck.css +++ b/src/autoskillit/assets/deck/deck.css @@ -82,12 +82,36 @@ svg { display: block; max-width: 100%; height: auto; } .hatch-line { stroke: var(--hatch); stroke-width: 2; } .bar-label, .bar-value { fill: var(--ink-2); font: 11px var(--mono); } .defs { position: absolute; } +.view-stack { display: grid; gap: 18px; } +.view-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 18px; } +.view-lede, .view-note { color: var(--ink-2); line-height: 1.55; } +.view-empty { color: var(--ink-3); font-style: italic; } +.view-links, .view-pills { display: flex; flex-wrap: wrap; gap: 8px; } +.view-pills span, .view-sample { color: var(--ink-3); font: 11px var(--mono); } +.view-measure { white-space: nowrap; } +.view-definition { border-top: 1px solid var(--rule); padding: 12px 0; } +.view-definition h3 { font-size: 13px; margin: 0 0 6px; } +.view-definition pre { white-space: pre-wrap; overflow-wrap: anywhere; color: var(--ink-2); } +.view-review-signal { display: grid; justify-items: start; gap: 4px; margin-top: 5px; } +.view-review__definitions { display: flex; flex-wrap: wrap; gap: 6px; font-size: 11px; } +.view-review__marker { color: var(--warn); font-size: 11px; font-weight: 600; } +.view-review { border: 1px solid var(--rule); border-radius: 6px; padding: 12px; } +.view-review button, .view-review-signal button { border: 1px solid var(--rule-2); + border-radius: 5px; padding: 5px 8px; + color: var(--ink); background: var(--surface-2); cursor: pointer; } +.view-review button:focus-visible, .view-review-signal button:focus-visible, +.view-links a:focus-visible { outline: 2px solid var(--s1); + outline-offset: 2px; } +.view-review button:disabled, .view-review-signal button:disabled { cursor: not-allowed; opacity: .6; } +.view-review__reason { color: var(--warn); font-size: 12px; } +.view-actor { color: var(--ink-3); font: 11px var(--mono); } .foot { color: var(--ink-3); font: 10px/1.5 var(--mono); padding: 0 28px 24px; } @media screen and (max-width: 860px) { .shell { grid-template-columns: minmax(0,1fr); } .rail { border-right: 0; border-bottom: 1px solid var(--rule); } .cohort { position: static; } .cohort, #deck-view { padding: 18px; } + .view-grid { grid-template-columns: minmax(0,1fr); } } @media print { @page { margin: 12mm; } @@ -95,6 +119,7 @@ svg { display: block; max-width: 100%; height: auto; } .rail { display: none; } .shell { display: block; } .cohort { position: static; } - .card, table, svg { break-inside: avoid; } + .card, table, svg, .view-definition, .view-review { break-inside: avoid; } + .view-review button, .view-review-signal button { display: none; } #deck-view { padding: 18px 0; } } diff --git a/src/autoskillit/assets/deck/shell.js b/src/autoskillit/assets/deck/shell.js index a42d05b890..e0fd0014ab 100644 --- a/src/autoskillit/assets/deck/shell.js +++ b/src/autoskillit/assets/deck/shell.js @@ -155,7 +155,7 @@ globalThis.DeckShell = (() => { } } - function renderCohort(chips, result, sentence) { + function renderCohort(chips, result, sentence, facets = model.facets) { const cohort = document.getElementById("deck-cohort"); cohort.replaceChildren(); cohort.appendChild(el("div", {class: "cohort-sentence"}, [ @@ -164,10 +164,11 @@ globalThis.DeckShell = (() => { cohort.appendChild(el("ul", {class: "cohort-notes"}, sentence.notes.map(note => el("li", {}, note)))); - for (const facet of model.facets) { + for (const facet of facets) { const facetChips = facet.kind === "window" ? chips.window : chips[facet.id]; - const selectedKeys = facet.kind === "window" ? - [DeckCore.windowSelection(chips.window, route.params.window ?? null).chip.key] : + const window = facet.kind === "window" ? + DeckCore.windowSelection(chips.window, route.params.window ?? null).chip : null; + const selectedKeys = facet.kind === "window" ? (window ? [window.key] : []) : DeckCore.effectiveSelection(chips[facet.id], route.params[facet.id] ?? null).keys; const group = el("div", {class: "cohort-group"}, [el("span", {}, facet.label)]); const unavailable = []; @@ -236,10 +237,21 @@ globalThis.DeckShell = (() => { notice = "No view named " + route.view; } - const table = model.tables.sessions; - const result = DeckCore.filterRows(model, table, chips, route); - const sentence = DeckCore.populationSentence(model, chips, route, result); - renderCohort(chips, result, sentence); + const selected = renderer && view.id !== model.landing ? + DeckCore.selectPrepared(model.prepared, view.id, chips, route) : null; + const primary = view?.id === "role" ? selected?.roleRows : selected?.skillRows; + const result = selected ? {rows: primary, untimed: 0} : + DeckCore.filterRows(model, model.tables.sessions, chips, route); + const populationModel = selected ? {...model, history: selected.viewHistory, + facets: model.facets.map(facet => view.id === "role" && facet.id === "level" ? + {...facet, label: "Spawning skill level"} : facet)} : model; + const sentence = DeckCore.populationSentence(populationModel, chips, route, result); + if (selected) { + const noun = view.id === "role" ? "role/provider row" : "skill cell"; + sentence.headline = DeckCore.formatCount(result.rows.length) + " " + noun + + (result.rows.length === 1 ? "" : "s"); + } + renderCohort(chips, result, sentence, populationModel.facets); const content = document.getElementById("deck-view"); content.replaceChildren(); @@ -247,6 +259,18 @@ globalThis.DeckShell = (() => { const ctx = {model, route, rows: result.rows, chips, href: target => DeckCore.hrefFor(route, target, cohortKeys), el, entityLink, sortableTable, availabilityCell, barChart}; + if (view.id !== model.landing) { + ctx.prepared = model.prepared; + ctx.metrics = view.id === "role" ? selected.roleRows : selected.skillRows; + ctx.relationships = selected.relationships; + ctx.definitions = selected.definitions; + ctx.viewHistory = selected.viewHistory; + ctx.selection = selected.selection; + if (view.id === "spend" || view.id === "efficiency") { + ctx.skillMetrics = selected.skillRows; + ctx.roleMetrics = selected.roleRows; + } + } content.appendChild(renderer(ctx)); } else { const noticeCard = el("section", {class: "card"}, [el("p", {}, notice)]); @@ -264,7 +288,7 @@ globalThis.DeckShell = (() => { DeckCore.formatDate(first) + " → " + DeckCore.formatDate(last); const generated = new Date(model.generated_at_ms).toISOString().slice(0, 16); document.getElementById("deck-foot").textContent = "report index v" + - model.index_schema_version + " · " + DeckCore.formatCount(table.length) + + model.index_schema_version + " · " + DeckCore.formatCount(model.tables.sessions.length) + " session rows · history " + history + " · generated " + generated + " UTC"; } diff --git a/src/autoskillit/assets/deck/views/efficiency.js b/src/autoskillit/assets/deck/views/efficiency.js new file mode 100644 index 0000000000..3f2970ca02 --- /dev/null +++ b/src/autoskillit/assets/deck/views/efficiency.js @@ -0,0 +1,75 @@ +DeckShell.registerView("efficiency", ctx => { + const skillRows = ctx.skillMetrics ?? []; + const roleRows = DeckCore.roleHarnessRows(ctx.roleMetrics ?? []); + + function ratioSignals(ratios = {}) { + const values = []; + if (ratios.input_output) values.push({label: "Input exposure per output", ratio: ratios.input_output}); + if (ratios.cache_share) values.push({label: "Cache-read share", ratio: ratios.cache_share, + percent: true}); + for (const [tool, ratio] of Object.entries(ratios.tool_mix ?? {})) { + values.push({label: "Tool mix · " + tool, ratio, percent: true}); + } + return values; + } + + const contributorRoles = new Set((ctx.relationships ?? []).map(edge => edge.role)); + for (const row of skillRows) { + for (const signal of ratioSignals(row.ratios)) { + for (const role of signal.ratio.definition_roles ?? []) contributorRoles.add(role); + } + } + for (const row of roleRows) { + for (const signal of ratioSignals(row.ratios)) { + for (const role of signal.ratio.definition_roles ?? []) contributorRoles.add(role); + } + } + const definitionCards = [...contributorRoles].sort().map(role => + DeckCore.definitionCard(ctx, role)); + const skillTable = skillRows.length ? ctx.sortableTable({ + columns: [ + {key: "skill", label: "Skill", cell: row => ctx.entityLink(row.skill, { + view: "skill", entity: row.skill + })}, + {key: "harness", label: "Harness"}, + {key: "provider", label: "Provider"}, + {key: "input_output", label: "Input / output", cell: row => + DeckCore.ratioCell(ctx, row.ratios?.input_output)}, + {key: "cache_share", label: "Cache-read share", cell: row => + DeckCore.ratioCell(ctx, row.ratios?.cache_share, true)}, + {key: "tool_mix", label: "Observed tool mix", cell: row => DeckCore.toolMix(ctx, row.ratios)} + ], + rows: skillRows, + defaultSort: {key: "skill", dir: "asc"} + }) : ctx.el("p", {class: "view-empty"}, "No skill-run ratios are available in this cohort."); + const roleTable = roleRows.length ? ctx.sortableTable({ + columns: [ + {key: "role", label: "Role", cell: row => ctx.entityLink(row.role, { + view: "role", entity: row.role + })}, + {key: "harness", label: "Harness"}, + {key: "provider", label: "Provider"}, + {key: "input_output", label: "Input / output", cell: row => + DeckCore.ratioCell(ctx, row.ratios?.input_output)}, + {key: "cache_share", label: "Cache-read share", cell: row => + DeckCore.ratioCell(ctx, row.ratios?.cache_share, true)}, + {key: "tool_mix", label: "Observed tool mix", cell: row => DeckCore.toolMix(ctx, row.ratios)} + ], + rows: roleRows, + defaultSort: {key: "role", dir: "asc"} + }) : ctx.el("p", {class: "view-empty"}, "No child-invocation ratios are available in this cohort."); + + return ctx.el("div", {class: "view-stack"}, [ + ctx.el("p", {class: "view-lede"}, + "Prepared ratios compare eligible input, output, cache-read, and observed tool totals. " + + "They are evidence signals, not quality diagnoses or automatic model recommendations."), + ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, "Contributor definitions"), + ...(definitionCards.length ? definitionCards : [ + ctx.el("p", {class: "view-empty"}, "No contributor definitions are linked in this cohort.") + ]) + ]), + ctx.el("section", {class: "card"}, [ctx.el("h2", {}, "Skill-run efficiency"), skillTable]), + ctx.el("section", {class: "card"}, [ctx.el("h2", {}, "Child-invocation efficiency"), roleTable]) + ]); +}); diff --git a/src/autoskillit/assets/deck/views/role.js b/src/autoskillit/assets/deck/views/role.js new file mode 100644 index 0000000000..553a635158 --- /dev/null +++ b/src/autoskillit/assets/deck/views/role.js @@ -0,0 +1,105 @@ +DeckShell.registerView("role", ctx => { + const roleName = ctx.route.entity; + const identities = ctx.model.tables?.roles ?? []; + const definitions = ctx.definitions ?? {}; + const rows = ctx.metrics ?? []; + const roleNames = new Set([ + ...identities.map(row => row.role), + ...Object.keys(definitions), + ...(roleName == null ? [] : [roleName]) + ]); + const roleRows = roleName == null ? [...roleNames].sort().map(role => ({ + role, + definition: definitions[role], + hasMetrics: rows.some(row => row.role === role) + })) : rows.filter(row => row.role === roleName); + + const spawningLevels = (ctx.selection?.level?.keys ?? []).map(key => + (ctx.chips?.level ?? []).find(chip => chip.key === key)?.label ?? key); + const edges = ctx.relationships ?? []; + const spawningTable = edges.length ? ctx.sortableTable({ + columns: [ + {key: "skill", label: "Spawning skill", cell: edge => ctx.entityLink(edge.skill, { + view: "skill", entity: edge.skill + })}, + {key: "harness", label: "Harness"}, + {key: "provider", label: "Child provider"} + ], + rows: edges, + defaultSort: {key: "skill", dir: "asc"} + }) : ctx.el("p", {class: "view-empty"}, "No verified spawning-skill links are recorded."); + + const harnessRows = DeckCore.roleHarnessRows(roleRows); + const usageTable = harnessRows.length ? ctx.sortableTable({ + columns: [ + {key: "harness", label: "Harness"}, + {key: "provider", label: "Provider"}, + {key: "models", label: "Observed models", cell: row => + (row.models ?? []).join(", ") || "No model recorded"}, + {key: "input_tokens", label: "Input tokens", numeric: true, + cell: row => DeckCore.measureCell(ctx, row.measures?.input_tokens)}, + {key: "output_tokens", label: "Output tokens", numeric: true, + cell: row => DeckCore.measureCell(ctx, row.measures?.output_tokens)}, + {key: "cache_read_tokens", label: "Cache-read tokens", numeric: true, + cell: row => DeckCore.measureCell(ctx, row.measures?.cache_read_tokens)}, + {key: "cache_write_tokens", label: "Cache-write tokens", numeric: true, + cell: row => DeckCore.measureCell(ctx, row.measures?.cache_write_tokens)}, + {key: "input_output", label: "Input / output", cell: row => + DeckCore.ratioCell(ctx, row.ratios?.input_output)}, + {key: "cache_share", label: "Cache-read share", cell: row => + DeckCore.ratioCell(ctx, row.ratios?.cache_share, true)}, + {key: "tool_mix", label: "Observed tool mix", cell: row => DeckCore.toolMix(ctx, row.ratios)} + ], + rows: harnessRows, + defaultSort: {key: "harness", dir: "asc"} + }) : ctx.el("p", {class: "view-empty"}, "No child-invocation metrics match these facets."); + + const definition = roleName == null ? null : definitions[roleName]; + const available = definition?.state === "available"; + const definitionCard = roleName == null ? null : ctx.el("section", {class: "card view-definition"}, [ + ctx.el("h2", {}, "Role definition · " + roleName), + available ? ctx.el("p", {}, definition.description ?? "Role definition loaded.") : + ctx.el("p", {class: "view-review__reason"}, "Definition unavailable for " + roleName + "."), + available ? ctx.el("p", {}, "Claude Code declared tools: " + + ((definition.tools ?? []).join(", ") || "none recorded")) : null, + available ? ctx.el("p", {}, "Claude Code declared model: " + + (definition.model ?? "not declared")) : null, + available ? ctx.el("p", {}, "Codex read-only tools: " + + ((definition.reader_tools ?? []).join(", ") || "none declared")) : null, + available ? ctx.el("p", {}, "Codex model: " + + (definition.codex_model ?? "not declared")) : null, + available ? ctx.el("pre", {}, definition.body ?? "") : null + ]); + const indexTable = roleName == null ? ctx.sortableTable({ + columns: [ + {key: "role", label: "Role", cell: row => ctx.entityLink(row.role, { + view: "role", entity: row.role + })}, + {key: "state", label: "Definition", cell: row => row.definition?.state ?? "unavailable"}, + {key: "hasMetrics", label: "Selected metrics", cell: row => + row.hasMetrics ? "available" : "none in this cohort"} + ], + rows: roleRows, + defaultSort: {key: "role", dir: "asc"} + }) : null; + + return ctx.el("div", {class: "view-stack"}, [ + ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, roleName ?? "Known specialized roles"), + ctx.el("p", {class: "view-actor"}, roleName == null ? + "Native specialized child actor · L0" : "Actor level: L0 · Spawning skill level: " + + (spawningLevels.join(" + ") || "all observed levels")), + ctx.el("p", {class: "view-lede"}, + "Observed child-invocation usage stays separate from its spawning skill's session spend."), + indexTable + ]), + definitionCard, + ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, "Observed role usage"), + ctx.el("p", {class: "view-note"}, + "Observed models and tool mix come from child evidence. Declared tools and models above come from the role definition."), + usageTable + ]), + ctx.el("section", {class: "card"}, [ctx.el("h2", {}, "Spawning skills"), spawningTable]) + ]); +}); diff --git a/src/autoskillit/assets/deck/views/skill.js b/src/autoskillit/assets/deck/views/skill.js new file mode 100644 index 0000000000..09e809c0f5 --- /dev/null +++ b/src/autoskillit/assets/deck/views/skill.js @@ -0,0 +1,139 @@ +DeckShell.registerView("skill", ctx => { + const rows = ctx.metrics ?? []; + const routeSkill = ctx.route.entity; + const relationships = ctx.relationships ?? []; + const selected = routeSkill == null ? [] : rows.filter(row => row.skill === routeSkill); + const skillNames = [...new Set(selected.map(row => row.skill))]; + if (routeSkill != null && !skillNames.includes(routeSkill)) skillNames.unshift(routeSkill); + + const roleNames = new Set(relationships.map(edge => edge.role)); + for (const row of selected) { + for (const child of row.child_roles ?? []) roleNames.add(child.role); + for (const role of row.ratios?.input_output?.definition_roles ?? []) roleNames.add(role); + for (const role of row.ratios?.cache_share?.definition_roles ?? []) roleNames.add(role); + } + const definitions = [...roleNames].sort().map(role => DeckCore.definitionCard(ctx, role)); + + const skillTable = rows.length ? ctx.sortableTable({ + columns: [ + {key: "skill", label: "Skill", cell: row => ctx.entityLink(row.skill, { + view: "skill", entity: row.skill + })}, + {key: "harness", label: "Harness"}, + {key: "provider", label: "Provider"}, + {key: "input_tokens", label: "Input tokens", numeric: true, + cell: row => DeckCore.measureCell(ctx, row.measures?.input_tokens)}, + {key: "output_tokens", label: "Output tokens", numeric: true, + cell: row => DeckCore.measureCell(ctx, row.measures?.output_tokens)}, + {key: "cache_read_tokens", label: "Cache-read tokens", numeric: true, + cell: row => DeckCore.measureCell(ctx, row.measures?.cache_read_tokens)}, + {key: "cache_write_tokens", label: "Cache-write tokens", numeric: true, + cell: row => DeckCore.measureCell(ctx, row.measures?.cache_write_tokens)}, + {key: "input_output", label: "Input / output", cell: row => + DeckCore.ratioCell(ctx, row.ratios?.input_output)}, + {key: "cache_share", label: "Cache-read share", cell: row => + DeckCore.ratioCell(ctx, row.ratios?.cache_share, true)}, + {key: "exact_retransmission", label: "Exact prompt retransmission", cell: row => + DeckCore.measureCell(ctx, row.exact_retransmission)} + ], + rows, + defaultSort: {key: "skill", dir: "asc"} + }) : ctx.el("p", {class: "view-empty"}, "No skill-run metrics match these facets."); + + if (routeSkill == null) { + return ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, "Skills in this cohort"), + ctx.el("p", {class: "view-lede"}, + "Select a skill to inspect its recipe-step and child-role attribution."), + skillTable + ]); + } + + const steps = selected.flatMap(row => (row.recipe_steps ?? []).map(step => ({ + skill: row.skill, + recipe: step.recipe, + step: step.step, + measures: step.measures, + ratios: step.ratios, + exact_retransmission: step.exact_retransmission + }))); + const stepTable = steps.length ? ctx.sortableTable({ + columns: [ + {key: "recipe", label: "Recipe"}, + {key: "step", label: "Step"}, + {key: "input_tokens", label: "Input tokens", cell: row => + DeckCore.measureCell(ctx, row.measures?.input_tokens)}, + {key: "output_tokens", label: "Output tokens", cell: row => + DeckCore.measureCell(ctx, row.measures?.output_tokens)}, + {key: "cache_read_tokens", label: "Cache-read tokens", cell: row => + DeckCore.measureCell(ctx, row.measures?.cache_read_tokens)}, + {key: "exact_retransmission", label: "Exact prompt retransmission", cell: row => + DeckCore.measureCell(ctx, row.exact_retransmission)} + ], + rows: steps, + defaultSort: {key: "recipe", dir: "asc"} + }) : ctx.el("p", {class: "view-empty"}, "No recipe-step attribution is available for this skill."); + + const childRows = selected.flatMap(row => (row.child_roles ?? []).map(child => ({ + skill: row.skill, + ...child + }))); + const childTable = childRows.length ? ctx.sortableTable({ + columns: [ + {key: "role", label: "Child role", cell: row => ctx.entityLink(row.role, { + view: "role", entity: row.role + })}, + {key: "harness", label: "Harness"}, + {key: "provider", label: "Provider"}, + {key: "input_tokens", label: "Child input tokens", cell: row => + DeckCore.measureCell(ctx, row.measures?.input_tokens)}, + {key: "output_tokens", label: "Child output tokens", cell: row => + DeckCore.measureCell(ctx, row.measures?.output_tokens)}, + {key: "input_output", label: "Input / output", cell: row => + DeckCore.ratioCell(ctx, row.ratios?.input_output)}, + {key: "cache_share", label: "Cache-read share", cell: row => + DeckCore.ratioCell(ctx, row.ratios?.cache_share, true)} + ], + rows: childRows, + defaultSort: {key: "role", dir: "asc"} + }) : ctx.el("p", {class: "view-empty"}, "No verified child-role attribution is linked to this skill."); + + const heading = routeSkill ?? "Skills in this cohort"; + const related = relationships.length ? ctx.sortableTable({ + columns: [ + {key: "role", label: "Spawned role", cell: edge => ctx.entityLink(edge.role, { + view: "role", entity: edge.role + })}, + {key: "harness", label: "Harness"}, + {key: "provider", label: "Child provider"} + ], + rows: relationships, + defaultSort: {key: "role", dir: "asc"} + }) : ctx.el("p", {class: "view-empty"}, "No verified child relationships are recorded."); + + return ctx.el("div", {class: "view-stack"}, [ + ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, heading), + routeSkill == null ? ctx.el("p", {class: "view-lede"}, + "Select a skill to inspect its recipe-step and child-role attribution.") : + ctx.el("p", {class: "view-lede"}, + "Input exposure and cache-read share are retransmission proxies. Exact repeated-prompt tokens are not reported when absent; cache-read is already included in input."), + skillTable + ]), + ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, "Contributor definitions"), + ...(definitions.length ? definitions : [ + ctx.el("p", {class: "view-empty"}, "No child-role definitions are linked to this skill.") + ]) + ]), + ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, "Recipe-step attribution"), + ctx.el("p", {class: "view-note"}, + "Step measures remain separate from child-role usage and from the skill-run total."), + stepTable + ]), + ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, "Child-role attribution"), childTable, related + ]) + ]); +}); diff --git a/src/autoskillit/assets/deck/views/spend.js b/src/autoskillit/assets/deck/views/spend.js new file mode 100644 index 0000000000..ce4c5a0dce --- /dev/null +++ b/src/autoskillit/assets/deck/views/spend.js @@ -0,0 +1,74 @@ +DeckShell.registerView("spend", ctx => { + const skillRows = ctx.skillMetrics ?? ctx.metrics ?? []; + const roleRows = ctx.roleMetrics ?? []; + const amount = measure => measure && + (measure.state === "measured" || measure.state === "measured_zero") ? measure.value : null; + const skillChart = skillRows.map(row => ({ + label: row.skill + " · " + row.harness + " · " + row.provider, + value: amount(row.measures?.input_tokens), + href: ctx.href({view: "skill", entity: row.skill}) + })); + const roleCells = DeckCore.roleHarnessRows(roleRows); + const roleChart = roleCells.map(row => ({ + label: row.role + " · " + row.harness + " · " + row.provider, + value: amount(row.measures?.input_tokens), + href: ctx.href({view: "role", entity: row.role}) + })); + const measureCell = measure => ctx.availabilityCell(measure ?? {state: "unavailable"}); + const skillTable = skillRows.length ? ctx.sortableTable({ + columns: [ + {key: "skill", label: "Skill", cell: row => ctx.entityLink(row.skill, { + view: "skill", entity: row.skill + })}, + {key: "harness", label: "Harness"}, + {key: "provider", label: "Provider"}, + {key: "input_tokens", label: "Input tokens", numeric: true, + cell: row => measureCell(row.measures?.input_tokens)}, + {key: "output_tokens", label: "Output tokens", numeric: true, + cell: row => measureCell(row.measures?.output_tokens)}, + {key: "cache_read_tokens", label: "Cache-read tokens", numeric: true, + cell: row => measureCell(row.measures?.cache_read_tokens)}, + {key: "cache_write_tokens", label: "Cache-write tokens", numeric: true, + cell: row => measureCell(row.measures?.cache_write_tokens)} + ], + rows: skillRows, + defaultSort: {key: "skill", dir: "asc"} + }) : ctx.el("p", {class: "view-empty"}, "No skill-run spend is available in this cohort."); + const roleTable = roleCells.length ? ctx.sortableTable({ + columns: [ + {key: "role", label: "Role", cell: row => ctx.entityLink(row.role, { + view: "role", entity: row.role + })}, + {key: "harness", label: "Harness"}, + {key: "provider", label: "Provider"}, + {key: "input_tokens", label: "Input tokens", numeric: true, + cell: row => measureCell(row.measures?.input_tokens)}, + {key: "output_tokens", label: "Output tokens", numeric: true, + cell: row => measureCell(row.measures?.output_tokens)}, + {key: "cache_read_tokens", label: "Cache-read tokens", numeric: true, + cell: row => measureCell(row.measures?.cache_read_tokens)}, + {key: "cache_write_tokens", label: "Cache-write tokens", numeric: true, + cell: row => measureCell(row.measures?.cache_write_tokens)} + ], + rows: roleCells, + defaultSort: {key: "role", dir: "asc"} + }) : ctx.el("p", {class: "view-empty"}, "No verified child-role spend is available in this cohort."); + + return ctx.el("div", {class: "view-stack"}, [ + ctx.el("p", {class: "view-lede"}, + "Input, output, cache-read, and cache-write tokens remain separate inclusive measures. " + + "Cache-read is already included in input. Child-role usage is shown as its own population."), + ctx.el("div", {class: "view-grid"}, [ + ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, "Skill-run spend"), + ctx.barChart(skillChart, {label: "Input tokens by skill run identity"}), + skillTable + ]), + ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, "Child-role spend"), + ctx.barChart(roleChart, {label: "Input tokens by child role identity"}), + roleTable + ]) + ]) + ]); +}); diff --git a/src/autoskillit/cli/ops/_sessions.py b/src/autoskillit/cli/ops/_sessions.py index ce3d17f0dd..5a0fcc4655 100644 --- a/src/autoskillit/cli/ops/_sessions.py +++ b/src/autoskillit/cli/ops/_sessions.py @@ -79,6 +79,9 @@ def sessions_deck(output: str) -> None: report = read_report_index(index_dir) html = render_deck( report.sessions.values(), + request_rows=report.requests.values(), + tool_rows=report.tools.values(), + subagent_rows=report.subagents.values(), generated_at=datetime.now(UTC), index_schema_version=REPORT_INDEX_SCHEMA_VERSION, ) diff --git a/src/autoskillit/execution/_report_index_rows.py b/src/autoskillit/execution/_report_index_rows.py index 45b467f630..eb5eb6ebd7 100644 --- a/src/autoskillit/execution/_report_index_rows.py +++ b/src/autoskillit/execution/_report_index_rows.py @@ -34,7 +34,7 @@ from autoskillit.execution.evidence.report_walk import OTLP_WALK_KIND, SESSION_WALK_KIND, WalkItem from autoskillit.execution.session.turn_usage import classify_token_measure -REPORT_INDEX_SCHEMA_VERSION: Final[int] = 1 +REPORT_INDEX_SCHEMA_VERSION: Final[int] = 2 SESSION_KIND: Final[str] = "session" REQUEST_KIND: Final[str] = "request" TOOL_KIND: Final[str] = "tool" @@ -121,7 +121,7 @@ class ReportToolRow(ReportRowBase): class ReportSubagentRow(ReportRowBase): - """Subagent facts derived from a ``subagent_completed`` log record.""" + """Native child or OTLP completion facts, kept in one subagent fact family.""" harness: str agent_type: str | None @@ -129,6 +129,20 @@ class ReportSubagentRow(ReportRowBase): final_model: str | None model_swapped: bool | None event_sequence: int | None + child_id: str | None + native_parent_session_id: str | None + parent_session_key: str | None + role: str | None + actor_level: str | None + provider: str + skill: str | None + recipe: str | None + step: str | None + level: str | None + token_usage: dict[str, SerializedTokenMeasure] + tool_counts: dict[str, int] | None + transcript_state: str | None + usage_state: str | None REPORT_ROW_TYPES: Mapping[str, type] = MappingProxyType( @@ -176,6 +190,26 @@ def _count_map(value: object) -> dict[str, int] | None: return counts +def _token_usage_map(value: object) -> dict[str, SerializedTokenMeasure]: + mapping = value if isinstance(value, Mapping) else {} + measures: dict[str, SerializedTokenMeasure] = {} + for field in CANONICAL_ACCOUNTING_FIELDS: + raw = mapping.get(field) + try: + measures[field] = ( + TokenMeasure.from_dict(raw).to_dict() + if isinstance(raw, dict) + else ( + TokenMeasure.observed(raw).to_dict() + if isinstance(raw, int) and not isinstance(raw, bool) and raw >= 0 + else TokenMeasure.unknown().to_dict() + ) + ) + except ValueError: + measures[field] = TokenMeasure.unknown().to_dict() + return measures + + def _pair_text(value: object) -> str: return _text(value) or UNKNOWN_SOURCE @@ -257,6 +291,20 @@ def _raw_measure(value: object) -> object: "final_model": _text, "model_swapped": _flag, "event_sequence": _count, + "child_id": _text, + "native_parent_session_id": _text, + "parent_session_key": _text, + "role": _text, + "actor_level": _text, + "provider": _pair_text, + "skill": _text, + "recipe": _text, + "step": _text, + "level": _text, + "token_usage": _token_usage_map, + "tool_counts": _count_map, + "transcript_state": _text, + "usage_state": _text, }, } @@ -294,7 +342,8 @@ def rows_for_walk_item(item: WalkItem) -> list[dict[str, Any]]: if item.record is None or item.source_id is None: return [] if item.kind == SESSION_WALK_KIND: - return [_session_row(item.source_id, item.record)] + session = _session_row(item.source_id, item.record) + return [session, *_native_subagent_rows(item.source_id, item.record, session)] if item.kind == OTLP_WALK_KIND: return _otlp_rows(item.source_id, item.record) return [] @@ -375,6 +424,13 @@ def _skill_name(value: object) -> str | None: return extract_skill_name(command) +def _attribution_skill(value: object) -> str | None: + skill = _text(value) + if skill is None: + return None + return _skill_name(skill) if skill.startswith(("/", "$")) else skill + + def _tool_counts(record: dict[str, Any]) -> dict[str, int] | None: if record.get("transcripts_available") is not True: return None @@ -390,6 +446,108 @@ def _tool_counts(record: dict[str, Any]) -> dict[str, int] | None: return dict(sorted(counts.items())) +def _native_subagent_rows( + key: str, + record: dict[str, Any], + parent: Mapping[str, Any], +) -> list[dict[str, Any]]: + source_row = record.get("row") + outcomes = record.get("child_outcomes") + if not isinstance(source_row, dict) or not isinstance(outcomes, (tuple, list)): + return [] + native_parent_id = _text(source_row.get("session_id")) + parent_key = _text(source_row.get("dir_name")) or key + if native_parent_id is None: + return [] + + rows: list[dict[str, Any]] = [] + seen_keys: set[str] = set() + for child in outcomes: + if not isinstance(child, Mapping): + continue + child_id = _text(child.get("child_id")) + role = _text(child.get("role")) + backend = _text(child.get("backend")) + expected_source = { + "claude_code": "transcript_metadata", + "codex": "codex_rollout_metadata", + }.get(backend or "") + if ( + child_id is None + or role is None + or expected_source is None + or child.get("evidence_source") != expected_source + or child.get("parent_session_id") != native_parent_id + or child.get("parent_session_key") not in (None, parent_key) + ): + continue + child_key = f"{parent_key}:native:{backend}:{native_parent_id}:{child_id}" + if child_key in seen_keys: + continue + seen_keys.add(child_key) + + effective_provider = _text(child.get("effective_provider")) + if effective_provider is not None and effective_provider.casefold() == UNKNOWN_SOURCE: + effective_provider = None + provider = _pair_text(effective_provider) + token_usage = ( + _token_usage_map(child.get("token_usage")) + if effective_provider is not None + else _unknown_child_usage_map() + ) + if child.get("usage_state") == "unknown": + token_usage = _unknown_child_usage_map() + + parent_skill = parent["skill"] + attribution_skill = _attribution_skill(child.get("attribution_skill")) + skill = ( + parent_skill + if parent["level"] in ("skill", "orchestrator") + and parent_skill is not None + and attribution_skill == parent_skill + else None + ) + transcript_state = _text(child.get("transcript_state")) or "unknown" + tool_counts = ( + _count_map(child.get("tool_counts")) if transcript_state == "observed" else None + ) + rows.append( + { + "schema_version": REPORT_INDEX_SCHEMA_VERSION, + "kind": SUBAGENT_KIND, + "key": child_key, + "session_id": native_parent_id, + "time_ms": parent["time_ms"], + "harness": parent["harness"], + "agent_type": None, + "model": _text(child.get("effective_model")), + "final_model": None, + "model_swapped": None, + "event_sequence": None, + "child_id": child_id, + "native_parent_session_id": native_parent_id, + "parent_session_key": parent_key, + "role": role, + "actor_level": "L0", + "provider": provider, + "skill": skill, + "recipe": parent["recipe"], + "step": parent["step"], + "level": parent["level"], + "token_usage": token_usage, + "tool_counts": tool_counts, + "transcript_state": transcript_state, + "usage_state": _text(child.get("usage_state")) or "unknown", + } + ) + rows.sort(key=lambda row: row["key"]) + return rows + + +def _unknown_child_usage_map() -> dict[str, SerializedTokenMeasure]: + return {field: TokenMeasure.unknown().to_dict() for field in CANONICAL_ACCOUNTING_FIELDS} + + def _otlp_rows(source_id: str, record: dict[str, Any]) -> list[dict[str, Any]]: if record.get("signal") != "logs": return [] @@ -475,6 +633,20 @@ def _subagent_fields( "final_model": unique_string_attribute(attributes, "final_model"), "model_swapped": unique_flag_attribute(attributes, "model_swapped"), "event_sequence": unique_count_attribute(attributes, "event.sequence"), + "child_id": None, + "native_parent_session_id": None, + "parent_session_key": None, + "role": None, + "actor_level": None, + "provider": UNKNOWN_SOURCE, + "skill": None, + "recipe": None, + "step": None, + "level": None, + "token_usage": _unknown_child_usage_map(), + "tool_counts": None, + "transcript_state": None, + "usage_state": None, } diff --git a/src/autoskillit/execution/backends/_codex_parse.py b/src/autoskillit/execution/backends/_codex_parse.py index df7e12a8ad..17a26d68cb 100644 --- a/src/autoskillit/execution/backends/_codex_parse.py +++ b/src/autoskillit/execution/backends/_codex_parse.py @@ -229,13 +229,17 @@ def _codex_turn_usage_entry( info: Mapping[str, Any], last_usage: Mapping[str, Any], effective_model: str | None, - start: datetime, - end: datetime, + start: datetime | None, + end: datetime | None, provider_used: str, ) -> TurnTokenEntry | None: timestamp = first_nonempty_string(record.get("timestamp")) event_time = _utc_datetime(timestamp) - if event_time is None or event_time < start or event_time > end: + if ( + event_time is None + or (start is not None and event_time < start) + or (end is not None and event_time > end) + ): return None input_tokens = first_valid_token_count(last_usage, "input_tokens") @@ -304,29 +308,68 @@ def extract_codex_turn_usage( if path is None: return [] + return _extract_codex_turn_usage(path, start, end, provider_used) + + +def extract_codex_child_turn_usage( + rollout_path: Path, + child_thread_id: str, + *, + provider_used: str = "codex", +) -> list[TurnTokenEntry]: + """Extract snapshots from a dedicated Codex rollout after verifying its thread.""" + if not child_thread_id or _thread_id(rollout_path) != child_thread_id: + return [] + return _extract_codex_turn_usage(rollout_path, None, None, provider_used) + + +def _codex_usage_records(reader: BinaryIO) -> Iterator[Mapping[str, Any]]: + for raw_line in reader: + try: + record = json.loads(raw_line) + except (UnicodeDecodeError, json.JSONDecodeError): + continue + if not isinstance(record, Mapping): + continue + if record.get("type") == "compacted": + yield record + continue + if not isinstance(record.get("payload"), Mapping): + continue + payload = record["payload"] + if record.get("type") == "event_msg" and payload.get("type") == "token_count": + info = payload.get("info") + if not isinstance(info, Mapping): + continue + if not isinstance(info.get("total_token_usage"), Mapping) or not isinstance( + info.get("last_token_usage"), Mapping + ): + continue + yield record + + +def _extract_codex_turn_usage( + path: Path, + start: datetime | None, + end: datetime | None, + provider_used: str, +) -> list[TurnTokenEntry]: + """Read advancing request snapshots within optional rollout time bounds.""" + rows: list[TurnTokenEntry] = [] current_model: str | None = None cumulative_high_water = 0 compacting = False try: with _logical_rollout_reader(path) as reader: - while raw_line := reader.readline(): - try: - record = json.loads(raw_line) - except (UnicodeDecodeError, json.JSONDecodeError): - continue - if not isinstance(record, Mapping): - continue - + for record in _codex_usage_records(reader): record_type = record.get("type") - payload = record.get("payload") - if record_type == "turn_context" and isinstance(payload, Mapping): - current_model = first_nonempty_string(payload.get("model")) - continue if record_type == "compacted": compacting = True continue - if not isinstance(payload, Mapping): + payload: Mapping[str, Any] = record["payload"] + if record_type == "turn_context" and isinstance(payload, Mapping): + current_model = first_nonempty_string(payload.get("model")) continue payload_type = payload.get("type") if record_type == "event_msg" and payload_type == "context_compacted": @@ -335,13 +378,9 @@ def extract_codex_turn_usage( if record_type != "event_msg" or payload_type != "token_count": continue - info = payload.get("info") - if not isinstance(info, Mapping): - continue - total_usage = info.get("total_token_usage") - last_usage = info.get("last_token_usage") - if not isinstance(total_usage, Mapping) or not isinstance(last_usage, Mapping): - continue + info = payload["info"] + total_usage = info["total_token_usage"] + last_usage = info["last_token_usage"] cumulative_total = valid_token_count(total_usage.get("total_tokens")) if cumulative_total is None or cumulative_total <= cumulative_high_water: continue diff --git a/src/autoskillit/execution/evidence/_native_child_projection.py b/src/autoskillit/execution/evidence/_native_child_projection.py new file mode 100644 index 0000000000..af20c85fd3 --- /dev/null +++ b/src/autoskillit/execution/evidence/_native_child_projection.py @@ -0,0 +1,428 @@ +"""Read and attribute structurally verified native child evidence for reports.""" + +from __future__ import annotations + +import hashlib +import json +import math +from collections import Counter +from collections.abc import Mapping, Sequence +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import zstandard + +from autoskillit.core import ( + AGENT_BACKEND_CLAUDE_CODE, + AGENT_BACKEND_CODEX, + CANONICAL_ACCOUNTING_FIELDS, + SerializedTokenMeasure, + TokenMeasure, + TurnTokenEntry, + iter_merged_assistant_turns, +) +from autoskillit.execution.backends._codex_execution_identity import ( + extract_codex_child_metadata, +) +from autoskillit.execution.backends._codex_parse import ( + _logical_rollout_reader, + extract_codex_child_turn_usage, +) +from autoskillit.execution.child_outcomes import ( + collect_child_outcomes, + enumerate_claude_subagent_transcripts, + normalize_backend_name, +) +from autoskillit.execution.session import extract_token_usage +from autoskillit.execution.session.turn_usage import ( + classify_token_measure, + claude_inclusive_input_tokens, + merge_turn_usage, +) + + +def project_child_outcomes( + row: dict[str, Any], + *, + log_root: Path, + parent_rows: Mapping[tuple[str, str], Sequence[dict[str, Any]]], +) -> tuple[dict[str, Any], ...]: + parent_id = row.get("session_id") + backend = row.get("backend") + if not isinstance(parent_id, str) or not parent_id or not isinstance(backend, str): + return () + normalized_backend = normalize_backend_name(backend) + candidates = parent_rows.get((normalized_backend, parent_id), ()) + outcomes = collect_child_outcomes( + backend=normalized_backend, + parent_session_id=parent_id, + log_root=log_root, + ) + projected = [] + for outcome in outcomes: + child = _project_child_outcome( + row, + dict(outcome), + parent_id=parent_id, + backend=normalized_backend, + candidates=candidates, + ) + if child is not None: + projected.append(child) + return tuple(projected) + + +def _project_child_outcome( + row: dict[str, Any], + child: dict[str, Any], + *, + parent_id: str, + backend: str, + candidates: Sequence[dict[str, Any]], +) -> dict[str, Any] | None: + if not _is_native_child(child, backend, parent_id): + return None + path, identity_verified = _child_transcript_path(row, child, backend, parent_id) + if not identity_verified: + return None + text = _read_child_transcript(path, backend) if path is not None and path.is_file() else None + complete = _complete_jsonl(text) + child_timestamp = _first_transcript_timestamp(text) if text is not None else None + if not _child_belongs_to_parent(row, candidates, child_timestamp): + return None + + provider = child.get("effective_provider") + provider_used = ( + provider + if isinstance(provider, str) and provider and provider.casefold() != "unknown" + else "" + ) + tool_counts, token_usage, usage_state = _child_measurements( + backend, + child["child_id"], + path, + text, + complete, + provider_used, + ) + child.update( + { + "native_parent_session_id": parent_id, + "parent_session_key": row.get("dir_name"), + "transcript_state": "observed" if complete else "unknown", + "usage_state": usage_state, + "tool_counts": tool_counts, + "token_usage": token_usage, + "_child_timestamp": child_timestamp, + "_transcript_fingerprint": _transcript_fingerprint(path, text), + } + ) + return child + + +def _is_native_child(child: Mapping[str, Any], backend: str, parent_id: str) -> bool: + expected_source = { + "claude_code": "transcript_metadata", + "codex": "codex_rollout_metadata", + }.get(backend) + return ( + expected_source is not None + and child.get("backend") == backend + and child.get("parent_session_id") == parent_id + and isinstance(child.get("child_id"), str) + and bool(child.get("child_id")) + and isinstance(child.get("role"), str) + and bool(child.get("role")) + and child.get("evidence_source") == expected_source + ) + + +def _child_transcript_path( + row: dict[str, Any], child: Mapping[str, Any], backend: str, parent_id: str +) -> tuple[Path | None, bool]: + child_id = child.get("child_id") + if not isinstance(child_id, str) or not child_id: + return None, False + if backend == normalize_backend_name(AGENT_BACKEND_CLAUDE_CODE): + return _claude_child_transcript_path(row, child_id, parent_id) + if backend == normalize_backend_name(AGENT_BACKEND_CODEX): + return _codex_child_transcript_path(child, child_id, parent_id) + return None, False + + +def _claude_child_transcript_path( + row: dict[str, Any], child_id: str, parent_id: str +) -> tuple[Path | None, bool]: + parent_log = row.get("claude_code_log") + if not isinstance(parent_log, str) or not Path(parent_log).is_absolute(): + return None, True + if Path(parent_log).stem != parent_id: + return None, False + try: + paths = enumerate_claude_subagent_transcripts(Path(parent_log)) + except OSError: + return None, True + return next((path for path in paths if path.name == f"agent-{child_id}.jsonl"), None), True + + +def _codex_child_transcript_path( + child: Mapping[str, Any], child_id: str, parent_id: str +) -> tuple[Path | None, bool]: + locator = child.get("transcript_locator") + if not isinstance(locator, str) or not Path(locator).is_absolute(): + return None, True + path = Path(locator) + if not path.exists(): + return path, True + try: + extract_codex_child_metadata( + path, + expected_parent_id=parent_id, + expected_child_id=child_id, + ) + except (OSError, RuntimeError, ValueError, zstandard.ZstdError): + return None, False + return path, True + + +def _child_belongs_to_parent( + row: dict[str, Any], candidates: Sequence[dict[str, Any]], child_timestamp: float | None +) -> bool: + row_key = row.get("dir_name") + if not isinstance(row_key, str): + return False + if len(candidates) == 1: + return candidates[0].get("dir_name") == row_key + if child_timestamp is None: + return False + matches = [ + candidate for candidate in candidates if _invocation_contains(candidate, child_timestamp) + ] + return len(matches) == 1 and matches[0].get("dir_name") == row_key + + +def _invocation_contains(row: dict[str, Any], timestamp: float) -> bool: + start = _timestamp_seconds(row.get("timestamp")) + duration = row.get("duration_seconds") + if ( + start is None + or isinstance(duration, bool) + or not isinstance(duration, (int, float)) + or not math.isfinite(float(duration)) + or duration < 0 + ): + return False + return start <= timestamp <= start + float(duration) + + +def _timestamp_seconds(value: object) -> float | None: + if not isinstance(value, str) or not value: + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=UTC) + return parsed.timestamp() + + +def _first_transcript_timestamp(text: str) -> float | None: + for line in text.splitlines(): + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(record, Mapping): + timestamp = _timestamp_seconds(record.get("timestamp")) + if timestamp is not None: + return timestamp + return None + + +def _read_child_transcript(path: Path, backend: str) -> str | None: + try: + if backend == normalize_backend_name(AGENT_BACKEND_CODEX): + with _logical_rollout_reader(path) as handle: + return handle.read().decode("utf-8") + return path.read_text(encoding="utf-8") + except (OSError, RuntimeError, UnicodeError, ValueError, zstandard.ZstdError): + return None + + +def _complete_jsonl(text: str | None) -> bool: + if not text or not text.endswith("\n"): + return False + for line in text.splitlines(): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError: + return False + if not isinstance(record, Mapping): + return False + return True + + +def _child_measurements( + backend: str, + child_id: str, + path: Path | None, + text: str | None, + complete: bool, + provider: str, +) -> tuple[dict[str, int] | None, dict[str, SerializedTokenMeasure], str]: + if not complete or text is None: + return None, _unknown_usage(), "unknown" + parser_backend = ( + "claude" if backend == normalize_backend_name(AGENT_BACKEND_CLAUDE_CODE) else backend + ) + turns = list(iter_merged_assistant_turns(text, backend=parser_backend)) + tool_counts = dict(Counter(name for turn in turns for name in turn.tool_names)) + token_usage, usage_state = _child_token_usage(backend, child_id, path, text, provider) + return tool_counts, token_usage, usage_state + + +def _child_token_usage( + backend: str, child_id: str, path: Path | None, text: str, provider: str +) -> tuple[dict[str, SerializedTokenMeasure], str]: + if not provider: + return _unknown_usage(), "unknown" + if backend == normalize_backend_name(AGENT_BACKEND_CLAUDE_CODE): + aggregate, _rows = extract_token_usage(text, provider_used=provider) + if aggregate is not None: + usage = { + field: aggregate.get(field, TokenMeasure.unknown().to_dict()) + for field in CANONICAL_ACCOUNTING_FIELDS + } + inclusive_input = claude_inclusive_input_tokens( + *( + TokenMeasure.from_dict(usage[field]).value + for field in ("input_tokens", "cache_read_tokens", "cache_write_tokens") + ) + ) + usage["input_tokens"] = classify_token_measure( + backend, provider, "input_tokens", inclusive_input + ).to_dict() + return ( + usage, + "observed", + ) + return _unknown_usage(), "unknown" + if backend == normalize_backend_name(AGENT_BACKEND_CODEX) and path is not None: + rows = extract_codex_child_turn_usage(path, child_id, provider_used=provider) + if rows: + return _codex_usage_measures(rows, provider), "observed" + return _unknown_usage(), "unknown" + + +def _codex_usage_measures( + rows: Sequence[TurnTokenEntry], provider: str +) -> dict[str, SerializedTokenMeasure]: + rows = merge_turn_usage(rows) + request_positions: dict[tuple[str, str, str], int] = {} + folded_rows: list[TurnTokenEntry] = [] + for row in rows: + request_id = row["request_id"] + identity = (row["backend"], row["provider_used"], request_id or "") + if not request_id or identity not in request_positions: + folded_rows.append(row.copy()) + if request_id: + request_positions[identity] = len(folded_rows) - 1 + continue + + current = folded_rows[request_positions[identity]] + for field in ( + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cache_creation_tokens", + "context_window_tokens", + ): + if row[field] is not None: + current[field] = row[field] + + rows = folded_rows + measures: dict[str, SerializedTokenMeasure] = {} + for field in CANONICAL_ACCOUNTING_FIELDS: + values = [row.get(field) for row in rows] + total = ( + sum( + value for value in values if isinstance(value, int) and not isinstance(value, bool) + ) + if values + and all( + isinstance(value, int) and not isinstance(value, bool) and value >= 0 + for value in values + ) + else None + ) + measures[field] = classify_token_measure("codex", provider, field, total).to_dict() + return measures + + +def _unknown_usage() -> dict[str, SerializedTokenMeasure]: + return {field: TokenMeasure.unknown().to_dict() for field in CANONICAL_ACCOUNTING_FIELDS} + + +def _transcript_fingerprint(path: Path | None, text: str | None) -> str | None: + if path is None: + return None + identity: list[int] | None = None + try: + stat = path.stat() + identity = [stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns] + except OSError: + pass + material = json.dumps( + { + "path": str(path), + "identity": identity, + "content": hashlib.sha256((text or "").encode("utf-8")).hexdigest(), + }, + sort_keys=True, + separators=(",", ":"), + ).encode() + return hashlib.sha256(material).hexdigest() + + +def child_evidence_fingerprint( + log_root: Path, + rows: Sequence[dict[str, Any]], + parent_rows: Mapping[tuple[str, str], Sequence[dict[str, Any]]], +) -> str: + """Digest owner-attributed child facts and the dedicated transcripts they use.""" + dependencies = [ + _child_dependency(row, child) + for row in rows + for child in project_child_outcomes(row, log_root=log_root, parent_rows=parent_rows) + ] + dependencies.sort(key=_canonical_json) + return hashlib.sha256(_canonical_json(dependencies).encode("utf-8")).hexdigest() + + +def _child_dependency(row: dict[str, Any], child: Mapping[str, Any]) -> dict[str, Any]: + return { + "parent_session_key": child.get("parent_session_key"), + "parent_timestamp": row.get("timestamp"), + "parent_duration_seconds": row.get("duration_seconds"), + "native_parent_session_id": child.get("native_parent_session_id"), + "child_id": child.get("child_id"), + "backend": child.get("backend"), + "role": child.get("role"), + "skill": child.get("attribution_skill"), + "provider": child.get("effective_provider"), + "model": child.get("effective_model"), + "evidence_source": child.get("evidence_source"), + "transcript_state": child.get("transcript_state"), + "usage_state": child.get("usage_state"), + "tool_counts": child.get("tool_counts"), + "token_usage": child.get("token_usage"), + "transcript_fingerprint": child.get("_transcript_fingerprint"), + } + + +def _canonical_json(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) diff --git a/src/autoskillit/execution/evidence/report_walk.py b/src/autoskillit/execution/evidence/report_walk.py index 56242f7695..126597b8b7 100644 --- a/src/autoskillit/execution/evidence/report_walk.py +++ b/src/autoskillit/execution/evidence/report_walk.py @@ -11,7 +11,7 @@ import hashlib import json import os -from collections.abc import Iterator +from collections.abc import Iterator, Mapping, Sequence from contextlib import ExitStack from dataclasses import dataclass from pathlib import Path @@ -19,7 +19,16 @@ from autoskillit.core import ArtifactLease, get_logger, iter_merged_assistant_turns from autoskillit.execution.backends._codex_parse import _logical_rollout_reader -from autoskillit.execution.child_outcomes import enumerate_claude_subagent_transcripts +from autoskillit.execution.child_outcomes import ( + enumerate_claude_subagent_transcripts, + normalize_backend_name, +) +from autoskillit.execution.evidence._native_child_projection import ( + child_evidence_fingerprint as _native_child_evidence_fingerprint, +) +from autoskillit.execution.evidence._native_child_projection import ( + project_child_outcomes, +) from autoskillit.execution.session_log.session_index import ( iter_tolerant_session_index_lines, read_tolerant_session_index_rows, @@ -269,11 +278,53 @@ def _transcript_turns(path: Path, backend: str) -> list[dict[str, Any]]: ] -def _session_record(row: dict[str, Any]) -> dict[str, Any]: +def _native_parent_rows( + rows: Sequence[dict[str, Any]], +) -> dict[tuple[str, str], list[dict[str, Any]]]: + by_key: dict[str, dict[str, Any]] = {} + for row in rows: + key = row.get("dir_name") + if isinstance(key, str) and key: + by_key[key] = row + parents: dict[tuple[str, str], list[dict[str, Any]]] = {} + for row in by_key.values(): + backend = row.get("backend") + session_id = row.get("session_id") + if not isinstance(backend, str) or not isinstance(session_id, str) or not session_id: + continue + identity = normalize_backend_name(backend), session_id + parents.setdefault(identity, []).append(row) + for candidates in parents.values(): + candidates.sort(key=lambda candidate: str(candidate.get("dir_name", ""))) + return parents + + +def _retained_session_rows(log_root: Path) -> list[dict[str, Any]]: + rows_by_key: dict[str, dict[str, Any]] = {} + for row in read_tolerant_session_index_rows(log_root / "sessions-archive.jsonl"): + key = row.get("dir_name") + if isinstance(key, str) and key: + rows_by_key[key] = row + with ArtifactLease.acquire_shared( + session_index_lock_path(log_root), timeout=_LEASE_TIMEOUT_SECONDS + ): + for row in read_tolerant_session_index_rows(log_root / "sessions.jsonl"): + key = row.get("dir_name") + if isinstance(key, str) and key: + rows_by_key[key] = row + return [rows_by_key[key] for key in sorted(rows_by_key)] + + +def _session_record( + row: dict[str, Any], + *, + log_root: Path, + parent_rows: Mapping[tuple[str, str], Sequence[dict[str, Any]]], +) -> dict[str, Any]: turns: list[dict[str, Any]] = [] unavailable_reasons: list[str] = [] has_transcript = False - for field, backend in (("claude_code_log", "claude"), ("codex_log", "codex")): + for field, transcript_backend in (("claude_code_log", "claude"), ("codex_log", "codex")): raw = row.get(field) if raw is None: if field in row: @@ -287,14 +338,16 @@ def _session_record(row: dict[str, Any]) -> dict[str, Any]: paths: tuple[Path, ...] try: paths = (path,) + ( - enumerate_claude_subagent_transcripts(path) if backend == "claude" else () + enumerate_claude_subagent_transcripts(path) + if transcript_backend == "claude" + else () ) except OSError as exc: unavailable_reasons.append(f"{field}:enumerate:{exc}") continue for transcript in paths: try: - turns.extend(_transcript_turns(transcript, backend)) + turns.extend(_transcript_turns(transcript, transcript_backend)) except (OSError, UnicodeError, ValueError) as exc: unavailable_reasons.append(f"{transcript}:{exc}") except RuntimeError as exc: @@ -303,16 +356,24 @@ def _session_record(row: dict[str, Any]) -> dict[str, Any]: # rather than masking the caller's intent. unavailable_reasons.append(f"{transcript}:runtime:{exc}") unavailable = not has_transcript or bool(unavailable_reasons) - return { + record: dict[str, Any] = { "row": row, "assistant_turn_count": None if unavailable else len(turns), "assistant_turns": turns, "transcripts_available": not unavailable, "transcript_unavailable_reasons": unavailable_reasons, } + record["child_outcomes"] = project_child_outcomes( + row, log_root=log_root, parent_rows=parent_rows + ) + return record -def _walk_archive(root: Path, state: dict[str, Any]) -> Iterator[WalkItem]: +def _walk_archive( + root: Path, + state: dict[str, Any], + parent_rows: Mapping[tuple[str, str], Sequence[dict[str, Any]]], +) -> Iterator[WalkItem]: # The session archive is an append-only retention file with no concurrent # writers in the live session workflow, so an ArtifactLease is not # required; the file handle is closed promptly and identity is captured @@ -350,12 +411,16 @@ def _walk_archive(root: Path, state: dict[str, Any]) -> Iterator[WalkItem]: SESSION_WALK_KIND, row["dir_name"], session_id, - _session_record(row), + _session_record(row, log_root=root, parent_rows=parent_rows), _copy(state), ) -def _walk_projection(root: Path, state: dict[str, Any]) -> Iterator[WalkItem]: +def _walk_projection( + root: Path, + state: dict[str, Any], + parent_rows: Mapping[tuple[str, str], Sequence[dict[str, Any]]], +) -> Iterator[WalkItem]: path = root / "sessions.jsonl" with ArtifactLease.acquire_shared( session_index_lock_path(root), timeout=_LEASE_TIMEOUT_SECONDS @@ -389,7 +454,13 @@ def _walk_projection(root: Path, state: dict[str, Any]) -> Iterator[WalkItem]: session_id = row.get("session_id") if session_id is not None and not isinstance(session_id, str): session_id = None - yield WalkItem(SESSION_WALK_KIND, name, session_id, _session_record(row), _copy(state)) + yield WalkItem( + SESSION_WALK_KIND, + name, + session_id, + _session_record(row, log_root=root, parent_rows=parent_rows), + _copy(state), + ) state["projection"] = current state["projection_identity"] = identity yield WalkItem(CHECKPOINT_WALK_KIND, None, None, None, _copy(state)) @@ -407,5 +478,14 @@ def iter_report_walk( should not assume one from ``_walk_otlp`` or ``_walk_archive``. """ state = _copy(watermark or {}) - for walk in (_walk_otlp, _walk_archive, _walk_projection): - yield from walk(log_root, state) + parent_rows = _native_parent_rows(_retained_session_rows(log_root)) + yield from _walk_otlp(log_root, state) + yield from _walk_archive(log_root, state, parent_rows) + yield from _walk_projection(log_root, state, parent_rows) + + +def child_evidence_fingerprint(log_root: Path) -> str: + """Return the current fingerprint of owner-attributed native child evidence.""" + rows = _retained_session_rows(log_root) + parent_rows = _native_parent_rows(rows) + return _native_child_evidence_fingerprint(log_root, rows, parent_rows) diff --git a/src/autoskillit/execution/report_index.py b/src/autoskillit/execution/report_index.py index fc02e6ea23..a3bfd207c0 100644 --- a/src/autoskillit/execution/report_index.py +++ b/src/autoskillit/execution/report_index.py @@ -7,9 +7,10 @@ the attributed harness/provider pair. For sources changed only in ways tracked by ``iter_report_walk``, an incremental -build and a rebuild have the same rows. This assumes transcripts are not modified -after their session row is walked; ``rebuild_report_index`` refreshes transcript -counts when that assumption no longer holds. +build and a rebuild have the same rows. Child snapshots and their dedicated +transcripts are fingerprinted on each update. The parent transcript is assumed not +to change after its session row is walked; ``rebuild_report_index`` refreshes its +derived counts when that assumption no longer holds. """ from __future__ import annotations @@ -52,6 +53,7 @@ VALID_SOURCE_KEYS, SourceGapError, WalkItem, + child_evidence_fingerprint, iter_report_walk, ) from autoskillit.execution.session_log.session_index import ( @@ -116,13 +118,29 @@ def rebuild_report_index(log_root: Path, index_dir: Path) -> ReportIndexUpdate: return _update(log_root, index_dir, rebuild=True) +def _clear_index_files(index_dir: Path) -> None: + (index_dir / _ROWS_FILE).unlink(missing_ok=True) + (index_dir / _STATE_FILE).unlink(missing_ok=True) + + def _update(log_root: Path, index_dir: Path, *, rebuild: bool) -> ReportIndexUpdate: index_dir.mkdir(mode=0o700, parents=True, exist_ok=True) with ArtifactLease.acquire_exclusive(index_dir / _LOCK_FILE, timeout=_LEASE_TIMEOUT_SECONDS): if rebuild: - (index_dir / _ROWS_FILE).unlink(missing_ok=True) - (index_dir / _STATE_FILE).unlink(missing_ok=True) - with _RowAppender.open(index_dir) as appender: + _clear_index_files(index_dir) + fingerprint = child_evidence_fingerprint(log_root) + state = read_versioned_json(index_dir / _STATE_FILE, _STATE_SCHEMA_VERSION) + row_schema_mismatch = ( + state.get("row_schema_version") != REPORT_INDEX_SCHEMA_VERSION + if state is not None + else (index_dir / _ROWS_FILE).exists() + ) + child_evidence_changed = ( + state is not None and state.get("child_evidence_fingerprint") != fingerprint + ) + if row_schema_mismatch or child_evidence_changed: + _clear_index_files(index_dir) + with _RowAppender.open(index_dir, child_evidence_fingerprint=fingerprint) as appender: return _walk_into(log_root, appender) @@ -271,10 +289,12 @@ def __init__( index_dir: Path, handle: BinaryIO, watermark: dict[str, Any] | None, + child_evidence_fingerprint: str, ) -> None: self.index_dir = index_dir self.handle = handle self.watermark = watermark + self.child_evidence_fingerprint = child_evidence_fingerprint self._pending = bytearray() self._dirty = False self.items_walked = 0 @@ -282,7 +302,7 @@ def __init__( @classmethod @contextmanager - def open(cls, index_dir: Path) -> Iterator[_RowAppender]: + def open(cls, index_dir: Path, *, child_evidence_fingerprint: str) -> Iterator[_RowAppender]: rows_path = index_dir / _ROWS_FILE watermark, committed = _committed_state(index_dir, rows_path) fd = os.open( @@ -293,7 +313,7 @@ def open(cls, index_dir: Path) -> Iterator[_RowAppender]: with os.fdopen(fd, "r+b") as handle: handle.truncate(committed) handle.seek(committed) - yield cls(index_dir, handle, watermark) + yield cls(index_dir, handle, watermark, child_evidence_fingerprint) def add(self, item: WalkItem) -> None: for row in rows_for_walk_item(item): @@ -311,15 +331,20 @@ def add(self, item: WalkItem) -> None: if item.kind == CHECKPOINT_WALK_KIND or len(self._pending) >= _COMMIT_BYTES: self.commit() - def commit(self) -> None: - if not self._dirty: + def commit(self, *, force: bool = False) -> None: + if not self._dirty and not force: return self.handle.write(bytes(self._pending)) self.handle.flush() os.fsync(self.handle.fileno()) write_versioned_json( self.index_dir / _STATE_FILE, - {"walk": self.watermark, "rows_bytes": self.handle.tell()}, + { + "walk": self.watermark, + "rows_bytes": self.handle.tell(), + "row_schema_version": REPORT_INDEX_SCHEMA_VERSION, + "child_evidence_fingerprint": self.child_evidence_fingerprint, + }, schema_version=_STATE_SCHEMA_VERSION, ) self._pending.clear() @@ -352,7 +377,7 @@ def _walk_into(log_root: Path, appender: _RowAppender) -> ReportIndexUpdate: logger.warning("report_index_source_gap", source=gap.source, detail=str(gap)) appender.reset_source(gap.source) continue - appender.commit() + appender.commit(force=True) return ReportIndexUpdate(appender.items_walked, appender.rows_written, tuple(gaps)) @@ -449,8 +474,23 @@ def _sorted_rows(rows: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]: return dict(sorted(rows.items())) +def _require_row_schema(index_dir: Path) -> None: + rows_path = index_dir / _ROWS_FILE + state_path = index_dir / _STATE_FILE + if not rows_path.exists() and not state_path.exists(): + return + state = read_versioned_json(state_path, _STATE_SCHEMA_VERSION) + found = state.get("row_schema_version") if state is not None else None + if found != REPORT_INDEX_SCHEMA_VERSION: + raise ValueError( + f"report index row schema mismatch: expected {REPORT_INDEX_SCHEMA_VERSION}, " + f"found {found!r}" + ) + + def read_report_index(index_dir: Path) -> ReportIndex: """Read the latest valid fact for each key and resolve its session join.""" + _require_row_schema(index_dir) by_kind = _read_rows(index_dir / _ROWS_FILE) sessions = by_kind[SESSION_KIND] requests = by_kind[REQUEST_KIND] diff --git a/src/autoskillit/execution/session/_session_model.py b/src/autoskillit/execution/session/_session_model.py index 008711e3dd..82ca21dd6f 100644 --- a/src/autoskillit/execution/session/_session_model.py +++ b/src/autoskillit/execution/session/_session_model.py @@ -26,6 +26,7 @@ from autoskillit.execution.session.turn_usage import ( build_turn_token_entry, classify_token_measure, + claude_inclusive_input_tokens, first_nonempty_string, first_valid_token_count, merge_turn_usage, @@ -410,11 +411,7 @@ def _build_turn_usage_and_model_buckets( raw_input = row["input_tokens"] cache_read = row["cache_read_tokens"] cache_creation = row["cache_creation_tokens"] - inclusive_input = ( - raw_input + cache_read + cache_creation - if raw_input is not None and cache_read is not None and cache_creation is not None - else None - ) + inclusive_input = claude_inclusive_input_tokens(raw_input, cache_read, cache_creation) windows = model_windows.get(model, set()) if model is not None else set() context_window = next(iter(windows)) if len(windows) == 1 else None turn_usage.append( diff --git a/src/autoskillit/execution/session/turn_usage.py b/src/autoskillit/execution/session/turn_usage.py index 7883e174a1..82b3f9c9c2 100644 --- a/src/autoskillit/execution/session/turn_usage.py +++ b/src/autoskillit/execution/session/turn_usage.py @@ -230,6 +230,17 @@ def first_valid_token_count(usage: Mapping[str, Any], *fields: str) -> int | Non return None +def claude_inclusive_input_tokens( + raw_input: int | None, + cache_read: int | None, + cache_write: int | None, +) -> int | None: + """Return inclusive Claude input only when every raw component was observed.""" + if raw_input is None or cache_read is None or cache_write is None: + return None + return raw_input + cache_read + cache_write + + def valid_context_window(value: Any) -> int | None: """Return a positive provider-supplied context capacity.""" count = valid_token_count(value) diff --git a/src/autoskillit/report/deck/_html.py b/src/autoskillit/report/deck/_html.py index bdcbbf2a68..0da3411f9a 100644 --- a/src/autoskillit/report/deck/_html.py +++ b/src/autoskillit/report/deck/_html.py @@ -67,10 +67,21 @@ def render_deck_html(payload: Mapping[str, Any]) -> str: def render_deck( - session_rows: Iterable[Mapping[str, Any]], *, generated_at: datetime, index_schema_version: int + session_rows: Iterable[Mapping[str, Any]], + *, + request_rows: Iterable[Mapping[str, Any]] = (), + tool_rows: Iterable[Mapping[str, Any]] = (), + subagent_rows: Iterable[Mapping[str, Any]] = (), + generated_at: datetime, + index_schema_version: int, ) -> str: return render_deck_html( build_deck_payload( - session_rows, generated_at=generated_at, index_schema_version=index_schema_version + session_rows, + request_rows=request_rows, + tool_rows=tool_rows, + subagent_rows=subagent_rows, + generated_at=generated_at, + index_schema_version=index_schema_version, ) ) diff --git a/src/autoskillit/report/deck/_payload.py b/src/autoskillit/report/deck/_payload.py index 8374f0ea00..6d7e963924 100644 --- a/src/autoskillit/report/deck/_payload.py +++ b/src/autoskillit/report/deck/_payload.py @@ -3,9 +3,18 @@ from collections import Counter from collections.abc import Iterable, Mapping, Sequence from datetime import UTC, datetime, timedelta +from itertools import combinations from typing import Any -from autoskillit.core import TokenMeasureState +from autoskillit.core import ( + MeasureRecord, + SourcePair, + TokenMeasure, + TokenMeasureState, + aggregate_measures, + load_bundled_agent_definitions, + measure_ratio, +) from ._registry import ( AVAILABILITY_VOCABULARY, @@ -24,6 +33,8 @@ ) DAY_MS = 86_400_000 +TOKEN_FIELDS = ("input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens") +SKILL_LEVELS = ("skill", "orchestrator", None) def encode_table(rows: Sequence[Mapping[str, Any]], columns: Sequence[str]) -> dict[str, Any]: @@ -104,6 +115,7 @@ def _window_chips( *, generated_at_ms: int, history: Mapping[str, Any], + population_label: str | None = None, ) -> list[dict[str, Any]]: chips = [] for window in WINDOWS: @@ -113,7 +125,11 @@ def _window_chips( state, count = ChipState.LIVE, len(rows) elif history["first_ms"] is None: state = ChipState.ABSENT - reason = ReasonDef("no session row in this index carries a timestamp") + reason = ReasonDef( + f"no {population_label} row in this view carries a timestamp" + if population_label + else "no session row in this index carries a timestamp" + ) else: first = history["first_ms"] retained = (generated_at_ms - first) // DAY_MS @@ -132,7 +148,11 @@ def _window_chips( ) state = ChipState.LIVE if count else ChipState.ABSENT if not count: - reason = ReasonDef(f"no rows in the last {window.label}") + reason = ReasonDef( + f"no {population_label} rows in the last {window.label}" + if population_label + else f"no rows in the last {window.label}" + ) chips.append(_chip(window.key, window.label, state, count, reason, days=window.days)) return chips @@ -143,15 +163,550 @@ def resolve_view_chips( *, generated_at_ms: int, history: Mapping[str, Any], + population_label: str | None = None, ) -> dict[str, list[dict[str, Any]]]: return {f.facet_id: _value_chips(view, f, rows) for f in FACETS} | { - WINDOW_FACET_ID: _window_chips(rows, generated_at_ms=generated_at_ms, history=history) + WINDOW_FACET_ID: _window_chips( + rows, + generated_at_ms=generated_at_ms, + history=history, + population_label=population_label, + ) + } + + +def _canonical_children( + sessions: Sequence[Mapping[str, Any]], + subagents: Iterable[Mapping[str, Any]], +) -> list[dict[str, Any]]: + parents = {row.get("key"): row for row in sessions if isinstance(row.get("key"), str)} + children = [] + for raw in subagents: + row = dict(raw) + parent_key = row.get("parent_session_key") + parent = parents.get(parent_key) + role = row.get("role") + if ( + row.get("actor_level") != "L0" + or not isinstance(row.get("child_id"), str) + or not row["child_id"] + or not isinstance(role, str) + or not role + or parent is None + or not isinstance(row.get("native_parent_session_id"), str) + or row["native_parent_session_id"] != parent.get("session_id") + ): + continue + children.append( + { + **row, + "harness": parent.get("harness") or "unknown", + "provider": row.get("provider") or "unknown", + "time_ms": parent.get("time_ms"), + "level": parent.get("level"), + "recipe": parent.get("recipe"), + "step": parent.get("step"), + "skill": row.get("skill") if isinstance(row.get("skill"), str) else None, + "_parent_provider": parent.get("provider") or "unknown", + } + ) + return children + + +def _window_level_blocks( + rows: Sequence[Mapping[str, Any]], *, generated_at_ms: int +) -> list[tuple[str, list[str | None], list[Mapping[str, Any]]]]: + blocks: list[tuple[str, list[str | None], list[Mapping[str, Any]]]] = [] + for window in WINDOWS: + in_window = [ + row + for row in rows + if window.days is None + or ( + row.get("time_ms") is not None + and row["time_ms"] >= generated_at_ms - window.days * DAY_MS + ) + ] + levels = sorted( + {row.get("level") for row in in_window}, + key=lambda level: (level is None, level or ""), + ) + for size in range(1, len(levels) + 1): + for selected in combinations(levels, size): + selected_set = set(selected) + members = [row for row in in_window if row.get("level") in selected_set] + if members: + blocks.append((window.key, list(selected), members)) + return blocks + + +def _measure_records( + rows: Sequence[Mapping[str, Any]], *, tool_names: Sequence[str] = () +) -> list[MeasureRecord]: + records = [] + for row in rows: + harness = row.get("harness") + provider = row.get("provider") + pair = SourcePair( + harness if isinstance(harness, str) and harness else "unknown", + provider if isinstance(provider, str) and provider else "unknown", + ) + child = row.get("actor_level") == "L0" + raw = row.get("token_usage") if child else row + raw = raw if isinstance(raw, Mapping) else {} + usage_known = not child or row.get("usage_state") == "observed" + measures = { + field: TokenMeasure.measure_from_raw(raw.get(field)) + if usage_known + else TokenMeasure.unknown() + for field in TOKEN_FIELDS + } + if child: + counts = row.get("tool_counts") + transcript_known = row.get("transcript_state") == "observed" and isinstance( + counts, Mapping + ) + counts = counts if isinstance(counts, Mapping) else {} + measures["tool_calls"] = ( + TokenMeasure.observed(sum(counts.values())) + if transcript_known + else TokenMeasure.unknown() + ) + for tool in tool_names: + measures[f"tool:{tool}"] = ( + TokenMeasure.observed(counts.get(tool, 0)) + if transcript_known + else TokenMeasure.unknown() + ) + records.append(MeasureRecord(pair, measures)) + return records + + +def _measure_payload(records: Sequence[MeasureRecord], fields: Sequence[str]) -> dict[str, Any]: + result = aggregate_measures(records, fields) + return { + field: { + "state": aggregate.value.state.value, + "value": aggregate.value.value, + "state_counts": { + state.value: count for state, count in aggregate.state_counts.items() + }, + } + for field, aggregate in result.fields.items() + } + + +def _ratio_payload( + records: Sequence[MeasureRecord], + numerator: str, + denominator: str, + *, + sample_unit: str, + definition_roles: Sequence[str], + definitions: Mapping[str, Mapping[str, Any]], +) -> dict[str, Any]: + result = measure_ratio(records, numerator, denominator) + roles = sorted(set(definition_roles)) + missing = [role for role in roles if definitions.get(role, {}).get("state") != "available"] + eligible = ( + result.state in (TokenMeasureState.MEASURED, TokenMeasureState.MEASURED_ZERO) + and result.value is not None + and result.sample_size > 0 + and bool(roles) + and not missing + ) + reasons = [] + if result.state is TokenMeasureState.UNKNOWN: + reasons.append("the ratio has unknown contributing measurements") + elif result.state is TokenMeasureState.UNAVAILABLE: + reasons.append("the ratio is unavailable for this source") + elif result.state is TokenMeasureState.NOT_APPLICABLE: + reasons.append("the ratio has no applicable positive denominator") + if not roles: + reasons.append("no contributor role definitions are linked") + if missing: + reasons.append("missing definitions: " + ", ".join(missing)) + return { + "state": result.state.value, + "value": result.value, + "numerator": result.numerator_field, + "denominator": result.denominator_field, + "numerator_total": result.numerator_total, + "denominator_total": result.denominator_total, + "sample_unit": sample_unit, + "sample_size": result.sample_size, + "excluded_runs": result.excluded_runs, + "unknown_runs": result.unknown_runs, + "definition_roles": roles, + "review_eligible": eligible, + "review_reason": None if eligible else "; ".join(reasons), } +def _metric_payload( + rows: Sequence[Mapping[str, Any]], + *, + sample_unit: str, + definition_roles: Sequence[str], + definitions: Mapping[str, Mapping[str, Any]], +) -> dict[str, Any]: + tool_names = sorted( + { + tool + for row in rows + if row.get("transcript_state") == "observed" + and isinstance(row.get("tool_counts"), Mapping) + for tool in row["tool_counts"] + if isinstance(tool, str) + } + ) + records = _measure_records(rows, tool_names=tool_names) + fields: tuple[str, ...] = TOKEN_FIELDS + has_tool_metrics = sample_unit == "child-invocation" + if has_tool_metrics: + fields += ("tool_calls",) + tuple(f"tool:{tool}" for tool in tool_names) + measures = _measure_payload(records, fields) + ratios = { + "input_output": _ratio_payload( + records, + "input_tokens", + "output_tokens", + sample_unit=sample_unit, + definition_roles=definition_roles, + definitions=definitions, + ), + "cache_share": _ratio_payload( + records, + "cache_read_tokens", + "input_tokens", + sample_unit=sample_unit, + definition_roles=definition_roles, + definitions=definitions, + ), + } + if has_tool_metrics: + ratios["tool_mix"] = { + tool: _ratio_payload( + records, + f"tool:{tool}", + "tool_calls", + sample_unit=sample_unit, + definition_roles=definition_roles, + definitions=definitions, + ) + for tool in tool_names + } + return {"measures": measures, "ratios": ratios} + + +def _skill_blocks( + sessions: Sequence[Mapping[str, Any]], + children: Sequence[Mapping[str, Any]], + *, + generated_at_ms: int, + definitions: Mapping[str, Mapping[str, Any]], +) -> list[dict[str, Any]]: + skill_rows: list[Mapping[str, Any]] = [ + row + for row in sessions + if row.get("level") in SKILL_LEVELS and isinstance(row.get("skill"), str) + ] + child_blocks = { + (window, tuple(levels)): rows + for window, levels, rows in _window_level_blocks(children, generated_at_ms=generated_at_ms) + } + blocks = [] + for window, levels, selected in _window_level_blocks( + skill_rows, generated_at_ms=generated_at_ms + ): + groups: dict[tuple[str, str, str], list[Mapping[str, Any]]] = {} + for row in selected: + identity = ( + row["skill"], + row.get("harness") or "unknown", + row.get("provider") or "unknown", + ) + groups.setdefault(identity, []).append(row) + rows_out = [] + selected_children = child_blocks.get((window, tuple(levels)), []) + for skill, harness, provider in sorted(groups): + runs = groups[(skill, harness, provider)] + related = [ + row + for row in selected_children + if row.get("skill") == skill + and row.get("harness") == harness + and row.get("_parent_provider") == provider + ] + related_roles = sorted({row["role"] for row in related}) + metric = _metric_payload( + runs, + sample_unit="skill-run", + definition_roles=related_roles, + definitions=definitions, + ) + step_groups: dict[tuple[str | None, str | None], list[Mapping[str, Any]]] = {} + for run in runs: + recipe = run.get("recipe") if isinstance(run.get("recipe"), str) else None + step = run.get("step") if isinstance(run.get("step"), str) else None + step_groups.setdefault((recipe, step), []).append(run) + recipe_steps = [] + for recipe, step in sorted( + step_groups, + key=lambda value: ( + value[0] is None, + value[0] or "", + value[1] is None, + value[1] or "", + ), + ): + step_runs = step_groups[(recipe, step)] + step_children = [ + row + for row in related + if row.get("recipe") == recipe and row.get("step") == step + ] + step_metric = _metric_payload( + step_runs, + sample_unit="skill-run", + definition_roles=sorted({row["role"] for row in step_children}), + definitions=definitions, + ) + recipe_steps.append( + { + "recipe": recipe, + "step": step, + **step_metric, + "exact_retransmission": TokenMeasure.unavailable().to_dict(), + } + ) + role_groups: dict[tuple[str, str, str], list[Mapping[str, Any]]] = {} + for child in related: + key = (child["role"], child["provider"], child["harness"]) + role_groups.setdefault(key, []).append(child) + child_roles = [] + for role, role_provider, role_harness in sorted(role_groups): + role_runs = role_groups[(role, role_provider, role_harness)] + role_metric = _metric_payload( + role_runs, + sample_unit="child-invocation", + definition_roles=[role], + definitions=definitions, + ) + child_roles.append( + { + "role": role, + "provider": role_provider, + "harness": role_harness, + "models": sorted( + { + row["model"] + for row in role_runs + if isinstance(row.get("model"), str) + } + ), + **role_metric, + } + ) + rows_out.append( + { + "skill": skill, + "harness": harness, + "provider": provider, + **metric, + "exact_retransmission": TokenMeasure.unavailable().to_dict(), + "recipe_steps": recipe_steps, + "child_roles": child_roles, + } + ) + blocks.append({"window": window, "levels": levels, "rows": rows_out}) + return blocks + + +def _role_blocks( + children: Sequence[Mapping[str, Any]], + *, + generated_at_ms: int, + definitions: Mapping[str, Mapping[str, Any]], +) -> list[dict[str, Any]]: + role_rows: list[Mapping[str, Any]] = [ + row for row in children if row.get("level") in SKILL_LEVELS + ] + blocks = [] + for window, levels, selected in _window_level_blocks( + role_rows, generated_at_ms=generated_at_ms + ): + groups: dict[tuple[str, str], list[Mapping[str, Any]]] = {} + for row in selected: + groups.setdefault((row["role"], row["provider"]), []).append(row) + rows_out = [] + for role, provider in sorted(groups): + harness_groups: dict[str, list[Mapping[str, Any]]] = {} + for row in groups[(role, provider)]: + harness_groups.setdefault(row["harness"], []).append(row) + harnesses = [] + for harness in sorted(harness_groups): + invocations = harness_groups[harness] + metric = _metric_payload( + invocations, + sample_unit="child-invocation", + definition_roles=[role], + definitions=definitions, + ) + harnesses.append( + { + "harness": harness, + "models": sorted( + { + row["model"] + for row in invocations + if isinstance(row.get("model"), str) + } + ), + **metric, + } + ) + rows_out.append({"role": role, "provider": provider, "harnesses": harnesses}) + blocks.append({"window": window, "levels": levels, "rows": rows_out}) + return blocks + + +def _prepare_deck_data( + sessions: Sequence[Mapping[str, Any]], + subagents: Iterable[Mapping[str, Any]], + *, + generated_at_ms: int, +) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]: + children = _canonical_children(sessions, subagents) + skill_rows: list[Mapping[str, Any]] = [ + row + for row in sessions + if row.get("level") in SKILL_LEVELS and isinstance(row.get("skill"), str) + ] + role_rows: list[Mapping[str, Any]] = [ + row for row in children if row.get("level") in SKILL_LEVELS + ] + bundled = {definition.name: definition for definition in load_bundled_agent_definitions()} + roles = sorted({row["role"] for row in role_rows}) + definitions: dict[str, dict[str, Any]] = {} + for role in roles: + definition = bundled.get(role) + definitions[role] = ( + { + "state": "available", + "description": definition.description, + "body": definition.body, + "tools": list(definition.tools), + "model": definition.model, + "reader_tools": list(definition.reader_tools), + "codex_model": getattr(getattr(definition, "codex", None), "model", None), + } + if definition is not None + else { + "state": "unavailable", + "description": None, + "body": None, + "tools": [], + "model": None, + "reader_tools": [], + "codex_model": None, + } + ) + skill_identities = sorted( + { + ( + row["skill"], + row.get("harness") or "unknown", + row.get("provider") or "unknown", + ) + for row in skill_rows + } + ) + role_identities = sorted({(row["role"], row["provider"]) for row in role_rows}) + relationships = [ + {"skill": skill, "role": role, "harness": harness, "provider": provider} + for skill, role, harness, provider in sorted( + { + (row["skill"], row["role"], row["harness"], row["provider"]) + for row in role_rows + if isinstance(row.get("skill"), str) and row["skill"] + } + ) + ] + view_rows: dict[str, list[Mapping[str, Any]]] = { + "cohort": list(sessions), + "spend": skill_rows, + "efficiency": [*skill_rows, *role_rows], + "skill": skill_rows, + "role": role_rows, + } + population_labels = { + "spend": "skill", + "efficiency": "contributor", + "skill": "skill", + "role": "child", + } + view_chips: dict[str, dict[str, list[dict[str, Any]]]] = {} + view_histories: dict[str, dict[str, Any]] = {} + for view in DECK_VIEWS: + if view.planned_issue is not None: + continue + rows = view_rows.get(view.view_id, list(sessions)) + history = _history(rows) + view_chips[view.view_id] = resolve_view_chips( + view, + rows, + generated_at_ms=generated_at_ms, + history=history, + population_label=population_labels.get(view.view_id), + ) + view_histories[view.view_id] = history + return ( + { + "skills": _skill_blocks( + skill_rows, + role_rows, + generated_at_ms=generated_at_ms, + definitions=definitions, + ), + "roles": _role_blocks( + role_rows, generated_at_ms=generated_at_ms, definitions=definitions + ), + "relationships": relationships, + "definitions": definitions, + "view_chips": view_chips, + "view_histories": view_histories, + }, + { + "skills": encode_table( + [ + {"skill": skill, "harness": harness, "provider": provider} + for skill, harness, provider in skill_identities + ], + ("skill", "harness", "provider"), + ), + "roles": encode_table( + [{"role": role, "provider": provider} for role, provider in role_identities], + ("role", "provider"), + ), + }, + ) + + def build_deck_payload( - session_rows: Iterable[Mapping[str, Any]], *, generated_at: datetime, index_schema_version: int + session_rows: Iterable[Mapping[str, Any]], + *, + request_rows: Iterable[Mapping[str, Any]] = (), + tool_rows: Iterable[Mapping[str, Any]] = (), + subagent_rows: Iterable[Mapping[str, Any]] = (), + generated_at: datetime, + index_schema_version: int, ) -> dict[str, Any]: + """Encode the session cohort and prepare attributed skill and child-role metrics. + + Request and tool rows remain available at this boundary, but contribute to no metric + without a verified child identity. Native child rows already carry folded usage. + """ if generated_at.utcoffset() is None: raise ValueError("generated_at must be timezone-aware") epoch = datetime(1970, 1, 1, tzinfo=UTC) @@ -161,12 +716,16 @@ def build_deck_payload( key=lambda r: (r.get("time_ms") is None, r.get("time_ms") or 0, r.get("key") or ""), ) history = _history(rows) + prepared, identity_tables = _prepare_deck_data( + rows, subagent_rows, generated_at_ms=generated_at_ms + ) + tables = {SESSION_TABLE: encode_table(rows, SESSION_COLUMNS), **identity_tables} return { "generated_at_ms": generated_at_ms, "index_schema_version": index_schema_version, "landing": LANDING_VIEW, "history": history, - "tables": {SESSION_TABLE: encode_table(rows, SESSION_COLUMNS)}, + "tables": tables, "facets": [ {"id": f.facet_id, "label": f.label, "column": f.column, "kind": "values"} for f in FACETS @@ -183,13 +742,8 @@ def build_deck_payload( } for v in DECK_VIEWS ], - "chips": { - v.view_id: resolve_view_chips( - v, rows, generated_at_ms=generated_at_ms, history=history - ) - for v in DECK_VIEWS - if v.planned_issue is None - }, + "chips": prepared["view_chips"], + "prepared": prepared, "availability": [ { "state": s.value, diff --git a/src/autoskillit/report/deck/_registry.py b/src/autoskillit/report/deck/_registry.py index c057f6dbf8..13d2cc8076 100644 --- a/src/autoskillit/report/deck/_registry.py +++ b/src/autoskillit/report/deck/_registry.py @@ -151,36 +151,36 @@ class DeckViewDef(NamedTuple): "Where do the tokens go?", "Where it is worth spending optimization time at all", "What do you need to know?", + "skills", + "views/spend.js", None, - None, - 4651, ), DeckViewDef( "efficiency", "What's inefficient?", "Which skill or role is wasteful, as opposed to merely big", "What do you need to know?", + "skills", + "views/efficiency.js", None, - None, - 4651, ), DeckViewDef( "skill", "What is this skill spending on?", "Which step, which subagent, how much is re-transmitted prompt", "What do you need to know?", + "skills", + "views/skill.js", None, - None, - 4651, ), DeckViewDef( "role", "Can this agent be cheaper?", "Whether to restrict a role to read-only on a smaller model", "What do you need to know?", + "roles", + "views/role.js", None, - None, - 4651, ), DeckViewDef( "context", diff --git a/tests/_test_filter.py b/tests/_test_filter.py index 6fc08e6b5a..8a49c6f20b 100644 --- a/tests/_test_filter.py +++ b/tests/_test_filter.py @@ -606,7 +606,7 @@ class SourceMapRejection(enum.StrEnum): "_type_figure_spec": frozenset({"core", "report"}), "_type_session_shape": frozenset({"core", "cli", "fleet", "pipeline", "server"}), "_type_capture": frozenset({"core", "fleet", "recipe", "cli"}), - "_type_token": frozenset({"cli", "core", "execution", "fleet", "pipeline"}), + "_type_token": frozenset({"cli", "core", "execution", "fleet", "pipeline", "report"}), "_type_constants_env": frozenset( { "cli", diff --git a/tests/arch/test_hook_flock_nonblocking.py b/tests/arch/test_hook_flock_nonblocking.py index 7dea6a01e1..8af3b237fe 100644 --- a/tests/arch/test_hook_flock_nonblocking.py +++ b/tests/arch/test_hook_flock_nonblocking.py @@ -103,6 +103,11 @@ "_open_otlp_handles", "ArtifactLease.acquire_shared", ), + ( + "execution/evidence/report_walk.py", + "_retained_session_rows", + "ArtifactLease.acquire_shared", + ), ( "execution/evidence/report_walk.py", "_walk_projection", diff --git a/tests/cli/test_sessions_deck_cmd.py b/tests/cli/test_sessions_deck_cmd.py index 174eeab213..c6a01cbb96 100644 --- a/tests/cli/test_sessions_deck_cmd.py +++ b/tests/cli/test_sessions_deck_cmd.py @@ -5,6 +5,8 @@ import json import re from pathlib import Path +from types import SimpleNamespace +from typing import Any import pytest @@ -14,6 +16,41 @@ pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium] +def test_sessions_deck_passes_all_fact_collections( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import autoskillit.execution as execution + import autoskillit.report as report + from autoskillit.cli.ops import _sessions + + log_root = tmp_path / "logs" + _configure_log_root(monkeypatch, log_root) + facts = SimpleNamespace( + sessions={"parent": {"key": "parent"}}, + requests={"request": {"key": "request"}}, + tools={"tool": {"key": "tool"}}, + subagents={"child": {"key": "child", "actor_level": "L0"}}, + ) + received: dict[str, Any] = {} + + def capture_deck(session_rows: Any, **kwargs: Any) -> str: + received["session_rows"] = list(session_rows) + received.update(kwargs) + return "" + + monkeypatch.setattr(_sessions, "_refresh_report_index", lambda *a, **kw: None) + monkeypatch.setattr(execution, "read_report_index", lambda *a: facts) + monkeypatch.setattr(report, "render_deck", capture_deck) + out = tmp_path / "deck.html" + + _sessions.sessions_deck(str(out)) + + assert received["session_rows"] == list(facts.sessions.values()) + for name in ("request", "tool", "subagent"): + assert list(received[f"{name}_rows"]) == list(getattr(facts, f"{name}s").values()) + assert out.read_text(encoding="utf-8") == "" + + def test_sessions_deck_refreshes_index_and_writes_session_data( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/contracts/test_package_gateways.py b/tests/contracts/test_package_gateways.py index ae21e2d528..a75911004e 100644 --- a/tests/contracts/test_package_gateways.py +++ b/tests/contracts/test_package_gateways.py @@ -506,6 +506,7 @@ def test_measure_aggregation_ships_as_projected_hook_asset() -> None: "execution_evidence": frozenset( { "anomaly_detection", + "_native_child_projection", "linux_tracing", "otlp_sink", "report_walk", diff --git a/tests/execution/backends/test_child_task_transcripts.py b/tests/execution/backends/test_child_task_transcripts.py index 3f5b3dd391..4cd47533fe 100644 --- a/tests/execution/backends/test_child_task_transcripts.py +++ b/tests/execution/backends/test_child_task_transcripts.py @@ -12,6 +12,10 @@ from autoskillit.execution.backends import CompositeSessionLocator from autoskillit.execution.backends._claude.child_task import parse_claude_child_task from autoskillit.execution.backends._codex.child_task import parse_codex_child_task +from autoskillit.execution.backends._codex_parse import ( + extract_codex_child_turn_usage, + extract_codex_turn_usage, +) pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] @@ -58,6 +62,25 @@ def _write_codex(path: Path, child_id: str, events: list[dict[str, object]]) -> return _write_jsonl(path, [_codex_meta(child_id), *events]) +def _codex_usage_snapshot(cumulative_total: int, timestamp: str) -> dict[str, object]: + return { + "type": "event_msg", + "timestamp": timestamp, + "payload": { + "type": "token_count", + "info": { + "last_token_usage": { + "input_tokens": 10, + "cached_input_tokens": 2, + "output_tokens": 3, + "total_tokens": 13, + }, + "total_token_usage": {"total_tokens": cumulative_total}, + }, + }, + } + + @pytest.mark.parametrize( "prompt", [ @@ -491,3 +514,92 @@ def test_composite_locator_preserves_transcript_read_errors() -> None: with pytest.raises(OSError, match="unreadable"): locator.read_child_task("child") + + +def test_codex_child_usage_scans_whole_verified_rollout_and_deduplicates_snapshots( + tmp_path: Path, +) -> None: + child_thread_id = "child-thread" + path = _write_codex( + tmp_path / "child-rollout.jsonl", + child_thread_id, + [ + {"type": "turn_context", "payload": {"model": "codex-child-model"}}, + _codex_usage_snapshot(13, "2026-09-01T10:00:01Z"), + _codex_usage_snapshot(13, "2026-09-01T10:00:02Z"), + ], + ) + locator = SimpleNamespace(locate_session=lambda _thread_id: path) + + dedicated = extract_codex_child_turn_usage( + path, + child_thread_id, + provider_used="codex", + ) + interval = extract_codex_turn_usage( + locator, + child_thread_id, + "2026-09-01T10:00:00Z", + "2026-09-01T10:00:03Z", + provider_used="codex", + ) + + assert dedicated == interval + assert len(dedicated) == 1 + assert dedicated[0]["model"] == "codex-child-model" + assert dedicated[0]["input_tokens"] == 10 + + +def test_codex_compaction_exclusion_is_preserved_for_both_extractors(tmp_path: Path) -> None: + child_id = "child-thread" + path = _write_codex( + tmp_path / "child.jsonl", + child_id, + [ + _codex_usage_snapshot(13, "2026-09-01T10:00:01Z"), + {"type": "compacted"}, + _codex_usage_snapshot(26, "2026-09-01T10:00:02Z"), + _codex_event("event_msg", {"type": "context_compacted"}), + _codex_usage_snapshot(39, "2026-09-01T10:00:03Z"), + ], + ) + locator = SimpleNamespace(locate_session=lambda _thread_id: path) + dedicated = extract_codex_child_turn_usage(path, child_id, provider_used="codex") + interval = extract_codex_turn_usage( + locator, child_id, "2026-09-01T10:00:00Z", "2026-09-01T10:00:04Z", provider_used="codex" + ) + assert dedicated == interval + assert len(dedicated) == 2 + + +def test_native_codex_usage_folds_advancing_snapshots_of_one_request(tmp_path: Path) -> None: + from autoskillit.execution.evidence._native_child_projection import _child_token_usage + + snapshots = [ + _codex_usage_snapshot(13, "2026-09-01T10:00:01Z"), + _codex_usage_snapshot(26, "2026-09-01T10:00:02Z"), + ] + for snapshot in snapshots: + payload = snapshot["payload"] + assert isinstance(payload, dict) + payload["request_id"] = "same-request" + path = _write_codex(tmp_path / "child.jsonl", "child-thread", snapshots) + usage, state = _child_token_usage( + "codex", "child-thread", path, path.read_text(encoding="utf-8"), "codex" + ) + assert state == "observed" + assert usage["input_tokens"] == {"state": "measured", "value": 10} + assert usage["output_tokens"] == {"state": "measured", "value": 3} + + +def test_codex_child_usage_rejects_mismatched_rollout_identity(tmp_path: Path) -> None: + path = _write_codex(tmp_path / "child-rollout.jsonl", "actual-child", []) + + assert ( + extract_codex_child_turn_usage( + path, + "different-child", + provider_used="codex", + ) + == [] + ) diff --git a/tests/execution/test_report_index.py b/tests/execution/test_report_index.py index 53992ad2b9..5f6359ad18 100644 --- a/tests/execution/test_report_index.py +++ b/tests/execution/test_report_index.py @@ -8,6 +8,7 @@ import pytest +import autoskillit.execution.child_outcomes as child_snapshot from autoskillit.core import ArtifactLease, ArtifactLeaseContention from autoskillit.execution import ( REPORT_INDEX_SCHEMA_VERSION, @@ -69,6 +70,138 @@ def _basic_session(dir_name: str, session_id: str, **fields: Any) -> dict[str, A return basic_session_row(dir_name, session_id, **fields) +def _write_native_child(log_root: Path, parent_id: str, child_id: str, *, model: str) -> Path: + snapshot_path = child_snapshot.resolve_snapshot_path( + log_root, backend="claude_code", parent_session_id=parent_id + ) + child_snapshot.observe_child( + snapshot_path, + backend="claude_code", + parent_session_id=parent_id, + child_id=child_id, + ) + child_snapshot.record_terminal_evidence( + snapshot_path, + backend="claude_code", + parent_session_id=parent_id, + child_id=child_id, + evidence_key=f"{child_id}:metadata", + evidence={ + "role": "Explore", + "attribution_skill": "implement-worktree-no-merge", + "effective_provider": "openai", + "effective_model": model, + "evidence_source": "transcript_metadata", + }, + ) + transcript = log_root / parent_id / "subagents" / f"agent-{child_id}.jsonl" + transcript.parent.mkdir(parents=True, exist_ok=True) + transcript.write_text( + json.dumps( + { + "type": "assistant", + "agentId": child_id, + "attributionAgent": "Explore", + "attributionSkill": "implement-worktree-no-merge", + "message": {"id": f"message-{child_id}", "model": model}, + "sessionId": parent_id, + } + ) + + "\n", + encoding="utf-8", + ) + return snapshot_path + + +def _remove_native_child(snapshot_path: Path, child_id: str) -> None: + document = json.loads(snapshot_path.read_text(encoding="utf-8")) + document["children"].pop(child_id) + snapshot_path.write_text(json.dumps(document, sort_keys=True), encoding="utf-8") + + +def _indexed_native_children(index_dir: Path) -> dict[str, dict[str, Any]]: + return { + row["child_id"]: row + for row in read_report_index(index_dir).subagents.values() + if row["actor_level"] == "L0" and row["child_id"] is not None + } + + +@pytest.mark.parametrize("cache_write", [0, None]) +def test_native_claude_input_is_inclusive_only_with_complete_components( + tmp_path: Path, cache_write: int | None +) -> None: + root = tmp_path / "logs" + parent_id, child_id = "native-parent", "native-child" + root.mkdir() + parent_log = root / f"{parent_id}.jsonl" + parent_log.write_text("", encoding="utf-8") + _write_jsonl( + root / "sessions.jsonl", + [ + _basic_session( + "parent-key", + parent_id, + claude_code_log=str(parent_log), + session_type="skill", + skill_command="/autoskillit:implement-worktree-no-merge", + ) + ], + ) + snapshot = _write_native_child(root, parent_id, child_id, model="child-model") + child_snapshot.record_terminal_evidence( + snapshot, + backend="claude_code", + parent_session_id=parent_id, + child_id=child_id, + evidence_key=f"{child_id}:metadata", + evidence={ + "role": "Explore", + "attribution_skill": "implement-worktree-no-merge", + "effective_provider": "anthropic", + "effective_model": "child-model", + "evidence_source": "transcript_metadata", + }, + ) + usage = {"input_tokens": 12, "output_tokens": 4, "cache_read_input_tokens": 3} + if cache_write is not None: + usage["cache_creation_input_tokens"] = cache_write + line = ( + json.dumps( + { + "type": "assistant", + "agentId": child_id, + "sessionId": parent_id, + "requestId": "request-1", + "message": { + "id": "message-1", + "model": "child-model", + "content": [], + "usage": usage, + }, + } + ) + + "\n" + ) + (root / parent_id / "subagents" / f"agent-{child_id}.jsonl").write_text(line * 2) + index_dir = tmp_path / "index" + + update_report_index(root, index_dir) + + child = _indexed_native_children(index_dir)[child_id] + expected = ( + {"state": "measured", "value": 15} + if cache_write is not None + else { + "state": "unknown", + "value": None, + } + ) + assert child["token_usage"]["input_tokens"] == expected + assert child["token_usage"]["output_tokens"] == {"state": "measured", "value": 4} + assert child["token_usage"]["cache_read_tokens"] == {"state": "measured", "value": 3} + + def test_update_then_read_joins_events_to_sessions(tmp_path: Path) -> None: root = tmp_path / "logs" session_dir = "session-a" @@ -301,6 +434,57 @@ def test_rebuild_from_scratch_equals_incremental_build(tmp_path: Path) -> None: assert read_report_index(inc) == read_report_index(full) +def test_child_only_changes_refresh_live_and_archive_rows_once(tmp_path: Path) -> None: + root = tmp_path / "logs" + index_dir = tmp_path / "index" + live_id, archive_id = "parent-live", "parent-archive" + live_transcript, archive_transcript = root / f"{live_id}.jsonl", root / f"{archive_id}.jsonl" + live_transcript.parent.mkdir(parents=True) + live_transcript.write_text("", encoding="utf-8") + archive_transcript.write_text("", encoding="utf-8") + _write_jsonl( + root / "sessions.jsonl", + [_basic_session("live", live_id, claude_code_log=str(live_transcript))], + ) + _write_jsonl( + root / "sessions-archive.jsonl", + [_basic_session("archive", archive_id, claude_code_log=str(archive_transcript))], + ) + + live_snapshot = _write_native_child(root, live_id, "child-live", model="model-v1") + _write_native_child(root, live_id, "child-removed", model="model-removed") + _write_native_child(root, archive_id, "child-archive", model="model-v1") + update_report_index(root, index_dir) + assert set(_indexed_native_children(index_dir)) == { + "child-live", + "child-removed", + "child-archive", + } + + _write_native_child(root, live_id, "child-live", model="model-v2") + _remove_native_child(live_snapshot, "child-removed") + (root / live_id / "subagents" / "agent-child-removed.jsonl").unlink() + _write_native_child(root, archive_id, "child-added", model="model-added") + + update_report_index(root, index_dir) + children = _indexed_native_children(index_dir) + assert set(children) == {"child-live", "child-archive", "child-added"} + assert children["child-live"]["model"] == "model-v2" + assert children["child-live"]["provider"] == "openai" + assert children["child-added"]["model"] == "model-added" + + rebuilt = tmp_path / "rebuilt" + rebuild_report_index(root, rebuilt) + expected = read_report_index(rebuilt) + assert read_report_index(index_dir) == expected + rows_path = index_dir / "rows.jsonl" + rows_size = rows_path.stat().st_size + repeated = update_report_index(root, index_dir) + assert repeated.rows_written == 0 + assert rows_path.stat().st_size == rows_size + assert read_report_index(index_dir) == expected + + def test_interrupted_update_resumes_to_the_same_index( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -396,6 +580,17 @@ def test_older_version_rows_parse_without_error(tmp_path: Path) -> None: } ) ) + (rows_path.parent / "state.json").write_text( + json.dumps( + { + "schema_version": 1, + "row_schema_version": REPORT_INDEX_SCHEMA_VERSION, + "walk": None, + "rows_bytes": rows_path.stat().st_size, + } + ), + encoding="utf-8", + ) indexed = read_report_index(rows_path.parent) @@ -410,7 +605,63 @@ def test_older_version_rows_parse_without_error(tmp_path: Path) -> None: assert len(indexed.sessions) == 2 -def test_lost_state_keeps_rows_whose_sources_rotated_away(tmp_path: Path) -> None: +@pytest.mark.parametrize( + ("row_schema_marker", "remove_rows"), + [(None, False), (1, False), (REPORT_INDEX_SCHEMA_VERSION + 1, True)], + ids=["absent", "mismatched", "future"], +) +def test_incompatible_row_schema_is_rejected_and_rebuilt( + tmp_path: Path, + row_schema_marker: int | None, + remove_rows: bool, +) -> None: + assert not read_report_index(tmp_path / "absent-index").sessions + + root = tmp_path / "logs" + _write_jsonl(root / "sessions.jsonl", [_basic_session("current", "sid-current")]) + index_dir = tmp_path / "index" + update_report_index(root, index_dir) + + state_path = index_dir / "state.json" + state = json.loads(state_path.read_text(encoding="utf-8")) + if row_schema_marker is None: + state.pop("row_schema_version", None) + else: + state["row_schema_version"] = row_schema_marker + rows_path = index_dir / "rows.jsonl" + with rows_path.open("ab") as handle: + handle.write( + _json_line( + { + "schema_version": REPORT_INDEX_SCHEMA_VERSION, + "kind": "session", + "key": "stale-child-generation", + "session_id": "sid-stale", + } + ) + ) + state["rows_bytes"] = rows_path.stat().st_size + state_path.write_text(json.dumps(state), encoding="utf-8") + if remove_rows: + rows_path.unlink() + + with pytest.raises(ValueError, match="row schema") as exc_info: + read_report_index(index_dir) + assert str(exc_info.value) == ( + f"report index row schema mismatch: expected {REPORT_INDEX_SCHEMA_VERSION}, " + f"found {row_schema_marker!r}" + ) + + update_report_index(root, index_dir) + + indexed = read_report_index(index_dir) + committed = json.loads(state_path.read_text(encoding="utf-8")) + assert set(indexed.sessions) == {"current"} + assert committed["schema_version"] == report_index._STATE_SCHEMA_VERSION + assert committed["row_schema_version"] == REPORT_INDEX_SCHEMA_VERSION + + +def test_lost_state_rebuilds_rows_from_retained_sources(tmp_path: Path) -> None: root = tmp_path / "logs" _write_jsonl(root / "sessions.jsonl", [_basic_session("a", "sid-a")]) (root / "otlp.jsonl").write_bytes( @@ -422,11 +673,14 @@ def test_lost_state_keeps_rows_whose_sources_rotated_away(tmp_path: Path) -> Non (root / "otlp.jsonl").unlink() (index_dir / "state.json").unlink() + with pytest.raises(ValueError, match="row schema"): + read_report_index(index_dir) + update_report_index(root, index_dir) indexed = read_report_index(index_dir) assert indexed.sessions == sessions_before - assert {row["request_id"] for row in indexed.requests.values()} == {"a"} + assert indexed.requests == {} def test_archive_gap_reindexes_archive_then_projection(tmp_path: Path) -> None: diff --git a/tests/execution/test_report_index_rows.py b/tests/execution/test_report_index_rows.py index ac09cb4005..26848e1042 100644 --- a/tests/execution/test_report_index_rows.py +++ b/tests/execution/test_report_index_rows.py @@ -55,6 +55,7 @@ def _session_item( *, turns: tuple[tuple[str, ...], ...] = (), available: bool = True, + child_outcomes: tuple[dict[str, Any], ...] = (), ) -> WalkItem: return WalkItem( "session", @@ -67,6 +68,7 @@ def _session_item( {"turn_id": "private-turn-id", "timestamp": None, "tool_names": list(names)} for names in turns ], + "child_outcomes": child_outcomes, "transcripts_available": available, "transcript_unavailable_reasons": [], }, @@ -149,6 +151,132 @@ def test_session_row_carries_facets_pair_and_resolved_measures() -> None: assert "private-turn-id" not in serialized +@pytest.mark.parametrize("session_type", ["skill", "orchestrator"]) +def test_session_item_projects_verified_native_child_under_index_parent_key( + session_type: str, +) -> None: + token_usage = { + "input_tokens": {"state": "measured", "value": 15}, + "output_tokens": {"state": "measured", "value": 4}, + "cache_read_tokens": {"state": "measured", "value": 3}, + "cache_write_tokens": {"state": "measured_zero", "value": 0}, + } + child = { + "child_id": "native-child-1", + "backend": "claude_code", + "parent_session_id": "native-parent-id", + "role": "audit-impl-slice-auditor", + "attribution_skill": "make-plan", + "effective_provider": "anthropic", + "effective_model": "claude-sonnet-child", + "evidence_source": "transcript_metadata", + "transcript_locator": "/private/logs/agent-native-child-1.jsonl", + "token_usage": token_usage, + "tool_counts": {"Bash": 1, "Read": 2}, + "transcript_state": "observed", + "usage_state": "observed", + } + managed_attempt = { + "child_id": "managed-attempt-1", + "launch_alias": "claude-managed-attempt", + "backend": "claude_code", + "parent_session_id": "native-parent-id", + "role": "audit-bugs", + "attribution_skill": "make-plan", + "evidence_source": "managed_attempt_reservation", + } + providerless_child = { + "child_id": "native-child-providerless", + "backend": "claude_code", + "parent_session_id": "native-parent-id", + "role": "audit-impl-slice-auditor", + "attribution_skill": "make-plan", + "effective_provider": "", + "effective_model": "", + "evidence_source": "transcript_metadata", + "transcript_locator": "/private/logs/agent-native-child-providerless.jsonl", + "token_usage": { + field: {"state": "unknown", "value": None} + for field in ( + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cache_write_tokens", + ) + }, + "tool_counts": {"Read": 1}, + "transcript_state": "observed", + "usage_state": "unknown", + } + mismatched_skill_child = { + "child_id": "native-child-mismatched-skill", + "backend": "claude_code", + "parent_session_id": "native-parent-id", + "role": "audit-impl-slice-auditor", + "attribution_skill": "other-skill", + "effective_provider": "anthropic", + "effective_model": "claude-sonnet-child", + "evidence_source": "transcript_metadata", + "token_usage": token_usage, + "tool_counts": {}, + "transcript_state": "observed", + "usage_state": "observed", + } + parent = { + "session_id": "native-parent-id", + "timestamp": "2026-09-01T00:00:00Z", + "backend": "claude-code", + "provider_used": "openai", + "model_identifier": "parent-model", + "skill_command": "/autoskillit:make-plan", + "recipe_name": "research", + "step_name": "scope", + "session_type": session_type, + } + + rows = rows_for_walk_item( + _session_item( + "report-index-key", + parent, + child_outcomes=(child, managed_attempt, providerless_child, mismatched_skill_child), + ) + ) + + assert [row["kind"] for row in rows] == [ + "session", + "subagent", + "subagent", + "subagent", + ] + projected_by_id = {row["child_id"]: row for row in rows if row["kind"] == "subagent"} + projected = projected_by_id["native-child-1"] + assert projected["key"].startswith("report-index-key:") + assert projected["child_id"] == "native-child-1" + assert projected["parent_session_key"] == "report-index-key" + assert projected["native_parent_session_id"] == "native-parent-id" + assert projected["role"] == "audit-impl-slice-auditor" + assert projected["actor_level"] == "L0" + assert projected["skill"] == "make-plan" + assert projected["level"] == session_type + assert projected["recipe"] == "research" + assert projected["step"] == "scope" + assert projected["provider"] == "anthropic" + assert projected["model"] == "claude-sonnet-child" + assert projected["time_ms"] == rows[0]["time_ms"] + assert projected["token_usage"] == token_usage + assert projected["tool_counts"] == {"Bash": 1, "Read": 2} + assert projected["transcript_state"] == projected["usage_state"] == "observed" + without_provider = projected_by_id["native-child-providerless"] + assert without_provider["provider"] == "unknown" + assert without_provider["model"] is None + assert without_provider["token_usage"]["input_tokens"]["state"] == "unknown" + assert without_provider["usage_state"] == "unknown" + mismatched = projected_by_id["native-child-mismatched-skill"] + assert mismatched["skill"] is None + assert mismatched["role"] == "audit-impl-slice-auditor" + assert mismatched["level"] == session_type + + def test_session_row_legacy_zero_and_codex_sigil() -> None: row_data = { "schema_version": 13, diff --git a/tests/execution/test_report_index_schema.py b/tests/execution/test_report_index_schema.py index eebef314d4..74f771d174 100644 --- a/tests/execution/test_report_index_schema.py +++ b/tests/execution/test_report_index_schema.py @@ -15,7 +15,7 @@ pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] -_EXPECTED = (1, "8418c2eb676180f7865ce20982478b76f175e0d7d3527f5353f71e4a26117d06") +_EXPECTED = (2, "e574d4e83971be001c1c92d29e807a421edbf705acbe5e72a03e7e513d43c4ba") # Locked snapshot of per-kind field name orderings that produced ``_EXPECTED``. # Bump ``_EXPECTED[0]`` and refresh this dict in lockstep when row fields change; # ``test_report_index_schema_version_matches_field_digest`` is what fails first @@ -73,17 +73,31 @@ "tool_counts", ], "subagent": [ + "actor_level", "agent_type", + "child_id", "event_sequence", "final_model", "harness", "key", "kind", + "level", "model", "model_swapped", + "native_parent_session_id", + "parent_session_key", + "provider", + "recipe", + "role", "schema_version", "session_id", + "skill", + "step", "time_ms", + "tool_counts", + "token_usage", + "transcript_state", + "usage_state", ], "tool": [ "agent_name", diff --git a/tests/execution/test_report_walk.py b/tests/execution/test_report_walk.py index e7a823405f..51ee45b5af 100644 --- a/tests/execution/test_report_walk.py +++ b/tests/execution/test_report_walk.py @@ -10,6 +10,7 @@ import pytest import zstandard +import autoskillit.execution.evidence._native_child_projection as native_child_projection from autoskillit.core import iter_merged_assistant_turns from autoskillit.execution.evidence.report_walk import ( SourceGapError, @@ -46,12 +47,21 @@ def _session( *, claude_code_log: object = ..., codex_log: object = ..., + backend: str | None = None, + timestamp: str | None = None, + duration_seconds: float | None = None, ) -> dict[str, Any]: row: dict[str, Any] = {"dir_name": dir_name, "session_id": session_id} if claude_code_log is not ...: row["claude_code_log"] = claude_code_log if codex_log is not ...: row["codex_log"] = codex_log + if backend is not None: + row["backend"] = backend + if timestamp is not None: + row["timestamp"] = timestamp + if duration_seconds is not None: + row["duration_seconds"] = duration_seconds return row @@ -128,6 +138,154 @@ def test_first_pass_emits_joinable_records_and_tolerates_old_index_rows( assert read_tolerant_session_index_rows(root / "sessions.jsonl") == [current, old_schema] +def test_session_walk_reads_child_outcomes_by_native_parent_id( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = tmp_path / "logs" + child = { + "child_id": "native-child", + "backend": "claude_code", + "parent_session_id": "native-parent-id", + "role": "audit-bugs", + "attribution_skill": "make-plan", + "effective_provider": "", + "effective_model": "", + "evidence_source": "transcript_metadata", + } + calls: list[tuple[str, str, Path]] = [] + + def collect(*, backend: str, parent_session_id: str, log_root: Path): + calls.append((backend, parent_session_id, log_root)) + return (child,) + + monkeypatch.setattr(native_child_projection, "collect_child_outcomes", collect) + _write_jsonl( + root / "sessions.jsonl", + [_session("report-index-key", "native-parent-id", backend="claude-code")], + ) + + (item,) = [candidate for candidate in iter_report_walk(root) if candidate.kind == "session"] + + assert calls == [("claude_code", "native-parent-id", root)] + assert item.source_id == "report-index-key" + assert item.session_id == "native-parent-id" + assert item.record is not None + (projected_child,) = item.record["child_outcomes"] + assert projected_child["child_id"] == "native-child" + assert projected_child["native_parent_session_id"] == "native-parent-id" + assert projected_child["parent_session_key"] == "report-index-key" + assert projected_child["transcript_state"] == "unknown" + assert projected_child["usage_state"] == "unknown" + assert item.watermark["projection"]["report-index-key"] + + +def test_resumed_native_parent_owns_only_timestamp_matched_child( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = tmp_path / "logs" + parent_log = tmp_path / "project" / "native-parent-id.jsonl" + parent_log.parent.mkdir(parents=True) + parent_log.write_text("{}\n", encoding="utf-8") + child_dir = parent_log.parent / "native-parent-id" / "subagents" + _write_jsonl( + child_dir / "agent-child-in-second-resume.jsonl", + [ + { + "type": "assistant", + "timestamp": "2026-10-05T10:00:25Z", + "message": {"id": "m1", "content": []}, + } + ], + ) + _write_jsonl( + child_dir / "agent-child-without-time.jsonl", + [{"type": "assistant", "message": {"id": "m2", "content": []}}], + ) + (child_dir / "agent-incomplete-child.jsonl").write_text( + json.dumps( + { + "type": "assistant", + "timestamp": "2026-10-05T10:00:26Z", + "message": {"id": "m3", "content": []}, + } + ), + encoding="utf-8", + ) + children = ( + { + "child_id": "child-in-second-resume", + "backend": "claude_code", + "parent_session_id": "native-parent-id", + "role": "audit-bugs", + "attribution_skill": "make-plan", + "effective_provider": "anthropic", + "effective_model": "claude-child", + "evidence_source": "transcript_metadata", + }, + { + "child_id": "child-without-time", + "backend": "claude_code", + "parent_session_id": "native-parent-id", + "role": "audit-bugs", + "attribution_skill": "make-plan", + "effective_provider": "anthropic", + "effective_model": "claude-child", + "evidence_source": "transcript_metadata", + }, + { + "child_id": "incomplete-child", + "backend": "claude_code", + "parent_session_id": "native-parent-id", + "role": "audit-bugs", + "attribution_skill": "make-plan", + "effective_provider": "anthropic", + "effective_model": "claude-child", + "evidence_source": "transcript_metadata", + }, + ) + monkeypatch.setattr( + native_child_projection, + "collect_child_outcomes", + lambda **_kwargs: children, + ) + _write_jsonl( + root / "sessions.jsonl", + [ + _session( + "resume-a", + "native-parent-id", + backend="claude-code", + claude_code_log=str(parent_log), + timestamp="2026-10-05T10:00:00Z", + duration_seconds=10, + ), + _session( + "resume-b", + "native-parent-id", + backend="claude-code", + claude_code_log=str(parent_log), + timestamp="2026-10-05T10:00:20Z", + duration_seconds=10, + ), + ], + ) + + sessions = { + item.source_id: item.record for item in iter_report_walk(root) if item.kind == "session" + } + + assert sessions["resume-a"]["child_outcomes"] == () + owned_children = {child["child_id"]: child for child in sessions["resume-b"]["child_outcomes"]} + tool_free = owned_children["child-in-second-resume"] + assert tool_free["tool_counts"] == {} + assert tool_free["transcript_state"] == "observed" + assert tool_free["usage_state"] == "unknown" + incomplete = owned_children["incomplete-child"] + assert incomplete["tool_counts"] is None + assert incomplete["transcript_state"] == "unknown" + assert incomplete["usage_state"] == "unknown" + + @pytest.mark.parametrize("pause", ("before", "after")) def test_committed_watermark_resumes_without_missing_or_duplicate_records( tmp_path: Path, pause: str diff --git a/tests/execution/test_session_parsing.py b/tests/execution/test_session_parsing.py index 8a97617b32..08043c4cdc 100644 --- a/tests/execution/test_session_parsing.py +++ b/tests/execution/test_session_parsing.py @@ -20,6 +20,7 @@ extract_token_usage, parse_session_result, ) +from autoskillit.execution.session.turn_usage import claude_inclusive_input_tokens from tests._helpers import UNKNOWN_MEASURE as _UNKNOWN from tests._helpers import _flush_structlog_proxy_caches as _flush_logger_proxy_caches from tests._helpers import observed_measure as _observed @@ -28,6 +29,24 @@ pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] +@pytest.mark.parametrize( + ("raw_input", "cache_read", "cache_write", "expected"), + [ + (10, 3, 2, 15), + (None, 3, 2, None), + (10, None, 2, None), + (10, 3, None, None), + ], +) +def test_claude_inclusive_input_requires_all_raw_components( + raw_input: int | None, + cache_read: int | None, + cache_write: int | None, + expected: int | None, +) -> None: + assert claude_inclusive_input_tokens(raw_input, cache_read, cache_write) == expected + + def _make_session_result( returncode: int = 0, stdout: str = "", diff --git a/tests/infra/test_plugin_source_ratchets.py b/tests/infra/test_plugin_source_ratchets.py index 5f7d484b8e..c909460787 100644 --- a/tests/infra/test_plugin_source_ratchets.py +++ b/tests/infra/test_plugin_source_ratchets.py @@ -55,13 +55,13 @@ "Removes only the unique physical-attempt directory created with exist_ok=False " "when its supervision-channel context exits.", ), - ("execution/report_index.py", "_update", "(index_dir / _ROWS_FILE).unlink"): ( + ("execution/report_index.py", "_clear_index_files", "(index_dir / _ROWS_FILE).unlink"): ( 1, - "Rebuild removes derived rows while holding the report-index writer lease.", + "Rebuilds and incompatible generations remove derived rows under the writer lease.", ), - ("execution/report_index.py", "_update", "(index_dir / _STATE_FILE).unlink"): ( + ("execution/report_index.py", "_clear_index_files", "(index_dir / _STATE_FILE).unlink"): ( 1, - "Rebuild removes the derived watermark while holding the report-index writer lease.", + "Rebuilds and incompatible generations remove the watermark under the writer lease.", ), ( "server/tools/tools_kitchen/_close_kitchen.py", diff --git a/tests/report/_fixtures.py b/tests/report/_fixtures.py index 8f4e304698..08bae77ac8 100644 --- a/tests/report/_fixtures.py +++ b/tests/report/_fixtures.py @@ -26,3 +26,50 @@ def session_row(key: str, **fields: Any) -> dict[str, Any]: ) row.update(fields) return row + + +def token_measure(value: int | None, *, state: str | None = None) -> dict[str, Any]: + if state is None: + state = "unknown" if value is None else ("measured_zero" if value == 0 else "measured") + return {"state": state, "value": value} + + +def subagent_row( + child_id: str, + *, + parent: dict[str, Any], + role: str, + skill: str, + provider: str, + input_tokens: int | None, + output_tokens: int | None, + cache_read_tokens: int | None, + cache_write_tokens: int | None, + tool_counts: dict[str, int], + transcript_state: str = "observed", + usage_state: str = "observed", +) -> dict[str, Any]: + """Build one canonical child fact linked to its parent cohort row.""" + return { + "child_id": child_id, + "role": role, + "parent_session_key": parent["key"], + "native_parent_session_id": parent["session_id"], + "skill": skill, + "provider": provider, + "model": "test-child-model", + "actor_level": "L0", + "time_ms": parent["time_ms"], + "level": parent["level"], + "recipe": parent["recipe"], + "step": parent["step"], + "token_usage": { + "input_tokens": token_measure(input_tokens), + "output_tokens": token_measure(output_tokens), + "cache_read_tokens": token_measure(cache_read_tokens), + "cache_write_tokens": token_measure(cache_write_tokens), + }, + "tool_counts": tool_counts, + "transcript_state": transcript_state, + "usage_state": usage_state, + } diff --git a/tests/report/test_deck_core_js.py b/tests/report/test_deck_core_js.py index ac9f8c6b98..51324fbe4f 100644 --- a/tests/report/test_deck_core_js.py +++ b/tests/report/test_deck_core_js.py @@ -80,7 +80,7 @@ def test_href_for_keeps_cohort_params_and_applies_target_changes(deck_js: Any) - ) -def test_facet_selection_drops_struck_keys_and_preserves_one_live_key( +def test_facet_selection_drops_stale_keys_without_widening_scope( deck_js: Any, deck_model: dict[str, Any] ) -> None: chips = deck_model["chips"]["cohort"] @@ -97,9 +97,9 @@ def test_facet_selection_drops_struck_keys_and_preserves_one_live_key( assert mixed["dropped"] == ["L0"] assert mixed["widened"] is False - widened = deck_js.call("DeckCore.effectiveSelection", levels, ["L0"]) - assert widened["keys"] == ["unrecorded"] - assert widened["widened"] is True + stale = deck_js.call("DeckCore.effectiveSelection", levels, ["L0"]) + assert stale["keys"] == [] + assert stale["dropped"] == ["L0"] selected = deck_js.call("DeckCore.toggleSelection", harness, None, "codex") assert selected == ["claude-code"] @@ -109,17 +109,257 @@ def test_facet_selection_drops_struck_keys_and_preserves_one_live_key( assert deck_js.call("DeckCore.toggleSelection", harness, selected, "codex") is None -def test_window_selection_falls_back_to_all_for_absent_window( +def test_window_selection_does_not_fall_back_for_absent_window( deck_js: Any, deck_model: dict[str, Any] ) -> None: windows = deck_model["chips"]["cohort"]["window"] assert deck_js.call("DeckCore.windowSelection", windows, None)["chip"]["key"] == "all" absent = deck_js.call("DeckCore.windowSelection", windows, ["28d"]) - assert absent["chip"]["key"] == "all" + assert absent["chip"] is None assert absent["dropped"] == ["28d"] assert deck_js.call("DeckCore.windowSelection", windows, ["7d"])["chip"]["key"] == "7d" +def test_prepared_selection_uses_exact_scope_and_keeps_metric_cells_distinct( + deck_js: Any, +) -> None: + skill = { + "skill": "review", + "harness": "codex", + "provider": "openai", + "measures": {"input_tokens": {"state": "measured", "value": 17}}, + } + other_skill = {**skill, "harness": "claude-code", "provider": "anthropic"} + role = { + "role": "reviewer", + "provider": "openai", + "harnesses": [ + {"harness": "codex", "measures": {"input_tokens": {"value": 5}}}, + {"harness": "claude-code", "measures": {"input_tokens": {"value": 9}}}, + ], + } + chips = { + "window": [ + {"key": "7d", "match": "7d", "label": "7d", "state": "live", "days": 7}, + {"key": "all", "match": "all", "label": "all", "state": "live", "days": None}, + ], + "level": [ + {"key": "L1", "match": "skill", "label": "L1", "state": "live"}, + {"key": "L2", "match": "orchestrator", "label": "L2", "state": "live"}, + ], + "harness": [ + {"key": "codex", "match": "codex", "label": "codex", "state": "live"}, + { + "key": "claude-code", + "match": "claude-code", + "label": "claude-code", + "state": "live", + }, + ], + "provider": [ + {"key": "openai", "match": "openai", "label": "openai", "state": "live"}, + { + "key": "anthropic", + "match": "anthropic", + "label": "anthropic", + "state": "live", + }, + ], + } + relationship = { + "skill": "review", + "role": "reviewer", + "harness": "codex", + "provider": "openai", + } + definition = {"state": "available", "description": "Reads code"} + history = {"first_ms": 1, "last_ms": 2, "untimed": 0} + prepared = { + "skills": [ + {"window": "7d", "levels": ["skill"], "rows": [{**skill, "skill": "wrong-level"}]}, + { + "window": "all", + "levels": ["orchestrator", "skill"], + "rows": [{**skill, "skill": "wrong-window"}], + }, + { + "window": "7d", + "levels": ["orchestrator", "skill"], + "rows": [skill, other_skill], + }, + ], + "roles": [ + { + "window": "7d", + "levels": ["orchestrator", "skill"], + "rows": [role], + } + ], + "relationships": [relationship], + "definitions": {"reviewer": definition}, + "view_chips": {"efficiency": chips}, + "view_histories": {"efficiency": history}, + } + route = { + "params": { + "window": ["7d"], + "level": ["L1", "L2"], + "harness": ["codex"], + "provider": ["openai"], + } + } + + selected = deck_js.call("DeckCore.selectPrepared", prepared, "efficiency", chips, route) + + assert selected["skillRows"] == [skill] + assert selected["roleRows"] == [ + { + **role, + "harnesses": [role["harnesses"][0]], + } + ] + assert selected["relationships"] == [relationship] + assert selected["definitions"] == {"reviewer": definition} + assert selected["viewHistory"] == history + + route["params"]["window"] = ["missing-window"] + empty = deck_js.call("DeckCore.selectPrepared", prepared, "efficiency", chips, route) + assert empty["skillRows"] == empty["roleRows"] == [] + assert empty["relationships"] == [relationship] + assert empty["definitions"] == {"reviewer": definition} + + route["params"]["window"] = ["7d"] + prepared["skills"] = [{"window": "7d", "levels": ["skill"], "rows": [skill]}] + selected = deck_js.call("DeckCore.selectPrepared", prepared, "skill", chips, route) + assert selected["skillRows"] == [skill] + route["params"]["level"] = ["L2"] + selected = deck_js.call("DeckCore.selectPrepared", prepared, "skill", chips, route) + assert selected["skillRows"] == [] + route["params"]["level"] = [] + selected = deck_js.call("DeckCore.selectPrepared", prepared, "skill", chips, route) + assert selected["skillRows"] == [skill] + + +def test_prepared_selection_returns_no_metrics_for_stale_scope(deck_js: Any) -> None: + chips = { + "window": [{"key": "all", "match": "all", "label": "all", "state": "live", "days": None}], + "level": [{"key": "L1", "match": "skill", "label": "L1", "state": "live"}], + "harness": [{"key": "codex", "match": "codex", "label": "codex", "state": "live"}], + "provider": [{"key": "openai", "match": "openai", "label": "openai", "state": "live"}], + } + prepared = { + "skills": [{"window": "all", "levels": ["skill"], "rows": [{"skill": "review"}]}], + "roles": [{"window": "all", "levels": ["skill"], "rows": []}], + "relationships": [], + "definitions": {}, + "view_chips": {"skill": chips}, + "view_histories": {"skill": {"first_ms": 1, "last_ms": 1, "untimed": 0}}, + } + + selected = deck_js.call( + "DeckCore.selectPrepared", + prepared, + "skill", + chips, + {"params": {"window": ["deleted-window"], "level": ["L2"]}}, + ) + + assert selected["skillRows"] == [] + assert selected["roleRows"] == [] + + +@pytest.mark.parametrize( + ("levels", "expected"), + [ + pytest.param(None, ["short-window-skill"], id="window-local-default"), + pytest.param(["L1", "L2"], ["short-window-skill"], id="mixed-levels-intersect"), + pytest.param(["L2"], [], id="unobserved-window-level-is-empty"), + ], +) +def test_prepared_selection_uses_the_selected_windows_level_domain( + deck_js: Any, + levels: list[str] | None, + expected: list[str], +) -> None: + chips = { + "window": [ + {"key": "7d", "match": "7d", "label": "7d", "state": "live", "days": 7}, + {"key": "all", "match": "all", "label": "all", "state": "live", "days": None}, + ], + "level": [ + {"key": "L1", "match": "skill", "label": "L1", "state": "live"}, + {"key": "L2", "match": "orchestrator", "label": "L2", "state": "live"}, + ], + "harness": [{"key": "codex", "match": "codex", "label": "codex", "state": "live"}], + "provider": [{"key": "openai", "match": "openai", "label": "openai", "state": "live"}], + } + short_row = {"skill": "short-window-skill", "harness": "codex", "provider": "openai"} + all_row = {"skill": "older-orchestrator", "harness": "codex", "provider": "openai"} + prepared = { + "skills": [ + {"window": "7d", "levels": ["skill"], "rows": [short_row]}, + {"window": "all", "levels": ["orchestrator", "skill"], "rows": [all_row]}, + ], + "roles": [], + "relationships": [], + "definitions": {}, + } + params = {"window": ["7d"], "harness": ["codex"], "provider": ["openai"]} + if levels is not None: + params["level"] = levels + + selected = deck_js.call( + "DeckCore.selectPrepared", prepared, "skill", chips, {"params": params} + ) + + assert [row["skill"] for row in selected["skillRows"]] == expected + + +def test_prepared_selection_keeps_skill_edges_with_unknown_child_provider( + deck_js: Any, +) -> None: + chips = { + "window": [{"key": "all", "match": "all", "label": "all", "state": "live", "days": None}], + "level": [{"key": "L1", "match": "skill", "label": "L1", "state": "live"}], + "harness": [{"key": "codex", "match": "codex", "label": "codex", "state": "live"}], + "provider": [{"key": "openai", "match": "openai", "label": "openai", "state": "live"}], + } + skill = { + "skill": "parent-skill", + "harness": "codex", + "provider": "openai", + "measures": {"input_tokens": {"state": "measured", "value": 42}}, + } + edge = { + "skill": "parent-skill", + "role": "native-reader", + "harness": "codex", + "provider": "unknown", + } + prepared = { + "skills": [{"window": "all", "levels": ["skill"], "rows": [skill]}], + "roles": [], + "relationships": [edge, {**edge, "skill": "another-skill"}], + "definitions": {"native-reader": {"state": "available"}}, + "view_histories": {"skill": {"first_ms": 1, "last_ms": 1, "untimed": 0}}, + } + route = { + "view": "skill", + "entity": "parent-skill", + "params": { + "window": ["all"], + "level": ["L1"], + "harness": ["codex"], + "provider": ["openai"], + }, + } + + selected = deck_js.call("DeckCore.selectPrepared", prepared, "skill", chips, route) + + assert selected["skillRows"] == [skill] + assert selected["relationships"] == [edge] + + @pytest.mark.parametrize( ("params", "expected_keys", "untimed"), [ @@ -127,7 +367,8 @@ def test_window_selection_falls_back_to_all_for_absent_window( pytest.param({"harness": ["codex"]}, ["s4"], 0, id="codex"), pytest.param({"window": ["7d"]}, ["s1", "s2"], 1, id="last-week"), pytest.param({"harness": ["codex"], "window": ["7d"]}, [], 1, id="untimed-codex"), - pytest.param({"window": ["28d"]}, ["s1", "s2", "s3", "s4"], 0, id="absent-window"), + pytest.param({"window": ["28d"]}, [], 0, id="absent-window"), + pytest.param({"harness": ["retired"]}, [], 0, id="stale-harness"), ], ) def test_filter_rows_applies_facets_windows_and_untimed_count( @@ -170,29 +411,31 @@ def test_filter_rows_applies_facets_windows_and_untimed_count( ), pytest.param( _route(level=["L0"]), - "4 runs", - ALL_POPULATION_DETAIL, + "0 runs", + ALL_POPULATION_DETAIL.replace("level unrecorded (all)", "level none selected"), [ "L0 is not selectable on this view — L0 leaf agents write no session row; " "they exist only as subagent transcripts", - "no selected level is selectable here — showing every selectable level", ], id="struck-level", ), pytest.param( _route(harness=["nope"]), - "4 runs", - ALL_POPULATION_DETAIL, + "0 runs", + ALL_POPULATION_DETAIL.replace( + "harness claude-code + codex (all)", "harness none selected" + ), [ "nope does not appear in this index", - "no selected harness is selectable here — showing every selectable harness", ], id="unknown-harness", ), pytest.param( _route(window=["28d"]), - "4 runs", - ALL_POPULATION_DETAIL, + "0 runs", + ALL_POPULATION_DETAIL.replace( + "window all history (2026-09-25 → 2026-10-04)", "window unavailable" + ), [ "28 days is not selectable on this view — index history begins 2026-09-25 — " "9 days retained (#4621)" diff --git a/tests/report/test_deck_html.py b/tests/report/test_deck_html.py index a9f7f10ff1..8f25fd71a5 100644 --- a/tests/report/test_deck_html.py +++ b/tests/report/test_deck_html.py @@ -4,21 +4,26 @@ import hashlib import json import re +import subprocess from html.parser import HTMLParser +from pathlib import Path +from types import SimpleNamespace from typing import Any +from zipfile import ZipFile import pytest from autoskillit.core import pkg_root from autoskillit.report import render_deck from autoskillit.report.deck import _html as deck_html +from autoskillit.report.deck import _payload as deck_payload_module from autoskillit.report.deck import build_deck_payload from autoskillit.report.deck import render_deck as render_deck_from_deck from autoskillit.report.deck._html import render_deck_html from autoskillit.report.deck._registry import DECK_VIEWS -from tests.report._fixtures import DECK_GENERATED_AT +from tests.report._fixtures import DECK_GENERATED_AT, session_row, subagent_row, token_measure -pytestmark = [pytest.mark.small] +pytestmark = [pytest.mark.medium] class _HTMLCollector(HTMLParser): @@ -204,6 +209,28 @@ def test_deck_asset_directory_contains_only_registered_shell_assets() -> None: assert actual == {"deck.html", "deck.css", "core.js", "shell.js"} | view_assets +def test_built_wheel_contains_the_four_new_view_scripts(tmp_path: Path) -> None: + # REQ-108/REQ-158: isolate the plan's wheel-check directory per test worker. + wheel_out = tmp_path / "wheel-check" + result = subprocess.run( + ["uv", "build", "--wheel", "--out-dir", str(wheel_out), "--offline"], + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + text=True, + timeout=120, + ) + assert result.returncode == 0, (result.stdout + result.stderr)[-4000:] + wheels = list(wheel_out.glob("*.whl")) + assert len(wheels) == 1, f"Expected one built wheel, found {wheels}" + with ZipFile(wheels[0]) as wheel: + required = { + f"autoskillit/assets/deck/views/{view}.js" + for view in ("spend", "efficiency", "skill", "role") + } + missing = required - set(wheel.namelist()) + assert not missing, f"Built wheel is missing view scripts: {sorted(missing)}" + + def test_embedded_cohort_chips_cover_all_states(deck_payload: dict[str, Any]) -> None: collector = _collect(render_deck_html(deck_payload)) data = next(source for attrs, source in collector.scripts if attrs.get("id") == "deck-data") @@ -222,9 +249,85 @@ def test_render_deck_facades_match_html_wrapper( expected = render_deck_html(deck_payload) assert ( - render_deck(deck_rows, generated_at=DECK_GENERATED_AT, index_schema_version=1) == expected + render_deck( + deck_rows, + request_rows=(), + tool_rows=(), + subagent_rows=(), + generated_at=DECK_GENERATED_AT, + index_schema_version=1, + ) + == expected ) assert ( - render_deck_from_deck(deck_rows, generated_at=DECK_GENERATED_AT, index_schema_version=1) + render_deck_from_deck( + deck_rows, + request_rows=(), + tool_rows=(), + subagent_rows=(), + generated_at=DECK_GENERATED_AT, + index_schema_version=1, + ) == expected ) + + +def test_render_deck_forwards_child_facts_and_embeds_definitions_as_literal_json( + monkeypatch: pytest.MonkeyPatch, +) -> None: + role = "audit-impl-slice-auditor" + parent = session_row( + "parent", + session_id="native-parent", + time_ms=1_791_000_000_000, + skill="planner", + recipe="planner", + step="inspect", + level="skill", + input_tokens=token_measure(12), + output_tokens=token_measure(3), + ) + child = subagent_row( + "native-child", + parent=parent, + role=role, + skill="planner", + provider="anthropic", + input_tokens=20, + output_tokens=4, + cache_read_tokens=2, + cache_write_tokens=1, + tool_counts={"Bash": 3944, "Read": 6}, + ) + hostile_body = "Run git show. " + definition = SimpleNamespace( + name=role, + description="Inspect a committed slice.", + body=hostile_body, + tools=("Bash", "Read"), + model="test-claude-model", + reader_tools=(), + codex=SimpleNamespace(model="test-codex-model"), + ) + monkeypatch.setattr( + deck_payload_module, + "load_bundled_agent_definitions", + lambda: (definition,), + raising=False, + ) + html = deck_html.render_deck( + [parent], + request_rows=[{"request_id": "unlinked", "agent_name": role}], + tool_rows=[{"agent_name": role, "tool_name": "Bash", "count": 999_999}], + subagent_rows=[child], + generated_at=DECK_GENERATED_AT, + index_schema_version=1, + ) + collector = _collect(html) + data = next(source for attrs, source in collector.scripts if attrs.get("id") == "deck-data") + payload = json.loads(data) + + assert payload["prepared"]["definitions"][role]["body"] == hostile_body + assert any(row[0] == role for row in payload["tables"]["roles"]["rows"]) + assert "\\u003c/script\\u003e" in data + assert not any(tag == "img" for tag, _ in collector.elements) diff --git a/tests/report/test_deck_payload.py b/tests/report/test_deck_payload.py index 8ca06df34d..06d1f0970b 100644 --- a/tests/report/test_deck_payload.py +++ b/tests/report/test_deck_payload.py @@ -2,11 +2,20 @@ import json from datetime import datetime +from types import SimpleNamespace from typing import Any import pytest -from autoskillit.core import TokenMeasureState +from autoskillit.core import ( + MeasureRecord, + SourcePair, + TokenMeasure, + TokenMeasureState, + aggregate_measures, + measure_ratio, +) +from autoskillit.report.deck import _payload as deck_payload_module from autoskillit.report.deck import build_deck_payload from autoskillit.report.deck._payload import encode_table from autoskillit.report.deck._registry import ( @@ -16,7 +25,7 @@ SESSION_COLUMNS, SESSION_TABLE, ) -from tests.report._fixtures import DECK_GENERATED_AT +from tests.report._fixtures import DECK_GENERATED_AT, subagent_row, token_measure from tests.report._fixtures import session_row as _row pytestmark = [pytest.mark.small] @@ -24,13 +33,275 @@ def _payload(rows: list[dict[str, Any]]) -> dict[str, Any]: - return build_deck_payload(rows, generated_at=DECK_GENERATED_AT, index_schema_version=7) + return build_deck_payload( + rows, + request_rows=(), + tool_rows=(), + subagent_rows=(), + generated_at=DECK_GENERATED_AT, + index_schema_version=7, + ) def _chip(chips: dict[str, list[dict[str, Any]]], facet: str, key: str) -> dict[str, Any]: return next(chip for chip in chips[facet] if chip["key"] == key) +_TOKEN_FIELDS = ("input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens") +_AUDITOR = "audit-impl-slice-auditor" + + +def _session_fact( + key: str, + *, + days_ago: int, + skill: str, + level: str, + harness: str = "claude-code", + provider: str = "anthropic", + input_tokens: int | None = 10, + output_tokens: int | None = 2, + cache_read_tokens: int | None = 0, + cache_write_tokens: int | None = 0, +) -> dict[str, Any]: + return _row( + key, + session_id=f"native-{key}", + time_ms=GEN_MS - days_ago * 86_400_000, + harness=harness, + provider=provider, + skill=skill, + recipe=f"recipe-{skill}", + step="inspect", + level=level, + **{ + field: token_measure(value) + for field, value in zip( + _TOKEN_FIELDS, + (input_tokens, output_tokens, cache_read_tokens, cache_write_tokens), + strict=True, + ) + }, + ) + + +def _full_index_facts() -> tuple[ + list[dict[str, Any]], + list[dict[str, Any]], + list[dict[str, Any]], + list[dict[str, Any]], +]: + sessions = [ + _session_fact("skill", days_ago=1, skill="planner", level="skill", input_tokens=10), + _session_fact( + "orchestrator", days_ago=2, skill="planner", level="orchestrator", input_tokens=20 + ), + _session_fact("review", days_ago=3, skill="review", level="skill", input_tokens=5), + _session_fact( + "other-provider", + days_ago=4, + skill="planner", + level="skill", + provider="minimax", + input_tokens=7, + ), + _session_fact( + "codex", + days_ago=9, + skill="planner", + level="skill", + harness="codex", + provider="codex", + input_tokens=4, + ), + _session_fact("no-child", days_ago=0, skill="maintenance", level="skill", input_tokens=0), + ] + by_key = {row["key"]: row for row in sessions} + children = [ + subagent_row( + "child-a", + parent=by_key["skill"], + role=_AUDITOR, + skill="planner", + provider="anthropic", + input_tokens=100, + output_tokens=20, + cache_read_tokens=10, + cache_write_tokens=2, + tool_counts={"Bash": 3944, "Read": 6}, + ), + subagent_row( + "child-b", + parent=by_key["orchestrator"], + role=_AUDITOR, + skill="planner", + provider="anthropic", + input_tokens=0, + output_tokens=10, + cache_read_tokens=0, + cache_write_tokens=0, + tool_counts={"Bash": 1, "Read": 1}, + ), + subagent_row( + "child-c", + parent=by_key["review"], + role=_AUDITOR, + skill="review", + provider="anthropic", + input_tokens=5, + output_tokens=5, + cache_read_tokens=2, + cache_write_tokens=0, + tool_counts={"Read": 2}, + ), + subagent_row( + "child-d", + parent=by_key["other-provider"], + role=_AUDITOR, + skill="planner", + provider="minimax", + input_tokens=500, + output_tokens=25, + cache_read_tokens=20, + cache_write_tokens=5, + tool_counts={"Bash": 3}, + ), + subagent_row( + "child-e", + parent=by_key["codex"], + role="codex-reviewer", + skill="planner", + provider="codex", + input_tokens=9, + output_tokens=3, + cache_read_tokens=0, + cache_write_tokens=0, + tool_counts={"Read": 1}, + ), + subagent_row( + "child-f", + parent=by_key["review"], + role="unresolved-observer", + skill="review", + provider="anthropic", + input_tokens=None, + output_tokens=None, + cache_read_tokens=None, + cache_write_tokens=None, + tool_counts={}, + transcript_state="unknown", + usage_state="unknown", + ), + subagent_row( + "child-g", + parent=by_key["review"], + role="zero-denominator", + skill="review", + provider="anthropic", + input_tokens=2, + output_tokens=0, + cache_read_tokens=0, + cache_write_tokens=0, + tool_counts={"Read": 1}, + ), + ] + children[4]["token_usage"]["cache_read_tokens"] = token_measure(None, state="unavailable") + request_rows = [ + {"request_id": "unlinked-request", "agent_name": _AUDITOR, "input_tokens": 999_999} + ] + tool_rows = [{"agent_name": _AUDITOR, "tool_name": "Bash", "count": 999_999}] + return sessions, request_rows, tool_rows, children + + +def _source_records( + rows: list[dict[str, Any]], + *, + child: bool = False, + parents: dict[str, dict[str, Any]] | None = None, +) -> list[MeasureRecord]: + if child and parents is None: + raise ValueError("child records require their parent rows") + parent_rows = parents or {} + records = [] + for row in rows: + raw = row["token_usage"] if child else row + harness = parent_rows[row["parent_session_key"]]["harness"] if child else row["harness"] + provider = row["provider"] + records.append( + MeasureRecord( + SourcePair(harness, provider), + {field: TokenMeasure.from_dict(raw[field]) for field in _TOKEN_FIELDS}, + ) + ) + return records + + +def _definition(name: str, body: str) -> SimpleNamespace: + return SimpleNamespace( + name=name, + description="Canonical reviewer role.", + body=body, + tools=("Bash", "Read"), + model="test-claude-model", + reader_tools=(), + codex=SimpleNamespace(model="test-codex-model"), + ) + + +def _metric_row( + blocks: list[dict[str, Any]], *, window: str, levels: set[str | None] +) -> dict[str, Any]: + block = next( + block for block in blocks if block["window"] == window and set(block["levels"]) == levels + ) + return block + + +def _assert_serialized_measure(actual: dict[str, Any], expected: Any) -> None: + assert actual["state"] == expected.value.state.value + assert actual["value"] == expected.value.value + assert actual["state_counts"] == { + state.value: count for state, count in expected.state_counts.items() + } + + +def _assert_serialized_ratio(actual: dict[str, Any], expected: Any, *, sample_unit: str) -> None: + assert actual["state"] == expected.state.value + assert actual["value"] == expected.value + assert actual["numerator"] == expected.numerator_field + assert actual["denominator"] == expected.denominator_field + assert actual["numerator_total"] == expected.numerator_total + assert actual["denominator_total"] == expected.denominator_total + assert actual["sample_unit"] == sample_unit + assert actual["sample_size"] == expected.sample_size + assert actual["excluded_runs"] == expected.excluded_runs + assert actual["unknown_runs"] == expected.unknown_runs + + +def _tool_records( + rows: list[dict[str, Any]], parents: dict[str, dict[str, Any]] +) -> list[MeasureRecord]: + tools = sorted({tool for row in rows for tool in row["tool_counts"]}) + base_records = _source_records(rows, child=True, parents=parents) + records = [] + for row, base in zip(rows, base_records, strict=True): + measures = dict(base.measures) + observed = row["transcript_state"] == "observed" + measures["tool_calls"] = ( + TokenMeasure.observed(sum(row["tool_counts"].values())) + if observed + else TokenMeasure.unknown() + ) + for tool in tools: + measures[f"tool:{tool}"] = ( + TokenMeasure.observed(row["tool_counts"].get(tool, 0)) + if observed + else TokenMeasure.unknown() + ) + records.append(MeasureRecord(base.pair, measures)) + return records + + def test_encode_table_aligns_rows_to_columns_and_fills_missing_keys() -> None: encoded = encode_table([{"key": "row-1", "time_ms": 42}], SESSION_COLUMNS) @@ -228,3 +499,237 @@ def test_empty_index_has_absent_declared_values_and_empty_observed_facets() -> N assert (chip["state"], chip["issue"]) == ("absent", 4622) assert all(chip["state"] == "absent" for chip in chips["window"][:-1]) assert (chips["window"][-1]["key"], chips["window"][-1]["state"]) == ("all", "live") + + +@pytest.fixture +def prepared_metrics( + monkeypatch: pytest.MonkeyPatch, +) -> SimpleNamespace: + sessions, requests, tools, children = _full_index_facts() + body = "Bash runs git show to inspect the committed slice." + loader_calls: list[None] = [] + + def definition_loader() -> tuple[SimpleNamespace, ...]: + loader_calls.append(None) + return (_definition(_AUDITOR, body),) + + monkeypatch.setattr( + deck_payload_module, + "load_bundled_agent_definitions", + definition_loader, + raising=False, + ) + aggregate_calls: list[int] = [] + ratio_calls: list[int] = [] + original_aggregate = getattr(deck_payload_module, "aggregate_measures", aggregate_measures) + original_ratio = getattr(deck_payload_module, "measure_ratio", measure_ratio) + + def aggregate_spy(records: Any, fields: Any, **kwargs: Any) -> Any: + rows = list(records) + aggregate_calls.append(len(rows)) + return original_aggregate(rows, fields, **kwargs) + + def ratio_spy(records: Any, numerator: str, denominator: str, **kwargs: Any) -> Any: + rows = list(records) + ratio_calls.append(len(rows)) + return original_ratio(rows, numerator, denominator, **kwargs) + + monkeypatch.setattr(deck_payload_module, "aggregate_measures", aggregate_spy, raising=False) + monkeypatch.setattr(deck_payload_module, "measure_ratio", ratio_spy, raising=False) + payload = build_deck_payload( + sessions, + request_rows=requests, + tool_rows=tools, + subagent_rows=children, + generated_at=DECK_GENERATED_AT, + index_schema_version=7, + ) + + return SimpleNamespace( + payload=payload, + sessions=sessions, + children=children, + definition_body=body, + aggregate_calls=aggregate_calls, + ratio_calls=ratio_calls, + loader_calls=loader_calls, + ) + + +def test_prepared_skill_and_child_metrics_use_shared_accounting( + prepared_metrics: SimpleNamespace, +) -> None: + payload = prepared_metrics.payload + sessions = prepared_metrics.sessions + children = prepared_metrics.children + assert prepared_metrics.aggregate_calls and prepared_metrics.ratio_calls + assert len(prepared_metrics.loader_calls) == 1 + prepared = payload["prepared"] + levels: set[str | None] = {"orchestrator", "skill"} + skill_block = _metric_row(prepared["skills"], window="all", levels=levels) + planner = next( + row + for row in skill_block["rows"] + if (row["skill"], row["harness"], row["provider"]) + == ("planner", "claude-code", "anthropic") + ) + planner_runs = [ + row + for row in sessions + if row["skill"] == "planner" + and row["harness"] == "claude-code" + and row["provider"] == "anthropic" + and row["level"] in levels + ] + planner_aggregate = aggregate_measures(_source_records(planner_runs), _TOKEN_FIELDS) + planner_ratio = measure_ratio(_source_records(planner_runs), "input_tokens", "output_tokens") + assert planner["measures"]["input_tokens"]["value"] == 30 + assert planner["exact_retransmission"] == {"state": "unavailable", "value": None} + assert planner["recipe_steps"][0]["exact_retransmission"] == { + "state": "unavailable", + "value": None, + } + _assert_serialized_measure( + planner["measures"]["input_tokens"], planner_aggregate.fields["input_tokens"] + ) + _assert_serialized_ratio( + planner["ratios"]["input_output"], planner_ratio, sample_unit="skill-run" + ) + + role_block = _metric_row(prepared["roles"], window="all", levels=levels) + auditor = next( + row + for row in role_block["rows"] + if (row["role"], row["provider"]) == (_AUDITOR, "anthropic") + ) + claude_metrics = next(row for row in auditor["harnesses"] if row["harness"] == "claude-code") + auditor_runs = [ + row + for row in children + if row["role"] == _AUDITOR and row["provider"] == "anthropic" and row["level"] in levels + ] + parents = {row["key"]: row for row in sessions} + child_records = _source_records(auditor_runs, child=True, parents=parents) + child_aggregate = aggregate_measures(child_records, _TOKEN_FIELDS) + child_ratio = measure_ratio(child_records, "input_tokens", "output_tokens") + tool_records = _tool_records(auditor_runs, parents) + bash_ratio = measure_ratio(tool_records, "tool:Bash", "tool_calls") + assert claude_metrics["measures"]["input_tokens"]["value"] == 105 + _assert_serialized_measure( + claude_metrics["measures"]["input_tokens"], child_aggregate.fields["input_tokens"] + ) + _assert_serialized_ratio( + claude_metrics["ratios"]["input_output"], child_ratio, sample_unit="child-invocation" + ) + _assert_serialized_ratio( + claude_metrics["ratios"]["tool_mix"]["Bash"], + bash_ratio, + sample_unit="child-invocation", + ) + assert claude_metrics["ratios"]["input_output"]["sample_size"] == 3 + assert claude_metrics["ratios"]["tool_mix"]["Bash"]["review_eligible"] is True + assert claude_metrics["ratios"]["tool_mix"]["Bash"]["definition_roles"] == [_AUDITOR] + + +def test_prepared_ratio_states_and_definition_availability( + prepared_metrics: SimpleNamespace, +) -> None: + prepared = prepared_metrics.payload["prepared"] + role_block = _metric_row(prepared["roles"], window="all", levels={"orchestrator", "skill"}) + zero_skill_block = _metric_row(prepared["skills"], window="all", levels={"skill"}) + maintenance = next(row for row in zero_skill_block["rows"] if row["skill"] == "maintenance") + zero_ratio = maintenance["ratios"]["input_output"] + assert (zero_ratio["state"], zero_ratio["value"], zero_ratio["sample_size"]) == ( + "measured_zero", + 0.0, + 1, + ) + + unresolved = next(row for row in role_block["rows"] if row["role"] == "unresolved-observer") + unresolved_ratio = unresolved["harnesses"][0]["ratios"]["input_output"] + assert unresolved_ratio["state"] == "unknown" + assert unresolved_ratio["review_eligible"] is False + assert unresolved_ratio["review_reason"] + assert prepared["definitions"][_AUDITOR]["state"] == "available" + assert prepared["definitions"][_AUDITOR]["body"] == prepared_metrics.definition_body + assert prepared["definitions"]["unresolved-observer"]["state"] == "unavailable" + + zero_denominator = next(row for row in role_block["rows"] if row["role"] == "zero-denominator") + zero_denominator_ratio = zero_denominator["harnesses"][0]["ratios"]["input_output"] + assert (zero_denominator_ratio["state"], zero_denominator_ratio["sample_size"]) == ( + "not_applicable", + 1, + ) + codex = next(row for row in role_block["rows"] if row["role"] == "codex-reviewer") + codex_cache = codex["harnesses"][0]["ratios"]["cache_share"] + assert (codex_cache["state"], codex_cache["excluded_runs"]) == ("unavailable", 1) + + +def test_prepared_identity_tables_and_view_projections(prepared_metrics: SimpleNamespace) -> None: + payload = prepared_metrics.payload + sessions = prepared_metrics.sessions + prepared = payload["prepared"] + built_views = {view.view_id for view in DECK_VIEWS if view.planned_issue is None} + assert set(prepared["view_chips"]) == built_views + assert set(prepared["view_histories"]) == built_views + assert set(payload["tables"]) == {SESSION_TABLE, "skills", "roles"} + + def table_rows(table: dict[str, Any]) -> list[dict[str, Any]]: + return [dict(zip(table["columns"], row, strict=True)) for row in table["rows"]] + + skill_identities = table_rows(payload["tables"]["skills"]) + role_identities = table_rows(payload["tables"]["roles"]) + assert {(row["skill"], row["harness"], row["provider"]) for row in skill_identities} >= { + ("planner", "claude-code", "anthropic"), + ("planner", "claude-code", "minimax"), + ("planner", "codex", "codex"), + } + assert {(row["role"], row["provider"]) for row in role_identities} >= { + (_AUDITOR, "anthropic"), + (_AUDITOR, "minimax"), + } + assert { + (edge["skill"], edge["role"], edge["harness"], edge["provider"]) + for edge in prepared["relationships"] + } >= { + ("planner", _AUDITOR, "claude-code", "anthropic"), + ("review", _AUDITOR, "claude-code", "anthropic"), + ("planner", _AUDITOR, "claude-code", "minimax"), + } + assert prepared["view_chips"] == payload["chips"] + assert prepared["view_histories"]["skill"]["last_ms"] == sessions[-1]["time_ms"] + assert ( + prepared["view_histories"]["role"]["last_ms"] + < prepared["view_histories"]["skill"]["last_ms"] + ) + json.dumps(payload, allow_nan=False) + + +def test_observed_zero_ratio_can_be_flagged_with_complete_definitions( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sessions, _, _, children = _full_index_facts() + for child in children: + if child.get("role") == _AUDITOR: + child["token_usage"]["cache_read_tokens"] = token_measure(0) + monkeypatch.setattr( + deck_payload_module, + "load_bundled_agent_definitions", + lambda: (_definition(_AUDITOR, "Inspect the slice with git show."),), + ) + payload = build_deck_payload( + sessions, + subagent_rows=children, + generated_at=DECK_GENERATED_AT, + index_schema_version=7, + ) + block = _metric_row(payload["prepared"]["roles"], window="all", levels={"skill"}) + auditor = next( + row for row in block["rows"] if row["role"] == _AUDITOR and row["provider"] == "anthropic" + ) + signal = auditor["harnesses"][0]["ratios"]["cache_share"] + assert (signal["state"], signal["value"]) == ("measured_zero", 0.0) + assert signal["sample_size"] > 0 + assert signal["definition_roles"] == [_AUDITOR] + assert signal["review_eligible"] is True + assert signal["review_reason"] is None diff --git a/tests/report/test_deck_registry.py b/tests/report/test_deck_registry.py index 7c33a70e6a..7c208808ae 100644 --- a/tests/report/test_deck_registry.py +++ b/tests/report/test_deck_registry.py @@ -6,6 +6,7 @@ from autoskillit.execution._report_index_rows import ReportSessionRow from autoskillit.report.deck._registry import ( AVAILABILITY_VOCABULARY, + DECK_VIEWS, FACETS, SESSION_COLUMNS, SESSION_TABLE, @@ -93,6 +94,21 @@ def test_session_columns_follow_report_session_row_schema() -> None: assert len(SESSION_COLUMNS) == len(set(SESSION_COLUMNS)) +def test_reserved_measure_views_have_primary_tables_and_registered_scripts() -> None: + views = {view.view_id: view for view in DECK_VIEWS} + expected = { + "spend": ("skills", "views/spend.js"), + "efficiency": ("skills", "views/efficiency.js"), + "skill": ("skills", "views/skill.js"), + "role": ("roles", "views/role.js"), + } + + assert { + view_id: (views[view_id].table, views[view_id].script) for view_id in expected + } == expected + assert all(views[view_id].planned_issue is None for view_id in expected) + + def test_availability_vocabulary_covers_every_token_measure_state() -> None: assert set(AVAILABILITY_VOCABULARY) == set(TokenMeasureState) diff --git a/tests/report/test_deck_views_js.py b/tests/report/test_deck_views_js.py new file mode 100644 index 0000000000..e6da2c5274 --- /dev/null +++ b/tests/report/test_deck_views_js.py @@ -0,0 +1,331 @@ +"""Browser-free renderer contracts for the observability deck's prepared views.""" + +from typing import Any + +import pytest +from py_mini_racer import MiniRacer + +pytestmark = [pytest.mark.small] + +VIEW_IDS = ("spend", "efficiency", "skill", "role") + +_RENDERER_HARNESS = r""" +globalThis.__deckViews = {}; +globalThis.DeckShell = { + registerView(id, render) { __deckViews[id] = render; } +}; +globalThis.location = {hash: "#/cohort"}; +globalThis.window = {location, addEventListener() {}}; + +function deckNode(tag, attrs = {}, children = null) { + const node = {tag, attrs: {...attrs}, children: [], events: {}, hidden: attrs.hidden === true, + textContent: ""}; + node.appendChild = child => { node.children.push(child); return child; }; + node.addEventListener = (name, callback) => { node.events[name] = callback; }; + node.setAttribute = (name, value) => { node.attrs[name] = String(value); }; + if (Array.isArray(children)) node.children.push(...children); + else if (children !== null && children !== undefined) node.children.push(children); + return node; +} + +function deckText(node) { + if (node === null || node === undefined) return ""; + if (Array.isArray(node)) return node.map(deckText).join(" "); + if (typeof node !== "object") return String(node); + return [node.textContent || "", ...node.children.map(deckText)].join(" "); +} + +function deckLinks(node) { + if (node === null || node === undefined) return []; + if (Array.isArray(node)) return node.flatMap(deckLinks); + if (typeof node !== "object") return []; + return [...(node.tag === "a" ? [node.attrs.href] : []), ...node.children.flatMap(deckLinks)]; +} + +function deckFind(node, predicate) { + if (node === null || node === undefined) return null; + if (Array.isArray(node)) { + for (const child of node) { + const found = deckFind(child, predicate); + if (found) return found; + } + return null; + } + if (typeof node !== "object") return null; + if (predicate(node)) return node; + for (const child of node.children) { + const found = deckFind(child, predicate); + if (found) return found; + } + return null; +} + +function deckFindAll(node, predicate) { + if (node === null || node === undefined) return []; + if (Array.isArray(node)) return node.flatMap(child => deckFindAll(child, predicate)); + if (typeof node !== "object") return []; + return [...(predicate(node) ? [node] : []), ...node.children.flatMap(child => + deckFindAll(child, predicate))]; +} + +function deckSnapshot(node) { + if (node === null || node === undefined || typeof node !== "object") return node; + return { + tag: node.tag, + attrs: node.attrs, + text: deckText(node), + links: deckLinks(node), + reviews: deckFindAll(node, child => + child.attrs && child.attrs["data-review-signal"] !== undefined).map(signal => { + const control = deckFind(signal, child => + child.attrs && child.attrs["data-review-flag"] !== undefined); + const marker = deckFind(signal, child => + child.attrs && child.attrs.class === "view-review__marker"); + const definitions = deckFind(signal, child => + child.attrs && child.attrs.class === "view-review__definitions"); + return { + attrs: control.attrs, + markerHidden: marker.hidden, + markerText: deckText(marker), + linksBeforeFlag: definitions !== null && signal.children.indexOf(definitions) < + signal.children.indexOf(control) + }; + }) + }; +} + +function deckTable({columns, rows, defaultSort}) { + return deckNode("table", {"data-default-sort": defaultSort && defaultSort.key}, [ + deckNode("thead", {}, columns.map(column => deckNode("th", {}, column.label))), + deckNode("tbody", {}, rows.map(row => deckNode("tr", {}, columns.map(column => + deckNode("td", {}, column.cell ? column.cell(row) : row[column.key]))))) + ]); +} + +globalThis.DeckTest = { + render(id, input) { + const route = input.route; + const cohortKeys = ["harness", "provider", "level", "window"]; + const context = { + ...input, + model: {prepared: input.prepared}, + el: deckNode, + href: target => DeckCore.hrefFor(route, target, cohortKeys), + entityLink: (text, target) => deckNode("a", { + href: DeckCore.hrefFor(route, target, cohortKeys) + }, text), + sortableTable: deckTable, + availabilityCell: measure => deckNode("span", {"data-state": measure.state}, + measure.state === "measured" ? String(measure.value) : measure.state), + barChart: (items, options) => deckNode("svg", {"aria-label": options.label}, + JSON.stringify(items)) + }; + location.hash = DeckCore.encodeRoute(route, cohortKeys); + globalThis.__deckLastTree = __deckViews[id](context); + return deckSnapshot(__deckLastTree); + }, + clickReview(index = 0) { + const signals = deckFindAll(__deckLastTree, + child => child.attrs && child.attrs["data-review-signal"] !== undefined); + const control = deckFind(signals[index], + child => child.attrs && child.attrs["data-review-flag"] !== undefined); + if (!control || !control.events.click) throw new Error("review flag has no click handler"); + control.events.click(); + return deckSnapshot(__deckLastTree).reviews[index]; + }, + snapshot() { return deckSnapshot(__deckLastTree); }, + currentHash() { return location.hash; } +}; +""" + + +def _load_renderers(deck_asset: Any) -> MiniRacer: + context = MiniRacer() + context.eval(deck_asset("core.js")) + context.eval(_RENDERER_HARNESS) + for view_id in VIEW_IDS: + context.eval(deck_asset(f"views/{view_id}.js")) + return context + + +def _view_context(view_id: str, *, definitions_available: bool = True) -> dict[str, Any]: + roles = ("reader", "reviewer") + definitions = { + role: { + "state": "available" if definitions_available or role == "reader" else "unavailable", + "description": f"Definition for {role}: definition-first marker", + "body": f"{role} role instructions", + "tools": ["read_file"], + "model": "small-model", + "reader_tools": ["read_file"], + "codex_model": "small-model", + } + for role in roles + } + ratio = { + "state": "measured", + "value": 2.5, + "sample_unit": "child-invocation", + "sample_size": 2, + "review_eligible": True, + "review_reason": "Definitions are available for all contributors.", + "definition_roles": list(roles), + } + skill_ratio = {**ratio, "sample_unit": "skill-run"} + skill_row = { + "skill": "summarizer", + "harness": "codex", + "provider": "openai", + "measures": { + "input_tokens": {"state": "measured", "value": 101, "runs": 2}, + "output_tokens": {"state": "measured", "value": 40, "runs": 2}, + "cache_read_tokens": {"state": "measured", "value": 20, "runs": 2}, + "cache_write_tokens": {"state": "measured", "value": 3, "runs": 2}, + }, + "ratios": {"input_output": skill_ratio, "cache_share": skill_ratio}, + "recipe_steps": [], + } + role_row = { + "role": "reviewer", + "provider": "openai", + "harnesses": [ + { + "harness": "codex", + "models": ["observed-model"], + "measures": {"input_tokens": {"state": "measured", "value": 11, "runs": 2}}, + "ratios": { + "input_output": ratio, + "cache_share": ratio, + "tool_mix": {"read_file": ratio}, + }, + } + ], + } + prepared = { + "skills": [], + "roles": [], + "relationships": [ + {"skill": "summarizer", "role": role, "harness": "codex", "provider": "openai"} + for role in roles + ], + "definitions": definitions, + "view_chips": {}, + "view_histories": {view_id: {"first_ms": 1, "last_ms": 2, "untimed": 0}}, + } + return { + "prepared": prepared, + "metrics": [skill_row] if view_id == "skill" else [role_row], + "skillMetrics": [skill_row], + "roleMetrics": [role_row], + "relationships": prepared["relationships"], + "definitions": definitions, + "viewHistory": prepared["view_histories"][view_id], + "chips": {"window": [], "level": [], "harness": [], "provider": []}, + "selection": {}, + "route": { + "view": view_id, + "entity": "summarizer" + if view_id == "skill" + else "reviewer" + if view_id == "role" + else None, + "params": { + "window": ["all"], + "level": ["L1"], + "harness": ["codex"], + "provider": ["openai"], + }, + }, + } + + +def test_each_prepared_view_renders_its_population_and_cohort_links(deck_asset: Any) -> None: + context = _load_renderers(deck_asset) + try: + markers = {"spend": "101", "efficiency": "2.5", "skill": "summarizer", "role": "reviewer"} + for view_id, marker in markers.items(): + rendered = context.call("DeckTest.render", view_id, _view_context(view_id)) + assert marker in rendered["text"] + assert rendered["links"] + assert any( + "harness=codex" in href and "provider=openai" in href for href in rendered["links"] + ) + if view_id in {"efficiency", "skill", "role"}: + assert rendered["reviews"] + assert all(control["linksBeforeFlag"] for control in rendered["reviews"]) + role_view = context.call("DeckTest.render", "role", _view_context("role")) + assert "L0" in role_view["text"] + assert "spawning skill level" in role_view["text"].lower() + assert "observed-model" in role_view["text"] + assert "Claude Code declared tools" in role_view["text"] + assert "Codex read-only tools" in role_view["text"] + efficiency_view = context.call( + "DeckTest.render", "efficiency", _view_context("efficiency") + ) + assert "2 skill runs" in efficiency_view["text"] + assert "2 child invocations" in efficiency_view["text"] + finally: + context.close() + + +def test_manual_review_flags_require_definitions_and_reset_on_rerender( + deck_asset: Any, +) -> None: + context = _load_renderers(deck_asset) + try: + available = context.call("DeckTest.render", "efficiency", _view_context("efficiency")) + assert len(available["reviews"]) >= 3 + assert all( + control["attrs"].get("disabled") is not True for control in available["reviews"] + ) + assert all( + control["attrs"].get("aria-pressed") == "false" for control in available["reviews"] + ) + assert all(control["markerHidden"] for control in available["reviews"]) + assert all(control["linksBeforeFlag"] for control in available["reviews"]) + assert available["text"].index("definition-first marker") < available["text"].index( + "Flag for review" + ) + initial_hash = context.call("DeckTest.currentHash") + + flagged = context.call("DeckTest.clickReview", 0) + after_one_flag = context.call("DeckTest.snapshot")["reviews"] + assert flagged["attrs"]["aria-pressed"] == "true" + assert flagged["markerHidden"] is False + assert flagged["markerText"].strip() == "Flagged for review" + assert after_one_flag[1]["attrs"]["aria-pressed"] == "false" + assert context.call("DeckTest.currentHash") == initial_hash + + unavailable = context.call( + "DeckTest.render", + "efficiency", + _view_context("efficiency", definitions_available=False), + ) + assert unavailable["reviews"] + assert all("disabled" in control["attrs"] for control in unavailable["reviews"]) + assert all(control["linksBeforeFlag"] for control in unavailable["reviews"]) + assert "Contributor definitions unavailable" in unavailable["text"] + + reset = context.call("DeckTest.render", "efficiency", _view_context("efficiency")) + assert all(control["attrs"]["aria-pressed"] == "false" for control in reset["reviews"]) + assert all(control["markerHidden"] for control in reset["reviews"]) + finally: + context.close() + + +def test_role_definition_and_spawning_links_survive_empty_metric_scope(deck_asset: Any) -> None: + context = _load_renderers(deck_asset) + try: + view = _view_context("role") + view["route"]["entity"] = "reviewer" + view["metrics"] = [] + view["roleMetrics"] = [] + + rendered = context.call("DeckTest.render", "role", view) + + assert "reviewer" in rendered["text"] + assert "reviewer role instructions" in rendered["text"] + assert "No child-invocation metrics match these facets." in rendered["text"] + assert any("#/skill/summarizer" in href for href in rendered["links"]) + finally: + context.close() diff --git a/tests/test_test_filter_core_cascade.py b/tests/test_test_filter_core_cascade.py index 71886eb818..0ccdcb42f1 100644 --- a/tests/test_test_filter_core_cascade.py +++ b/tests/test_test_filter_core_cascade.py @@ -350,7 +350,7 @@ def test_type_capture_cascade(self) -> None: def test_type_token_cascade(self) -> None: assert MODULE_CASCADE_CORE["_type_token"] == frozenset( - {"cli", "core", "execution", "fleet", "pipeline"} + {"cli", "core", "execution", "fleet", "pipeline", "report"} ) def test_type_protocols_backend_cascade(self) -> None: