diff --git a/docs/developer/diagnostics.md b/docs/developer/diagnostics.md index 8168f94bd0..2d373df397 100644 --- a/docs/developer/diagnostics.md +++ b/docs/developer/diagnostics.md @@ -303,7 +303,7 @@ its rotated `otlp.jsonl.1` generation. It stores report facts separately from the retained `sessions.jsonl` projection. -Every v2 row carries `schema_version`, `kind`, `key`, `session_id`, and +Every v3 row carries `schema_version`, `kind`, `key`, `session_id`, and `time_ms`. `time_ms` is epoch milliseconds: session rows use the source session timestamp; OTLP rows use `timeUnixNano`, falling back to `observedTimeUnixNano`. Rows are append-only upserts keyed by `(kind, key)`; @@ -315,6 +315,16 @@ when a key is written again, readers keep its last row. | `request` | `{session_id}:{request_id}` | `harness`, `request_id`, `agent_name`, `query_source`, `model`, `event_sequence`, the four token fields, `cost_usd`, `duration_ms` | | `tool` | `{session_id}:{tool_use_id}`, or `{source_id}#{ordinal}` when there is no tool-use ID | `harness`, `agent_name`, `tool_name`, `tool_use_id`, `error_type`, `success`, `duration_ms`, `tool_input_size_bytes`, `tool_result_size_bytes`, `event_sequence` | | `subagent` | `{source_id}#{ordinal}` | `harness`, `agent_type`, `model`, `final_model`, `model_swapped`, `event_sequence` | +| `turn` | `{source_id}:turn:{ordinal}` | Exact owning `session_key`, source identity and ordinal, request/message IDs, timestamp, harness/provider/resolved model, four structured token measures, `context_window_tokens`, `context_fraction` | + +Completed sessions admit turns through their versioned `token_usage.json` descriptor. +The descriptor declares the ledger schema, basename and row count; ledger rows do not +repeat that schema marker. Only a complete, validated `turn_usage.jsonl` is admitted. +Session rows retain `turn_usage_state` and `turn_usage_reason`, including when no turn +rows exist. A null ledger reference with count zero means no published ledger, not +measured-zero counters. Missing, malformed or unsupported descriptors and ledgers +retain explicit coverage reasons. Turns keep their emitting attempt's exact key; +untimed turns and reused session IDs never require temporal attribution. Native child `subagent` rows use `{parent_session_key}:native:{backend}:{native_parent_session_id}:{child_id}`. @@ -326,6 +336,17 @@ invocation observations. A managed attempt or launch alias alone cannot establis native actor. Resumed parents sharing a native session ID require unique invocation ownership; ambiguous children remain unattributed. +`parent_context_spans` retains counts of exact linked parent-visible invocation prompts +and returned text, with source-record/turn identity, timestamps, the resolved parent +invocation model and tokenizer provenance. These counts exclude hidden instructions, +message framing and unrelated parent work; child billing and final transcript text do +not establish what the parent received. Verified OpenAI parent models recognized by +`tiktoken` use that model's encoder for both sides. Missing links, incomplete text, +unresolved models, unsupported providers and encoder initialization failures retain +explicit unavailable reasons. Explicit empty text is a measured zero. Raw prompt and +result text are not copied into the derived index. After resolving tokenizer asset +initialization failures, use `sessions index --rebuild` to refresh unavailable counts. + Native child tokens and tools come from that child's verified dedicated transcript, with native message/request snapshots deduplicated. Tokens require the child's own effective provider; missing provider/model identities never inherit parent values. @@ -357,7 +378,7 @@ memory; it is not persisted. Events join to session attempts by `session_id` and `time_ms`. With a timestamp, an event is attributed to the attempt running at that time: if it predates all -attempts, the earliest attempt is used; otherwise the latest attempt that began +attempts, the event remains unattributed; otherwise the latest attempt that began by that time is used. Equal starts are resolved by the lexicographically greater session key, which gives a deterministic attempt order. An event without a time is attributed only when its session ID has exactly one attempt; otherwise @@ -366,14 +387,15 @@ is attributed only when its session ID has exactly one attempt; otherwise The writer holds an exclusive lease for the report index. Each commit appends and fsyncs `rows.jsonl` before atomically replacing the versioned `state.json` with the committed walk watermark, byte offset, `row_schema_version`, and -`child_evidence_fingerprint`. On the next open, bytes past that offset are truncated. +`report_evidence_fingerprint`. On the next open, bytes past that offset are truncated. The row-schema marker is an application-controlled compatibility boundary, distinct from the state envelope version. Missing/incompatible row-schema evidence resets the generation before append admission. Recoverable byte/watermark corruption within a known current schema retains the existing complete-line recovery behavior. -The child-evidence fingerprint covers canonical native child facts and dedicated -transcript dependencies, including changes, additions and removals. Drift selects a +The report-evidence fingerprint covers canonical native child facts, child and parent +transcript dependencies, tokenizer version, and retained turn descriptors and ledgers, +including changes, additions and removals. Drift selects a full replay under the existing exclusive index lease before unchanged live or consumed archive parents can be skipped. Markers commit with the successful row/watermark state; this reset does not preserve rows whose original sources are no longer retained. @@ -389,28 +411,31 @@ Rebuilding reads only the sources still retained on disk. Rows from OTLP data that has rotated out survive through incremental updates, but a rebuild cannot re-derive those older facts. A rebuild equals an incremental index when source changes are limited to those tracked by the walk: appended OTLP, OTLP rotation, -and changed, added, or evicted session rows, plus native child evidence changes. -Parent transcript changes still require `--rebuild` when their session row is unchanged. +and changed, added, or evicted session rows, plus report-evidence dependency changes. +Changed parent transcripts, turn descriptors and ledgers refresh the index even when +their session projection is unchanged. Codex OTLP token events are not projected: they do not carry a stable request identity for deduplication. Codex token measures enter the report index through -the session rows. +the session rows and admitted turn ledgers. ## Observability deck `autoskillit sessions deck ` incrementally refreshes the report index and writes one self-contained HTML deck. It can be opened from `file://` without a server or network -connection. It receives all four index fact collections as mapping iterables; the +connection. It receives sessions, requests, tools, subagents and turns as mapping iterables; the report layer imports core types and helpers rather than the execution index type. The cohort view retains its session table. Spend, efficiency, skill and role views select metric blocks prepared by the shared aggregation library, without browser -aggregation of measures. +aggregation of measures. Context, errors, trend, gaps and parity also select prepared +populations with their own availability coverage, history and cohort filters. ADR-0017 records why charts use first-party SVG. ### Payload contract The payload keys are `generated_at_ms`, `index_schema_version`, `landing`, and `history`; -column tables `tables.sessions`, `tables.skills`, and `tables.roles`; `facets[{id,label,column,kind}]`; +column tables for sessions, skills, roles, turns, errors, trends, gaps and parity; +`facets[{id,label,column,kind}]`; `views[{id,question,decision,group,status,issue}]`; `chips{view:{facet:[{key,label,state,count,reason,issue,match|days}]}}`; and `availability[{state,label,description}]`. `match` is used for value facets and `days` @@ -433,6 +458,27 @@ Role level selections mean the spawning skill level, while the actor remains L0. Identity details, definitions and known cross-references remain available in empty usage cohorts. A stale explicit window or facet selection yields no measurements. +Context shows each turn's cache-read occupancy proxy as a percentage of its resolved +model window. Missing windows or counters leave gaps in the series. Cache read, cache +write and output are visible by default; inclusive input is an optional separate +series. Claude's ledger input already includes cache read and cache creation, so adding +it to that stack would double count. Parent prompting and returned text are a separate +comparison of recorded spans tokenized at the parent invocation model, with matched +sample counts and explicit missing-source coverage. + +Errors distinguish session failures from failed indexed tool events and identify +symptom, skill and recipe step. Session failure rates use known outcomes; missing tool +events do not establish zero errors or a complete tool history. Trends use UTC daily +buckets and compare the earlier and later halves of the selected interval, retaining +model, skill, step and source-pair identity. Sample counts, missing buckets and untimed +exclusions accompany the descriptive changes; absent observations establish no trend. + +Gaps describe unanswered questions and their missing prerequisites. Parity compares +observed harness/provider fields against the deck's target fields. Both derive coverage +from selected eligible records, including absent ledgers, unknown windows, unattributed +tools and missing or unsupported parent spans. Mixed coverage and no observations are +derived labels, not additional primitive token states; measured zero remains distinct. + Canonical role routes provide offline definition links. The artifact preserves Claude tools/model separately from Codex reader tools/model and renders original definition body text literally. Manual ratio/tool-mix review controls require observed eligible @@ -453,7 +499,9 @@ the retained history cite [#4621](https://github.com/TalonT-Org/AutoSkillit/issu Routes use `#/[/][?k=v1,v2]`. Cohort selections survive navigation between views and entities; the `sort` choice applies only to its view. Every state change is a -hash navigation, so browser back and forward restore prior selections. Planned views +hash navigation, so browser back and forward restore prior selections. Context, gaps +and parity preserve an optional exact owning-session key in the `session` parameter. +Planned views show a notice while keeping the cohort bar available. ### Offline rendering and client code @@ -491,8 +539,9 @@ assistive technology. 4. Confirm L0 is struck with its reason. On a legacy index, L1–L3 should show ✕ with #4622; windows longer than retained history should show ✕ with #4621. 5. Follow a harness link, then use back and forward; confirm the cohort remains selected. -6. Open a planned route such as `#/spend`; confirm its notice appears and the cohort bar - remains visible. +6. Visit context, errors, trend, gaps and parity; change windows and cohort filters. + Confirm unavailable counters remain missing, observed zeros are labeled, input is + initially hidden, and session links preserve the selection across context/gaps/parity. 7. Emulate `prefers-color-scheme: dark`, then `light`; confirm both palettes render. 8. In print preview, confirm the light palette, vector bars, printed cohort sentence, and hidden navigation rail. diff --git a/pyproject.toml b/pyproject.toml index 06678b409b..f65d384336 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ dependencies = [ "cyclopts>=4.0", "regex>=2026.1", "structlog>=25.0,<26.0", + "tiktoken>=0.14.0", "zstandard>=0.23", "uvloop>=0.21.0; sys_platform != 'win32'", ] diff --git a/src/autoskillit/assets/deck/core.js b/src/autoskillit/assets/deck/core.js index cbcdf8b49d..b85bfd42ec 100644 --- a/src/autoskillit/assets/deck/core.js +++ b/src/autoskillit/assets/deck/core.js @@ -2,8 +2,21 @@ globalThis.DeckCore = (() => { "use strict"; const CHIP_STATES = Object.freeze({LIVE: "live", STRUCK: "struck", ABSENT: "absent"}); const DAY_MS = 86400000; + const SESSION_VIEWS = ["context", "gaps", "parity"]; const enc = encodeURIComponent; const formatCount = n => new Intl.NumberFormat("en-US").format(n); + const isPrimitiveState = state => ["measured", "measured_zero", "unknown", + "unavailable", "not_applicable"].includes(state); + const inspectParams = row => Object.fromEntries([ + ["harness", row?.harness], ["provider", row?.provider], ["skill", row?.skill], + ["recipe", row?.recipe], ["step", row?.step], ["model", row?.model], + ["session", row?.session_key] + ].filter(([, value]) => value != null).map(([key, value]) => [key, [value]])); + const countLabel = value => value == null ? "count unavailable" : formatCount(value); + const measureValue = measure => measure?.state === "measured" ? measure.value : + measure?.state === "measured_zero" ? 0 : null; + const formatStateCounts = counts => Object.entries(counts ?? {}).map(([state, count]) => + state + " " + formatCount(count)).join(" · "); const formatDate = ms => new Date(ms).toISOString().slice(0, 10); function formatRatio(ratio, percent = false) { @@ -84,6 +97,15 @@ globalThis.DeckCore = (() => { return ctx.availabilityCell(measure ?? {state: "unavailable"}); } + function coverageStateCell(ctx, measure, className) { + const state = measure?.state; + if (isPrimitiveState(state)) return ctx.availabilityCell(measure); + return ctx.el("span", {class: className ?? + "coverage-state coverage-state--" + (state ?? "unknown")}, + state === "no_observations" ? "no observations" : + state === "mixed" ? "mixed coverage" : state ?? "unknown"); + } + function ratioCell(ctx, ratio, percent = false) { const value = formatRatio(ratio, percent); return ctx.el("div", {class: "view-measure"}, [ @@ -175,9 +197,17 @@ globalThis.DeckCore = (() => { } function hrefFor(route, target, cohortKeys) { - const params = Object.fromEntries(cohortKeys.filter(k => + const identityViews = ["context", "errors", "trend", "gaps", "parity", "skill", "cohort"]; + const preserve = [...cohortKeys]; + if (identityViews.includes(target.view)) preserve.push("skill", "recipe", "step", "model"); + if (SESSION_VIEWS.includes(target.view)) preserve.push("session"); + const params = Object.fromEntries(preserve.filter(k => Object.hasOwn(route.params, k)).map(k => [k, [...route.params[k]]])); for (const [key, value] of Object.entries(target.params || {})) { + if (key === "session" && !SESSION_VIEWS.includes(target.view)) { + delete params[key]; + continue; + } if (value === null) delete params[key]; else Object.defineProperty(params, key, {value, enumerable: true, writable: true, configurable: true}); @@ -227,17 +257,20 @@ globalThis.DeckCore = (() => { const providers = new Set((chips.provider ?? []) .filter(chip => provider.keys.includes(chip.key)).map(chip => chip.match)); - function selectedRows(blocks) { - if (!window.chip) return []; + function selectedBlock(blocks) { + if (!window.chip) return null; const inWindow = blocks.filter(block => block.window === window.chip.key); const domain = [...new Set(inWindow.flatMap(block => block.levels))]; const selectedLevels = new Set(level.keys .map(key => (chips.level ?? []).find(chip => chip.key === key)?.match) ); const requested = domain.filter(value => selectedLevels.has(value)); - if (!requested.length) return []; - const block = inWindow.find(item => sameValues(item.levels, requested)); - return block?.rows ?? []; + if (!requested.length) return null; + return inWindow.find(item => sameValues(item.levels, requested)) ?? null; + } + + function selectedRows(blocks) { + return selectedBlock(blocks)?.rows ?? []; } let skillRows = viewId === "role" ? [] : selectedRows(prepared.skills ?? []).filter(row => @@ -252,6 +285,31 @@ globalThis.DeckCore = (() => { if (viewId === "role" && route.entity != null) { roleRows = roleRows.filter(row => row.role === route.entity); } + const viewData = prepared[viewId]; + const hasPreparedPopulation = viewData !== null && typeof viewData === "object"; + const block = Array.isArray(viewData?.blocks) ? selectedBlock(viewData.blocks) : null; + let rows = hasPreparedPopulation ? [] : viewId === "role" ? roleRows : skillRows; + let metrics = rows; + if (Array.isArray(viewData?.blocks)) { + const sessionKey = SESSION_VIEWS.includes(viewId) ? + params.session?.[0] ?? null : null; + const sessionBlock = sessionKey == null ? null : + (block?.sessions && Object.hasOwn(block.sessions, sessionKey) ? + block.sessions[sessionKey] : null); + const sourceRows = sessionKey == null ? block?.rows : sessionBlock?.rows; + const sourceMetrics = sessionKey == null ? block?.metrics : sessionBlock?.metrics; + const identityKeys = ["skill", "recipe", "step", "model"]; + const matchesSelection = row => + (row.harness == null || harnesses.has(row.harness)) && + (row.provider == null || providers.has(row.provider)) && + identityKeys.every(key => !Array.isArray(params[key]) || params[key].length === 0 || + row[key] == null || params[key].includes(row[key])); + rows = (sourceRows ?? []).filter(matchesSelection); + const filterMetricRows = value => Array.isArray(value) ? value.filter(matchesSelection) : + value && typeof value === "object" ? Object.fromEntries(Object.entries(value).map( + ([key, rows]) => [key, Array.isArray(rows) ? rows.filter(matchesSelection) : rows])) : value; + metrics = filterMetricRows(sourceMetrics) ?? []; + } const relationships = (prepared.relationships ?? []).filter(edge => { if (viewId === "skill" && route.entity != null) return edge.skill === route.entity; if (viewId === "role" && route.entity != null) return edge.role === route.entity; @@ -259,12 +317,15 @@ globalThis.DeckCore = (() => { }); return { + rows, + metrics, skillRows, roleRows, relationships, definitions: prepared.definitions ?? {}, viewHistory: prepared.view_histories?.[viewId] ?? null, - selection: {window, level, harness, provider} + selection: {window, level, harness, provider, + session: SESSION_VIEWS.includes(viewId) ? params.session?.[0] ?? null : null} }; } @@ -390,7 +451,7 @@ globalThis.DeckCore = (() => { return Object.freeze({decodeTable, encodeRoute, decodeRoute, hrefFor, effectiveSelection, toggleSelection, windowSelection, selectPrepared, filterRows, populationSentence, summarizePairs, sortRows, parseSort, chipPresentation, availabilityPresentation, - barLayout, formatCount, formatRatio, ratioSample, reviewEligibility, reviewSignal, - measureCell, ratioCell, definitionCard, toolMix, roleHarnessRows, formatDate, + isPrimitiveState, inspectParams, countLabel, measureValue, formatStateCounts, barLayout, formatCount, formatRatio, ratioSample, reviewEligibility, reviewSignal, + measureCell, coverageStateCell, ratioCell, definitionCard, toolMix, roleHarnessRows, formatDate, CHIP_STATES, DAY_MS}); })(); diff --git a/src/autoskillit/assets/deck/deck.css b/src/autoskillit/assets/deck/deck.css index 33b6ebb74e..4267c901e6 100644 --- a/src/autoskillit/assets/deck/deck.css +++ b/src/autoskillit/assets/deck/deck.css @@ -81,6 +81,49 @@ svg { display: block; max-width: 100%; height: auto; } .bar--absent { fill: url(#deck-hatch); } .hatch-line { stroke: var(--hatch); stroke-width: 2; } .bar-label, .bar-value { fill: var(--ink-2); font: 11px var(--mono); } +.deck-chart { width: 100%; margin: 10px 0 16px; overflow: visible; } +.deck-chart__title { fill: var(--ink); font: 600 12px var(--sans); } +.deck-chart__label, .deck-chart__legend { fill: var(--ink-2); font: 10px var(--mono); } +.deck-chart__axis-label { fill: var(--ink-3); font: 9px var(--mono); } +.deck-chart__axis { stroke: var(--rule-2); stroke-width: 1; } +.deck-chart__track { fill: var(--surface-2); stroke: var(--rule); stroke-width: .5; } +.deck-chart__bar { fill: var(--s1); } +.deck-chart__bar--cache_read_tokens, .deck-chart__bar--prompt { fill: var(--s1); } +.deck-chart__bar--cache_write_tokens, .deck-chart__bar--returned { fill: var(--s2); } +.deck-chart__bar--output_tokens { fill: var(--s3); } +.deck-chart__bar--input_tokens { fill: var(--s4); } +.deck-chart__line { fill: none; stroke: var(--s1); stroke-width: 2; } +.deck-chart__line--s1 { stroke: var(--s1); } +.deck-chart__line--s2 { stroke: var(--s2); stroke-dasharray: 6 3; } +.deck-chart__line--s3 { stroke: var(--s3); stroke-dasharray: 2 2; } +.deck-chart__line--s4 { stroke: var(--s4); stroke-dasharray: 8 2 2 2; } +.deck-chart__point { fill: var(--surface); stroke: var(--s1); stroke-width: 2; } +.deck-chart__point--s1 { stroke: var(--s1); } +.deck-chart__point--s2 { stroke: var(--s2); } +.deck-chart__point--s3 { stroke: var(--s3); } +.deck-chart__point--s4 { stroke: var(--s4); } +.deck-chart__zero { fill: var(--surface); stroke: var(--ink-2); stroke-width: 2; } +.deck-chart__missing { stroke: var(--ink-2); stroke-dasharray: 1 2; } +.deck-chart__empty { fill: var(--ink-2); font: 11px var(--sans); } +.deck-series-legend { display: flex; flex-wrap: wrap; gap: 8px 14px; margin: 4px 0 10px; } +.deck-series-legend__item { display: inline-flex; align-items: center; gap: 5px; + color: var(--ink-2); font: 10px var(--mono); } +.deck-series-swatch { display: inline-block; width: 12px; height: 10px; border: 1px solid var(--ink-2); } +.deck-series-swatch--cache_read_tokens, .deck-series-swatch--prompt { background: var(--s1); } +.deck-series-swatch--cache_write_tokens, .deck-series-swatch--returned { background: var(--s2); } +.deck-series-swatch--output_tokens { background: var(--s3); } +.deck-series-swatch--input_tokens { background: var(--s4); } +.view-legend, .view-note { border-left: 3px solid var(--rule-2); padding: 5px 9px; } +.coverage-state { display: inline-block; padding: 2px 5px; border: 1px solid var(--rule-2); + border-radius: 4px; white-space: nowrap; } +.coverage-state--mixed { border-style: double; font-weight: 600; } +.coverage-state--no_observations { border-style: dashed; font-style: italic; } +.parity-cell { display: grid; gap: 3px; min-width: 104px; } +.parity-cell__coverage { color: var(--ink-3); font: 9px/1.35 var(--mono); white-space: normal; } +.parity-cell-link { color: inherit; text-decoration-thickness: 1px; text-underline-offset: 2px; } +.context-input-toggle { border: 1px solid var(--rule-2); border-radius: 5px; padding: 5px 9px; + color: var(--ink); background: var(--surface-2); cursor: pointer; } +.context-input-toggle[aria-pressed="true"]::before { content: "✓ "; } .defs { position: absolute; } .view-stack { display: grid; gap: 18px; } .view-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 18px; } @@ -116,10 +159,27 @@ svg { display: block; max-width: 100%; height: auto; } @media print { @page { margin: 12mm; } * { print-color-adjust: exact; -webkit-print-color-adjust: exact; } + :root { + color-scheme: light; + --paper: #ffffff; --surface: #ffffff; --surface-2: #f1f3f5; --surface-3: #e4e7eb; + --ink: #111318; --ink-2: #303640; --ink-3: #505967; --rule: #aeb5c0; --rule-2: #7e8794; + --s1: #155ca8; --s2: #a83c16; --s3: #087247; --s4: #855b00; + --hatch: rgba(40,48,60,.55); + } + body { background: #fff; color: #111318; } .rail { display: none; } .shell { display: block; } .cohort { position: static; } + .card { overflow: visible; } .card, table, svg, .view-definition, .view-review { break-inside: avoid; } .view-review button, .view-review-signal button { display: none; } + .context-input-toggle { display: none; } + .deck-chart__track { fill: #f1f3f5; stroke: #7e8794; } + .deck-chart__zero { fill: #fff; stroke: #111318; } + .deck-chart__missing { stroke: #111318; } + .view-parity table { width: 100%; table-layout: fixed; font-size: 8px; } + .view-parity th, .view-parity td { padding: 3px; overflow-wrap: anywhere; } + .view-parity .parity-cell { min-width: 0; } + .view-parity .parity-cell__coverage { font-size: 7px; } #deck-view { padding: 18px 0; } } diff --git a/src/autoskillit/assets/deck/shell.js b/src/autoskillit/assets/deck/shell.js index e0fd0014ab..7533cb0faa 100644 --- a/src/autoskillit/assets/deck/shell.js +++ b/src/autoskillit/assets/deck/shell.js @@ -2,6 +2,16 @@ globalThis.DeckShell = (() => { "use strict"; const renderers = new Map(); + const SVG_NS = "http://www.w3.org/2000/svg"; + + function appendChild(parent, child) { + if (child === null || child === undefined) return; + if (typeof child === "string" || typeof child === "number" || typeof child === "boolean") { + parent.appendChild(document.createTextNode(String(child))); + return; + } + parent.appendChild(child); + } function el(tag, attrs = {}, children = null) { const node = document.createElement(tag); @@ -10,9 +20,7 @@ globalThis.DeckShell = (() => { node.setAttribute(name, value === true ? "" : String(value)); } if (Array.isArray(children)) { - for (const child of children) { - if (child !== null && child !== undefined) node.appendChild(child); - } + for (const child of children) appendChild(node, child); } else if (children !== null && children !== undefined) { if (typeof children === "object") node.appendChild(children); else node.textContent = String(children); @@ -29,6 +37,21 @@ globalThis.DeckShell = (() => { return Array.from(renderers.keys()); } + function svgElement(tag, attrs = {}, children = null) { + const node = document.createElementNS(SVG_NS, tag); + for (const [name, value] of Object.entries(attrs)) { + if (value === null || value === undefined || value === false) continue; + node.setAttribute(name, value === true ? "" : String(value)); + } + if (Array.isArray(children)) { + for (const child of children) appendChild(node, child); + } else if (children !== null && children !== undefined) { + if (typeof children === "object") node.appendChild(children); + else node.textContent = String(children); + } + return node; + } + function boot(payload) { const model = {...payload, tables: Object.fromEntries( Object.entries(payload.tables).map(([key, table]) => [key, DeckCore.decodeTable(table)]))}; @@ -75,26 +98,25 @@ globalThis.DeckShell = (() => { } function barChart(items, {label, width = 640}) { - const NS = "http://www.w3.org/2000/svg"; const labelWidth = 200, valueWidth = 80, rowHeight = 18, gap = 6; const layout = DeckCore.barLayout(items, {width, labelWidth, valueWidth, rowHeight, gap}); const height = layout.length ? layout[layout.length - 1].y + rowHeight : rowHeight; - const svg = document.createElementNS(NS, "svg"); + const svg = document.createElementNS(SVG_NS, "svg"); svg.setAttribute("role", "list"); svg.setAttribute("aria-label", label); svg.setAttribute("viewBox", "0 0 " + width + " " + height); layout.forEach(bar => { - const group = document.createElementNS(NS, "g"); + const group = document.createElementNS(SVG_NS, "g"); group.setAttribute("role", "listitem"); - const labelText = document.createElementNS(NS, "text"); + const labelText = document.createElementNS(SVG_NS, "text"); labelText.setAttribute("class", "bar-label"); labelText.setAttribute("x", "0"); labelText.setAttribute("y", String(bar.y + 13)); labelText.textContent = bar.label; if (bar.href) { - const link = document.createElementNS(NS, "a"); + const link = document.createElementNS(SVG_NS, "a"); link.setAttribute("href", bar.href); link.appendChild(labelText); group.appendChild(link); @@ -102,7 +124,7 @@ globalThis.DeckShell = (() => { group.appendChild(labelText); } - const rect = document.createElementNS(NS, "rect"); + const rect = document.createElementNS(SVG_NS, "rect"); rect.setAttribute("class", bar.w === null ? "bar bar--absent" : "bar"); rect.setAttribute("aria-hidden", "true"); rect.setAttribute("x", String(bar.x)); @@ -111,7 +133,7 @@ globalThis.DeckShell = (() => { rect.setAttribute("height", String(bar.h)); group.appendChild(rect); - const valueText = document.createElementNS(NS, "text"); + const valueText = document.createElementNS(SVG_NS, "text"); valueText.setAttribute("class", "bar-value"); valueText.setAttribute("x", String(width - valueWidth + 4)); valueText.setAttribute("y", String(bar.y + 13)); @@ -211,7 +233,8 @@ globalThis.DeckShell = (() => { } } - const resetParams = Object.fromEntries(cohortKeys.map(key => [key, null])); + const resetParams = Object.fromEntries([...cohortKeys, "skill", "recipe", "step", "model"] + .map(key => [key, null])); cohort.appendChild(el("a", { class: "cohort-reset", href: DeckCore.hrefFor(route, { @@ -239,15 +262,24 @@ globalThis.DeckShell = (() => { const selected = renderer && view.id !== model.landing ? DeckCore.selectPrepared(model.prepared, view.id, chips, route) : null; - const primary = view?.id === "role" ? selected?.roleRows : selected?.skillRows; + const primary = selected?.rows; const result = selected ? {rows: primary, untimed: 0} : DeckCore.filterRows(model, model.tables.sessions, chips, route); - const populationModel = selected ? {...model, history: selected.viewHistory, + const populationModel = selected ? {...model, history: selected.viewHistory ?? model.history, facets: model.facets.map(facet => view.id === "role" && facet.id === "level" ? {...facet, label: "Spawning skill level"} : facet)} : model; const sentence = DeckCore.populationSentence(populationModel, chips, route, result); if (selected) { - const noun = view.id === "role" ? "role/provider row" : "skill cell"; + const labels = {skill: "skill", recipe: "recipe", step: "step", model: "model"}; + const identityFilters = Object.entries(labels).flatMap(([key, label]) => + (route.params[key] ?? []).map(value => label + " " + value)); + if (selected.selection.session != null) { + identityFilters.push("session " + selected.selection.session); + } + if (identityFilters.length) sentence.detail += " · " + identityFilters.join(" · "); + const noun = ({context: "context record", errors: "symptom group", trend: "daily observation", + gaps: "gap", parity: "coverage cell"})[view.id] ?? + (view.id === "role" ? "role/provider row" : "skill cell"); sentence.headline = DeckCore.formatCount(result.rows.length) + " " + noun + (result.rows.length === 1 ? "" : "s"); } @@ -258,10 +290,10 @@ globalThis.DeckShell = (() => { if (renderer) { const ctx = {model, route, rows: result.rows, chips, href: target => DeckCore.hrefFor(route, target, cohortKeys), - el, entityLink, sortableTable, availabilityCell, barChart}; + el, entityLink, sortableTable, availabilityCell, barChart, svgElement}; if (view.id !== model.landing) { ctx.prepared = model.prepared; - ctx.metrics = view.id === "role" ? selected.roleRows : selected.skillRows; + ctx.metrics = selected.metrics; ctx.relationships = selected.relationships; ctx.definitions = selected.definitions; ctx.viewHistory = selected.viewHistory; diff --git a/src/autoskillit/assets/deck/views/context.js b/src/autoskillit/assets/deck/views/context.js new file mode 100644 index 0000000000..7a9ff7bbfb --- /dev/null +++ b/src/autoskillit/assets/deck/views/context.js @@ -0,0 +1,312 @@ +DeckShell.registerView("context", ctx => { + const turnRows = ctx.rows.filter(row => row.kind === "turn"); + const coverageRows = ctx.rows.filter(row => row.kind === "coverage"); + const previousModels = new Map(); + const modelChanges = new Map(); + turnRows.forEach(row => { + const prior = previousModels.get(row.session_key); + modelChanges.set(row.key, prior === undefined ? "first recorded model" : + prior === row.model ? "same as previous turn" : "changed from " + (prior ?? "unknown")); + previousModels.set(row.session_key, row.model); + }); + const stateName = measure => measure?.state ?? "unknown"; + const countCell = measure => ctx.availabilityCell(measure ?? {state: "unknown"}); + const stateCell = measure => DeckCore.coverageStateCell(ctx, measure); + const countText = measure => { + const value = DeckCore.measureValue(measure); + return value == null ? stateName(measure) : DeckCore.formatCount(value); + }; + const turnLabel = row => "turn " + (row.ordinal ?? "?") + " · " + + (row.model ?? "model unavailable"); + + function tokenChart(rows, fields, label, title) { + const width = 760, left = 230, right = 16, rowHeight = 22, gap = 5; + const plotWidth = width - left - right; + const totals = rows.map(row => fields.reduce((sum, field) => { + const value = DeckCore.measureValue(row[field.key]); + return sum + (value == null ? 0 : value); + }, 0)); + const maximum = Math.max(1, totals.reduce((current, total) => Math.max(current, total), 0)); + const height = Math.max(44, rows.length * (rowHeight + gap) + 28); + const children = [ctx.svgElement("text", {x: 0, y: 14, class: "deck-chart__title"}, title)]; + rows.forEach((row, index) => { + const y = 24 + index * (rowHeight + gap); + children.push(ctx.svgElement("text", {x: 0, y: y + 15, class: "deck-chart__label"}, + turnLabel(row))); + children.push(ctx.svgElement("rect", {x: left, y, width: plotWidth, height: 15, + class: "deck-chart__track"})); + let x = left; + fields.forEach(field => { + const measure = row[field.key]; + const value = DeckCore.measureValue(measure); + if (value === 0) { + children.push(ctx.svgElement("circle", {cx: x, cy: y + 7.5, r: 3, + class: "deck-chart__zero", "aria-label": field.label + " measured zero"})); + } else if (value > 0) { + const barWidth = value / maximum * plotWidth; + children.push(ctx.svgElement("rect", {x, y, width: barWidth, height: 15, + class: "deck-chart__bar deck-chart__bar--" + field.key}, [ + ctx.svgElement("title", {}, field.label + ": " + DeckCore.formatCount(value) + " tokens") + ])); + x += barWidth; + } + }); + }); + return ctx.svgElement("svg", {class: "deck-chart", role: "img", "aria-label": label, + viewBox: "0 0 " + width + " " + height}, children); + } + + function occupancyChart(rows) { + const width = 760, height = 230, left = 52, right = 18, top = 24, bottom = 40; + const points = rows.map((row, index) => ({row, index, + value: DeckCore.measureValue(row.context_fraction_percent)})); + const observed = points.filter(point => point.value !== null); + const maximum = Math.max(1, observed.reduce((current, point) => + Math.max(current, point.value), 0)); + const plotWidth = width - left - right, plotHeight = height - top - bottom; + const xFor = index => left + (rows.length < 2 ? plotWidth / 2 : + index / (rows.length - 1) * plotWidth); + const yFor = value => top + plotHeight - value / maximum * plotHeight; + const children = [ + ctx.svgElement("text", {x: left, y: 14, class: "deck-chart__title"}, + "Cache-read proxy as percent of each turn model's context window"), + ctx.svgElement("line", {x1: left, y1: top + plotHeight, x2: width - right, + y2: top + plotHeight, class: "deck-chart__axis"}), + ctx.svgElement("text", {x: 3, y: top + 4, class: "deck-chart__axis-label"}, + Math.round(maximum) + "%"), + ctx.svgElement("text", {x: 3, y: top + plotHeight, class: "deck-chart__axis-label"}, "0%") + ]; + let segment = []; + let priorSession = null; + const flush = () => { + if (segment.length > 1) { + children.push(ctx.svgElement("polyline", {points: segment.map(point => + point.x + "," + point.y).join(" "), class: "deck-chart__line"})); + } + segment = []; + }; + points.forEach(point => { + if (point.index > 0 && point.row.session_key !== priorSession) flush(); + priorSession = point.row.session_key; + if (point.value === null) { + flush(); + const x = xFor(point.index); + children.push(ctx.svgElement("rect", {x: x - 4, y: top + plotHeight - 8, + width: 8, height: 8, class: "deck-chart__missing", fill: "url(#deck-hatch)"}, [ + ctx.svgElement("title", {}, "Occupancy unavailable for " + turnLabel(point.row)) + ])); + return; + } + const x = xFor(point.index), y = yFor(point.value); + segment.push({x, y}); + children.push(ctx.svgElement("circle", {cx: x, cy: y, r: 3.5, + class: point.value === 0 ? "deck-chart__point deck-chart__zero" : "deck-chart__point"}, [ + ctx.svgElement("title", {}, turnLabel(point.row) + ": " + + (point.value === 0 ? "0% observed" : String(Math.round(point.value * 100) / 100) + "%")) + ])); + }); + flush(); + if (rows.length) { + children.push(ctx.svgElement("text", {x: left, y: height - 8, class: "deck-chart__axis-label"}, + turnLabel(rows[0]))); + if (rows.length > 1) children.push(ctx.svgElement("text", {x: width - right, y: height - 8, + "text-anchor": "end", class: "deck-chart__axis-label"}, turnLabel(rows[rows.length - 1]))); + } + if (!observed.length) children.push(ctx.svgElement("text", {x: left + 8, y: top + 28, + class: "deck-chart__empty"}, "No measured occupancy points in this selection.")); + return ctx.svgElement("svg", {class: "deck-chart", role: "img", + "aria-label": "Per-turn cache-read proxy occupancy; missing values break the line", + viewBox: "0 0 " + width + " " + height}, children); + } + + function pairedSpanChart(rows) { + const width = 760, left = 210, right = 135, rowHeight = 38, gap = 8; + const plotWidth = width - left - right; + const maxValue = Math.max(1, rows.reduce((current, row) => [ + row.prompt?.observed_subset ?? row.prompt, + row.returned?.observed_subset ?? row.returned + ] + .reduce((maximum, measure) => { + const value = DeckCore.measureValue(measure); + return value === null ? maximum : Math.max(maximum, value); + }, current), 0)); + const height = Math.max(46, rows.length * (rowHeight + gap) + 28); + const children = [ctx.svgElement("text", {x: 0, y: 14, class: "deck-chart__title"}, + "Observed-subset parent prompt and subagent returned text tokens")]; + rows.forEach((row, index) => { + const y = 24 + index * (rowHeight + gap); + children.push(ctx.svgElement("text", {x: 0, y: y + 13, class: "deck-chart__label"}, + [row.skill, row.recipe, row.step, row.harness, row.provider, row.model] + .filter(Boolean).join(" · "))); + for (const [key, measure, offset] of [["prompt", row.prompt, 0], + ["returned", row.returned, 18]]) { + const observedSubset = measure?.observed_subset ?? measure; + const value = DeckCore.measureValue(observedSubset); + const barY = y + offset; + if (value !== null && value > 0) { + children.push(ctx.svgElement("rect", {x: left, y: barY, width: value / maxValue * plotWidth, + height: 11, class: "deck-chart__bar deck-chart__bar--" + key}, [ + ctx.svgElement("title", {}, key + " observed subset: " + + DeckCore.formatCount(value) + " tokens") + ])); + } else if (value === 0) { + children.push(ctx.svgElement("circle", {cx: left, cy: barY + 5.5, r: 3, + class: "deck-chart__zero"})); + } + children.push(ctx.svgElement("text", {x: width - right + 8, y: barY + 10, + class: "deck-chart__axis-label"}, key + " observed subset · " + countText(observedSubset))); + } + }); + return ctx.svgElement("svg", {class: "deck-chart", role: "img", + "aria-label": "Paired parent prompt and subagent returned text token counts", + viewBox: "0 0 " + width + " " + height}, children); + } + + const tableRows = [...ctx.rows]; + const columns = [ + {key: "session_key", label: "Owning session", cell: row => row.session_key == null ? "unattributed" : + ctx.el("a", {href: ctx.href({view: "context", params: {session: [row.session_key]}})}, + row.session_key)}, + {key: "ordinal", label: "Turn", cell: row => row.kind === "coverage" ? + "Ledger coverage" : row.ordinal ?? "—"}, + {key: "time_ms", label: "Recorded (UTC)", cell: row => row.time_ms == null ? "untimed" : + new Date(row.time_ms).toISOString()}, + {key: "model", label: "Resolved model", cell: row => row.model ?? "unknown"}, + {key: "model_change", label: "Model history", cell: row => row.kind === "coverage" ? "—" : + modelChanges.get(row.key) ?? "first recorded model"}, + {key: "cache_read_tokens", label: "Cache read", cell: row => countCell(row.cache_read_tokens)}, + {key: "cache_write_tokens", label: "Cache write", cell: row => countCell(row.cache_write_tokens)}, + {key: "output_tokens", label: "Output", cell: row => countCell(row.output_tokens)}, + {key: "context_window_tokens", label: "Model window", cell: row => typeof + row.context_window_tokens === "number" ? DeckCore.formatCount(row.context_window_tokens) : + row.context_window_tokens == null ? "unknown" : stateCell(row.context_window_tokens)}, + {key: "context_fraction", label: "Stored fraction (validation)", cell: row => + typeof row.context_fraction === "number" ? + String(Math.round(row.context_fraction * 10000) / 100) + "%" : "not recorded"}, + {key: "fraction_disagreement", label: "Stored fraction check", cell: row => + row.fraction_disagreement === true ? "gap: differs from recomputed value" : + row.fraction_disagreement === false ? "agrees" : "not compared"}, + {key: "context_fraction_percent", label: "Cache-read window", cell: row => + row.context_fraction_percent ? stateCell(row.context_fraction_percent) : "not applicable"} + ]; + const inputFields = [{key: "input_tokens", label: "Input"}]; + const tokenFields = [ + {key: "cache_read_tokens", label: "Cache read"}, + {key: "cache_write_tokens", label: "Cache write"}, + {key: "output_tokens", label: "Output"} + ]; + const tokenLegend = tokenFields.map(field => ctx.el("span", { + class: "deck-series-legend__item" + }, [ctx.el("span", {class: "deck-series-swatch deck-series-swatch--" + field.key, + "aria-hidden": "true"}), ctx.el("span", {}, field.label)])); + const inputChart = tokenChart(turnRows, inputFields, "Inclusive input tokens by turn", + "Input tokens (inclusive; separate scale)"); + inputChart.setAttribute("style", "display:none"); + const inputToggle = ctx.el("button", {type: "button", class: "context-input-toggle", + "aria-pressed": "false"}, "Show inclusive input series"); + let inputShown = false; + inputToggle.addEventListener("click", () => { + inputShown = !inputShown; + inputChart.setAttribute("style", inputShown ? "display:block" : "display:none"); + inputToggle.setAttribute("aria-pressed", String(inputShown)); + inputToggle.textContent = inputShown ? "Hide inclusive input series" : + "Show inclusive input series"; + }); + const parentRows = ctx.metrics?.parent_context ?? []; + function coverageText(measure) { + const states = DeckCore.formatStateCounts(measure?.state_counts); + const reasons = Object.entries(measure?.reason_counts ?? {}).map(([reason, count]) => + reason + " " + DeckCore.formatCount(count)); + const population = measure?.observation_count == null ? null : + DeckCore.formatCount(measure.observation_count) + " observed / " + + (measure.eligible_count == null ? "eligible count unavailable" : + DeckCore.formatCount(measure.eligible_count) + " eligible"); + return [population, states, ...reasons] + .filter(Boolean).join(" · ") || "coverage unavailable"; + } + const provenanceText = provenance => (provenance ?? []).map(entry => typeof entry === "string" ? + entry : [entry.source_id, entry.encoding, entry.tokenizer_version, + [entry.child_role, entry.child_harness, entry.child_provider, entry.child_model] + .filter(Boolean).join(" / ")].filter(Boolean).join(" · ")) + .filter(Boolean).join("; ") || "no source span"; + const comparisonCell = row => { + const value = DeckCore.measureValue(row.comparison); + const samples = row.comparison?.sample_size ?? row.matched_invocations; + return value === null ? stateCell(row.comparison) : ctx.el("span", {class: "view-measure"}, [ + ctx.el("span", {}, (value === 0 ? "0" : String(Math.round(value * 100) / 100)) + "×"), + ctx.el("small", {class: "view-sample"}, samples == null ? "sample unavailable" : + DeckCore.formatCount(samples) + " matched invocations") + ]); + }; + const parentColumns = [ + {key: "skill", label: "Parent skill", cell: row => row.skill ?? "unknown"}, + {key: "recipe", label: "Recipe", cell: row => row.recipe ?? "unknown"}, + {key: "step", label: "Step", cell: row => row.step ?? "unknown"}, + {key: "harness", label: "Parent harness"}, + {key: "provider", label: "Parent provider"}, + {key: "model", label: "Parent invocation model"}, + {key: "prompt", label: "Prompt tokens", cell: row => stateCell(row.prompt)}, + {key: "prompt_observed_subset", label: "Prompt observed subset", cell: row => + stateCell(row.prompt?.observed_subset)}, + {key: "prompt_coverage", label: "Prompt coverage", cell: row => coverageText(row.prompt)}, + {key: "returned", label: "Returned text tokens", cell: row => stateCell(row.returned)}, + {key: "returned_observed_subset", label: "Return observed subset", cell: row => + stateCell(row.returned?.observed_subset)}, + {key: "returned_coverage", label: "Return coverage", cell: row => coverageText(row.returned)}, + {key: "eligible_invocations", label: "Eligible", numeric: true}, + {key: "matched_invocations", label: "Matched", numeric: true}, + {key: "untimed_excluded", label: "Untimed excluded", numeric: true}, + {key: "provenance", label: "Source / encoder provenance", cell: row => + provenanceText(row.provenance)}, + {key: "source_basis", label: "Count basis"}, + {key: "comparison", label: "Returned / prompt ratio", cell: comparisonCell} + ]; + const links = ["gaps", "parity"].map(view => ctx.el("a", {href: ctx.href({view})}, + "Open " + view + " coverage")); + + return ctx.el("div", {class: "view-stack view-context"}, [ + ctx.el("section", {class: "card"}, [ + ctx.el("h1", {}, "Context occupancy and per-turn tokens"), + ctx.el("p", {class: "view-lede"}, "Occupancy is cache-read tokens divided by the " + + "resolved turn model's context window, shown as a percent without clamping. The token " + + "stack uses cache read, cache write, and output. Input is an inclusive count, so it is " + + "shown only as a separate optional series and never added to that stack."), + turnRows.length ? tokenChart(turnRows, tokenFields, "Cache read, cache write, and output by turn", + "Per-turn token breakdown") : ctx.el("p", {class: "view-empty"}, + "No observed turn rows match this selection."), + ctx.el("div", {class: "deck-series-legend", "aria-label": "Token series legend"}, + tokenLegend), + inputToggle, inputChart, + ctx.el("p", {class: "view-legend"}, "Measured values are amounts; measured_zero is an " + + "explicit zero marker. Unknown, unavailable, and not_applicable remain labeled in the " + + "table and are not drawn as zero-height observations."), + occupancyChart(turnRows), + ctx.el("p", {class: "view-note"}, "Hatched marks identify missing occupancy. The line " + + "breaks at every missing or unsupported turn; the adjacent table spells out each " + + "availability state."), + ctx.sortableTable({columns, rows: tableRows, defaultSort: {key: "time_ms", dir: "asc"}}) + ]), + coverageRows.length ? ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, "Turn ledger coverage"), + ctx.sortableTable({columns: [ + {key: "session_key", label: "Owning session"}, + {key: "turn_usage_state", label: "Ledger state", cell: row => row.turn_usage_state === + "observed" ? "observed" : ctx.availabilityCell({state: "unavailable"})}, + {key: "turn_usage_reason", label: "Reason", cell: row => row.turn_usage_reason ?? "—"}, + {key: "request_model", label: "Session request model", cell: row => row.request_model ?? "unknown"} + ], rows: coverageRows, defaultSort: {key: "session_key", dir: "asc"}}) + ]) : null, + ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, "Parent prompt and subagent returned text"), + ctx.el("p", {class: "view-lede"}, "These are tokenizer counts of the recorded prompt " + + "and returned text at the parent invocation model. They exclude argument metadata, " + + "hidden/system/framing tokens, and unrelated parent work. Partial source coverage is " + + "shown beside observed-subset comparisons."), + parentRows.length ? pairedSpanChart(parentRows) : ctx.el("p", {class: "view-empty"}, + "No verified parent prompt/return spans match this selection."), + parentRows.length ? ctx.sortableTable({columns: parentColumns, rows: parentRows, + defaultSort: {key: "harness", dir: "asc"}}) : null, + ctx.el("div", {class: "view-links"}, links) + ]) + ]); +}); diff --git a/src/autoskillit/assets/deck/views/errors.js b/src/autoskillit/assets/deck/views/errors.js new file mode 100644 index 0000000000..1f54cb236b --- /dev/null +++ b/src/autoskillit/assets/deck/views/errors.js @@ -0,0 +1,105 @@ +DeckShell.registerView("errors", ctx => { + const stateCell = measure => DeckCore.coverageStateCell(ctx, measure, "coverage-state"); + const countCoverage = row => DeckCore.formatStateCounts(row.failures?.state_counts) || + "outcome coverage unavailable"; + const rateText = rate => { + const value = rate?.state === "measured" ? rate.value : + rate?.state === "measured_zero" ? 0 : null; + const sample = rate?.sample_size; + return value == null ? stateCell(rate) : ctx.el("span", {class: "view-measure"}, [ + ctx.el("span", {}, String(Math.round(value * 10000) / 100) + "%"), + ctx.el("small", {class: "view-sample"}, sample == null ? "sample unavailable" : + DeckCore.formatCount(sample) + " known outcomes") + ]); + }; + const inspectLink = row => { + if (row.skill != null) return ctx.el("a", {href: ctx.href({view: "skill", + entity: row.skill})}, row.skill); + if (row.session_key != null) return ctx.el("a", {href: ctx.href({view: "context", + params: {session: [row.session_key]}})}, "session evidence"); + return ctx.el("a", {href: ctx.href({view: "cohort"})}, "unattributed cohort"); + }; + const columns = [ + {key: "symptom", label: "Symptom", cell: row => row.symptom ?? "unknown symptom"}, + {key: "population", label: "Source population", cell: row => ({session_failure: + "failed session", tool_failure: "failed tool_result event", unattributed_tool: + "unattributed tool event", session_outcome: "unknown session outcome"})[row.population] ?? + row.population}, + {key: "skill", label: "Skill", cell: row => row.skill ?? "unknown skill"}, + {key: "recipe", label: "Recipe", cell: row => row.recipe ?? "unknown recipe"}, + {key: "step", label: "Step", cell: row => row.step ?? "unknown step"}, + {key: "harness", label: "Harness", cell: row => row.harness ?? "unattributed"}, + {key: "provider", label: "Provider", cell: row => row.provider ?? "unattributed"}, + {key: "failures", label: "Observed failures", cell: row => stateCell(row.failures)}, + {key: "failure_rate", label: "Known-outcome failure share", cell: row => rateText(row.failure_rate)}, + {key: "failure_coverage", label: "Outcome coverage", cell: row => + DeckCore.formatCount(row.observed_count ?? 0) + " observed / " + + DeckCore.formatCount(row.eligible_count ?? 0) + " eligible · " + + DeckCore.formatCount(row.unknown_count ?? 0) + " unknown · " + countCoverage(row)}, + {key: "tool_result_event_count", label: "Indexed tool_result events", numeric: true, + cell: row => row.tool_result_event_count == null ? "unobserved" : + DeckCore.formatCount(row.tool_result_event_count)}, + {key: "tool_event_coverage", label: "Tool event coverage", cell: row => + rateText(row.tool_event_coverage)}, + {key: "attribution_state", label: "Attribution", cell: row => row.attribution_state ?? + (row.population === "unattributed_tool" ? "unattributed" : "owner indexed")}, + {key: "inspect", label: "Inspect", cell: inspectLink} + ]; + const tableRows = [...ctx.rows]; + const populationRows = ctx.metrics?.populations ?? []; + const outcomeCoverage = summary => DeckCore.countLabel(summary?.observed_count) + + " observed / " + DeckCore.countLabel(summary?.eligible_count) + " eligible · " + + DeckCore.countLabel(summary?.unknown_count) + " unknown · " + + DeckCore.formatStateCounts(summary?.state_counts); + const coverageColumns = [ + {key: "skill", label: "Skill", cell: row => row.skill ?? "unknown skill"}, + {key: "recipe", label: "Recipe", cell: row => row.recipe ?? "unknown recipe"}, + {key: "step", label: "Step", cell: row => row.step ?? "unknown step"}, + {key: "harness", label: "Harness", cell: row => row.harness ?? "unattributed"}, + {key: "provider", label: "Provider", cell: row => row.provider ?? "unattributed"}, + {key: "eligible_sessions", label: "Eligible sessions", numeric: true}, + {key: "session_failures", label: "Session failures", cell: row => + stateCell(row.session_outcomes?.failures)}, + {key: "session_failure_share", label: "Known-outcome session failure share", cell: row => + rateText(row.session_outcomes?.failure_rate)}, + {key: "session_outcome_coverage", label: "Session outcome coverage", cell: row => + outcomeCoverage(row.session_outcomes)}, + {key: "tool_result_event_count", label: "Indexed tool_result events", numeric: true, + cell: row => row.tool_result_event_count == null ? "unobserved" : + DeckCore.formatCount(row.tool_result_event_count)}, + {key: "tool_event_coverage", label: "Indexed event presence", cell: row => + rateText(row.tool_event_coverage)}, + {key: "tool_failures", label: "Observed tool failures", cell: row => + stateCell(row.tool_outcomes?.failures)}, + {key: "tool_failure_share", label: "Known-outcome tool failure share", cell: row => + rateText(row.tool_outcomes?.failure_rate)}, + {key: "tool_outcome_coverage", label: "Tool outcome coverage", cell: row => + outcomeCoverage(row.tool_outcomes)}, + {key: "attribution_state", label: "Attribution", cell: row => row.attribution_state} + ]; + + return ctx.el("section", {class: "card view-errors"}, [ + ctx.el("h1", {}, "Failure symptoms by skill and step"), + ctx.el("p", {class: "view-lede"}, "Session failures and explicit failed tool_result events " + + "remain separate populations, even when they belong to one attempt. Unknown outcomes stay " + + "in the eligible coverage counts and are not counted as failures."), + ctx.el("p", {class: "view-note"}, "A tool failure is counted only when indexed success is " + + "explicitly false. Tool event coverage means indexed tool_result event presence among " + + "eligible sessions; the observed events do not establish a complete tool history."), + tableRows.length ? ctx.sortableTable({columns, rows: tableRows, + defaultSort: {key: "symptom", dir: "asc"}}) : + ctx.el("p", {class: "view-empty"}, "No observed failure symptoms match this cohort."), + populationRows.length ? ctx.el("section", {class: "view-error-coverage"}, [ + ctx.el("h2", {}, "Outcome and tool-event coverage"), + ctx.el("p", {class: "view-note"}, "Outcome state counts use all eligible sessions. Tool " + + "event presence describes only indexed tool_result rows and does not prove complete " + + "tool history. Unattributed events keep their own population row."), + ctx.sortableTable({columns: coverageColumns, rows: populationRows, + defaultSort: {key: "eligible_sessions", dir: "desc"}}) + ]) : null, + ctx.el("p", {class: "view-legend"}, "Unknown symptom, skill, step, or event attribution " + + "is retained as an explicit row instead of being assigned to a nearby attempt. " + + "A measured_zero failure share is an observed 0%; unknown, unavailable, and " + + "not_applicable remain separate states.") + ]); +}); diff --git a/src/autoskillit/assets/deck/views/gaps.js b/src/autoskillit/assets/deck/views/gaps.js new file mode 100644 index 0000000000..6ff8f28b34 --- /dev/null +++ b/src/autoskillit/assets/deck/views/gaps.js @@ -0,0 +1,52 @@ +DeckShell.registerView("gaps", ctx => { + const viewForField = field => { + const name = String(field ?? ""); + if (name.includes("session_outcome") || name.includes("tool_error") || + name.includes("attribution")) return "errors"; + if (name.includes("turn") || name.includes("context") || name.includes("window") || + name.includes("prompt") || name.includes("return")) return "context"; + return "trend"; + }; + const stateCell = row => DeckCore.coverageStateCell(ctx, + DeckCore.isPrimitiveState(row.state) ? row.measure ?? {state: row.state} : {state: row.state}); + const stateCounts = row => DeckCore.formatStateCounts(row.state_counts) || "state counts unavailable"; + const tableRows = [...ctx.rows]; + const columns = [ + {key: "question", label: "Unanswered question", cell: row => row.question ?? + "What evidence is missing for " + row.field + "?"}, + {key: "prerequisite", label: "Prerequisite", cell: row => row.prerequisite ?? row.reason ?? "—"}, + {key: "field", label: "Affected field"}, + {key: "harness", label: "Harness", cell: row => row.harness ?? "unattributed"}, + {key: "provider", label: "Provider", cell: row => row.provider ?? "unattributed"}, + {key: "skill", label: "Skill", cell: row => row.skill ?? "unknown"}, + {key: "recipe", label: "Recipe", cell: row => row.recipe ?? "unknown"}, + {key: "step", label: "Step", cell: row => row.step ?? "unknown"}, + {key: "model", label: "Resolved model", cell: row => row.model ?? "unknown"}, + {key: "population", label: "Population"}, + {key: "state", label: "Coverage state", cell: stateCell}, + {key: "coverage_counts", label: "Observed / eligible", cell: row => + DeckCore.countLabel(row.observation_count) + " observed / " + DeckCore.countLabel(row.eligible_count) + + " eligible · " + stateCounts(row)}, + {key: "source", label: "Evidence source", cell: row => row.source ?? "not recorded"}, + {key: "reason", label: "Reason", cell: row => row.reason ?? "—"}, + {key: "affected_view", label: "Inspect", cell: row => ctx.el("a", { + href: ctx.href({view: viewForField(row.field), params: DeckCore.inspectParams(row)}) + }, "Open evidence")} + ]; + + return ctx.el("section", {class: "card view-gaps"}, [ + ctx.el("h1", {}, "Live evidence gaps"), + ctx.el("p", {class: "view-lede"}, "Each entry states the unanswered question, its missing " + + "prerequisite, the affected field and population, and the indexed source. Counts describe " + + "observed records among eligible records; they do not turn absent evidence into zero."), + ctx.el("p", {class: "view-note"}, "Tool coverage means indexed tool_result event presence " + + "among eligible sessions. Observed events do not prove complete tool history, and absent " + + "rows mean unobserved evidence."), + tableRows.length ? ctx.sortableTable({columns, rows: tableRows, + defaultSort: {key: "observation_count", dir: "desc"}}) : + ctx.el("p", {class: "view-empty"}, "No uncovered fields match this selected cohort."), + ctx.el("p", {class: "view-legend"}, "Measured, measured_zero, unknown, unavailable, and " + + "not_applicable are primitive states. Mixed coverage and no observations are summaries " + + "with their state counts shown; they are not measurements.") + ]); +}); diff --git a/src/autoskillit/assets/deck/views/parity.js b/src/autoskillit/assets/deck/views/parity.js new file mode 100644 index 0000000000..0c3c7703a0 --- /dev/null +++ b/src/autoskillit/assets/deck/views/parity.js @@ -0,0 +1,61 @@ +DeckShell.registerView("parity", ctx => { + const groupFields = ["session_key", "harness", "provider", "skill", "recipe", "step", + "model", "level", "population"]; + const groupKey = row => JSON.stringify(groupFields.map(field => row[field] ?? null)); + const stateCell = row => { + const state = row?.state ?? "no_observations"; + const body = DeckCore.coverageStateCell(ctx, + DeckCore.isPrimitiveState(state) ? row.measure ?? {state} : {state}); + const counts = row ? DeckCore.formatStateCounts(row.state_counts) : "no row in this selection"; + const coverage = ctx.el("small", {class: "parity-cell__coverage"}, row ? + DeckCore.countLabel(row.observation_count) + " observations · " + + DeckCore.countLabel(row.eligible_count) + " eligible source records · " + counts : counts); + const cell = ctx.el("span", {class: "parity-cell"}, [body, coverage]); + if (!row || !["measured", "measured_zero", "not_applicable"].includes(state)) { + return ctx.el("a", {class: "parity-cell-link", + href: ctx.href({view: "gaps", params: DeckCore.inspectParams(row)})}, cell); + } + return cell; + }; + const groups = new Map(); + const fields = []; + ctx.rows.forEach(row => { + const key = groupKey(row); + if (!groups.has(key)) { + groups.set(key, {...Object.fromEntries(groupFields.map(field => [field, row[field] ?? null])), + fields: {}}); + } + groups.get(key).fields[row.field] = row; + if (!fields.includes(row.field)) fields.push(row.field); + }); + const matrixRows = [...groups.values()]; + const columns = [ + {key: "harness", label: "Harness", cell: row => row.harness ?? "unattributed"}, + {key: "provider", label: "Provider", cell: row => row.provider ?? "unattributed"}, + {key: "skill", label: "Skill", cell: row => row.skill ?? "unknown"}, + {key: "recipe", label: "Recipe", cell: row => row.recipe ?? "unknown"}, + {key: "step", label: "Step", cell: row => row.step ?? "unknown"}, + {key: "model", label: "Resolved model", cell: row => row.model ?? "unknown"}, + {key: "level", label: "Level", cell: row => row.level ?? "unrecorded"}, + {key: "population", label: "Population"}, + {key: "session_key", label: "Session", cell: row => row.session_key ?? "all eligible"}, + ...fields.map(field => ({key: field, label: field.replace(/_/g, " "), + cell: row => stateCell(row.fields[field])})) + ]; + + return ctx.el("section", {class: "card view-parity"}, [ + ctx.el("h1", {}, "Harness and provider coverage parity"), + ctx.el("p", {class: "view-lede"}, "Each row shows the same target fields for one selected " + + "harness/provider population. Observation and eligibility counts are separate: " + + "eligibility counts owner sessions for attributed evidence and events for " + + "unattributed tools. Text accompanies every state; color is supplementary."), + ctx.el("p", {class: "view-legend"}, "Measured and measured_zero are observed states. " + + "Unknown, unavailable, and not_applicable remain distinct. Mixed coverage and no " + + "observations are summaries of the primitive states shown in each cell."), + matrixRows.length ? ctx.sortableTable({columns, rows: matrixRows, + defaultSort: {key: "harness", dir: "asc"}}) : + ctx.el("p", {class: "view-empty"}, "No parity rows match this selected cohort."), + ctx.el("p", {class: "view-note"}, "Links on partial or missing cells open the gap register " + + "with this cohort and exact session selection retained.") + ]); +}); diff --git a/src/autoskillit/assets/deck/views/trend.js b/src/autoskillit/assets/deck/views/trend.js new file mode 100644 index 0000000000..2455171d89 --- /dev/null +++ b/src/autoskillit/assets/deck/views/trend.js @@ -0,0 +1,212 @@ +DeckShell.registerView("trend", ctx => { + const tokenFields = ["input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens"]; + const colors = ["s1", "s2", "s3", "s4"]; + const groupText = row => [row.skill ?? "unknown skill", row.recipe ?? "unknown recipe", + row.step ?? "unknown step", row.harness ?? "unknown harness", + row.provider ?? "unknown provider", row.model ?? "unknown model"].join(" · "); + const groupKey = row => JSON.stringify([row.skill, row.recipe, row.step, + row.harness, row.provider, row.model]); + const timeOf = row => row.time_ms ?? (row.day == null ? null : Date.parse(row.day + "T00:00:00Z")); + + function seriesGroups(rows) { + const groups = new Map(); + rows.forEach(row => { + const key = groupKey(row); + if (!groups.has(key)) groups.set(key, {label: groupText(row), rows: []}); + groups.get(key).rows.push(row); + }); + return [...groups.values()].map(group => ({...group, rows: [...group.rows].sort((a, b) => + (timeOf(a) ?? Infinity) - (timeOf(b) ?? Infinity))})); + } + + function lineChart(rows, field, label, percent = false) { + const groups = seriesGroups(rows); + const timed = rows.map(timeOf).filter(value => value != null); + const first = timed.reduce((minimum, value) => Math.min(minimum, value), Infinity); + const last = timed.reduce((maximum, value) => Math.max(maximum, value), -Infinity); + const values = rows.map(row => DeckCore.measureValue( + field === "failure_share" ? row.failure_share : row.measures?.[field] + )).filter(value => value !== null); + const minValue = values.reduce((minimum, value) => Math.min(minimum, value), 0); + const maxValue = Math.max(1, values.reduce((maximum, value) => Math.max(maximum, value), 0)); + const width = 820, height = Math.max(250, groups.length * 40 + 150); + const left = 58, right = 18, top = 52 + groups.length * 13, bottom = 52; + const plotWidth = width - left - right, plotHeight = height - top - bottom; + const xFor = time => left + (last <= first ? plotWidth / 2 : + (time - first) / (last - first) * plotWidth); + const yFor = value => top + plotHeight - (value - minValue) / (maxValue - minValue) * plotHeight; + const rendered = [ + ctx.svgElement("text", {x: left, y: 18, class: "deck-chart__title"}, label), + ctx.svgElement("line", {x1: left, y1: top + plotHeight, x2: width - right, + y2: top + plotHeight, class: "deck-chart__axis"}), + ctx.svgElement("text", {x: 3, y: top + 4, class: "deck-chart__axis-label"}, + percent ? Math.round(maxValue * 100) + "%" : DeckCore.formatCount(maxValue)), + ctx.svgElement("text", {x: 3, y: top + plotHeight, class: "deck-chart__axis-label"}, + percent ? Math.round(minValue * 100) + "%" : DeckCore.formatCount(minValue)) + ]; + if (timed.length) { + rendered.push(ctx.svgElement("text", {x: left, y: height - 8, + class: "deck-chart__axis-label"}, DeckCore.formatDate(first))); + if (last !== first) rendered.push(ctx.svgElement("text", {x: width - right, y: height - 8, + "text-anchor": "end", class: "deck-chart__axis-label"}, DeckCore.formatDate(last))); + } + groups.forEach((group, groupIndex) => { + const color = colors[groupIndex % colors.length]; + let segment = []; + let priorTime = null; + const flush = () => { + if (segment.length > 1) rendered.push(ctx.svgElement("polyline", { + points: segment.map(point => point.x + "," + point.y).join(" "), + class: "deck-chart__line deck-chart__line--" + color + })); + segment = []; + }; + group.rows.forEach(row => { + const time = timeOf(row); + const measure = field === "failure_share" ? row.failure_share : row.measures?.[field]; + const value = DeckCore.measureValue(measure); + if (time == null || value == null || (priorTime != null && time - priorTime > DeckCore.DAY_MS)) { + flush(); + if (time != null && value == null) rendered.push(ctx.svgElement("rect", { + x: xFor(time) - 4, y: top + plotHeight - 8, width: 8, height: 8, + class: "deck-chart__missing", fill: "url(#deck-hatch)"}, [ + ctx.svgElement("title", {}, group.label + " · " + (row.day ?? "day") + + ": no measured observation") + ])); + priorTime = time; + if (value == null || time == null) return; + } + const x = xFor(time), y = yFor(value); + segment.push({x, y}); + const sample = field === "failure_share" ? measure?.sample_size : + measure?.observed_sessions; + rendered.push(ctx.svgElement("circle", {cx: x, cy: y, r: 3.5, + class: "deck-chart__point deck-chart__point--" + color}, [ + ctx.svgElement("title", {}, group.label + " · " + (row.day ?? "day") + ": " + + (percent ? String(Math.round(value * 10000) / 100) + "%" : + DeckCore.formatCount(value) + " tokens/session") + " · " + + (sample == null ? "sample unavailable" : DeckCore.formatCount(sample) + + (field === "failure_share" ? " known outcomes" : " observed sessions"))) + ])); + priorTime = time; + }); + flush(); + const legendY = 36 + groupIndex * 13; + rendered.push(ctx.svgElement("line", {x1: left, y1: legendY - 3, x2: left + 16, + y2: legendY - 3, class: "deck-chart__line deck-chart__line--" + color})); + rendered.push(ctx.svgElement("text", {x: left + 22, y: legendY, + class: "deck-chart__legend"}, group.label)); + }); + if (!timed.length || !groups.length) rendered.push(ctx.svgElement("text", { + x: left + 8, y: top + 28, class: "deck-chart__empty" + }, "No timed daily buckets match this measure.")); + return ctx.svgElement("svg", {class: "deck-chart", role: "img", "aria-label": label, + viewBox: "0 0 " + width + " " + height}, rendered); + } + + function measureText(measure, unit = "tokens/session") { + const value = DeckCore.measureValue(measure); + if (value === null) return [measure?.state === "no_observations" ? "no observations" : + measure?.state ?? "unknown", measure?.reason].filter(Boolean).join(" · "); + const sample = measure.observed_sessions ?? measure.sample_size; + const eligible = measure.eligible_sessions ?? measure.eligible_count; + const states = Object.entries(measure.state_counts ?? {}).map(([state, count]) => + state + " " + DeckCore.formatCount(count)).join(", "); + return (value === 0 ? "0" : DeckCore.formatCount(value)) + " " + + (measure.unit ?? unit) + + (sample == null ? " · sample unavailable" : " · n=" + DeckCore.formatCount(sample)) + + (eligible == null ? "" : " / " + DeckCore.formatCount(eligible) + " eligible") + + (states ? " · " + states : ""); + } + + function dateRange(interval) { + const start = interval?.covered_start_ms ?? interval?.start_ms; + const end = interval?.covered_end_ms ?? interval?.end_ms; + return start == null || end == null ? "no covered dates" : + DeckCore.formatDate(start) + " → " + DeckCore.formatDate(end); + } + + const comparisons = ctx.metrics?.comparisons ?? []; + const comparisonRows = comparisons.flatMap(comparison => [ + ...tokenFields.map(field => ({...comparison, field, earlierMeasure: comparison.earlier?.measures?.[field], + laterMeasure: comparison.later?.measures?.[field], delta: comparison.deltas?.[field]})), + {...comparison, field: "failure_share", earlierMeasure: comparison.earlier?.failure_share, + laterMeasure: comparison.later?.failure_share, delta: comparison.deltas?.failure_share} + ]); + const comparisonColumns = [ + {key: "skill", label: "Skill", cell: row => row.skill ?? "unknown"}, + {key: "step", label: "Step", cell: row => row.step ?? "unknown"}, + {key: "harness", label: "Harness"}, + {key: "provider", label: "Provider"}, + {key: "model", label: "Model", cell: row => row.model ?? "unknown"}, + {key: "field", label: "Measure"}, + {key: "earlier_range", label: "Earlier covered dates", cell: row => dateRange(row.earlier)}, + {key: "earlierMeasure", label: "Earlier", cell: row => measureText(row.earlierMeasure, + row.field === "failure_share" ? "failure share" : "tokens/session")}, + {key: "later_range", label: "Later covered dates", cell: row => dateRange(row.later)}, + {key: "laterMeasure", label: "Later", cell: row => measureText(row.laterMeasure, + row.field === "failure_share" ? "failure share" : "tokens/session")}, + {key: "delta", label: "Descriptive delta", cell: row => { + const value = DeckCore.measureValue(row.delta); + return value == null ? ["no observations", "unknown", "unavailable"].includes(row.delta?.state) ? + row.delta.state + (row.delta.reason ? ": " + row.delta.reason : "") : + row.delta?.reason ?? "insufficient observations" : + (value === 0 ? "0" : String(Math.round(value * 100) / 100)) + " " + + (row.delta.unit ?? "tokens/session") + " · n=" + + (row.delta.earlier_samples == null ? "unavailable" : + DeckCore.formatCount(row.delta.earlier_samples)) + "/" + + (row.delta.later_samples == null ? "unavailable" : + DeckCore.formatCount(row.delta.later_samples)); + }} + ]; + const tableColumns = [ + {key: "day", label: "UTC day"}, + {key: "skill", label: "Skill", cell: row => row.skill ?? "unknown"}, + {key: "step", label: "Step", cell: row => row.step ?? "unknown"}, + {key: "harness", label: "Harness"}, + {key: "provider", label: "Provider"}, + {key: "model", label: "Resolved model", cell: row => row.model ?? "unknown"}, + ...tokenFields.map(field => ({key: field, label: field.replaceAll("_", " "), + cell: row => measureText(row.measures?.[field])})), + {key: "failure_share", label: "Failure share", cell: row => measureText( + row.failure_share, "failure share")}, + {key: "eligible_sessions", label: "Eligible sessions", numeric: true}, + {key: "untimed_sessions", label: "Untimed", numeric: true}, + {key: "future_sessions", label: "Future excluded", numeric: true} + ]; + const history = ctx.rows.find(row => row.retained_from_ms != null)?.retained_from_ms; + + return ctx.el("div", {class: "view-stack view-trend"}, [ + ctx.el("section", {class: "card"}, [ + ctx.el("h1", {}, "Daily token usage and failure share"), + ctx.el("p", {class: "view-lede"}, "Each line keeps its skill, step, harness, provider, " + + "and resolved model identity. Points show field-specific measured session samples; missing " + + "or unsupported days break the line instead of becoming zero."), + history == null ? null : ctx.el("p", {class: "view-note"}, "Retained history starts " + + DeckCore.formatDate(history) + ". Untimed sessions remain outside daily buckets; future " + + "timestamps are excluded."), + ...tokenFields.map(field => lineChart(ctx.rows, field, + field.replaceAll("_", " ") + " per observed session")), + lineChart(ctx.rows, "failure_share", "Known-outcome session failure share", true), + ctx.el("p", {class: "view-legend"}, "Hatched marks identify missing observations. " + + "measured_zero is an explicit zero; unknown, unavailable, and not_applicable remain " + + "textually distinct in the table. Failure share uses known outcomes only; the comparison " + + "below is descriptive, without a causal or sustained-change claim."), + ctx.sortableTable({columns: tableColumns, rows: [...ctx.rows], + defaultSort: {key: "day", dir: "asc"}}) + ]), + ctx.el("section", {class: "card"}, [ + ctx.el("h2", {}, "Earlier and later intervals"), + ctx.el("p", {class: "view-note"}, "The selected interval is split at its exact temporal " + + "midpoint; midpoint records belong to the later half. Each value carries its own observed " + + "sample count and covered date range."), + comparisonRows.length ? ctx.sortableTable({columns: comparisonColumns, + rows: comparisonRows, defaultSort: {key: "field", dir: "asc"}}) : + ctx.el("p", {class: "view-empty"}, "No earlier/later comparison has two observed sides."), + ctx.el("div", {class: "view-links"}, [ + ctx.el("a", {href: ctx.href({view: "errors"})}, "Inspect failure symptoms"), + ctx.el("a", {href: ctx.href({view: "gaps"})}, "Inspect evidence gaps") + ]) + ]) + ]); +}); diff --git a/src/autoskillit/cli/ops/_sessions.py b/src/autoskillit/cli/ops/_sessions.py index 5a0fcc4655..412bfc3c5c 100644 --- a/src/autoskillit/cli/ops/_sessions.py +++ b/src/autoskillit/cli/ops/_sessions.py @@ -56,7 +56,7 @@ def sessions_index(*, update: bool = False, rebuild: bool = False) -> None: print( f"report index v{REPORT_INDEX_SCHEMA_VERSION} at {index_dir}: " f"sessions={len(report.sessions)} requests={len(report.requests)} " - f"tools={len(report.tools)} subagents={len(report.subagents)}" + f"tools={len(report.tools)} subagents={len(report.subagents)} turns={len(report.turns)}" ) @@ -82,6 +82,7 @@ def sessions_deck(output: str) -> None: request_rows=report.requests.values(), tool_rows=report.tools.values(), subagent_rows=report.subagents.values(), + turn_rows=report.turns.values(), generated_at=datetime.now(UTC), index_schema_version=REPORT_INDEX_SCHEMA_VERSION, ) diff --git a/src/autoskillit/execution/_report_index_rows.py b/src/autoskillit/execution/_report_index_rows.py index eb5eb6ebd7..2dc8cfabb9 100644 --- a/src/autoskillit/execution/_report_index_rows.py +++ b/src/autoskillit/execution/_report_index_rows.py @@ -10,8 +10,7 @@ from collections import Counter from collections.abc import Callable, Mapping from datetime import UTC, datetime, timedelta -from types import MappingProxyType -from typing import Any, Final, TypedDict +from typing import Any, Final from autoskillit.core import ( AGENT_BACKEND_CLAUDE_CODE, @@ -21,6 +20,13 @@ extract_skill_name, get_logger, ) +from autoskillit.execution._report_index_types import ( + REQUEST_KIND, + SESSION_KIND, + SUBAGENT_KIND, + TOOL_KIND, + TURN_KIND, +) from autoskillit.execution.evidence.otlp_tokens import ( CLAUDE_CODE_SCOPE_NAME, claude_request_usage, @@ -34,127 +40,20 @@ from autoskillit.execution.evidence.report_walk import OTLP_WALK_KIND, SESSION_WALK_KIND, WalkItem from autoskillit.execution.session.turn_usage import classify_token_measure -REPORT_INDEX_SCHEMA_VERSION: Final[int] = 2 -SESSION_KIND: Final[str] = "session" -REQUEST_KIND: Final[str] = "request" -TOOL_KIND: Final[str] = "tool" -SUBAGENT_KIND: Final[str] = "subagent" +REPORT_INDEX_SCHEMA_VERSION: Final[int] = 3 UNKNOWN_SOURCE: Final[str] = "unknown" _STRUCTURED_MEASURE_SESSION_VERSION = 14 +_TURN_MEASURE_FIELDS = ( + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cache_write_tokens", +) +_PARENT_CONTEXT_FIELDS = frozenset({"parent_prompt_tokens", "subagent_return_tokens"}) logger = get_logger(__name__) -class ReportRowBase(TypedDict): - """Common fields carried by each report-index fact row.""" - - schema_version: int - kind: str - key: str - session_id: str | None - time_ms: int | None - - -class ReportSessionRow(ReportRowBase): - """Session facts derived from one session or archive walk item. - - The provider is persisted verbatim from ``provider_used``; token - classification casefolds it only for the source-pair lookup. - """ - - harness: str - provider: str - model: str | None - skill: str | None - recipe: str | None - step: str | None - level: str | None - kitchen_id: str | None - order_id: str | None - dispatch_id: str | None - campaign_id: str | None - caller_session_id: str | None - parent_session_id: str | None - success: bool | None - subtype: str | None - adjudication_reason: str | None - adjudication_subtype: str | None - duration_seconds: float | None - input_tokens: SerializedTokenMeasure - output_tokens: SerializedTokenMeasure - cache_read_tokens: SerializedTokenMeasure - cache_write_tokens: SerializedTokenMeasure - assistant_turn_count: int | None - tool_counts: dict[str, int] | None - - -class ReportRequestRow(ReportRowBase): - """Request facts derived from a Claude Code ``api_request`` log record.""" - - harness: str - request_id: str - agent_name: str | None - query_source: str | None - model: str | None - event_sequence: int | None - input_tokens: int | None - output_tokens: int | None - cache_read_tokens: int | None - cache_write_tokens: int | None - cost_usd: float | None - duration_ms: int | None - - -class ReportToolRow(ReportRowBase): - """Tool facts derived from a Claude Code ``tool_result`` log record.""" - - harness: str - agent_name: str | None - tool_name: str | None - tool_use_id: str | None - error_type: str | None - success: bool | None - duration_ms: int | None - tool_input_size_bytes: int | None - tool_result_size_bytes: int | None - event_sequence: int | None - - -class ReportSubagentRow(ReportRowBase): - """Native child or OTLP completion facts, kept in one subagent fact family.""" - - harness: str - agent_type: str | None - model: str | None - final_model: str | None - model_swapped: bool | None - event_sequence: int | None - child_id: str | None - native_parent_session_id: str | None - parent_session_key: str | None - role: str | None - actor_level: str | None - provider: str - skill: str | None - recipe: str | None - step: str | None - level: str | None - token_usage: dict[str, SerializedTokenMeasure] - tool_counts: dict[str, int] | None - transcript_state: str | None - usage_state: str | None - - -REPORT_ROW_TYPES: Mapping[str, type] = MappingProxyType( - { - SESSION_KIND: ReportSessionRow, - REQUEST_KIND: ReportRequestRow, - TOOL_KIND: ReportToolRow, - SUBAGENT_KIND: ReportSubagentRow, - } -) - - def _text(value: object) -> str | None: return value if isinstance(value, str) and value else None @@ -210,6 +109,46 @@ def _token_usage_map(value: object) -> dict[str, SerializedTokenMeasure]: return measures +def _serialized_measure(value: object) -> SerializedTokenMeasure: + try: + if isinstance(value, TokenMeasure): + return value.to_dict() + if isinstance(value, Mapping): + return TokenMeasure.from_dict(dict(value)).to_dict() + except ValueError: + pass + return TokenMeasure.unknown().to_dict() + + +def _parent_context_spans(value: object) -> list[dict[str, Any]]: + if not isinstance(value, (tuple, list)): + return [] + spans: list[dict[str, Any]] = [] + for raw in value: + if not isinstance(raw, Mapping): + continue + field = _text(raw.get("field")) + if field not in _PARENT_CONTEXT_FIELDS: + continue + spans.append( + { + "field": field, + "measure": _serialized_measure(raw.get("measure")), + "invocation_id": _text(raw.get("invocation_id")), + "turn_id": _text(raw.get("turn_id")), + "source_id": _text(raw.get("source_id")), + "timestamp": _text(raw.get("timestamp")), + "harness": _text(raw.get("harness")), + "provider": _text(raw.get("provider")), + "model": _text(raw.get("model")), + "tokenizer_version": _text(raw.get("tokenizer_version")), + "encoding": _text(raw.get("encoding")), + "reason": _text(raw.get("reason")), + } + ) + return spans + + def _pair_text(value: object) -> str: return _text(value) or UNKNOWN_SOURCE @@ -251,6 +190,8 @@ def _raw_measure(value: object) -> object: **{field: _raw_measure for field in CANONICAL_ACCOUNTING_FIELDS}, "assistant_turn_count": _count, "tool_counts": _count_map, + "turn_usage_state": _text, + "turn_usage_reason": _text, }, REQUEST_KIND: { "schema_version": _count, @@ -305,6 +246,27 @@ def _raw_measure(value: object) -> object: "tool_counts": _count_map, "transcript_state": _text, "usage_state": _text, + "parent_context_spans": _parent_context_spans, + }, + TURN_KIND: { + "schema_version": _count, + "time_ms": _count, + "session_id": _text, + "source_id": _text, + "session_key": _text, + "ordinal": _count, + "request_id": _text, + "message_id": _text, + "harness": _pair_text, + "provider": _pair_text, + "model": _text, + "skill": _text, + "recipe": _text, + "step": _text, + "level": _text, + **{field: _serialized_measure for field in _TURN_MEASURE_FIELDS}, + "context_window_tokens": _count, + "context_fraction": _number, }, } @@ -343,7 +305,11 @@ def rows_for_walk_item(item: WalkItem) -> list[dict[str, Any]]: return [] if item.kind == SESSION_WALK_KIND: session = _session_row(item.source_id, item.record) - return [session, *_native_subagent_rows(item.source_id, item.record, session)] + return [ + session, + *_turn_rows(item.source_id, item.record, session), + *_native_subagent_rows(item.source_id, item.record, session), + ] if item.kind == OTLP_WALK_KIND: return _otlp_rows(item.source_id, item.record) return [] @@ -387,9 +353,48 @@ def _session_row(key: str, record: dict[str, Any]) -> dict[str, Any]: **measures, "assistant_turn_count": _count(record.get("assistant_turn_count")), "tool_counts": _tool_counts(record), + "turn_usage_state": _text(record.get("turn_usage_state")), + "turn_usage_reason": _text(record.get("turn_usage_reason")), } +def _turn_rows( + source_id: str, record: dict[str, Any], parent: Mapping[str, Any] +) -> list[dict[str, Any]]: + raw_rows = record.get("turn_usage_rows") + if not isinstance(raw_rows, (tuple, list)): + return [] + rows: list[dict[str, Any]] = [] + for ordinal, turn in enumerate(raw_rows): + if not isinstance(turn, Mapping): + continue + rows.append( + { + "schema_version": REPORT_INDEX_SCHEMA_VERSION, + "kind": TURN_KIND, + "key": f"{source_id}:turn:{ordinal}", + "session_id": parent["session_id"], + "time_ms": _iso_to_ms(turn.get("timestamp")), + "source_id": source_id, + "session_key": source_id, + "ordinal": ordinal, + "request_id": _text(turn.get("request_id")), + "message_id": _text(turn.get("message_id")), + "harness": _pair_text(turn.get("backend")), + "provider": _pair_text(turn.get("provider_used")), + "model": _text(turn.get("model")), + "skill": parent["skill"], + "recipe": parent["recipe"], + "step": parent["step"], + "level": parent["level"], + **{field: _serialized_measure(turn.get(field)) for field in _TURN_MEASURE_FIELDS}, + "context_window_tokens": _count(turn.get("context_window_tokens")), + "context_fraction": _number(turn.get("context_fraction")), + } + ) + return rows + + def _session_source_measure(row: dict[str, Any], field: str) -> object: value = row.get(field) version = _count(row.get("schema_version")) @@ -538,6 +543,7 @@ def _native_subagent_rows( "tool_counts": tool_counts, "transcript_state": transcript_state, "usage_state": _text(child.get("usage_state")) or "unknown", + "parent_context_spans": _parent_context_spans(child.get("parent_context_spans")), } ) rows.sort(key=lambda row: row["key"]) @@ -647,6 +653,23 @@ def _subagent_fields( "tool_counts": None, "transcript_state": None, "usage_state": None, + "parent_context_spans": [ + { + "field": field, + "measure": TokenMeasure.unavailable().to_dict(), + "invocation_id": None, + "turn_id": None, + "source_id": position_key, + "timestamp": None, + "harness": AGENT_BACKEND_CLAUDE_CODE, + "provider": None, + "model": None, + "tokenizer_version": None, + "encoding": None, + "reason": "parent_invocation_unverified", + } + for field in ("parent_prompt_tokens", "subagent_return_tokens") + ], } diff --git a/src/autoskillit/execution/_report_index_types.py b/src/autoskillit/execution/_report_index_types.py new file mode 100644 index 0000000000..692b3decf5 --- /dev/null +++ b/src/autoskillit/execution/_report_index_types.py @@ -0,0 +1,152 @@ +"""Private typed row contracts for the derived report index.""" + +from __future__ import annotations + +from collections.abc import Mapping +from types import MappingProxyType +from typing import Any, Final, TypedDict + +from autoskillit.core import SerializedTokenMeasure + +SESSION_KIND: Final[str] = "session" +REQUEST_KIND: Final[str] = "request" +TOOL_KIND: Final[str] = "tool" +SUBAGENT_KIND: Final[str] = "subagent" +TURN_KIND: Final[str] = "turn" + + +class ReportRowBase(TypedDict): + """Common fields carried by each report-index fact row.""" + + schema_version: int + kind: str + key: str + session_id: str | None + time_ms: int | None + + +class ReportSessionRow(ReportRowBase): + """Session facts from one session or archive walk item. + + The provider is persisted verbatim from ``provider_used``; token + classification casefolds it only for the source-pair lookup. + """ + + harness: str + provider: str + model: str | None + skill: str | None + recipe: str | None + step: str | None + level: str | None + kitchen_id: str | None + order_id: str | None + dispatch_id: str | None + campaign_id: str | None + caller_session_id: str | None + parent_session_id: str | None + success: bool | None + subtype: str | None + adjudication_reason: str | None + adjudication_subtype: str | None + duration_seconds: float | None + input_tokens: SerializedTokenMeasure + output_tokens: SerializedTokenMeasure + cache_read_tokens: SerializedTokenMeasure + cache_write_tokens: SerializedTokenMeasure + assistant_turn_count: int | None + tool_counts: dict[str, int] | None + turn_usage_state: str | None + turn_usage_reason: str | None + + +class ReportRequestRow(ReportRowBase): + """Request facts derived from a Claude Code ``api_request`` log record.""" + + harness: str + request_id: str + agent_name: str | None + query_source: str | None + model: str | None + event_sequence: int | None + input_tokens: int | None + output_tokens: int | None + cache_read_tokens: int | None + cache_write_tokens: int | None + cost_usd: float | None + duration_ms: int | None + + +class ReportToolRow(ReportRowBase): + """Tool facts derived from a Claude Code ``tool_result`` log record.""" + + harness: str + agent_name: str | None + tool_name: str | None + tool_use_id: str | None + error_type: str | None + success: bool | None + duration_ms: int | None + tool_input_size_bytes: int | None + tool_result_size_bytes: int | None + event_sequence: int | None + + +class ReportSubagentRow(ReportRowBase): + """Native child or OTLP completion facts, kept in one subagent fact family.""" + + harness: str + agent_type: str | None + model: str | None + final_model: str | None + model_swapped: bool | None + event_sequence: int | None + child_id: str | None + native_parent_session_id: str | None + parent_session_key: str | None + role: str | None + actor_level: str | None + provider: str + skill: str | None + recipe: str | None + step: str | None + level: str | None + token_usage: dict[str, SerializedTokenMeasure] + tool_counts: dict[str, int] | None + transcript_state: str | None + usage_state: str | None + parent_context_spans: list[dict[str, Any]] + + +class ReportTurnRow(ReportRowBase): + """One owner-attributed, already-normalized session turn usage fact.""" + + source_id: str + session_key: str + ordinal: int + request_id: str | None + message_id: str | None + harness: str + provider: str + model: str | None + skill: str | None + recipe: str | None + step: str | None + level: str | None + input_tokens: SerializedTokenMeasure + output_tokens: SerializedTokenMeasure + cache_read_tokens: SerializedTokenMeasure + cache_write_tokens: SerializedTokenMeasure + context_window_tokens: int | None + context_fraction: float | None + + +REPORT_ROW_TYPES: Mapping[str, type] = MappingProxyType( + { + SESSION_KIND: ReportSessionRow, + REQUEST_KIND: ReportRequestRow, + TOOL_KIND: ReportToolRow, + SUBAGENT_KIND: ReportSubagentRow, + TURN_KIND: ReportTurnRow, + } +) diff --git a/src/autoskillit/execution/evidence/_native_child_projection.py b/src/autoskillit/execution/evidence/_native_child_projection.py index af20c85fd3..cf1abb5d35 100644 --- a/src/autoskillit/execution/evidence/_native_child_projection.py +++ b/src/autoskillit/execution/evidence/_native_child_projection.py @@ -34,6 +34,13 @@ enumerate_claude_subagent_transcripts, normalize_backend_name, ) +from autoskillit.execution.evidence._native_parent_context import ( + parent_context_spans, + parent_transcript_records, +) +from autoskillit.execution.evidence._native_parent_context import ( + tokenizer_version as _tokenizer_version, +) from autoskillit.execution.session import extract_token_usage from autoskillit.execution.session.turn_usage import ( classify_token_measure, @@ -59,6 +66,18 @@ def project_child_outcomes( parent_session_id=parent_id, log_root=log_root, ) + if not outcomes: + return () + parent_path = _parent_transcript_path(row, normalized_backend) + parent_text = ( + _read_child_transcript(parent_path, normalized_backend) + if parent_path is not None and parent_path.is_file() + else None + ) + parent_records = parent_transcript_records(parent_text, normalized_backend) + parent_complete = _complete_jsonl(parent_text) + parent_fingerprint = _transcript_fingerprint(parent_path, parent_text) + tokenizer_version = _tokenizer_version() projected = [] for outcome in outcomes: child = _project_child_outcome( @@ -67,6 +86,11 @@ def project_child_outcomes( parent_id=parent_id, backend=normalized_backend, candidates=candidates, + parent_records=parent_records, + parent_complete=parent_complete, + parent_transcript_reason=_parent_transcript_reason(parent_path, parent_text), + parent_fingerprint=parent_fingerprint, + tokenizer_version=tokenizer_version, ) if child is not None: projected.append(child) @@ -80,6 +104,11 @@ def _project_child_outcome( parent_id: str, backend: str, candidates: Sequence[dict[str, Any]], + parent_records: Sequence[dict[str, Any]], + parent_complete: bool, + parent_transcript_reason: str | None, + parent_fingerprint: str | None, + tokenizer_version: str | None, ) -> dict[str, Any] | None: if not _is_native_child(child, backend, parent_id): return None @@ -114,13 +143,44 @@ def _project_child_outcome( "usage_state": usage_state, "tool_counts": tool_counts, "token_usage": token_usage, + "parent_context_spans": parent_context_spans( + row, + child["child_id"], + backend, + parent_records, + parent_complete=parent_complete, + parent_transcript_reason=parent_transcript_reason, + tokenizer_version=tokenizer_version, + ), "_child_timestamp": child_timestamp, "_transcript_fingerprint": _transcript_fingerprint(path, text), + "_parent_transcript_fingerprint": parent_fingerprint, + "_tokenizer_version": tokenizer_version, } ) return child +def _parent_transcript_path(row: Mapping[str, Any], backend: str) -> Path | None: + field = "claude_code_log" if backend == "claude_code" else "codex_log" + raw_path = row.get(field) + if not isinstance(raw_path, str) or not raw_path or not Path(raw_path).is_absolute(): + return None + return Path(raw_path) + + +def _parent_transcript_reason(path: Path | None, text: str | None) -> str | None: + if path is None: + return "parent_transcript_path_unavailable" + if not path.is_file(): + return "parent_transcript_missing" + if text is None: + return "parent_transcript_unreadable" + if not text: + return "parent_transcript_empty" + return None + + def _is_native_child(child: Mapping[str, Any], backend: str, parent_id: str) -> bool: expected_source = { "claude_code": "transcript_metadata", @@ -421,6 +481,8 @@ def _child_dependency(row: dict[str, Any], child: Mapping[str, Any]) -> dict[str "tool_counts": child.get("tool_counts"), "token_usage": child.get("token_usage"), "transcript_fingerprint": child.get("_transcript_fingerprint"), + "parent_transcript_fingerprint": child.get("_parent_transcript_fingerprint"), + "tokenizer_version": child.get("_tokenizer_version"), } diff --git a/src/autoskillit/execution/evidence/_native_parent_context.py b/src/autoskillit/execution/evidence/_native_parent_context.py new file mode 100644 index 0000000000..7499b65557 --- /dev/null +++ b/src/autoskillit/execution/evidence/_native_parent_context.py @@ -0,0 +1,667 @@ +"""Parent-side text spans delivered to verified native child invocations.""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping, Sequence +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as package_version +from typing import Any + +from autoskillit._parent_assistant_turns import ( + _codex_assistant_payload, + _resolve_codex_turn_id, + _resolve_turn_id, + is_parent_assistant_record, +) +from autoskillit.core import BackendCapabilities, TokenMeasure, get_logger +from autoskillit.execution.backends import get_backend + +logger = get_logger(__name__) + + +def parent_transcript_records(text: str | None, backend: str) -> tuple[dict[str, Any], ...]: + if text is None: + return () + raw_records = _raw_records(text) + return ( + _claude_records(raw_records) if backend == "claude_code" else _codex_records(raw_records) + ) + + +def _raw_records(text: str) -> tuple[dict[str, Any], ...]: + records = [] + for line in text.splitlines(keepends=True): + if not line.endswith("\n") or not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(record, dict): + records.append(record) + return tuple(records) + + +def _claude_records(raw_records: Sequence[dict[str, Any]]) -> tuple[dict[str, Any], ...]: + entries = [] + for record in raw_records: + parent = is_parent_assistant_record(record) + message = record.get("message") + model = message.get("model") if parent and isinstance(message, dict) else None + turn_id = _resolve_turn_id(record) if parent else "" + entries.append(_record_entry(record, "claude_code", parent, turn_id, model)) + return tuple(entries) + + +def _codex_records(raw_records: Sequence[dict[str, Any]]) -> tuple[dict[str, Any], ...]: + entries = [] + active_turn_id = "" + active_model: str | None = None + context_seen = False + for record in raw_records: + payload = record.get("payload") + if record.get("type") == "turn_context" and isinstance(payload, dict): + context_seen = True + active_model = _model_text(payload.get("model")) + active_turn_id = _model_text(payload.get("turn_id")) or "" + continue + if not isinstance(payload, dict) or not _is_parent_codex_record(record, payload): + continue + turn_id = _resolve_codex_turn_id(record, active_turn_id) + model = _codex_parent_model( + record, payload, turn_id, active_turn_id, active_model, context_seen + ) + entries.append(_record_entry(record, "codex", True, turn_id, model)) + return tuple(entries) + + +def _is_parent_codex_record(record: dict[str, Any], payload: dict[str, Any]) -> bool: + if record.get("type") != "response_item": + return False + if payload.get("agent_id") or payload.get("agentId"): + return False + return _codex_assistant_payload(record) is not None or payload.get("type") in { + "function_call_output", + "custom_tool_call_output", + } + + +def _codex_parent_model( + record: Mapping[str, Any], + payload: Mapping[str, Any], + resolved_turn_id: str, + context_turn_id: str, + context_model: str | None, + context_seen: bool, +) -> str | None: + models = { + value + for value in (_model_text(record.get("model")), _model_text(payload.get("model"))) + if value is not None + } + if len(models) > 1: + return None + explicit_model = next(iter(models)) if models else None + explicit_turn_id = ( + _resolve_turn_id(dict(record)) + or _model_text(record.get("turn_id")) + or _model_text(payload.get("turn_id")) + ) + if context_turn_id: + if resolved_turn_id != context_turn_id: + return None + if context_model and explicit_model and context_model != explicit_model: + return None + return context_model or explicit_model + if not explicit_turn_id or not explicit_model: + return None + if context_seen and context_model and context_model != explicit_model: + return None + return explicit_model + + +def _model_text(value: object) -> str | None: + return value if isinstance(value, str) and value else None + + +def _record_entry( + record: dict[str, Any], backend: str, parent: bool, turn_id: str, model: object +) -> dict[str, Any]: + timestamp = record.get("timestamp") + return { + "record": record, + "parent_record": parent, + "turn_id": turn_id or None, + "model": model if isinstance(model, str) and model else None, + "timestamp": timestamp if isinstance(timestamp, str) else None, + "source_id": _source_id(record, backend), + } + + +def _source_id(record: Mapping[str, Any], backend: str) -> str: + serialized = json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + digest = hashlib.sha256(serialized.encode("utf-8")).hexdigest()[:20] + record_id = record.get("uuid", record.get("id")) + return ( + f"{backend}:{record_id if isinstance(record_id, str) and record_id else digest}:{digest}" + ) + + +def parent_context_spans( + row: Mapping[str, Any], + child_id: str, + backend: str, + parent_records: Sequence[dict[str, Any]], + *, + parent_complete: bool, + parent_transcript_reason: str | None, + tokenizer_version: str | None, +) -> list[dict[str, Any]]: + invocations = _linked_parent_invocations(parent_records, child_id, backend) + if not invocations: + reason = parent_transcript_reason or ( + "parent_transcript_incomplete" if not parent_complete else "child_identity_unlinked" + ) + return _unlinked_spans(row, tokenizer_version, reason) + return [ + span + for invocation in invocations + for span in _invocation_spans(row, invocation, parent_complete, tokenizer_version) + ] + + +def _unlinked_spans( + row: Mapping[str, Any], tokenizer_version: str | None, reason: str +) -> list[dict[str, Any]]: + return [ + _span( + field, + {}, + row, + None, + None, + tokenizer_version, + None, + None, + None, + None, + reason, + None, + None, + ) + for field in ("parent_prompt_tokens", "subagent_return_tokens") + ] + + +def _linked_parent_invocations( + records: Sequence[dict[str, Any]], child_id: str, backend: str +) -> list[dict[str, Any]]: + capabilities = get_backend(backend.replace("_", "-")).capabilities + if backend == "claude_code": + return _claude_invocations(records, child_id, capabilities) + return _codex_invocations(records, child_id, capabilities) + + +def _claude_invocations( + records: Sequence[dict[str, Any]], child_id: str, capabilities: BackendCapabilities +) -> list[dict[str, Any]]: + calls: dict[str, dict[str, Any]] = {} + results: dict[str, list[dict[str, Any]]] = {} + seen_results: set[str] = set() + for entry in records: + if entry["parent_record"]: + _claude_calls(entry, calls, child_id) + if entry["record"].get("type") == "user": + _claude_results(entry, results, seen_results) + return _linked_invocations(calls, results, child_id, capabilities) + + +def _claude_calls( + entry: Mapping[str, Any], calls: dict[str, dict[str, Any]], child_id: str +) -> None: + record = entry["record"] + message = record.get("message") + content = message.get("content") if isinstance(message, dict) else None + if not isinstance(content, list): + return + for index, block in enumerate(content): + if not isinstance(block, Mapping) or block.get("type") != "tool_use": + continue + if str(block.get("name", "")).casefold() not in {"agent", "task"}: + continue + invocation_id = block.get("id") + if not isinstance(invocation_id, str) or invocation_id in calls: + continue + call_input = block.get("input") + prompt = call_input.get("prompt") if isinstance(call_input, Mapping) else None + identities = _child_ids(call_input) | _child_ids(block) + calls[invocation_id] = { + "invocation_id": invocation_id, + "turn_id": entry.get("turn_id"), + "model": entry.get("model"), + "prompt_text": prompt if isinstance(prompt, str) else None, + "prompt_reason": None if isinstance(prompt, str) else "parent_prompt_text_missing", + "prompt_source_id": f"{entry['source_id']}:tool-{index}", + "prompt_timestamp": entry.get("timestamp"), + "linked": identities == {child_id}, + } + + +def _claude_results( + entry: Mapping[str, Any], + results: dict[str, list[dict[str, Any]]], + seen_results: set[str], +) -> None: + record = entry["record"] + message = record.get("message") + content = message.get("content") if isinstance(message, Mapping) else None + if not isinstance(content, list): + return + for index, block in enumerate(content): + if not isinstance(block, Mapping) or block.get("type") != "tool_result": + continue + invocation_id = block.get("tool_use_id") + if not isinstance(invocation_id, str): + continue + source_id = f"{entry['source_id']}:tool-result-{index}" + if source_id in seen_results: + continue + seen_results.add(source_id) + raw_text = block.get("content") + result_text, reason = _claude_result_text(raw_text) + results.setdefault(invocation_id, []).append( + { + "text": result_text, + "reason": reason, + "source_id": source_id, + "timestamp": entry.get("timestamp"), + "identities": _child_ids(block) | _child_ids(raw_text), + } + ) + + +def _claude_result_text(value: object) -> tuple[str | None, str | None]: + if isinstance(value, str): + return value, None + if isinstance(value, list): + parts = [ + block["text"] + for block in value + if isinstance(block, Mapping) + and block.get("type") == "text" + and isinstance(block.get("text"), str) + ] + if parts: + return "".join(parts), None + return None, "parent_return_text_missing" + + +def _codex_invocations( + records: Sequence[dict[str, Any]], child_id: str, capabilities: BackendCapabilities +) -> list[dict[str, Any]]: + calls: dict[str, dict[str, Any]] = {} + results: dict[str, list[dict[str, Any]]] = {} + seen_results: set[str] = set() + for entry in records: + if not entry["parent_record"]: + continue + payload = entry["record"].get("payload") + if not isinstance(payload, Mapping): + continue + call_id = payload.get("call_id") + if not isinstance(call_id, str) or not call_id: + continue + payload_type = payload.get("type") + if payload_type in {"function_call", "custom_tool_call"}: + _codex_call(entry, payload, call_id, calls, child_id) + elif payload_type in {"function_call_output", "custom_tool_call_output"}: + _codex_result(entry, payload, call_id, results, seen_results) + return _linked_invocations(calls, results, child_id, capabilities) + + +def _codex_call( + entry: Mapping[str, Any], + payload: Mapping[str, Any], + call_id: str, + calls: dict[str, dict[str, Any]], + child_id: str, +) -> None: + if call_id in calls: + return + arguments = payload.get("arguments", payload.get("args", payload.get("input"))) + prompt = _codex_prompt_text(arguments) + identities = _child_ids(payload) | _child_ids(arguments) + calls[call_id] = { + "invocation_id": call_id, + "turn_id": entry.get("turn_id"), + "model": entry.get("model"), + "prompt_text": prompt, + "prompt_reason": None if prompt is not None else "parent_prompt_text_missing", + "prompt_source_id": entry["source_id"], + "prompt_timestamp": entry.get("timestamp"), + "linked": identities == {child_id}, + } + + +def _codex_result( + entry: Mapping[str, Any], + payload: Mapping[str, Any], + call_id: str, + results: dict[str, list[dict[str, Any]]], + seen_results: set[str], +) -> None: + source_id = entry["source_id"] + if source_id in seen_results: + return + seen_results.add(source_id) + text, reason = _codex_result_text(payload.get("output")) + results.setdefault(call_id, []).append( + { + "text": text, + "reason": reason, + "source_id": source_id, + "timestamp": entry.get("timestamp"), + "identities": _child_ids(payload) | _child_ids(payload.get("output")), + "turn_id": entry.get("turn_id"), + } + ) + + +def _codex_result_text(output: object) -> tuple[str | None, str | None]: + if isinstance(output, Mapping): + structured = output + elif isinstance(output, str): + try: + decoded = json.loads(output) + except json.JSONDecodeError: + return output, None + if not isinstance(decoded, Mapping): + return output, None + structured = decoded + else: + return None, "parent_return_text_missing" + + identity_status_fields = {"child_id", "agent_id", "agentId", "agent_thread_id", "status"} + text_fields = {"text", "response_text"} + if set(structured) <= identity_status_fields | text_fields: + texts = [structured[key] for key in text_fields if isinstance(structured.get(key), str)] + if texts: + if len(set(texts)) != 1: + return None, "parent_return_text_ambiguous" + return texts[0], None + if set(structured) & identity_status_fields: + return None, "parent_return_acknowledgement_only" + if isinstance(output, str): + return output, None + return None, "parent_return_text_missing" + + +def _linked_invocations( + calls: Mapping[str, dict[str, Any]], + results: Mapping[str, list[dict[str, Any]]], + child_id: str, + capabilities: BackendCapabilities, +) -> list[dict[str, Any]]: + linked = [] + for invocation_id, call in calls.items(): + candidates = results.get(invocation_id, []) + if not _has_child_link(call, candidates, child_id): + continue + accepted = _linked_results(candidates, call, child_id, capabilities) + call["results"] = accepted + if candidates and not accepted: + call["return_reason"] = "parent_return_identity_mismatch" + linked.append(call) + return linked + + +def _has_child_link( + call: Mapping[str, Any], candidates: Sequence[Mapping[str, Any]], child_id: str +) -> bool: + return bool(call.get("linked")) or any( + result.get("identities") == {child_id} for result in candidates + ) + + +def _linked_results( + candidates: Sequence[dict[str, Any]], + call: Mapping[str, Any], + child_id: str, + capabilities: BackendCapabilities, +) -> list[dict[str, Any]]: + return [ + result + for result in candidates + if (not result["identities"] or result["identities"] == {child_id}) + and _same_invocation_turn(result, call, capabilities) + ] + + +def _same_invocation_turn( + result: Mapping[str, Any], call: Mapping[str, Any], capabilities: BackendCapabilities +) -> bool: + result_turn = result.get("turn_id") + call_turn = call.get("turn_id") + return ( + capabilities.supports_claude_format_stdout + or not result_turn + or not call_turn + or result_turn == call_turn + ) + + +def _child_ids(value: object) -> set[str]: + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError: + return set() + if not isinstance(value, Mapping): + return set() + return { + item + for key in ("child_id", "agent_id", "agentId", "agent_thread_id") + if isinstance((item := value.get(key)), str) and item + } + + +def _codex_prompt_text(arguments: object) -> str | None: + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except json.JSONDecodeError: + return None + if not isinstance(arguments, Mapping): + return None + texts = [ + arguments[key] for key in ("message", "prompt") if isinstance(arguments.get(key), str) + ] + return texts[0] if texts and len(set(texts)) == 1 else None + + +def _invocation_spans( + row: Mapping[str, Any], + invocation: Mapping[str, Any], + parent_complete: bool, + tokenizer_version: str | None, +) -> list[dict[str, Any]]: + provider = row.get("provider_used") + provider = provider if isinstance(provider, str) and provider else None + model = invocation.get("model") + encoding, encoding_name, encoding_reason = _parent_encoding(provider, model) + prompt_span = _span( + "parent_prompt_tokens", + invocation, + row, + provider, + model, + tokenizer_version, + encoding, + encoding_name, + encoding_reason, + invocation.get("prompt_text"), + invocation.get("prompt_reason"), + invocation.get("prompt_source_id"), + invocation.get("prompt_timestamp"), + ) + results = invocation.get("results") + if not isinstance(results, list) or not results: + return [ + prompt_span, + _missing_return_span( + row, + invocation, + provider, + model, + tokenizer_version, + encoding, + encoding_name, + encoding_reason, + parent_complete, + ), + ] + return [prompt_span] + [ + _span( + "subagent_return_tokens", + invocation, + row, + provider, + model, + tokenizer_version, + encoding, + encoding_name, + encoding_reason, + result.get("text"), + result.get("reason"), + result.get("source_id"), + result.get("timestamp"), + ) + for result in results + ] + + +def _missing_return_span( + row: Mapping[str, Any], + invocation: Mapping[str, Any], + provider: str | None, + model: object, + tokenizer_version: str | None, + encoding: Any | None, + encoding_name: str | None, + encoding_reason: str | None, + parent_complete: bool, +) -> dict[str, Any]: + reason = invocation.get("return_reason") or ( + "parent_transcript_incomplete" if not parent_complete else "parent_return_missing" + ) + return _span( + "subagent_return_tokens", + invocation, + row, + provider, + model, + tokenizer_version, + encoding, + encoding_name, + encoding_reason, + None, + reason, + None, + None, + ) + + +def _parent_encoding(provider: object, model: object) -> tuple[Any | None, str | None, str | None]: + if not isinstance(provider, str) or provider.casefold() != "openai": + return None, None, "unsupported_provider" + if not isinstance(model, str) or not model: + return None, None, "parent_model_missing" + try: + import tiktoken + + encoding = tiktoken.encoding_for_model(model) + except KeyError as exc: + logger.warning( + "parent_context_model_unsupported", + provider=provider, + model=model, + error=type(exc).__name__, + ) + return None, None, "unsupported_model" + except Exception as exc: + logger.warning( + "parent_context_tokenizer_unavailable", + provider=provider, + model=model, + error=type(exc).__name__, + exc_info=True, + ) + return None, None, "tokenizer_initialization_failed" + name = getattr(encoding, "name", None) + if not isinstance(name, str) or not name: + logger.warning("parent_context_encoding_name_missing", provider=provider, model=model) + return None, None, "tokenizer_initialization_failed" + return encoding, name, None + + +def _span( + field: str, + invocation: Mapping[str, Any], + row: Mapping[str, Any], + provider: str | None, + model: object, + tokenizer_version: str | None, + encoding: Any | None, + encoding_name: str | None, + encoding_reason: str | None, + text: object, + missing_reason: object, + source_id: object, + timestamp: object, +) -> dict[str, Any]: + reason = missing_reason if isinstance(missing_reason, str) else None + measure = TokenMeasure.unavailable() + if reason is None: + if not isinstance(text, str): + reason = "parent_text_missing" + elif encoding_reason: + reason = encoding_reason + elif encoding is None: + reason = "tokenizer_initialization_failed" + else: + try: + measure = TokenMeasure.observed(len(encoding.encode_ordinary(text))) + except Exception as exc: + logger.warning( + "parent_context_text_encoding_failed", + provider=provider, + model=model, + error=type(exc).__name__, + exc_info=True, + ) + reason = "tokenizer_encoding_failed" + invocation_id = invocation.get("invocation_id") + turn_id = invocation.get("turn_id") + return { + "field": field, + "measure": measure.to_dict(), + "invocation_id": invocation_id if isinstance(invocation_id, str) else None, + "turn_id": turn_id if isinstance(turn_id, str) else None, + "source_id": source_id if isinstance(source_id, str) else None, + "timestamp": timestamp if isinstance(timestamp, str) else None, + "harness": row.get("backend") if isinstance(row.get("backend"), str) else None, + "provider": provider, + "model": model if isinstance(model, str) else None, + "tokenizer_version": tokenizer_version, + "encoding": encoding_name, + "reason": reason, + } + + +def tokenizer_version() -> str | None: + try: + return package_version("tiktoken") + except PackageNotFoundError: + return None diff --git a/src/autoskillit/execution/evidence/report_walk.py b/src/autoskillit/execution/evidence/report_walk.py index 126597b8b7..e85676ef86 100644 --- a/src/autoskillit/execution/evidence/report_walk.py +++ b/src/autoskillit/execution/evidence/report_walk.py @@ -10,14 +10,23 @@ import copy import hashlib import json +import math import os +import stat from collections.abc import Iterator, Mapping, Sequence from contextlib import ExitStack from dataclasses import dataclass from pathlib import Path -from typing import Any, BinaryIO, Final - -from autoskillit.core import ArtifactLease, get_logger, iter_merged_assistant_turns +from typing import Any, BinaryIO, Final, TypeGuard + +from autoskillit.core import ( + TOKEN_USAGE_SCHEMA_VERSION, + TURN_USAGE_SCHEMA_VERSION, + ArtifactLease, + TokenMeasure, + get_logger, + iter_merged_assistant_turns, +) from autoskillit.execution.backends._codex_parse import _logical_rollout_reader from autoskillit.execution.child_outcomes import ( enumerate_claude_subagent_transcripts, @@ -48,6 +57,27 @@ _ARCHIVE_BOUNDARY_CHANGED = "Session archive boundary changed" _PROJECTION_DELETED_NO_RECORDS = "Session archive disappeared" +_TURN_USAGE_FILENAME = "turn_usage.jsonl" +_TURN_USAGE_MEASURE_FIELDS = ( + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cache_write_tokens", + "peak_context", +) +_TURN_USAGE_REQUIRED_FIELDS = frozenset( + { + "backend", + "provider_used", + "message_id", + "request_id", + "timestamp", + "model", + *_TURN_USAGE_MEASURE_FIELDS, + "context_window_tokens", + "context_fraction", + } +) # Watermark keys identifying the sources the report walker tracks. Shared with # ``report_index.py`` so the gap source and the reset table agree on the same @@ -363,12 +393,190 @@ def _session_record( "transcripts_available": not unavailable, "transcript_unavailable_reasons": unavailable_reasons, } + turn_usage_rows, turn_usage_state, turn_usage_reason = _read_turn_usage(row, log_root) + record.update( + { + "turn_usage_rows": turn_usage_rows, + "turn_usage_state": turn_usage_state, + "turn_usage_reason": turn_usage_reason, + } + ) record["child_outcomes"] = project_child_outcomes( row, log_root=log_root, parent_rows=parent_rows ) return record +def _read_turn_usage( + row: Mapping[str, Any], log_root: Path +) -> tuple[list[dict[str, Any]], str, str | None]: + dir_name = row.get("dir_name") + if not _valid_session_dir_name(dir_name): + return [], "unavailable", "descriptor-invalid-session-directory" + + session_dir = log_root / "sessions" / dir_name + count, filename, reason = _read_turn_usage_descriptor(session_dir / "token_usage.json") + if reason is not None: + return [], "unavailable", reason + if filename is None or count is None: + return [], "unavailable", "descriptor-malformed" + rows, reason = _read_turn_usage_rows(session_dir / filename, count) + if reason is not None: + return [], "unavailable", reason + return rows, "observed", None + + +def _valid_session_dir_name(value: object) -> TypeGuard[str]: + return ( + isinstance(value, str) + and bool(value) + and "\x00" not in value + and value not in (".", "..") + and Path(value).name == value + ) + + +def _read_turn_usage_descriptor( + path: Path, +) -> tuple[int | None, str | None, str | None]: + try: + with path.open("r", encoding="utf-8") as handle: + descriptor = json.load(handle) + except FileNotFoundError: + return None, None, "descriptor-missing" + except (OSError, UnicodeError, json.JSONDecodeError): + return None, None, "descriptor-malformed" + return _admit_turn_usage_descriptor(descriptor) + + +def _admit_turn_usage_descriptor( + descriptor: object, +) -> tuple[int | None, str | None, str | None]: + if not isinstance(descriptor, dict): + return None, None, "descriptor-malformed" + reason = _schema_version_reason( + descriptor.get("schema_version"), + TOKEN_USAGE_SCHEMA_VERSION, + "descriptor-unsupported-version", + ) + if reason is not None: + return None, None, reason + reason = _schema_version_reason( + descriptor.get("turn_usage_schema_version"), + TURN_USAGE_SCHEMA_VERSION, + "ledger-unsupported-version", + ) + if reason is not None: + return None, None, reason + count = descriptor.get("turn_usage_count") + if not _nonnegative_integer(count): + return None, None, "descriptor-malformed" + filename = descriptor.get("turn_usage_file") + if filename is None: + reason = "ledger-not-published" if count == 0 else "descriptor-malformed" + return count, None, reason + if not _valid_turn_usage_filename(filename): + return None, None, "descriptor-invalid-ledger-file" + return count, filename, None + + +def _schema_version_reason(value: object, expected: int, unsupported_reason: str) -> str | None: + if not _nonnegative_integer(value): + return "descriptor-malformed" + return unsupported_reason if value != expected else None + + +def _nonnegative_integer(value: object) -> TypeGuard[int]: + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def _valid_turn_usage_filename(value: object) -> TypeGuard[str]: + return isinstance(value, str) and Path(value).name == value and value == _TURN_USAGE_FILENAME + + +def _read_turn_usage_rows(path: Path, count: int) -> tuple[list[dict[str, Any]], str | None]: + rows: list[dict[str, Any]] = [] + try: + with path.open("r", encoding="utf-8") as handle: + for line in handle: + if not line.endswith("\n") or not line.strip(): + return [], "ledger-malformed" + raw = json.loads(line) + decoded = _decode_turn_usage_row(raw) + if decoded is None: + return [], "ledger-malformed" + rows.append(decoded) + except FileNotFoundError: + return [], "ledger-missing" + except (OSError, UnicodeError): + return [], "ledger-unreadable" + except (json.JSONDecodeError, ValueError, TypeError): + return [], "ledger-malformed" + if len(rows) != count: + return [], "ledger-count-mismatch" + return rows, None + + +def _decode_turn_usage_row(raw: object) -> dict[str, Any] | None: + if not isinstance(raw, dict): + return None + if _TURN_USAGE_REQUIRED_FIELDS.difference(raw): + return None + if not _valid_turn_usage_metadata(raw): + return None + measures = _decode_turn_usage_measures(raw) + if measures is None: + return None + decoded = dict(raw) + decoded.update(measures) + return decoded + + +def _valid_turn_usage_metadata(raw: Mapping[str, Any]) -> bool: + text_fields = ("backend", "provider_used") + nullable_text_fields = ("message_id", "request_id", "timestamp", "model") + return ( + all(_nonempty_text(raw[field]) for field in text_fields) + and all(_nullable_text(raw[field]) for field in nullable_text_fields) + and _positive_integer_or_none(raw["context_window_tokens"]) + and _finite_nonnegative_number_or_none(raw["context_fraction"]) + ) + + +def _nonempty_text(value: object) -> bool: + return isinstance(value, str) and bool(value) + + +def _nullable_text(value: object) -> bool: + return value is None or isinstance(value, str) + + +def _positive_integer_or_none(value: object) -> bool: + return value is None or (isinstance(value, int) and not isinstance(value, bool) and value > 0) + + +def _finite_nonnegative_number_or_none(value: object) -> bool: + return value is None or ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and _finite_nonnegative_number(value) + ) + + +def _decode_turn_usage_measures(raw: Mapping[str, Any]) -> dict[str, TokenMeasure] | None: + try: + return {field: TokenMeasure.from_dict(raw[field]) for field in _TURN_USAGE_MEASURE_FIELDS} + except ValueError: + return None + + +def _finite_nonnegative_number(value: int | float) -> bool: + try: + return math.isfinite(float(value)) and value >= 0 + except (OverflowError, ValueError): + return False + + def _walk_archive( root: Path, state: dict[str, Any], @@ -484,8 +692,64 @@ def iter_report_walk( yield from _walk_projection(log_root, state, parent_rows) -def child_evidence_fingerprint(log_root: Path) -> str: - """Return the current fingerprint of owner-attributed native child evidence.""" +def _fingerprint_file(digest: Any, path: Path) -> None: + try: + file_stat = path.lstat() + except FileNotFoundError: + digest.update(b"missing\0") + return + except OSError as exc: + digest.update(f"stat-error:{exc.errno}\0".encode()) + return + if not stat.S_ISREG(file_stat.st_mode): + digest.update(f"non-regular:{file_stat.st_mode}\0".encode()) + return + digest.update( + f"file:{file_stat.st_dev}:{file_stat.st_ino}:{file_stat.st_mode}:" + f"{file_stat.st_size}:{file_stat.st_mtime_ns}:" + f"{file_stat.st_ctime_ns}\0".encode() + ) + try: + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(64 * 1024), b""): + digest.update(chunk) + except OSError as exc: + digest.update(f"read-error:{exc.errno}\0".encode()) + + +def _descriptor_admits_turn_ledger(descriptor_path: Path) -> bool: + _, filename, reason = _read_turn_usage_descriptor(descriptor_path) + return filename is not None and reason is None + + +def report_evidence_fingerprint(log_root: Path) -> str: + """Fingerprint native-child evidence and retained per-turn usage sources.""" rows = _retained_session_rows(log_root) parent_rows = _native_parent_rows(rows) - return _native_child_evidence_fingerprint(log_root, rows, parent_rows) + digest = hashlib.sha256() + digest.update(_native_child_evidence_fingerprint(log_root, rows, parent_rows).encode("ascii")) + sessions_dir = log_root / "sessions" + for row in rows: + dir_name = row.get("dir_name") + if not isinstance(dir_name, str): + continue + digest.update(b"\0session\0") + digest.update(dir_name.encode("utf-8", errors="surrogatepass")) + if ( + not dir_name + or "\x00" in dir_name + or dir_name in (".", "..") + or Path(dir_name).name != dir_name + ): + digest.update(b"\0invalid-directory\0") + continue + session_dir = sessions_dir / dir_name + digest.update(b"\0descriptor\0") + descriptor_path = session_dir / "token_usage.json" + _fingerprint_file(digest, descriptor_path) + if _descriptor_admits_turn_ledger(descriptor_path): + digest.update(b"\0admitted-turn-ledger\0") + _fingerprint_file(digest, session_dir / _TURN_USAGE_FILENAME) + else: + digest.update(b"\0no-admitted-turn-ledger\0") + return digest.hexdigest() diff --git a/src/autoskillit/execution/report_index.py b/src/autoskillit/execution/report_index.py index a3bfd207c0..a381ea7e68 100644 --- a/src/autoskillit/execution/report_index.py +++ b/src/autoskillit/execution/report_index.py @@ -33,18 +33,22 @@ ) from autoskillit.execution._report_index_rows import ( REPORT_INDEX_SCHEMA_VERSION, + UNKNOWN_SOURCE, + normalize_report_row, + resolve_token_measure, + rows_for_walk_item, +) +from autoskillit.execution._report_index_types import ( REQUEST_KIND, SESSION_KIND, SUBAGENT_KIND, TOOL_KIND, - UNKNOWN_SOURCE, + TURN_KIND, ReportRequestRow, ReportSessionRow, ReportSubagentRow, ReportToolRow, - normalize_report_row, - resolve_token_measure, - rows_for_walk_item, + ReportTurnRow, ) from autoskillit.execution.evidence.report_walk import ( CHECKPOINT_WALK_KIND, @@ -53,8 +57,8 @@ VALID_SOURCE_KEYS, SourceGapError, WalkItem, - child_evidence_fingerprint, iter_report_walk, + report_evidence_fingerprint, ) from autoskillit.execution.session_log.session_index import ( iter_tolerant_session_index_lines, @@ -101,6 +105,7 @@ class ReportIndex: requests: dict[str, ReportRequestRow] tools: dict[str, ReportToolRow] subagents: dict[str, ReportSubagentRow] + turns: dict[str, ReportTurnRow] def report_index_dir(log_root: Path) -> Path: @@ -128,19 +133,19 @@ def _update(log_root: Path, index_dir: Path, *, rebuild: bool) -> ReportIndexUpd with ArtifactLease.acquire_exclusive(index_dir / _LOCK_FILE, timeout=_LEASE_TIMEOUT_SECONDS): if rebuild: _clear_index_files(index_dir) - fingerprint = child_evidence_fingerprint(log_root) + fingerprint = report_evidence_fingerprint(log_root) state = read_versioned_json(index_dir / _STATE_FILE, _STATE_SCHEMA_VERSION) row_schema_mismatch = ( state.get("row_schema_version") != REPORT_INDEX_SCHEMA_VERSION if state is not None else (index_dir / _ROWS_FILE).exists() ) - child_evidence_changed = ( - state is not None and state.get("child_evidence_fingerprint") != fingerprint + report_evidence_changed = ( + state is not None and state.get("report_evidence_fingerprint") != fingerprint ) - if row_schema_mismatch or child_evidence_changed: + if row_schema_mismatch or report_evidence_changed: _clear_index_files(index_dir) - with _RowAppender.open(index_dir, child_evidence_fingerprint=fingerprint) as appender: + with _RowAppender.open(index_dir, report_evidence_fingerprint=fingerprint) as appender: return _walk_into(log_root, appender) @@ -289,12 +294,12 @@ def __init__( index_dir: Path, handle: BinaryIO, watermark: dict[str, Any] | None, - child_evidence_fingerprint: str, + report_evidence_fingerprint: str, ) -> None: self.index_dir = index_dir self.handle = handle self.watermark = watermark - self.child_evidence_fingerprint = child_evidence_fingerprint + self.report_evidence_fingerprint = report_evidence_fingerprint self._pending = bytearray() self._dirty = False self.items_walked = 0 @@ -302,7 +307,7 @@ def __init__( @classmethod @contextmanager - def open(cls, index_dir: Path, *, child_evidence_fingerprint: str) -> Iterator[_RowAppender]: + def open(cls, index_dir: Path, *, report_evidence_fingerprint: str) -> Iterator[_RowAppender]: rows_path = index_dir / _ROWS_FILE watermark, committed = _committed_state(index_dir, rows_path) fd = os.open( @@ -313,7 +318,7 @@ def open(cls, index_dir: Path, *, child_evidence_fingerprint: str) -> Iterator[_ with os.fdopen(fd, "r+b") as handle: handle.truncate(committed) handle.seek(committed) - yield cls(index_dir, handle, watermark, child_evidence_fingerprint) + yield cls(index_dir, handle, watermark, report_evidence_fingerprint) def add(self, item: WalkItem) -> None: for row in rows_for_walk_item(item): @@ -343,7 +348,7 @@ def commit(self, *, force: bool = False) -> None: "walk": self.watermark, "rows_bytes": self.handle.tell(), "row_schema_version": REPORT_INDEX_SCHEMA_VERSION, - "child_evidence_fingerprint": self.child_evidence_fingerprint, + "report_evidence_fingerprint": self.report_evidence_fingerprint, }, schema_version=_STATE_SCHEMA_VERSION, ) @@ -387,6 +392,7 @@ def _read_rows(rows_path: Path) -> dict[str, dict[str, dict[str, Any]]]: REQUEST_KIND: {}, TOOL_KIND: {}, SUBAGENT_KIND: {}, + TURN_KIND: {}, } for _, raw in iter_tolerant_session_index_lines(rows_path, complete_only=True): row = normalize_report_row(raw) @@ -423,7 +429,7 @@ def session_for(self, session_id: str | None, time_ms: int | None) -> str | None if time_ms is None: return entries[0][1] if len(entries) == 1 else None index = bisect_right(self._starts[session_id], time_ms) - 1 - return entries[0][1] if index < 0 else entries[index][1] + return None if index < 0 else entries[index][1] def _resolve_session_measures(sessions: dict[str, dict[str, Any]]) -> None: @@ -496,6 +502,7 @@ def read_report_index(index_dir: Path) -> ReportIndex: requests = by_kind[REQUEST_KIND] tools = by_kind[TOOL_KIND] subagents = by_kind[SUBAGENT_KIND] + turns = by_kind[TURN_KIND] _resolve_session_measures(sessions) attribution = _SessionAttribution(sessions) _join_request_rows(requests, sessions, attribution) @@ -506,4 +513,5 @@ def read_report_index(index_dir: Path) -> ReportIndex: requests=cast("dict[str, ReportRequestRow]", _sorted_rows(requests)), tools=cast("dict[str, ReportToolRow]", _sorted_rows(tools)), subagents=cast("dict[str, ReportSubagentRow]", _sorted_rows(subagents)), + turns=cast("dict[str, ReportTurnRow]", _sorted_rows(turns)), ) diff --git a/src/autoskillit/report/deck/_context.py b/src/autoskillit/report/deck/_context.py new file mode 100644 index 0000000000..77996ba81d --- /dev/null +++ b/src/autoskillit/report/deck/_context.py @@ -0,0 +1,547 @@ +from __future__ import annotations + +import math +from collections import Counter +from collections.abc import Mapping, Sequence +from typing import Any + +from autoskillit.core import ( + MeasureRecord, + SourcePair, + TokenMeasure, + TokenMeasureState, +) + +from ._measure_helpers import ( + TOKEN_FIELDS, + _aggregate_fields, + _decode_measure, + _ratio_field, + _source_pair, +) +from ._view_common import _time_in_window, _timestamp_ms + +_SpanGroupKey = tuple[str, str, str | None, str | None, str | None, str | None] + + +def _context_turn_rows( + sessions: Sequence[Mapping[str, Any]], + turns: Sequence[Mapping[str, Any]], + requests: Sequence[Mapping[str, Any]], + *, + generated_at_ms: int, +) -> list[dict[str, Any]]: + parents: dict[str, Mapping[str, Any]] = { + key: row for row in sessions if isinstance((key := row.get("key")), str) + } + request_lookup: dict[tuple[str, str], Mapping[str, Any]] = { + (session_key, request_id): row + for row in requests + if isinstance((session_key := row.get("session_key")), str) + and isinstance((request_id := row.get("request_id")), str) + } + out: list[dict[str, Any]] = [] + turn_owners: set[str] = set() + for raw in turns: + owner_key = raw.get("session_key") + if not isinstance(owner_key, str): + continue + turn_owners.add(owner_key) + parent = parents.get(owner_key) + request_id = raw.get("request_id") + request = ( + request_lookup.get((owner_key, request_id)) if isinstance(request_id, str) else None + ) + pair = _source_pair(raw) + cache_read = _decode_measure(raw, "cache_read_tokens") + window = raw.get("context_window_tokens") + denominator = ( + TokenMeasure.observed(window) + if isinstance(window, int) and not isinstance(window, bool) and window > 0 + else TokenMeasure.unknown() + ) + ratio = _ratio_field( + [ + MeasureRecord( + pair, + { + "cache_read_tokens": cache_read, + "context_window_tokens": denominator, + }, + ) + ], + "cache_read_tokens", + "context_window_tokens", + eligible_count=1, + ) + stored_fraction = raw.get("context_fraction") + disagreement = ( + not math.isclose(stored_fraction, ratio["value"], rel_tol=1e-6, abs_tol=1e-9) + if isinstance(stored_fraction, (float, int)) + and not isinstance(stored_fraction, bool) + and ratio["value"] is not None + else None + ) + out.append( + { + **dict(raw), + "kind": "turn", + "owner_time_ms": parent.get("time_ms") if parent else None, + "level": raw.get("level", parent.get("level") if parent else None), + "skill": raw.get("skill", parent.get("skill") if parent else None), + "recipe": raw.get("recipe", parent.get("recipe") if parent else None), + "step": raw.get("step", parent.get("step") if parent else None), + "context_fraction_percent": { + "state": ratio["state"], + "value": ratio["value"] * 100 if ratio["value"] is not None else None, + "sample_size": ratio["sample_size"], + "excluded_runs": ratio["excluded_runs"], + "unknown_runs": ratio["unknown_runs"], + }, + "fraction_disagreement": disagreement, + "turn_usage_state": "observed", + "turn_usage_reason": parent.get("turn_usage_reason") if parent else None, + "request_recorded": request is not None, + "request_model": request.get("model") if request else None, + } + ) + for owner_key, parent in parents.items(): + if owner_key in turn_owners: + continue + out.append( + { + "key": f"{owner_key}:turn-coverage", + "session_key": owner_key, + "ordinal": None, + "time_ms": parent.get("time_ms"), + "owner_time_ms": parent.get("time_ms"), + "harness": parent.get("harness"), + "provider": parent.get("provider"), + "model": parent.get("model"), + "skill": parent.get("skill"), + "recipe": parent.get("recipe"), + "step": parent.get("step"), + "level": parent.get("level"), + "kind": "coverage", + **{field: TokenMeasure.unknown().to_dict() for field in TOKEN_FIELDS}, + "context_window_tokens": None, + "context_fraction": None, + "context_fraction_percent": { + "state": "no_observations", + "value": None, + "sample_size": 0, + "excluded_runs": 0, + "unknown_runs": 0, + }, + "fraction_disagreement": None, + "turn_usage_state": parent.get("turn_usage_state") or "unavailable", + "turn_usage_reason": parent.get("turn_usage_reason"), + "request_recorded": False, + "request_model": None, + } + ) + return sorted( + out, + key=lambda row: ( + row.get("owner_time_ms") is None, + row.get("owner_time_ms") or 0, + row.get("session_key") or "", + row.get("ordinal") if isinstance(row.get("ordinal"), int) else -1, + ), + ) + + +def _group_for_span( + groups: dict[_SpanGroupKey, dict[str, Any]], + owner: Mapping[str, Any], + model: object, + *, + harness: object = None, + provider: object = None, +) -> dict[str, Any]: + owner_pair = _source_pair(owner) + pair = SourcePair( + harness if isinstance(harness, str) and harness else owner_pair.harness, + provider if isinstance(provider, str) and provider else owner_pair.provider, + ) + resolved_model = model if isinstance(model, str) and model else None + skill = owner.get("skill") if isinstance(owner.get("skill"), str) else None + recipe = owner.get("recipe") if isinstance(owner.get("recipe"), str) else None + step = owner.get("step") if isinstance(owner.get("step"), str) else None + key = (pair.harness, pair.provider, resolved_model, skill, recipe, step) + if key not in groups: + groups[key] = { + "pair": pair, + "model": resolved_model, + "skill": skill, + "recipe": recipe, + "step": step, + "level": owner.get("level"), + "spans": {"parent_prompt_tokens": [], "subagent_return_tokens": []}, + "invocations": {}, + "eligible_invocations": set(), + "eligible_parent_keys": set(), + "untimed_excluded": 0, + "provenance": [], + } + return groups[key] + + +def _span_provenance( + measure: TokenMeasure, + raw_span: Mapping[str, Any], + child: Mapping[str, Any], + parent_key: str, + child_key: str, + span_time: int | None, +) -> dict[str, Any]: + reason = raw_span.get("reason") + if not isinstance(reason, str) and measure.state not in ( + TokenMeasureState.MEASURED, + TokenMeasureState.MEASURED_ZERO, + ): + reason = "source_reason_unrecorded" + return { + "measure": measure.to_dict(), + "invocation_id": raw_span.get("invocation_id"), + "parent_session_key": parent_key, + "child_key": child_key, + "turn_id": raw_span.get("turn_id"), + "source_id": raw_span.get("source_id"), + "timestamp": raw_span.get("timestamp"), + "timestamp_ms": span_time, + "field": raw_span.get("field"), + "reason": reason, + "tokenizer_version": raw_span.get("tokenizer_version"), + "encoding": raw_span.get("encoding"), + "child_role": child.get("role"), + "child_harness": child.get("harness"), + "child_provider": child.get("provider"), + "child_model": child.get("model"), + } + + +def _add_untimed_span( + groups: dict[_SpanGroupKey, dict[str, Any]], + *, + owner: Mapping[str, Any], + raw_span: Mapping[str, Any], + child: Mapping[str, Any], + parent_key: str, + child_key: str, + measure: TokenMeasure, +) -> None: + group = _group_for_span( + groups, + owner, + raw_span.get("model"), + harness=raw_span.get("harness"), + provider=raw_span.get("provider"), + ) + group["untimed_excluded"] += 1 + provenance = _span_provenance(measure, raw_span, child, parent_key, child_key, None) + provenance["reason"] = "untimed_observation_excluded" + group["provenance"].append(provenance) + + +def _add_selected_span( + groups: dict[_SpanGroupKey, dict[str, Any]], + *, + owners_with_spans: set[str], + eligible_keys: set[str], + owner: Mapping[str, Any], + raw_span: Mapping[str, Any], + child: Mapping[str, Any], + parent_key: str, + child_key: str, + measure: TokenMeasure, + span_time: int | None, +) -> None: + field = raw_span["field"] + group = _group_for_span( + groups, + owner, + raw_span.get("model"), + harness=raw_span.get("harness"), + provider=raw_span.get("provider"), + ) + owners_with_spans.add(parent_key) + if parent_key in eligible_keys: + group["eligible_parent_keys"].add(parent_key) + invocation_id = raw_span.get("invocation_id") + group["eligible_invocations"].add((parent_key, child_key, invocation_id)) + provenance = _span_provenance(measure, raw_span, child, parent_key, child_key, span_time) + group["spans"][field].append({**provenance, "measure": measure}) + group["provenance"].append(provenance) + if isinstance(invocation_id, str) and invocation_id: + invocation_key = (parent_key, child_key, invocation_id) + invocation = group["invocations"].setdefault( + invocation_key, + {"parent_prompt_tokens": [], "subagent_return_tokens": []}, + ) + invocation[field].append(measure) + + +def _collect_candidate_span( + groups: dict[_SpanGroupKey, dict[str, Any]], + *, + owners_with_spans: set[str], + eligible_keys: set[str], + owner: Mapping[str, Any], + child: Mapping[str, Any], + raw_span: object, + parent_key: str, + child_key: str, + window: str, + generated_at_ms: int, +) -> None: + if not isinstance(raw_span, Mapping): + return + field = raw_span.get("field") + if field not in ("parent_prompt_tokens", "subagent_return_tokens"): + return + measure = _decode_measure(raw_span, "measure") + span_time = _timestamp_ms(raw_span.get("timestamp")) + if span_time is not None and not _time_in_window( + span_time, window, generated_at_ms=generated_at_ms + ): + return + if span_time is None and measure.state in ( + TokenMeasureState.MEASURED, + TokenMeasureState.MEASURED_ZERO, + ): + _add_untimed_span( + groups, + owner=owner, + raw_span=raw_span, + child=child, + parent_key=parent_key, + child_key=child_key, + measure=measure, + ) + return + if span_time is None and parent_key not in eligible_keys: + return + _add_selected_span( + groups, + owners_with_spans=owners_with_spans, + eligible_keys=eligible_keys, + owner=owner, + raw_span=raw_span, + child=child, + parent_key=parent_key, + child_key=child_key, + measure=measure, + span_time=span_time, + ) + + +def _child_key(child: Mapping[str, Any]) -> str: + value = child.get("key") + if isinstance(value, str): + return value + child_id = child.get("child_id") + return child_id if isinstance(child_id, str) else "child" + + +def _collect_child_spans( + groups: dict[_SpanGroupKey, dict[str, Any]], + *, + owners_with_spans: set[str], + eligible_keys: set[str], + owner: Mapping[str, Any], + child: Mapping[str, Any], + parent_key: str, + window: str, + generated_at_ms: int, +) -> None: + child_key = _child_key(child) + raw_spans = child.get("parent_context_spans") + spans = raw_spans if isinstance(raw_spans, (list, tuple)) else () + if not spans: + if parent_key in eligible_keys: + group = _group_for_span(groups, owner, None) + group["eligible_parent_keys"].add(parent_key) + group["eligible_invocations"].add((parent_key, child_key, None)) + return + for raw_span in spans: + _collect_candidate_span( + groups, + owners_with_spans=owners_with_spans, + eligible_keys=eligible_keys, + owner=owner, + child=child, + raw_span=raw_span, + parent_key=parent_key, + child_key=child_key, + window=window, + generated_at_ms=generated_at_ms, + ) + + +def _collect_parent_groups( + eligible_sessions: Sequence[Mapping[str, Any]], + owners: Mapping[str, Mapping[str, Any]], + children: Sequence[Mapping[str, Any]], + *, + window: str, + generated_at_ms: int, +) -> dict[_SpanGroupKey, dict[str, Any]]: + eligible_keys = {key for row in eligible_sessions if isinstance((key := row.get("key")), str)} + groups: dict[_SpanGroupKey, dict[str, Any]] = {} + owners_with_spans: set[str] = set() + for child in children: + parent_key = child.get("parent_session_key") + if not isinstance(parent_key, str): + continue + owner = owners.get(parent_key) + if owner is None: + continue + _collect_child_spans( + groups, + owners_with_spans=owners_with_spans, + eligible_keys=eligible_keys, + owner=owner, + child=child, + parent_key=parent_key, + window=window, + generated_at_ms=generated_at_ms, + ) + for owner in eligible_sessions: + owner_key = owner.get("key") + if isinstance(owner_key, str) and owner_key not in owners_with_spans: + _group_for_span(groups, owner, None)["eligible_parent_keys"].add(owner_key) + return groups + + +def _field_span_summary( + group: Mapping[str, Any], + field: str, + *, + eligible_invocations: int, +) -> dict[str, Any]: + records = [ + MeasureRecord(group["pair"], {field: span["measure"]}) for span in group["spans"][field] + ] + observed = [ + record + for record in records + if record.measures[field].state + in (TokenMeasureState.MEASURED, TokenMeasureState.MEASURED_ZERO) + ] + summary = _aggregate_fields(records, (field,), eligible_count=eligible_invocations)[field] + summary["observed_subset"] = _aggregate_fields( + observed, (field,), eligible_count=eligible_invocations + )[field] + summary["reason_counts"] = dict( + Counter( + span["reason"] + for span in group["provenance"] + if span.get("field") == field and isinstance(span.get("reason"), str) + ) + ) + return summary + + +def _invocation_measure( + pair: SourcePair, values: Sequence[TokenMeasure], field: str +) -> TokenMeasure | None: + if not values: + return None + summary = _aggregate_fields( + [MeasureRecord(pair, {field: value}) for value in values], (field,) + )[field] + return TokenMeasure.measure_from_raw({"state": summary["state"], "value": summary["value"]}) + + +def _matched_invocation( + pair: SourcePair, invocation: Mapping[str, Sequence[TokenMeasure]] +) -> MeasureRecord | None: + prompt = _invocation_measure(pair, invocation["parent_prompt_tokens"], "parent_prompt_tokens") + returned = _invocation_measure( + pair, invocation["subagent_return_tokens"], "subagent_return_tokens" + ) + if prompt is None or returned is None: + return None + if prompt.state not in (TokenMeasureState.MEASURED, TokenMeasureState.MEASURED_ZERO): + return None + if returned.state not in (TokenMeasureState.MEASURED, TokenMeasureState.MEASURED_ZERO): + return None + return MeasureRecord( + pair, + { + "parent_prompt_tokens": prompt, + "subagent_return_tokens": returned, + }, + ) + + +def _matched_invocations(group: Mapping[str, Any]) -> list[MeasureRecord]: + pair: SourcePair = group["pair"] + records = [] + for invocation in group["invocations"].values(): + record = _matched_invocation(pair, invocation) + if record is not None: + records.append(record) + return records + + +def _parent_metric_row(key: _SpanGroupKey, group: Mapping[str, Any]) -> dict[str, Any]: + harness, provider, model, skill, recipe, step = key + eligible = len(group["eligible_invocations"]) + prompt = _field_span_summary(group, "parent_prompt_tokens", eligible_invocations=eligible) + returned = _field_span_summary(group, "subagent_return_tokens", eligible_invocations=eligible) + matched = _matched_invocations(group) + comparison = _ratio_field( + matched, + "subagent_return_tokens", + "parent_prompt_tokens", + eligible_count=eligible, + ) + return { + "harness": harness, + "provider": provider, + "model": model, + "skill": skill, + "recipe": recipe, + "step": step, + "level": group["level"], + "prompt": prompt, + "returned": returned, + "comparison": comparison, + "eligible_invocations": eligible, + "matched_invocations": len(matched), + "eligible_parent_sessions": len(group["eligible_parent_keys"]), + "untimed_excluded": group["untimed_excluded"], + "provenance": group["provenance"], + "source_basis": ( + "Tokenizer counts of explicit recorded prompt/message and returned text " + "at the parent invocation model; argument metadata, hidden/system/framing " + "tokens, and unrelated parent work are excluded." + ), + } + + +def _parent_context_metrics( + eligible_sessions: Sequence[Mapping[str, Any]], + all_sessions: Sequence[Mapping[str, Any]], + children: Sequence[Mapping[str, Any]], + *, + levels: Sequence[str | None], + window: str, + generated_at_ms: int, +) -> list[dict[str, Any]]: + owners: dict[str, Mapping[str, Any]] = { + key: row + for row in all_sessions + if isinstance((key := row.get("key")), str) and row.get("level") in set(levels) + } + groups = _collect_parent_groups( + eligible_sessions, + owners, + children, + window=window, + generated_at_ms=generated_at_ms, + ) + ordered_keys = sorted(groups, key=lambda key: tuple(str(value or "") for value in key)) + return [_parent_metric_row(key, groups[key]) for key in ordered_keys] diff --git a/src/autoskillit/report/deck/_coverage.py b/src/autoskillit/report/deck/_coverage.py new file mode 100644 index 0000000000..742369ae7f --- /dev/null +++ b/src/autoskillit/report/deck/_coverage.py @@ -0,0 +1,763 @@ +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Mapping, Sequence +from typing import Any + +from autoskillit.core import ( + MeasureRecord, + SourcePair, + TokenMeasure, + TokenMeasureState, +) + +from ._context import _parent_context_metrics +from ._coverage_support import ( + _categorical_measure, + _coverage_row, +) +from ._errors import _selected_tool_events +from ._measure_helpers import ( + TOKEN_FIELDS, + _aggregate_fields, + _decode_measure, + _measure_records, + _outcome_record, + _ratio_field, + _ratio_state_measure, + _source_pair, +) +from ._view_common import _latest_time, _session_identity, _time_in_window + +_Identity = tuple[str, str, str | None, str | None, str | None] + + +def _time_basis( + observations: Sequence[Mapping[str, Any]], + owners: Sequence[Mapping[str, Any]], +) -> tuple[int | None, str]: + if observations: + time_ms = _latest_time(observations) + return time_ms, "observation_timestamp" if time_ms is not None else "untimed_observation" + return _latest_time(owners), "owner_session_eligibility" + + +def _session_coverage_rows( + identity: _Identity, + pair: SourcePair, + level: str | None, + sessions: Sequence[Mapping[str, Any]], + *, + session_key: str | None, +) -> list[dict[str, Any]]: + skill, recipe, step = identity[2:] + eligible = len(sessions) + time_ms = _latest_time(sessions) + rows: list[dict[str, Any]] = [] + session_fields = _aggregate_fields( + _measure_records(sessions), TOKEN_FIELDS, eligible_count=eligible + ) + for field in TOKEN_FIELDS: + rows.append( + _coverage_row( + pair=pair, + model=None, + level=level, + skill=skill, + recipe=recipe, + step=step, + population="session", + field=field, + measure=session_fields[field], + eligible_count=eligible, + observation_count=eligible, + source="indexed session token usage at owner session time", + time_ms=time_ms, + session_key=session_key, + ) + ) + + model_measure, model_counts, model_reason = _categorical_measure( + [row.get("model") for row in sessions] + ) + rows.append( + _coverage_row( + pair=pair, + model=None, + level=level, + skill=skill, + recipe=recipe, + step=step, + population="session", + field="resolved_model", + measure=model_measure, + eligible_count=eligible, + observation_count=eligible, + source="indexed session model at owner session time", + time_ms=time_ms, + reason=model_reason, + state_counts=model_counts, + session_key=session_key, + ) + ) + + outcomes = [_outcome_record(row) for row in sessions] + outcome = _aggregate_fields(outcomes, ("failure",), eligible_count=eligible)["failure"] + known = [record for record in outcomes if record.measures["known_attempt"].value == 1] + outcome_ratio = _ratio_field(known, "failure", "known_attempt", eligible_count=eligible) + rows.append( + _coverage_row( + pair=pair, + model=None, + level=level, + skill=skill, + recipe=recipe, + step=step, + population="session", + field="session_outcomes", + measure=outcome, + eligible_count=eligible, + observation_count=eligible, + source="indexed session success outcome at owner session time", + time_ms=time_ms, + reason=( + f"{outcome_ratio['unknown_runs']} session outcomes are unknown" + if outcome_ratio.get("unknown_runs") + else None + ), + session_key=session_key, + ) + ) + return rows + + +def _turn_records( + rows: Sequence[Mapping[str, Any]], pair: SourcePair, eligible_count: int +) -> list[MeasureRecord]: + records = [] + for row in rows: + measures = {field: _decode_measure(row, field) for field in TOKEN_FIELDS} + window = row.get("context_window_tokens") + measures["context_window_tokens"] = ( + TokenMeasure.observed(window) + if isinstance(window, int) and not isinstance(window, bool) and window > 0 + else TokenMeasure.unknown() + ) + records.append(MeasureRecord(pair, measures)) + return records + + +def _turn_measure_rows( + identity: _Identity, + pair: SourcePair, + level: str | None, + sessions: Sequence[Mapping[str, Any]], + turns: Sequence[Mapping[str, Any]], + *, + session_key: str | None, +) -> list[dict[str, Any]]: + skill, recipe, step = identity[2:] + eligible = len(sessions) + time_ms, time_basis = _time_basis(turns, sessions) + records = _turn_records(turns, pair, eligible) + aggregates = _aggregate_fields( + records, (*TOKEN_FIELDS, "context_window_tokens"), eligible_count=eligible + ) + rows = [] + for field in (*TOKEN_FIELDS, "context_window_tokens"): + reasons = sorted( + { + str(row.get("turn_usage_reason")) + for row in sessions + if isinstance(row.get("turn_usage_reason"), str) + } + ) + rows.append( + _coverage_row( + pair=pair, + model=None, + level=level, + skill=skill, + recipe=recipe, + step=step, + population="turn", + field=field, + measure=aggregates[field], + eligible_count=eligible, + observation_count=len(records), + source=( + f"turn_usage.jsonl model window at {time_basis.replace('_', ' ')}" + if field == "context_window_tokens" + else f"turn_usage.jsonl at {time_basis.replace('_', ' ')}" + ), + time_ms=time_ms, + reason="; ".join(reasons) if reasons else None, + session_key=session_key, + ) + ) + return rows + + +def _ledger_row( + identity: _Identity, + pair: SourcePair, + level: str | None, + sessions: Sequence[Mapping[str, Any]], + turns: Sequence[Mapping[str, Any]], + all_sessions: Sequence[Mapping[str, Any]], + *, + session_key: str | None, +) -> dict[str, Any]: + time_ms, time_basis = _time_basis(turns, sessions) + ledger_rows: dict[str, Mapping[str, Any]] = { + key: row for row in sessions if isinstance((key := row.get("key")), str) + } + owners: dict[str, Mapping[str, Any]] = { + key: row for row in all_sessions if isinstance((key := row.get("key")), str) + } + for turn in turns: + owner_key = turn.get("session_key") + if not isinstance(owner_key, str) or owner_key in ledger_rows: + continue + owner = owners.get(owner_key, turn) + ledger_rows[owner_key] = { + **dict(owner), + "turn_usage_state": "observed", + "turn_usage_reason": None, + } + records = [] + for row in ledger_rows.values(): + state = row.get("turn_usage_state") + measure = ( + TokenMeasure.observed(1) + if state == "observed" + else TokenMeasure.unavailable() + if state == "unavailable" + else TokenMeasure.unknown() + ) + records.append(MeasureRecord(pair, {"ledger": measure})) + ledger = _aggregate_fields(records, ("ledger",), eligible_count=len(sessions))["ledger"] + reasons = sorted( + { + str(row.get("turn_usage_reason")) + for row in sessions + if isinstance(row.get("turn_usage_reason"), str) + } + ) + return _coverage_row( + pair=pair, + model=None, + level=level, + skill=identity[2], + recipe=identity[3], + step=identity[4], + population="turn", + field="turn_series_coverage", + measure=ledger, + eligible_count=len(sessions), + observation_count=len(records), + source=f"turn_usage_state and turn_usage_reason at {time_basis.replace('_', ' ')}", + time_ms=time_ms, + reason="; ".join(reasons) if reasons else None, + session_key=session_key, + ) + + +def _occupancy_rows( + identity: _Identity, + pair: SourcePair, + level: str | None, + sessions: Sequence[Mapping[str, Any]], + turns: Sequence[Mapping[str, Any]], + *, + session_key: str | None, +) -> list[dict[str, Any]]: + skill, recipe, step = identity[2:] + eligible = len(sessions) + time_ms, time_basis = _time_basis(turns, sessions) + fraction_records = [] + disagreement_records = [] + ratio_records = [] + for row in turns: + cache_read = _decode_measure(row, "cache_read_tokens") + raw_window = row.get("context_window_tokens") + denominator = ( + TokenMeasure.observed(raw_window) + if isinstance(raw_window, int) and not isinstance(raw_window, bool) and raw_window > 0 + else TokenMeasure.unknown() + ) + ratio_record = MeasureRecord( + pair, + {"cache_read_tokens": cache_read, "context_window_tokens": denominator}, + ) + ratio = _ratio_field( + [ratio_record], + "cache_read_tokens", + "context_window_tokens", + eligible_count=1, + ) + fraction_records.append( + MeasureRecord( + pair, + {"context_fraction": _ratio_state_measure(TokenMeasureState(ratio["state"]))}, + ) + ) + ratio_records.append(ratio_record) + disagreement = row.get("fraction_disagreement") + disagreement_measure = ( + TokenMeasure.observed(1) + if disagreement is True + else TokenMeasure.observed(0) + if disagreement is False + else TokenMeasure.unknown() + ) + disagreement_records.append( + MeasureRecord(pair, {"fraction_disagreement": disagreement_measure}) + ) + fraction_states = _aggregate_fields( + fraction_records, ("context_fraction",), eligible_count=eligible + )["context_fraction"] + pooled = _ratio_field( + ratio_records, "cache_read_tokens", "context_window_tokens", eligible_count=eligible + ) + disagreement = _aggregate_fields( + disagreement_records, ("fraction_disagreement",), eligible_count=eligible + )["fraction_disagreement"] + reason = ( + "stored context_fraction disagrees with the recomputed per-turn ratio" + if isinstance(disagreement.get("value"), int) and disagreement["value"] > 0 + else None + ) + rows = [ + _coverage_row( + pair=pair, + model=None, + level=level, + skill=skill, + recipe=recipe, + step=step, + population="turn", + field="context_fraction", + measure={ + "state": pooled.get("state"), + "value": pooled.get("value"), + "state_counts": fraction_states["state_counts"], + }, + eligible_count=eligible, + observation_count=len(turns), + source=( + "per-turn cache_read_tokens/context_window_tokens ratio at " + f"{time_basis.replace('_', ' ')}" + ), + time_ms=time_ms, + session_key=session_key, + ), + _coverage_row( + pair=pair, + model=None, + level=level, + skill=skill, + recipe=recipe, + step=step, + population="turn", + field="context_fraction_validation", + measure=disagreement, + eligible_count=eligible, + observation_count=len(turns), + source=( + "stored context_fraction compared with recomputed ratio at " + f"{time_basis.replace('_', ' ')}" + ), + time_ms=time_ms, + reason=reason, + session_key=session_key, + ), + ] + return rows + + +def _tool_coverage_rows( + identity: _Identity, + pair: SourcePair, + level: str | None, + sessions: Sequence[Mapping[str, Any]], + events: Sequence[Mapping[str, Any]], + *, + session_key: str | None, +) -> list[dict[str, Any]]: + skill, recipe, step = identity[2:] + eligible = len(sessions) + time_ms, time_basis = _time_basis(events, sessions) + outcome_records = [_outcome_record(event) for event in events] + outcomes = _aggregate_fields(outcome_records, ("failure",), eligible_count=eligible)["failure"] + attribution_records = [ + MeasureRecord( + pair, + { + "skill_step": ( + TokenMeasure.observed(1) + if isinstance(event.get("skill"), str) and isinstance(event.get("step"), str) + else TokenMeasure.unknown() + ) + }, + ) + for event in events + ] + attribution = _aggregate_fields(attribution_records, ("skill_step",), eligible_count=eligible)[ + "skill_step" + ] + return [ + _coverage_row( + pair=pair, + model=None, + level=level, + skill=skill, + recipe=recipe, + step=step, + population="tool_result", + field="tool_errors", + measure=outcomes, + eligible_count=eligible, + observation_count=len(events), + session_key=session_key, + source=( + f"indexed tool_result success and error_type at {time_basis.replace('_', ' ')}" + ), + time_ms=time_ms, + reason=( + "no indexed tool_result events; absence does not establish zero errors" + if not events + else None + ), + ), + _coverage_row( + pair=pair, + model=None, + level=level, + skill=skill, + recipe=recipe, + step=step, + population="tool_result", + field="tool_skill_step_attribution", + measure=attribution, + eligible_count=eligible, + observation_count=len(events), + session_key=session_key, + source=( + "tool session_key joined to indexed session skill and step at " + f"{time_basis.replace('_', ' ')}" + ), + time_ms=time_ms, + reason="no attributed tool events" if not events else None, + ), + ] + + +def _parent_time_basis( + item: Mapping[str, Any], sessions: Sequence[Mapping[str, Any]] +) -> tuple[int | None, str]: + provenance = item.get("provenance", ()) + timestamped = [ + span + for span in provenance + if isinstance(span, Mapping) + and isinstance(span.get("timestamp_ms"), int) + and not isinstance(span.get("timestamp_ms"), bool) + ] + if timestamped: + return _latest_time(timestamped, "timestamp_ms"), "parent_span_timestamp" + if item.get("untimed_excluded"): + return None, "untimed_observation_excluded" + return _latest_time(sessions), "owner_session_eligibility" + + +def _parent_coverage_rows( + identity: _Identity, + pair: SourcePair, + level: str | None, + sessions: Sequence[Mapping[str, Any]], + metrics: Sequence[Mapping[str, Any]], + *, + session_key: str | None, +) -> list[dict[str, Any]]: + rows = [] + for item in metrics: + time_ms, time_basis = _parent_time_basis(item, sessions) + for field, measure in ( + ("parent_prompt_tokens", item["prompt"]), + ("subagent_return_tokens", item["returned"]), + ): + reasons = sorted( + { + str(span["reason"]) + for span in item.get("provenance", ()) + if isinstance(span.get("reason"), str) + } + ) + rows.append( + _coverage_row( + pair=pair, + model=item.get("model"), + level=level, + skill=identity[2], + recipe=identity[3], + step=identity[4], + population="parent_context", + field=field, + measure=measure, + eligible_count=item.get("eligible_parent_sessions", len(sessions)), + observation_count=measure.get("observation_count", 0), + source=(f"indexed parent_context_spans at {time_basis.replace('_', ' ')}"), + time_ms=time_ms, + reason=( + "; ".join(reasons) + if reasons + else "no verified parent-context invocation spans" + if measure.get("state") == "no_observations" + else None + ), + session_key=session_key, + ) + ) + return rows + + +def _identity_coverage_rows( + identity: _Identity, + sessions: Sequence[Mapping[str, Any]], + turns: Sequence[Mapping[str, Any]], + events: Sequence[Mapping[str, Any]], + parent_metrics: Sequence[Mapping[str, Any]], + all_sessions: Sequence[Mapping[str, Any]], + *, + session_key: str | None, +) -> list[dict[str, Any]]: + pair = SourcePair(identity[0], identity[1]) + level = next( + ( + row.get("level") + for row in (*sessions, *turns, *events) + if isinstance(row.get("level"), str) + ), + next( + (item.get("level") for item in parent_metrics if isinstance(item.get("level"), str)), + None, + ), + ) + return [ + *_session_coverage_rows(identity, pair, level, sessions, session_key=session_key), + *_turn_measure_rows(identity, pair, level, sessions, turns, session_key=session_key), + _ledger_row( + identity, + pair, + level, + sessions, + turns, + all_sessions, + session_key=session_key, + ), + *_occupancy_rows(identity, pair, level, sessions, turns, session_key=session_key), + *_tool_coverage_rows(identity, pair, level, sessions, events, session_key=session_key), + *_parent_coverage_rows( + identity, pair, level, sessions, parent_metrics, session_key=session_key + ), + ] + + +def _parent_identity(item: Mapping[str, Any]) -> _Identity: + harness = item.get("harness") + provider = item.get("provider") + return ( + harness if isinstance(harness, str) and harness else "unknown", + provider if isinstance(provider, str) and provider else "unknown", + item.get("skill") if isinstance(item.get("skill"), str) else None, + item.get("recipe") if isinstance(item.get("recipe"), str) else None, + item.get("step") if isinstance(item.get("step"), str) else None, + ) + + +def _unattributed_tool_rows( + events: Sequence[Mapping[str, Any]], +) -> list[dict[str, Any]]: + if not events: + return [] + groups: dict[SourcePair, list[Mapping[str, Any]]] = defaultdict(list) + for event in events: + groups[_source_pair(event)].append(event) + rows = [] + for pair, pair_events in groups.items(): + time_ms = _latest_time(pair_events) + outcome_records = [_outcome_record(event) for event in pair_events] + outcome = _aggregate_fields( + outcome_records, ("failure",), eligible_count=len(pair_events) + )["failure"] + attribution_records = [ + MeasureRecord(pair, {"skill_step": TokenMeasure.unavailable()}) for _ in pair_events + ] + attribution = _aggregate_fields( + attribution_records, ("skill_step",), eligible_count=len(pair_events) + )["skill_step"] + for field, field_measure, source in ( + ("tool_errors", outcome, "indexed unattributed tool_result outcomes"), + ( + "tool_skill_step_attribution", + attribution, + "tool_result rows without an exact owning session_key", + ), + ): + row = _coverage_row( + pair=pair, + model=None, + level=None, + skill=None, + recipe=None, + step=None, + population="unattributed_tool", + field=field, + measure=field_measure, + eligible_count=len(pair_events), + observation_count=len(pair_events), + source=( + f"{source} at event timestamp" + if time_ms is not None + else f"{source} as an untimed observation" + ), + time_ms=time_ms, + reason="tool event has no exact indexed owner; no pair or skill/step is inferred", + state_counts=field_measure["state_counts"], + ) + rows.append( + { + **row, + "key": f"unattributed_tool:{pair.harness}:{pair.provider}:{field}", + } + ) + return rows + + +def _coverage_selection( + eligible_sessions: Sequence[Mapping[str, Any]], + all_sessions: Sequence[Mapping[str, Any]], + turns: Sequence[Mapping[str, Any]], + tools: Sequence[Mapping[str, Any]], + children: Sequence[Mapping[str, Any]], + *, + levels: Sequence[str | None], + window: str, + generated_at_ms: int, + session_key: str | None, +) -> tuple[ + Sequence[Mapping[str, Any]], + list[Mapping[str, Any]], + list[Mapping[str, Any]], + Sequence[Mapping[str, Any]], +]: + level_set = set(levels) + if session_key is not None: + eligible_sessions = [ + row + for row in all_sessions + if row.get("key") == session_key + and row.get("level") in level_set + and _time_in_window(row.get("time_ms"), window, generated_at_ms=generated_at_ms) + ] + children = [row for row in children if row.get("parent_session_key") == session_key] + selected_turns = [ + row + for row in turns + if row.get("level") in level_set + and _time_in_window(row.get("time_ms"), window, generated_at_ms=generated_at_ms) + ] + selected_tools = _selected_tool_events( + tools, + all_sessions, + levels=levels, + window=window, + generated_at_ms=generated_at_ms, + ) + if session_key is not None: + selected_turns = [row for row in selected_turns if row.get("session_key") == session_key] + selected_tools = [row for row in selected_tools if row.get("session_key") == session_key] + return eligible_sessions, selected_turns, selected_tools, children + + +def _coverage_rows( + eligible_sessions: Sequence[Mapping[str, Any]], + all_sessions: Sequence[Mapping[str, Any]], + turns: Sequence[Mapping[str, Any]], + tools: Sequence[Mapping[str, Any]], + children: Sequence[Mapping[str, Any]], + *, + levels: Sequence[str | None], + window: str, + generated_at_ms: int, + session_key: str | None = None, +) -> list[dict[str, Any]]: + level_set = set(levels) + eligible_sessions, selected_turns, selected_tools, children = _coverage_selection( + eligible_sessions, + all_sessions, + turns, + tools, + children, + levels=levels, + window=window, + generated_at_ms=generated_at_ms, + session_key=session_key, + ) + + sessions_by_identity: dict[_Identity, list[Mapping[str, Any]]] = defaultdict(list) + turns_by_identity: dict[_Identity, list[Mapping[str, Any]]] = defaultdict(list) + events_by_identity: dict[_Identity, list[Mapping[str, Any]]] = defaultdict(list) + for row in eligible_sessions: + sessions_by_identity[_session_identity(row)].append(row) + for row in selected_turns: + turns_by_identity[_session_identity(row)].append(row) + for row in selected_tools: + if row.get("attribution_state") == "attributed": + events_by_identity[_session_identity(row)].append(row) + + parent_metrics = _parent_context_metrics( + eligible_sessions, + all_sessions, + children, + levels=levels, + window=window, + generated_at_ms=generated_at_ms, + ) + parents_by_identity: dict[_Identity, list[Mapping[str, Any]]] = defaultdict(list) + for item in parent_metrics: + parents_by_identity[_parent_identity(item)].append(item) + identities = ( + set(sessions_by_identity) + | set(turns_by_identity) + | set(events_by_identity) + | set(parents_by_identity) + ) + ordered = sorted( + identities, key=lambda identity: tuple(str(value or "") for value in identity) + ) + rows = [] + for identity in ordered: + rows.extend( + _identity_coverage_rows( + identity, + sessions_by_identity.get(identity, ()), + turns_by_identity.get(identity, ()), + events_by_identity.get(identity, ()), + parents_by_identity.get(identity, ()), + all_sessions, + session_key=session_key, + ) + ) + if session_key is None and None in level_set: + rows.extend( + _unattributed_tool_rows( + [ + event + for event in selected_tools + if event.get("attribution_state") == "unattributed" + ] + ) + ) + return rows diff --git a/src/autoskillit/report/deck/_coverage_support.py b/src/autoskillit/report/deck/_coverage_support.py new file mode 100644 index 0000000000..e53b2ff6a0 --- /dev/null +++ b/src/autoskillit/report/deck/_coverage_support.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Any + +from autoskillit.core import SourcePair + +from ._measure_helpers import _coverage_state, _state_counts_empty + +_COVERAGE_LABELS: Mapping[str, tuple[str, str]] = { + "input_tokens": ( + "How many input tokens are recorded?", + "A session usage record with an input token measure", + ), + "output_tokens": ( + "How many output tokens are recorded?", + "A session usage record with an output token measure", + ), + "cache_read_tokens": ( + "How many cache-read tokens are recorded?", + "A session usage record with a cache-read token measure", + ), + "cache_write_tokens": ( + "How many cache-write tokens are recorded?", + "A session usage record with a cache-write token measure", + ), + "resolved_model": ( + "Which resolved model was recorded?", + "A session row with a resolved model", + ), + "context_window_tokens": ( + "Is the per-turn model window known?", + "A turn with a positive resolved model window", + ), + "context_fraction": ( + "Can cache-read occupancy be computed?", + "A turn with cache-read tokens and a positive model window", + ), + "context_fraction_validation": ( + "Does the stored fraction match the recomputed ratio?", + "A turn with both stored and recomputed fractions", + ), + "turn_series_coverage": ( + "Is turn-level usage available?", + "A session turn-usage ledger state", + ), + "session_outcomes": ( + "Are session outcomes known?", + "A selected session with an explicit success outcome", + ), + "tool_errors": ( + "Are tool failure outcomes observable?", + "An indexed tool_result event with a known success value", + ), + "tool_skill_step_attribution": ( + "Can tool events be attributed to a skill and step?", + "An indexed tool_result event with an exact owning session and skill/step", + ), + "parent_prompt_tokens": ( + "Are parent prompt token counts available?", + "A verified parent invocation with a timestamped prompt span", + ), + "subagent_return_tokens": ( + "Are returned text token counts available?", + "A verified parent invocation with a timestamped return span", + ), +} + + +def _coverage_row( + *, + pair: SourcePair, + model: str | None, + level: str | None, + skill: str | None, + recipe: str | None, + step: str | None, + population: str, + field: str, + measure: Mapping[str, Any], + eligible_count: int, + observation_count: int, + source: str, + time_ms: int | None = None, + reason: str | None = None, + state_counts: Mapping[str, int] | None = None, + session_key: str | None = None, +) -> dict[str, Any]: + counts = dict(state_counts or measure.get("state_counts") or _state_counts_empty()) + state = _coverage_state(counts, observation_count) + measure_state = measure.get("state") + if measure_state == "no_observations": + state = "no_observations" + elif measure_state == "mixed": + state = "mixed" + question, prerequisite = _COVERAGE_LABELS.get( + field, (field.replace("_", " ").capitalize(), "A corresponding indexed observation") + ) + return { + "key": ( + f"{pair.harness}:{pair.provider}:{model}:{skill}:{recipe}:{step}:" + f"{population}:{field}:{session_key or 'cohort'}" + ), + "session_key": session_key, + "time_ms": time_ms, + "harness": pair.harness, + "provider": pair.provider, + "model": model, + "level": level, + "skill": skill, + "recipe": recipe, + "step": step, + "population": population, + "field": field, + "state": state, + "measure": {"state": measure.get("state"), "value": measure.get("value")}, + "state_counts": counts, + "eligible_count": eligible_count, + "observation_count": observation_count, + "question": question, + "prerequisite": prerequisite, + "reason": reason, + "source": source, + } + + +def _categorical_measure( + values: Sequence[object], +) -> tuple[dict[str, Any], dict[str, int], str | None]: + known = [value for value in values if isinstance(value, str) and value] + unknown = len(values) - len(known) + counts = {"measured": len(known), "unknown": unknown} + distinct = sorted(set(known)) + if not values: + state = "no_observations" + elif len(distinct) > 1 or (known and unknown): + state = "mixed" + elif known: + state = "measured" + else: + state = "unknown" + reason = ", ".join(distinct) if distinct else "no resolved value was recorded" + return {"state": state, "value": distinct[0] if len(distinct) == 1 else None}, counts, reason + + +def _gap_rows(coverage_rows: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: + return [ + dict(row) + for row in coverage_rows + if row.get("state") not in ("measured", "measured_zero") + or ( + row.get("field") == "context_fraction_validation" + and isinstance(row.get("measure"), Mapping) + and row["measure"].get("value", 0) not in (None, 0) + ) + ] diff --git a/src/autoskillit/report/deck/_errors.py b/src/autoskillit/report/deck/_errors.py new file mode 100644 index 0000000000..00bdec2bfc --- /dev/null +++ b/src/autoskillit/report/deck/_errors.py @@ -0,0 +1,427 @@ +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Mapping, Sequence +from typing import Any + +from autoskillit.core import MeasureRecord, SourcePair, TokenMeasure + +from ._measure_helpers import ( + _aggregate_fields, + _outcome_summary, + _ratio_field, + _source_pair, +) +from ._view_common import _latest_time, _session_identity, _time_in_window + +_ErrorIdentity = tuple[str, str, str | None, str | None, str | None] + + +def _selected_tool_events( + tools: Sequence[Mapping[str, Any]], + all_sessions: Sequence[Mapping[str, Any]], + *, + levels: Sequence[str | None], + window: str, + generated_at_ms: int, +) -> list[Mapping[str, Any]]: + level_set = set(levels) + owners: dict[str, Mapping[str, Any]] = { + key: row + for row in all_sessions + if isinstance((key := row.get("key")), str) and row.get("level") in level_set + } + selected: list[Mapping[str, Any]] = [] + for raw in tools: + if not _time_in_window(raw.get("time_ms"), window, generated_at_ms=generated_at_ms): + continue + owner_key = raw.get("session_key") + owner = owners.get(owner_key) if isinstance(owner_key, str) else None + if owner is None: + event_pair = _source_pair(raw) + selected.append( + { + **dict(raw), + "session_key": None, + "harness": event_pair.harness, + "provider": event_pair.provider, + "skill": None, + "recipe": None, + "step": None, + "level": None, + "attribution_state": "unattributed", + } + ) + else: + owner_pair = _source_pair(owner) + selected.append( + { + **dict(raw), + "harness": owner_pair.harness, + "provider": owner_pair.provider, + "skill": owner.get("skill"), + "recipe": owner.get("recipe"), + "step": owner.get("step"), + "level": owner.get("level"), + "attribution_state": "attributed", + } + ) + return selected + + +def _session_symptoms( + rows: Sequence[Mapping[str, Any]], +) -> list[tuple[str, list[Mapping[str, Any]]]]: + groups: dict[str, list[Mapping[str, Any]]] = defaultdict(list) + for row in rows: + if row.get("success") is False: + symptom = next( + ( + value + for field in ("adjudication_reason", "adjudication_subtype", "subtype") + if isinstance((value := row.get(field)), str) and value + ), + "Unknown symptom", + ) + groups[symptom].append(row) + elif row.get("success") is None: + groups["Outcome unknown"].append(row) + return list(groups.items()) + + +def _tool_symptoms( + events: Sequence[Mapping[str, Any]], +) -> list[tuple[str, list[Mapping[str, Any]]]]: + groups: dict[str, list[Mapping[str, Any]]] = defaultdict(list) + for event in events: + error_type = event.get("error_type") + if event.get("success") is not False and not isinstance(error_type, str): + continue + symptom = error_type if isinstance(error_type, str) else "Unknown symptom" + groups[symptom].append(event) + return list(groups.items()) + + +def _session_error_row( + identity: _ErrorIdentity, + symptom: str, + affected: Sequence[Mapping[str, Any]], + eligible: Sequence[Mapping[str, Any]], + tool_events: Sequence[Mapping[str, Any]], +) -> dict[str, Any]: + harness, provider, skill, recipe, step = identity + pair = SourcePair(harness, provider) + failure_records = [ + MeasureRecord( + pair, + { + "failures": ( + TokenMeasure.observed(1) + if row.get("success") is False + else TokenMeasure.unknown() + ) + }, + ) + for row in affected + ] + failure_count = _aggregate_fields( + failure_records, ("failures",), eligible_count=len(eligible) + )["failures"] + outcome = _outcome_summary(eligible) + session_keys = sorted(str(key) for row in affected if isinstance((key := row.get("key")), str)) + return { + "key": f"session:{harness}:{provider}:{skill}:{recipe}:{step}:{symptom}", + "time_ms": _latest_time(affected), + "session_key": session_keys[0] if len(session_keys) == 1 else None, + "session_keys": session_keys, + "harness": harness, + "provider": provider, + "skill": skill, + "recipe": recipe, + "step": step, + "level": affected[0].get("level"), + "population": "session_outcome" if symptom == "Outcome unknown" else "session_failure", + "symptom": symptom, + "failures": failure_count, + "failure_rate": outcome["failure_rate"], + "eligible_count": outcome["eligible_count"], + "observed_count": outcome["observed_count"], + "unknown_count": outcome["unknown_count"], + "tool_result_event_count": len(tool_events), + "tool_event_coverage": _tool_event_coverage(eligible, tool_events), + "attribution_state": "attributed", + } + + +def _session_error_rows( + sessions: Sequence[Mapping[str, Any]], + tool_groups: Mapping[_ErrorIdentity, Sequence[Mapping[str, Any]]], +) -> list[dict[str, Any]]: + groups: dict[_ErrorIdentity, list[Mapping[str, Any]]] = defaultdict(list) + for row in sessions: + groups[_session_identity(row)].append(row) + return [ + _session_error_row( + identity, + symptom, + affected, + eligible, + tool_groups.get(identity, ()), + ) + for identity, eligible in groups.items() + for symptom, affected in _session_symptoms(eligible) + ] + + +def _tool_error_row( + identity: _ErrorIdentity, + symptom: str, + affected: Sequence[Mapping[str, Any]], + eligible_events: Sequence[Mapping[str, Any]], + eligible_sessions: Sequence[Mapping[str, Any]], +) -> dict[str, Any]: + harness, provider, skill, recipe, step = identity + pair = SourcePair(harness, provider) + failure_records = [ + MeasureRecord( + pair, + { + "failures": ( + TokenMeasure.observed(1) + if event.get("success") is False + else TokenMeasure.unknown() + if event.get("success") is None + else TokenMeasure.not_applicable() + ) + }, + ) + for event in affected + ] + failure_count = _aggregate_fields( + failure_records, ("failures",), eligible_count=len(eligible_events) + )["failures"] + outcome = _outcome_summary(eligible_events) + session_keys = sorted( + {str(key) for event in affected if isinstance((key := event.get("session_key")), str)} + ) + return { + "key": f"tool:{harness}:{provider}:{skill}:{recipe}:{step}:{symptom}", + "time_ms": _latest_time(affected), + "session_key": session_keys[0] if len(session_keys) == 1 else None, + "session_keys": session_keys, + "harness": harness, + "provider": provider, + "skill": skill, + "recipe": recipe, + "step": step, + "level": affected[0].get("level"), + "population": "tool_failure", + "symptom": symptom, + "failures": failure_count, + "failure_rate": outcome["failure_rate"], + "eligible_count": outcome["eligible_count"], + "observed_count": outcome["observed_count"], + "unknown_count": outcome["unknown_count"], + "tool_result_event_count": len(eligible_events), + "tool_event_coverage": _tool_event_coverage(eligible_sessions, eligible_events), + "attribution_state": "attributed", + } + + +def _tool_error_rows( + sessions: Sequence[Mapping[str, Any]], + tool_groups: Mapping[_ErrorIdentity, Sequence[Mapping[str, Any]]], +) -> list[dict[str, Any]]: + session_groups: dict[_ErrorIdentity, list[Mapping[str, Any]]] = defaultdict(list) + for row in sessions: + session_groups[_session_identity(row)].append(row) + rows = [] + for identity, eligible_events in tool_groups.items(): + eligible_sessions = session_groups.get(identity, []) + for symptom, affected in _tool_symptoms(eligible_events): + rows.append( + _tool_error_row( + identity, + symptom, + affected, + eligible_events, + eligible_sessions, + ) + ) + return rows + + +def _unattributed_error_rows(events: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: + if not events: + return [] + groups: dict[SourcePair, list[Mapping[str, Any]]] = defaultdict(list) + for event in events: + groups[_source_pair(event)].append(event) + rows = [] + for pair, pair_events in groups.items(): + outcome = _outcome_summary(pair_events) + for symptom, affected in _tool_symptoms(pair_events): + records = [ + MeasureRecord( + pair, + { + "failures": TokenMeasure.observed(1) + if event.get("success") is False + else TokenMeasure.unknown() + }, + ) + for event in affected + ] + rows.append( + { + "key": f"unattributed:{pair.harness}:{pair.provider}:{symptom}", + "time_ms": _latest_time(affected), + "session_key": None, + "session_keys": [], + "harness": pair.harness, + "provider": pair.provider, + "skill": None, + "recipe": None, + "step": None, + "level": None, + "population": "unattributed_tool", + "symptom": symptom, + "failures": _aggregate_fields(records, ("failures",))["failures"], + "failure_rate": outcome["failure_rate"], + "eligible_count": len(pair_events), + "observed_count": outcome["observed_count"], + "unknown_count": outcome["unknown_count"], + "tool_result_event_count": len(pair_events), + "tool_event_coverage": { + "state": "unattributed", + "value": None, + "sample_size": 0, + "excluded_runs": 0, + "unknown_runs": len(pair_events), + "eligible_count": len(pair_events), + "observation_count": len(pair_events), + }, + "attribution_state": "unattributed", + } + ) + return rows + + +def _error_rows( + sessions: Sequence[Mapping[str, Any]], + all_sessions: Sequence[Mapping[str, Any]], + tools: Sequence[Mapping[str, Any]], + *, + levels: Sequence[str | None], + window: str, + generated_at_ms: int, +) -> tuple[list[dict[str, Any]], list[Mapping[str, Any]]]: + events = _selected_tool_events( + tools, + all_sessions, + levels=levels, + window=window, + generated_at_ms=generated_at_ms, + ) + tool_groups: dict[_ErrorIdentity, list[Mapping[str, Any]]] = defaultdict(list) + unattributed = [] + for event in events: + if event.get("attribution_state") == "attributed": + tool_groups[_session_identity(event)].append(event) + else: + unattributed.append(event) + rows = [ + *_session_error_rows(sessions, tool_groups), + *_tool_error_rows(sessions, tool_groups), + *_unattributed_error_rows(unattributed), + ] + rows.sort(key=lambda row: (row.get("time_ms") is None, row.get("time_ms") or 0, row["key"])) + return rows, events + + +def _tool_event_coverage( + eligible_sessions: Sequence[Mapping[str, Any]], events: Sequence[Mapping[str, Any]] +) -> dict[str, Any]: + event_owners = {key for event in events if isinstance((key := event.get("session_key")), str)} + records = [ + MeasureRecord( + _source_pair(session), + { + "tool_result_event": ( + TokenMeasure.observed(1) + if session.get("key") in event_owners + else TokenMeasure.unknown() + ), + "eligible_sessions": TokenMeasure.observed(1), + }, + ) + for session in eligible_sessions + ] + if not records: + return _ratio_field([], "tool_result_event", "eligible_sessions") + result = _ratio_field(records, "tool_result_event", "eligible_sessions") + state_counts = _aggregate_fields(records, ("tool_result_event",))["tool_result_event"][ + "state_counts" + ] + return { + **result, + "observation_count": state_counts["measured"] + state_counts["measured_zero"], + "state_counts": state_counts, + } + + +def _error_population_metrics( + sessions: Sequence[Mapping[str, Any]], + events: Sequence[Mapping[str, Any]], +) -> list[dict[str, Any]]: + session_groups: dict[_ErrorIdentity, list[Mapping[str, Any]]] = defaultdict(list) + event_groups: dict[_ErrorIdentity, list[Mapping[str, Any]]] = defaultdict(list) + for row in sessions: + session_groups[_session_identity(row)].append(row) + for event in events: + if event.get("attribution_state") == "attributed": + event_groups[_session_identity(event)].append(event) + identities = sorted( + set(session_groups) | set(event_groups), + key=lambda identity: tuple(str(value or "") for value in identity), + ) + result = [] + for identity in identities: + eligible = session_groups.get(identity, []) + tool_events = event_groups.get(identity, []) + result.append( + { + "harness": identity[0], + "provider": identity[1], + "skill": identity[2], + "recipe": identity[3], + "step": identity[4], + "eligible_sessions": len(eligible), + "session_outcomes": _outcome_summary(eligible), + "tool_result_event_count": len(tool_events), + "tool_event_coverage": _tool_event_coverage(eligible, tool_events), + "tool_outcomes": _outcome_summary(tool_events), + "attribution_state": "attributed", + } + ) + unattributed_groups: dict[SourcePair, list[Mapping[str, Any]]] = defaultdict(list) + for event in events: + if event.get("attribution_state") == "unattributed": + unattributed_groups[_source_pair(event)].append(event) + for pair, unattributed in unattributed_groups.items(): + result.append( + { + "harness": pair.harness, + "provider": pair.provider, + "skill": None, + "recipe": None, + "step": None, + "eligible_sessions": 0, + "session_outcomes": _outcome_summary([]), + "tool_result_event_count": len(unattributed), + "tool_event_coverage": _tool_event_coverage([], unattributed), + "tool_outcomes": _outcome_summary(unattributed), + "attribution_state": "unattributed", + } + ) + return result diff --git a/src/autoskillit/report/deck/_html.py b/src/autoskillit/report/deck/_html.py index 0da3411f9a..c06fc0fe84 100644 --- a/src/autoskillit/report/deck/_html.py +++ b/src/autoskillit/report/deck/_html.py @@ -72,6 +72,7 @@ def render_deck( request_rows: Iterable[Mapping[str, Any]] = (), tool_rows: Iterable[Mapping[str, Any]] = (), subagent_rows: Iterable[Mapping[str, Any]] = (), + turn_rows: Iterable[Mapping[str, Any]] = (), generated_at: datetime, index_schema_version: int, ) -> str: @@ -81,6 +82,7 @@ def render_deck( request_rows=request_rows, tool_rows=tool_rows, subagent_rows=subagent_rows, + turn_rows=turn_rows, generated_at=generated_at, index_schema_version=index_schema_version, ) diff --git a/src/autoskillit/report/deck/_measure_helpers.py b/src/autoskillit/report/deck/_measure_helpers.py new file mode 100644 index 0000000000..ed405b03e0 --- /dev/null +++ b/src/autoskillit/report/deck/_measure_helpers.py @@ -0,0 +1,307 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Any + +from autoskillit.core import ( + MeasureRecord, + SourcePair, + TokenMeasure, + TokenMeasureState, + aggregate_measures, + measure_ratio, +) + +TOKEN_FIELDS = ("input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens") + + +def _source_pair(row: Mapping[str, Any]) -> SourcePair: + harness, provider = row.get("harness"), row.get("provider") + return SourcePair( + harness if isinstance(harness, str) and harness else "unknown", + provider if isinstance(provider, str) and provider else "unknown", + ) + + +def _state_counts_empty() -> dict[str, int]: + return {state.value: 0 for state in TokenMeasureState} + + +def _aggregate_fields( + records: Sequence[MeasureRecord], + fields: Sequence[str], + *, + eligible_count: int | None = None, +) -> dict[str, dict[str, Any]]: + eligible = len(records) if eligible_count is None else eligible_count + if not records: + return { + field: { + "state": "no_observations", + "value": None, + "state_counts": _state_counts_empty(), + "eligible_count": eligible, + "observation_count": 0, + } + for field in fields + } + result = aggregate_measures(records, fields) + return { + field: { + "state": aggregate.value.state.value, + "value": aggregate.value.value, + "state_counts": { + state.value: count for state, count in aggregate.state_counts.items() + }, + "eligible_count": eligible, + "observation_count": result.runs, + } + for field, aggregate in result.fields.items() + } + + +def _ratio_field( + records: Sequence[MeasureRecord], + numerator: str, + denominator: str, + *, + eligible_count: int | None = None, +) -> dict[str, Any]: + eligible = len(records) if eligible_count is None else eligible_count + if not records: + return { + "state": "no_observations", + "value": None, + "sample_size": 0, + "excluded_runs": 0, + "unknown_runs": 0, + "eligible_count": eligible, + "observation_count": 0, + } + result = measure_ratio(records, numerator, denominator) + return { + "state": result.state.value, + "value": result.value, + "sample_size": result.sample_size, + "excluded_runs": result.excluded_runs, + "unknown_runs": result.unknown_runs, + "eligible_count": eligible, + "observation_count": len(records), + } + + +def _coverage_state(state_counts: Mapping[str, int], observation_count: int) -> str: + if observation_count == 0: + return "no_observations" + active = [state for state, count in state_counts.items() if count] + return active[0] if len(active) == 1 else "mixed" + + +def _decode_measure(row: Mapping[str, Any], field: str) -> TokenMeasure: + return TokenMeasure.measure_from_raw(row.get(field)) + + +def _measure_records( + rows: Sequence[Mapping[str, Any]], *, tool_names: Sequence[str] = () +) -> list[MeasureRecord]: + records = [] + for row in rows: + harness = row.get("harness") + provider = row.get("provider") + pair = SourcePair( + harness if isinstance(harness, str) and harness else "unknown", + provider if isinstance(provider, str) and provider else "unknown", + ) + child = row.get("actor_level") == "L0" + raw = row.get("token_usage") if child else row + raw = raw if isinstance(raw, Mapping) else {} + usage_known = not child or row.get("usage_state") == "observed" + measures = { + field: TokenMeasure.measure_from_raw(raw.get(field)) + if usage_known + else TokenMeasure.unknown() + for field in TOKEN_FIELDS + } + if child: + counts = row.get("tool_counts") + transcript_known = row.get("transcript_state") == "observed" and isinstance( + counts, Mapping + ) + counts = counts if isinstance(counts, Mapping) else {} + measures["tool_calls"] = ( + TokenMeasure.observed(sum(counts.values())) + if transcript_known + else TokenMeasure.unknown() + ) + for tool in tool_names: + measures[f"tool:{tool}"] = ( + TokenMeasure.observed(counts.get(tool, 0)) + if transcript_known + else TokenMeasure.unknown() + ) + records.append(MeasureRecord(pair, measures)) + return records + + +def _measure_payload(records: Sequence[MeasureRecord], fields: Sequence[str]) -> dict[str, Any]: + result = aggregate_measures(records, fields) + return { + field: { + "state": aggregate.value.state.value, + "value": aggregate.value.value, + "state_counts": { + state.value: count for state, count in aggregate.state_counts.items() + }, + } + for field, aggregate in result.fields.items() + } + + +def _ratio_payload( + records: Sequence[MeasureRecord], + numerator: str, + denominator: str, + *, + sample_unit: str, + definition_roles: Sequence[str], + definitions: Mapping[str, Mapping[str, Any]], +) -> dict[str, Any]: + result = measure_ratio(records, numerator, denominator) + roles = sorted(set(definition_roles)) + missing = [role for role in roles if definitions.get(role, {}).get("state") != "available"] + eligible = ( + result.state in (TokenMeasureState.MEASURED, TokenMeasureState.MEASURED_ZERO) + and result.value is not None + and result.sample_size > 0 + and bool(roles) + and not missing + ) + reasons = [] + if result.state is TokenMeasureState.UNKNOWN: + reasons.append("the ratio has unknown contributing measurements") + elif result.state is TokenMeasureState.UNAVAILABLE: + reasons.append("the ratio is unavailable for this source") + elif result.state is TokenMeasureState.NOT_APPLICABLE: + reasons.append("the ratio has no applicable positive denominator") + if not roles: + reasons.append("no contributor role definitions are linked") + if missing: + reasons.append("missing definitions: " + ", ".join(missing)) + return { + "state": result.state.value, + "value": result.value, + "numerator": result.numerator_field, + "denominator": result.denominator_field, + "numerator_total": result.numerator_total, + "denominator_total": result.denominator_total, + "sample_unit": sample_unit, + "sample_size": result.sample_size, + "excluded_runs": result.excluded_runs, + "unknown_runs": result.unknown_runs, + "definition_roles": roles, + "review_eligible": eligible, + "review_reason": None if eligible else "; ".join(reasons), + } + + +def _metric_payload( + rows: Sequence[Mapping[str, Any]], + *, + sample_unit: str, + definition_roles: Sequence[str], + definitions: Mapping[str, Mapping[str, Any]], +) -> dict[str, Any]: + tool_names = sorted( + { + tool + for row in rows + if row.get("transcript_state") == "observed" + and isinstance(row.get("tool_counts"), Mapping) + for tool in row["tool_counts"] + if isinstance(tool, str) + } + ) + records = _measure_records(rows, tool_names=tool_names) + fields: tuple[str, ...] = TOKEN_FIELDS + has_tool_metrics = sample_unit == "child-invocation" + if has_tool_metrics: + fields += ("tool_calls",) + tuple(f"tool:{tool}" for tool in tool_names) + measures = _measure_payload(records, fields) + ratios = { + "input_output": _ratio_payload( + records, + "input_tokens", + "output_tokens", + sample_unit=sample_unit, + definition_roles=definition_roles, + definitions=definitions, + ), + "cache_share": _ratio_payload( + records, + "cache_read_tokens", + "input_tokens", + sample_unit=sample_unit, + definition_roles=definition_roles, + definitions=definitions, + ), + } + if has_tool_metrics: + ratios["tool_mix"] = { + tool: _ratio_payload( + records, + f"tool:{tool}", + "tool_calls", + sample_unit=sample_unit, + definition_roles=definition_roles, + definitions=definitions, + ) + for tool in tool_names + } + return {"measures": measures, "ratios": ratios} + + +def _outcome_record(row: Mapping[str, Any], field: str = "failure") -> MeasureRecord: + success = row.get("success") + outcome = ( + TokenMeasure.observed(0 if success else 1) + if isinstance(success, bool) + else TokenMeasure.unknown() + ) + denominator = TokenMeasure.observed(1) if isinstance(success, bool) else TokenMeasure.unknown() + return MeasureRecord(_source_pair(row), {field: outcome, "known_attempt": denominator}) + + +def _outcome_summary(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + records = [_outcome_record(row) for row in rows] + if not records: + return { + "failures": _aggregate_fields([], ("failure",))["failure"], + "failure_rate": _ratio_field([], "failure", "known_attempt"), + "eligible_count": 0, + "observed_count": 0, + "unknown_count": 0, + } + all_counts = _aggregate_fields(records, ("failure",))["failure"]["state_counts"] + known = [record for record in records if record.measures["known_attempt"].value == 1] + known_failure = _aggregate_fields(known, ("failure",))["failure"] + rate = _ratio_field(known, "failure", "known_attempt", eligible_count=len(rows)) + return { + "failures": known_failure, + "failure_rate": rate, + "state_counts": all_counts, + "eligible_count": len(rows), + "observed_count": len(known), + "unknown_count": all_counts.get(TokenMeasureState.UNKNOWN.value, 0), + } + + +def _ratio_state_measure(state: TokenMeasureState) -> TokenMeasure: + if state is TokenMeasureState.MEASURED: + return TokenMeasure.observed(1) + if state is TokenMeasureState.MEASURED_ZERO: + return TokenMeasure.observed(0) + if state is TokenMeasureState.UNAVAILABLE: + return TokenMeasure.unavailable() + if state is TokenMeasureState.NOT_APPLICABLE: + return TokenMeasure.not_applicable() + return TokenMeasure.unknown() diff --git a/src/autoskillit/report/deck/_payload.py b/src/autoskillit/report/deck/_payload.py index 6d7e963924..7d139d02c5 100644 --- a/src/autoskillit/report/deck/_payload.py +++ b/src/autoskillit/report/deck/_payload.py @@ -1,21 +1,16 @@ """Build the static deck model from resolved report-index session rows.""" +from __future__ import annotations + from collections import Counter from collections.abc import Iterable, Mapping, Sequence from datetime import UTC, datetime, timedelta -from itertools import combinations from typing import Any -from autoskillit.core import ( - MeasureRecord, - SourcePair, - TokenMeasure, - TokenMeasureState, - aggregate_measures, - load_bundled_agent_definitions, - measure_ratio, -) +from autoskillit.core import TokenMeasure, TokenMeasureState, load_bundled_agent_definitions +from ._measure_helpers import _metric_payload +from ._preparation import _prepare_additional_views from ._registry import ( AVAILABILITY_VOCABULARY, DECK_VIEWS, @@ -31,14 +26,13 @@ FacetDef, ReasonDef, ) - -DAY_MS = 86_400_000 -TOKEN_FIELDS = ("input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens") -SKILL_LEVELS = ("skill", "orchestrator", None) - - -def encode_table(rows: Sequence[Mapping[str, Any]], columns: Sequence[str]) -> dict[str, Any]: - return {"columns": list(columns), "rows": [[row.get(c) for c in columns] for row in rows]} +from ._view_common import ( + DAY_MS, + SKILL_LEVELS, + _canonical_children, + _window_level_blocks, + encode_table, +) def _history(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: @@ -175,228 +169,213 @@ def resolve_view_chips( } -def _canonical_children( +def _prepare_deck_data( sessions: Sequence[Mapping[str, Any]], + requests: Sequence[Mapping[str, Any]], + tools: Sequence[Mapping[str, Any]], subagents: Iterable[Mapping[str, Any]], -) -> list[dict[str, Any]]: - parents = {row.get("key"): row for row in sessions if isinstance(row.get("key"), str)} - children = [] - for raw in subagents: - row = dict(raw) - parent_key = row.get("parent_session_key") - parent = parents.get(parent_key) - role = row.get("role") - if ( - row.get("actor_level") != "L0" - or not isinstance(row.get("child_id"), str) - or not row["child_id"] - or not isinstance(role, str) - or not role - or parent is None - or not isinstance(row.get("native_parent_session_id"), str) - or row["native_parent_session_id"] != parent.get("session_id") - ): - continue - children.append( + turns: Sequence[Mapping[str, Any]], + *, + generated_at_ms: int, +) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]: + children = _canonical_children(sessions, subagents) + additional_prepared, additional_tables, additional_view_rows = _prepare_additional_views( + sessions, + requests, + tools, + children, + turns, + generated_at_ms=generated_at_ms, + ) + skill_rows: list[Mapping[str, Any]] = [ + row + for row in sessions + if row.get("level") in SKILL_LEVELS and isinstance(row.get("skill"), str) + ] + role_rows: list[Mapping[str, Any]] = [ + row for row in children if row.get("level") in SKILL_LEVELS + ] + bundled = {definition.name: definition for definition in load_bundled_agent_definitions()} + roles = sorted({row["role"] for row in role_rows}) + definitions: dict[str, dict[str, Any]] = {} + for role in roles: + definition = bundled.get(role) + definitions[role] = ( { - **row, - "harness": parent.get("harness") or "unknown", - "provider": row.get("provider") or "unknown", - "time_ms": parent.get("time_ms"), - "level": parent.get("level"), - "recipe": parent.get("recipe"), - "step": parent.get("step"), - "skill": row.get("skill") if isinstance(row.get("skill"), str) else None, - "_parent_provider": parent.get("provider") or "unknown", + "state": "available", + "description": definition.description, + "body": definition.body, + "tools": list(definition.tools), + "model": definition.model, + "reader_tools": list(definition.reader_tools), + "codex_model": getattr(getattr(definition, "codex", None), "model", None), + } + if definition is not None + else { + "state": "unavailable", + "description": None, + "body": None, + "tools": [], + "model": None, + "reader_tools": [], + "codex_model": None, } ) - return children - - -def _window_level_blocks( - rows: Sequence[Mapping[str, Any]], *, generated_at_ms: int -) -> list[tuple[str, list[str | None], list[Mapping[str, Any]]]]: - blocks: list[tuple[str, list[str | None], list[Mapping[str, Any]]]] = [] - for window in WINDOWS: - in_window = [ - row - for row in rows - if window.days is None - or ( - row.get("time_ms") is not None - and row["time_ms"] >= generated_at_ms - window.days * DAY_MS - ) - ] - levels = sorted( - {row.get("level") for row in in_window}, - key=lambda level: (level is None, level or ""), - ) - for size in range(1, len(levels) + 1): - for selected in combinations(levels, size): - selected_set = set(selected) - members = [row for row in in_window if row.get("level") in selected_set] - if members: - blocks.append((window.key, list(selected), members)) - return blocks - - -def _measure_records( - rows: Sequence[Mapping[str, Any]], *, tool_names: Sequence[str] = () -) -> list[MeasureRecord]: - records = [] - for row in rows: - harness = row.get("harness") - provider = row.get("provider") - pair = SourcePair( - harness if isinstance(harness, str) and harness else "unknown", - provider if isinstance(provider, str) and provider else "unknown", - ) - child = row.get("actor_level") == "L0" - raw = row.get("token_usage") if child else row - raw = raw if isinstance(raw, Mapping) else {} - usage_known = not child or row.get("usage_state") == "observed" - measures = { - field: TokenMeasure.measure_from_raw(raw.get(field)) - if usage_known - else TokenMeasure.unknown() - for field in TOKEN_FIELDS - } - if child: - counts = row.get("tool_counts") - transcript_known = row.get("transcript_state") == "observed" and isinstance( - counts, Mapping - ) - counts = counts if isinstance(counts, Mapping) else {} - measures["tool_calls"] = ( - TokenMeasure.observed(sum(counts.values())) - if transcript_known - else TokenMeasure.unknown() + skill_identities = sorted( + { + ( + row["skill"], + row.get("harness") or "unknown", + row.get("provider") or "unknown", ) - for tool in tool_names: - measures[f"tool:{tool}"] = ( - TokenMeasure.observed(counts.get(tool, 0)) - if transcript_known - else TokenMeasure.unknown() - ) - records.append(MeasureRecord(pair, measures)) - return records - - -def _measure_payload(records: Sequence[MeasureRecord], fields: Sequence[str]) -> dict[str, Any]: - result = aggregate_measures(records, fields) - return { - field: { - "state": aggregate.value.state.value, - "value": aggregate.value.value, - "state_counts": { - state.value: count for state, count in aggregate.state_counts.items() - }, + for row in skill_rows } - for field, aggregate in result.fields.items() - } - - -def _ratio_payload( - records: Sequence[MeasureRecord], - numerator: str, - denominator: str, - *, - sample_unit: str, - definition_roles: Sequence[str], - definitions: Mapping[str, Mapping[str, Any]], -) -> dict[str, Any]: - result = measure_ratio(records, numerator, denominator) - roles = sorted(set(definition_roles)) - missing = [role for role in roles if definitions.get(role, {}).get("state") != "available"] - eligible = ( - result.state in (TokenMeasureState.MEASURED, TokenMeasureState.MEASURED_ZERO) - and result.value is not None - and result.sample_size > 0 - and bool(roles) - and not missing ) - reasons = [] - if result.state is TokenMeasureState.UNKNOWN: - reasons.append("the ratio has unknown contributing measurements") - elif result.state is TokenMeasureState.UNAVAILABLE: - reasons.append("the ratio is unavailable for this source") - elif result.state is TokenMeasureState.NOT_APPLICABLE: - reasons.append("the ratio has no applicable positive denominator") - if not roles: - reasons.append("no contributor role definitions are linked") - if missing: - reasons.append("missing definitions: " + ", ".join(missing)) - return { - "state": result.state.value, - "value": result.value, - "numerator": result.numerator_field, - "denominator": result.denominator_field, - "numerator_total": result.numerator_total, - "denominator_total": result.denominator_total, - "sample_unit": sample_unit, - "sample_size": result.sample_size, - "excluded_runs": result.excluded_runs, - "unknown_runs": result.unknown_runs, - "definition_roles": roles, - "review_eligible": eligible, - "review_reason": None if eligible else "; ".join(reasons), + role_identities = sorted({(row["role"], row["provider"]) for row in role_rows}) + relationships = [ + {"skill": skill, "role": role, "harness": harness, "provider": provider} + for skill, role, harness, provider in sorted( + { + (row["skill"], row["role"], row["harness"], row["provider"]) + for row in role_rows + if isinstance(row.get("skill"), str) and row["skill"] + } + ) + ] + view_rows: dict[str, list[Mapping[str, Any]]] = { + "cohort": list(sessions), + "spend": skill_rows, + "efficiency": [*skill_rows, *role_rows], + "skill": skill_rows, + "role": role_rows, + **additional_view_rows, + } + population_labels = { + "spend": "skill", + "efficiency": "contributor", + "skill": "skill", + "role": "child", + "context": "turn", + "errors": "error observation", + "trend": "session", + "gaps": "indexed evidence", + "parity": "indexed evidence", } + view_chips: dict[str, dict[str, list[dict[str, Any]]]] = {} + view_histories: dict[str, dict[str, Any]] = {} + for view in DECK_VIEWS: + if view.planned_issue is not None: + continue + rows = view_rows.get(view.view_id, list(sessions)) + history = _history(rows) + view_chips[view.view_id] = resolve_view_chips( + view, + rows, + generated_at_ms=generated_at_ms, + history=history, + population_label=population_labels.get(view.view_id), + ) + view_histories[view.view_id] = history + return ( + { + "skills": _skill_blocks( + skill_rows, + role_rows, + generated_at_ms=generated_at_ms, + definitions=definitions, + ), + "roles": _role_blocks( + role_rows, generated_at_ms=generated_at_ms, definitions=definitions + ), + "relationships": relationships, + "definitions": definitions, + "view_chips": view_chips, + "view_histories": view_histories, + **additional_prepared, + }, + { + "skills": encode_table( + [ + {"skill": skill, "harness": harness, "provider": provider} + for skill, harness, provider in skill_identities + ], + ("skill", "harness", "provider"), + ), + "roles": encode_table( + [{"role": role, "provider": provider} for role, provider in role_identities], + ("role", "provider"), + ), + **additional_tables, + }, + ) -def _metric_payload( - rows: Sequence[Mapping[str, Any]], +def build_deck_payload( + session_rows: Iterable[Mapping[str, Any]], *, - sample_unit: str, - definition_roles: Sequence[str], - definitions: Mapping[str, Mapping[str, Any]], + request_rows: Iterable[Mapping[str, Any]] = (), + tool_rows: Iterable[Mapping[str, Any]] = (), + subagent_rows: Iterable[Mapping[str, Any]] = (), + turn_rows: Iterable[Mapping[str, Any]] = (), + generated_at: datetime, + index_schema_version: int, ) -> dict[str, Any]: - tool_names = sorted( - { - tool - for row in rows - if row.get("transcript_state") == "observed" - and isinstance(row.get("tool_counts"), Mapping) - for tool in row["tool_counts"] - if isinstance(tool, str) - } + """Encode the report populations and prepare each built view's selected metrics.""" + if generated_at.utcoffset() is None: + raise ValueError("generated_at must be timezone-aware") + epoch = datetime(1970, 1, 1, tzinfo=UTC) + generated_at_ms = (generated_at - epoch) // timedelta(milliseconds=1) + rows = sorted( + (dict(r) for r in session_rows), + key=lambda r: (r.get("time_ms") is None, r.get("time_ms") or 0, r.get("key") or ""), ) - records = _measure_records(rows, tool_names=tool_names) - fields: tuple[str, ...] = TOKEN_FIELDS - has_tool_metrics = sample_unit == "child-invocation" - if has_tool_metrics: - fields += ("tool_calls",) + tuple(f"tool:{tool}" for tool in tool_names) - measures = _measure_payload(records, fields) - ratios = { - "input_output": _ratio_payload( - records, - "input_tokens", - "output_tokens", - sample_unit=sample_unit, - definition_roles=definition_roles, - definitions=definitions, - ), - "cache_share": _ratio_payload( - records, - "cache_read_tokens", - "input_tokens", - sample_unit=sample_unit, - definition_roles=definition_roles, - definitions=definitions, - ), + requests = [dict(row) for row in request_rows] + tools = [dict(row) for row in tool_rows] + subagents = [dict(row) for row in subagent_rows] + turns = [dict(row) for row in turn_rows] + history = _history(rows) + prepared, identity_tables = _prepare_deck_data( + rows, + requests, + tools, + subagents, + turns, + generated_at_ms=generated_at_ms, + ) + tables = {SESSION_TABLE: encode_table(rows, SESSION_COLUMNS), **identity_tables} + return { + "generated_at_ms": generated_at_ms, + "index_schema_version": index_schema_version, + "landing": LANDING_VIEW, + "history": history, + "tables": tables, + "facets": [ + {"id": f.facet_id, "label": f.label, "column": f.column, "kind": "values"} + for f in FACETS + ] + + [{"id": WINDOW_FACET_ID, "label": "window", "column": "time_ms", "kind": "window"}], + "views": [ + { + "id": v.view_id, + "question": v.question, + "decision": v.decision, + "group": v.group, + "status": "built" if v.planned_issue is None else "planned", + "issue": v.planned_issue, + } + for v in DECK_VIEWS + ], + "chips": prepared["view_chips"], + "prepared": prepared, + "availability": [ + { + "state": s.value, + "label": AVAILABILITY_VOCABULARY[s][0], + "description": AVAILABILITY_VOCABULARY[s][1], + } + for s in TokenMeasureState + ], } - if has_tool_metrics: - ratios["tool_mix"] = { - tool: _ratio_payload( - records, - f"tool:{tool}", - "tool_calls", - sample_unit=sample_unit, - definition_roles=definition_roles, - definitions=definitions, - ) - for tool in tool_names - } - return {"measures": measures, "ratios": ratios} def _skill_blocks( @@ -569,187 +548,3 @@ def _role_blocks( rows_out.append({"role": role, "provider": provider, "harnesses": harnesses}) blocks.append({"window": window, "levels": levels, "rows": rows_out}) return blocks - - -def _prepare_deck_data( - sessions: Sequence[Mapping[str, Any]], - subagents: Iterable[Mapping[str, Any]], - *, - generated_at_ms: int, -) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]: - children = _canonical_children(sessions, subagents) - skill_rows: list[Mapping[str, Any]] = [ - row - for row in sessions - if row.get("level") in SKILL_LEVELS and isinstance(row.get("skill"), str) - ] - role_rows: list[Mapping[str, Any]] = [ - row for row in children if row.get("level") in SKILL_LEVELS - ] - bundled = {definition.name: definition for definition in load_bundled_agent_definitions()} - roles = sorted({row["role"] for row in role_rows}) - definitions: dict[str, dict[str, Any]] = {} - for role in roles: - definition = bundled.get(role) - definitions[role] = ( - { - "state": "available", - "description": definition.description, - "body": definition.body, - "tools": list(definition.tools), - "model": definition.model, - "reader_tools": list(definition.reader_tools), - "codex_model": getattr(getattr(definition, "codex", None), "model", None), - } - if definition is not None - else { - "state": "unavailable", - "description": None, - "body": None, - "tools": [], - "model": None, - "reader_tools": [], - "codex_model": None, - } - ) - skill_identities = sorted( - { - ( - row["skill"], - row.get("harness") or "unknown", - row.get("provider") or "unknown", - ) - for row in skill_rows - } - ) - role_identities = sorted({(row["role"], row["provider"]) for row in role_rows}) - relationships = [ - {"skill": skill, "role": role, "harness": harness, "provider": provider} - for skill, role, harness, provider in sorted( - { - (row["skill"], row["role"], row["harness"], row["provider"]) - for row in role_rows - if isinstance(row.get("skill"), str) and row["skill"] - } - ) - ] - view_rows: dict[str, list[Mapping[str, Any]]] = { - "cohort": list(sessions), - "spend": skill_rows, - "efficiency": [*skill_rows, *role_rows], - "skill": skill_rows, - "role": role_rows, - } - population_labels = { - "spend": "skill", - "efficiency": "contributor", - "skill": "skill", - "role": "child", - } - view_chips: dict[str, dict[str, list[dict[str, Any]]]] = {} - view_histories: dict[str, dict[str, Any]] = {} - for view in DECK_VIEWS: - if view.planned_issue is not None: - continue - rows = view_rows.get(view.view_id, list(sessions)) - history = _history(rows) - view_chips[view.view_id] = resolve_view_chips( - view, - rows, - generated_at_ms=generated_at_ms, - history=history, - population_label=population_labels.get(view.view_id), - ) - view_histories[view.view_id] = history - return ( - { - "skills": _skill_blocks( - skill_rows, - role_rows, - generated_at_ms=generated_at_ms, - definitions=definitions, - ), - "roles": _role_blocks( - role_rows, generated_at_ms=generated_at_ms, definitions=definitions - ), - "relationships": relationships, - "definitions": definitions, - "view_chips": view_chips, - "view_histories": view_histories, - }, - { - "skills": encode_table( - [ - {"skill": skill, "harness": harness, "provider": provider} - for skill, harness, provider in skill_identities - ], - ("skill", "harness", "provider"), - ), - "roles": encode_table( - [{"role": role, "provider": provider} for role, provider in role_identities], - ("role", "provider"), - ), - }, - ) - - -def build_deck_payload( - session_rows: Iterable[Mapping[str, Any]], - *, - request_rows: Iterable[Mapping[str, Any]] = (), - tool_rows: Iterable[Mapping[str, Any]] = (), - subagent_rows: Iterable[Mapping[str, Any]] = (), - generated_at: datetime, - index_schema_version: int, -) -> dict[str, Any]: - """Encode the session cohort and prepare attributed skill and child-role metrics. - - Request and tool rows remain available at this boundary, but contribute to no metric - without a verified child identity. Native child rows already carry folded usage. - """ - if generated_at.utcoffset() is None: - raise ValueError("generated_at must be timezone-aware") - epoch = datetime(1970, 1, 1, tzinfo=UTC) - generated_at_ms = (generated_at - epoch) // timedelta(milliseconds=1) - rows = sorted( - (dict(r) for r in session_rows), - key=lambda r: (r.get("time_ms") is None, r.get("time_ms") or 0, r.get("key") or ""), - ) - history = _history(rows) - prepared, identity_tables = _prepare_deck_data( - rows, subagent_rows, generated_at_ms=generated_at_ms - ) - tables = {SESSION_TABLE: encode_table(rows, SESSION_COLUMNS), **identity_tables} - return { - "generated_at_ms": generated_at_ms, - "index_schema_version": index_schema_version, - "landing": LANDING_VIEW, - "history": history, - "tables": tables, - "facets": [ - {"id": f.facet_id, "label": f.label, "column": f.column, "kind": "values"} - for f in FACETS - ] - + [{"id": WINDOW_FACET_ID, "label": "window", "column": "time_ms", "kind": "window"}], - "views": [ - { - "id": v.view_id, - "question": v.question, - "decision": v.decision, - "group": v.group, - "status": "built" if v.planned_issue is None else "planned", - "issue": v.planned_issue, - } - for v in DECK_VIEWS - ], - "chips": prepared["view_chips"], - "prepared": prepared, - "availability": [ - { - "state": s.value, - "label": AVAILABILITY_VOCABULARY[s][0], - "description": AVAILABILITY_VOCABULARY[s][1], - } - for s in TokenMeasureState - ], - } diff --git a/src/autoskillit/report/deck/_preparation.py b/src/autoskillit/report/deck/_preparation.py new file mode 100644 index 0000000000..a578fdec9c --- /dev/null +++ b/src/autoskillit/report/deck/_preparation.py @@ -0,0 +1,611 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Any + +from autoskillit.core import TokenMeasureState + +from ._context import _context_turn_rows, _parent_context_metrics +from ._coverage import _coverage_rows +from ._coverage_support import _gap_rows +from ._errors import _error_population_metrics, _error_rows, _selected_tool_events +from ._measure_helpers import _decode_measure +from ._registry import ( + COVERAGE_COLUMNS, + ERROR_COLUMNS, + ERROR_TABLE, + GAP_TABLE, + PARITY_TABLE, + TREND_COLUMNS, + TREND_TABLE, + TURN_COLUMNS, + TURN_TABLE, + WINDOWS, +) +from ._trend import _trend_view +from ._view_common import ( + _level_value, + _population_blocks, + _prepared_view_block, + _time_in_window, + _timestamp_ms, + _window_level_rows, + encode_table, +) + +_BlockKey = tuple[str, tuple[str | None, ...]] + + +def _owner_rows(sessions: Sequence[Mapping[str, Any]]) -> dict[str, Mapping[str, Any]]: + return {key: row for row in sessions if isinstance((key := row.get("key")), str)} + + +def _context_session_keys( + selected_sessions: Sequence[Mapping[str, Any]], + selected_rows: Sequence[Mapping[str, Any]], + parent_metrics: Sequence[Mapping[str, Any]], +) -> set[str]: + keys: set[str] = set() + for row in selected_sessions: + key = row.get("key") + if isinstance(key, str): + keys.add(key) + for row in selected_rows: + key = row.get("session_key") + if isinstance(key, str): + keys.add(key) + for group in parent_metrics: + for span in group.get("provenance", ()): + key = span.get("parent_session_key") + if isinstance(key, str): + keys.add(key) + return keys + + +def _block_keys( + sessions: Sequence[Mapping[str, Any]], + context_rows: Sequence[Mapping[str, Any]], + event_rows: Sequence[Mapping[str, Any]], + span_rows: Sequence[Mapping[str, Any]], + *, + generated_at_ms: int, +) -> tuple[list[_BlockKey], list[str | None], list[str | None]]: + session_blocks = _population_blocks(sessions, generated_at_ms=generated_at_ms) + context_blocks = _population_blocks(context_rows, generated_at_ms=generated_at_ms) + error_blocks = _population_blocks( + [*sessions, *event_rows, *span_rows], generated_at_ms=generated_at_ms + ) + keys: set[_BlockKey] = { + (window, tuple(levels)) + for window, levels, _ in (*session_blocks, *context_blocks, *error_blocks) + } + order = {definition.key: index for index, definition in enumerate(WINDOWS)} + ordered = sorted( + keys, + key=lambda item: ( + order[item[0]], + tuple((level is None, level or "") for level in item[1]), + ), + ) + session_levels = sorted( + {_level_value(row.get("level")) for row in sessions}, + key=lambda level: (level is None, level or ""), + ) + view_levels = sorted( + { + _level_value(row.get("level")) + for row in (*sessions, *context_rows, *event_rows, *span_rows) + }, + key=lambda level: (level is None, level or ""), + ) + return ordered, session_levels, view_levels + + +def _context_session_views( + window: str, + selected_sessions: Sequence[Mapping[str, Any]], + selected_rows: Sequence[Mapping[str, Any]], + parent_metrics: Sequence[Mapping[str, Any]], + all_context_rows: Sequence[Mapping[str, Any]], + sessions: Sequence[Mapping[str, Any]], + children: Sequence[Mapping[str, Any]], + owners: Mapping[str, Mapping[str, Any]], + cache: dict[tuple[str, str], dict[str, Any]], + *, + generated_at_ms: int, +) -> dict[str, dict[str, Any]]: + keys = _context_session_keys(selected_sessions, selected_rows, parent_metrics) + prepared: dict[str, dict[str, Any]] = {} + for key in sorted(keys): + owner = owners.get(key) + if owner is None: + continue + cache_key = (window, key) + if cache_key not in cache: + owner_level = [_level_value(owner.get("level"))] + rows = _window_level_rows( + [row for row in all_context_rows if row.get("session_key") == key], + window=window, + levels=owner_level, + generated_at_ms=generated_at_ms, + ) + eligible = ( + [owner] + if _time_in_window(owner.get("time_ms"), window, generated_at_ms=generated_at_ms) + else [] + ) + spans = [child for child in children if child.get("parent_session_key") == key] + metrics = _parent_context_metrics( + eligible, + sessions, + spans, + levels=owner_level, + window=window, + generated_at_ms=generated_at_ms, + ) + cache[cache_key] = {"rows": rows, "metrics": {"parent_context": metrics}} + prepared[key] = cache[cache_key] + return prepared + + +def _context_block( + window: str, + levels: list[str | None], + sessions_for_window: Sequence[Mapping[str, Any]], + context_rows: Sequence[Mapping[str, Any]], + sessions: Sequence[Mapping[str, Any]], + children: Sequence[Mapping[str, Any]], + owners: Mapping[str, Mapping[str, Any]], + cache: dict[tuple[str, str], dict[str, Any]], + *, + generated_at_ms: int, +) -> dict[str, Any]: + selected = _window_level_rows( + context_rows, + window=window, + levels=levels, + generated_at_ms=generated_at_ms, + ) + parent_metrics = _parent_context_metrics( + sessions_for_window, + sessions, + children, + levels=levels, + window=window, + generated_at_ms=generated_at_ms, + ) + session_views = _context_session_views( + window, + sessions_for_window, + selected, + parent_metrics, + context_rows, + sessions, + children, + owners, + cache, + generated_at_ms=generated_at_ms, + ) + return { + "window": window, + "levels": levels, + "rows": selected, + "metrics": { + "parent_context": parent_metrics, + "eligible_sessions": len(sessions_for_window), + }, + "sessions": session_views, + } + + +def _error_block( + window: str, + levels: list[str | None], + selected_sessions: Sequence[Mapping[str, Any]], + sessions: Sequence[Mapping[str, Any]], + tools: Sequence[Mapping[str, Any]], + *, + generated_at_ms: int, +) -> tuple[dict[str, Any], list[Mapping[str, Any]]]: + rows, events = _error_rows( + selected_sessions, + sessions, + tools, + levels=levels, + window=window, + generated_at_ms=generated_at_ms, + ) + return ( + { + "window": window, + "levels": levels, + "rows": rows, + "metrics": {"populations": _error_population_metrics(selected_sessions, events)}, + }, + events, + ) + + +def _trend_block( + window: str, + levels: list[str | None], + sessions: Sequence[Mapping[str, Any]], + selected_sessions: Sequence[Mapping[str, Any]], + *, + generated_at_ms: int, +) -> dict[str, Any]: + rows, comparisons, metadata = _trend_view( + sessions, + selected_sessions, + levels=levels, + window=window, + generated_at_ms=generated_at_ms, + ) + return { + "window": window, + "levels": levels, + "rows": rows, + "metrics": {"comparisons": comparisons, **metadata}, + } + + +def _coverage_session_views( + window: str, + session_keys: set[str], + owners: Mapping[str, Mapping[str, Any]], + sessions: Sequence[Mapping[str, Any]], + turns: Sequence[Mapping[str, Any]], + tools: Sequence[Mapping[str, Any]], + children: Sequence[Mapping[str, Any]], + cache: dict[tuple[str, str], dict[str, Any]], + *, + generated_at_ms: int, +) -> tuple[dict[str, dict[str, Any]], dict[str, dict[str, Any]]]: + gaps: dict[str, dict[str, Any]] = {} + parity: dict[str, dict[str, Any]] = {} + for key in sorted(session_keys): + owner = owners.get(key) + if owner is None: + continue + cache_key = (window, key) + if cache_key not in cache: + owner_level = [_level_value(owner.get("level"))] + coverage = _coverage_rows( + [owner], + sessions, + turns, + tools, + children, + levels=owner_level, + window=window, + generated_at_ms=generated_at_ms, + session_key=key, + ) + missing = _gap_rows(coverage) + cache[cache_key] = { + "coverage": coverage, + "gaps": missing, + "metrics": { + "eligible_sessions": 1, + "gap_count": len(missing), + "coverage_count": len(coverage), + }, + } + entry = cache[cache_key] + gaps[key] = {"rows": entry["gaps"], "metrics": entry["metrics"]} + parity[key] = {"rows": entry["coverage"], "metrics": entry["metrics"]} + return gaps, parity + + +def _coverage_block( + window: str, + levels: list[str | None], + selected_sessions: Sequence[Mapping[str, Any]], + selected_events: Sequence[Mapping[str, Any]], + parent_context: Sequence[Mapping[str, Any]], + sessions: Sequence[Mapping[str, Any]], + turns: Sequence[Mapping[str, Any]], + tools: Sequence[Mapping[str, Any]], + children: Sequence[Mapping[str, Any]], + owners: Mapping[str, Mapping[str, Any]], + cache: dict[tuple[str, str], dict[str, Any]], + *, + generated_at_ms: int, +) -> tuple[dict[str, Any], dict[str, Any]]: + coverage = _coverage_rows( + selected_sessions, + sessions, + turns, + tools, + children, + levels=levels, + window=window, + generated_at_ms=generated_at_ms, + ) + gaps = _gap_rows(coverage) + observed_turns = _window_level_rows( + turns, + window=window, + levels=levels, + generated_at_ms=generated_at_ms, + ) + session_keys: set[str] = set() + for row in selected_sessions: + key = row.get("key") + if isinstance(key, str): + session_keys.add(key) + for row in observed_turns: + key = row.get("session_key") + if isinstance(key, str): + session_keys.add(key) + for event in selected_events: + key = event.get("session_key") + if event.get("attribution_state") == "attributed" and isinstance(key, str): + session_keys.add(key) + for group in parent_context: + for span in group.get("provenance", ()): + key = span.get("parent_session_key") + if isinstance(key, str): + session_keys.add(key) + per_session_gaps, per_session_parity = _coverage_session_views( + window, + session_keys, + owners, + sessions, + turns, + tools, + children, + cache, + generated_at_ms=generated_at_ms, + ) + metrics = { + "eligible_sessions": len(selected_sessions), + "gap_count": len(gaps), + "coverage_count": len(coverage), + } + return ( + { + "window": window, + "levels": levels, + "rows": gaps, + "metrics": metrics, + "sessions": per_session_gaps, + }, + { + "window": window, + "levels": levels, + "rows": coverage, + "metrics": metrics, + "sessions": per_session_parity, + }, + ) + + +def _span_population_rows( + sessions: Sequence[Mapping[str, Any]], + children: Sequence[Mapping[str, Any]], + *, + generated_at_ms: int, +) -> list[Mapping[str, Any]]: + owners: dict[str, Mapping[str, Any]] = { + key: row for row in sessions if isinstance((key := row.get("key")), str) + } + rows: list[Mapping[str, Any]] = [] + for child in children: + parent_key = child.get("parent_session_key") + owner = owners.get(parent_key) if isinstance(parent_key, str) else None + if owner is None: + continue + spans = child.get("parent_context_spans") + if not isinstance(spans, (list, tuple)): + continue + for span in spans: + if not isinstance(span, Mapping): + continue + time_ms = _timestamp_ms(span.get("timestamp")) + if time_ms is not None and time_ms > generated_at_ms: + continue + measure = _decode_measure(span, "measure") + if time_ms is None and measure.state in ( + TokenMeasureState.MEASURED, + TokenMeasureState.MEASURED_ZERO, + ): + continue + owner_time = owner.get("time_ms") + if time_ms is None and not _time_in_window( + owner_time, "all", generated_at_ms=generated_at_ms + ): + continue + rows.append( + { + "key": span.get("source_id") or child.get("key"), + "session_key": parent_key, + "time_ms": time_ms if time_ms is not None else owner_time, + "harness": span.get("harness") or owner.get("harness"), + "provider": span.get("provider") or owner.get("provider"), + "model": span.get("model"), + "skill": owner.get("skill"), + "recipe": owner.get("recipe"), + "step": owner.get("step"), + "level": owner.get("level"), + } + ) + return rows + + +def _all_history_projections( + sessions: Sequence[Mapping[str, Any]], + tools: Sequence[Mapping[str, Any]], + context_rows: Sequence[Mapping[str, Any]], + span_rows: Sequence[Mapping[str, Any]], + blocks: Mapping[str, Sequence[Mapping[str, Any]]], + session_levels: Sequence[str | None], + view_levels: Sequence[str | None], + *, + generated_at_ms: int, +) -> tuple[dict[str, Any], dict[str, list[Mapping[str, Any]]]]: + all_context = [ + row + for row in context_rows + if _time_in_window(row.get("time_ms"), "all", generated_at_ms=generated_at_ms) + ] + _, all_events = _error_rows( + sessions, + sessions, + tools, + levels=session_levels, + window="all", + generated_at_ms=generated_at_ms, + ) + session_population = [ + row + for row in sessions + if _time_in_window(row.get("time_ms"), "all", generated_at_ms=generated_at_ms) + ] + error_population = [*session_population, *all_events] + gap_population: list[Mapping[str, Any]] = [ + *session_population, + *[row for row in all_context if row.get("kind") == "turn"], + *all_events, + *span_rows, + ] + view_rows: dict[str, list[Mapping[str, Any]]] = { + "context": [*all_context, *span_rows], + "errors": error_population, + "trend": session_population, + "gaps": gap_population, + "parity": gap_population, + } + + def table_rows(view: str) -> list[Mapping[str, Any]]: + block = _prepared_view_block(blocks[view], window="all", levels=view_levels) + return list(block.get("rows", ())) if block else [] + + tables = { + TURN_TABLE: encode_table( + [row for row in all_context if row.get("kind") == "turn"], + TURN_COLUMNS, + ), + ERROR_TABLE: encode_table(table_rows("errors"), ERROR_COLUMNS), + TREND_TABLE: encode_table(table_rows("trend"), TREND_COLUMNS), + GAP_TABLE: encode_table(table_rows("gaps"), COVERAGE_COLUMNS), + PARITY_TABLE: encode_table(table_rows("parity"), COVERAGE_COLUMNS), + } + return tables, view_rows + + +def _prepare_additional_views( + sessions: Sequence[Mapping[str, Any]], + requests: Sequence[Mapping[str, Any]], + tools: Sequence[Mapping[str, Any]], + children: Sequence[Mapping[str, Any]], + turns: Sequence[Mapping[str, Any]], + *, + generated_at_ms: int, +) -> tuple[dict[str, Any], dict[str, Any], dict[str, list[Mapping[str, Any]]]]: + context_rows = _context_turn_rows(sessions, turns, requests, generated_at_ms=generated_at_ms) + session_levels = sorted( + {_level_value(row.get("level")) for row in sessions}, + key=lambda level: (level is None, level or ""), + ) + event_rows = _selected_tool_events( + tools, + sessions, + levels=session_levels, + window="all", + generated_at_ms=generated_at_ms, + ) + span_rows = _span_population_rows(sessions, children, generated_at_ms=generated_at_ms) + block_keys, session_levels, view_levels = _block_keys( + sessions, + context_rows, + event_rows, + span_rows, + generated_at_ms=generated_at_ms, + ) + owners = _owner_rows(sessions) + context_cache: dict[tuple[str, str], dict[str, Any]] = {} + coverage_cache: dict[tuple[str, str], dict[str, Any]] = {} + context_blocks: list[dict[str, Any]] = [] + error_blocks: list[dict[str, Any]] = [] + trend_blocks: list[dict[str, Any]] = [] + gap_blocks: list[dict[str, Any]] = [] + parity_blocks: list[dict[str, Any]] = [] + for window, levels in block_keys: + selected_sessions = _window_level_rows( + sessions, + window=window, + levels=levels, + generated_at_ms=generated_at_ms, + ) + context_blocks.append( + _context_block( + window, + list(levels), + selected_sessions, + context_rows, + sessions, + children, + owners, + context_cache, + generated_at_ms=generated_at_ms, + ) + ) + error_block, selected_events = _error_block( + window, + list(levels), + selected_sessions, + sessions, + tools, + generated_at_ms=generated_at_ms, + ) + error_blocks.append(error_block) + trend_blocks.append( + _trend_block( + window, + list(levels), + sessions, + selected_sessions, + generated_at_ms=generated_at_ms, + ) + ) + gap_block, parity_block = _coverage_block( + window, + list(levels), + selected_sessions, + selected_events, + context_blocks[-1]["metrics"]["parent_context"], + sessions, + turns, + tools, + children, + owners, + coverage_cache, + generated_at_ms=generated_at_ms, + ) + gap_blocks.append(gap_block) + parity_blocks.append(parity_block) + blocks: dict[str, Sequence[Mapping[str, Any]]] = { + "errors": error_blocks, + "trend": trend_blocks, + "gaps": gap_blocks, + "parity": parity_blocks, + } + additional_tables, view_rows = _all_history_projections( + sessions, + tools, + context_rows, + span_rows, + blocks, + session_levels, + view_levels, + generated_at_ms=generated_at_ms, + ) + prepared = { + "context": {"blocks": context_blocks}, + "errors": {"blocks": error_blocks}, + "trend": {"blocks": trend_blocks}, + "gaps": {"blocks": gap_blocks}, + "parity": {"blocks": parity_blocks}, + } + return prepared, additional_tables, view_rows diff --git a/src/autoskillit/report/deck/_registry.py b/src/autoskillit/report/deck/_registry.py index 13d2cc8076..e681e94e53 100644 --- a/src/autoskillit/report/deck/_registry.py +++ b/src/autoskillit/report/deck/_registry.py @@ -80,6 +80,103 @@ class DeckViewDef(NamedTuple): "cache_read_tokens", "assistant_turn_count", "tool_counts", + "turn_usage_state", + "turn_usage_reason", +) +TURN_TABLE = "turns" +TURN_COLUMNS: tuple[str, ...] = ( + "key", + "session_key", + "ordinal", + "time_ms", + "owner_time_ms", + "harness", + "provider", + "model", + "skill", + "recipe", + "step", + "level", + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cache_write_tokens", + "context_window_tokens", + "context_fraction", + "context_fraction_percent", + "turn_usage_state", + "turn_usage_reason", + "request_recorded", + "request_model", +) +ERROR_TABLE = "errors" +ERROR_COLUMNS: tuple[str, ...] = ( + "key", + "time_ms", + "session_key", + "session_keys", + "harness", + "provider", + "skill", + "recipe", + "step", + "level", + "population", + "symptom", + "failures", + "failure_rate", + "eligible_count", + "observed_count", + "unknown_count", + "tool_result_event_count", + "tool_event_coverage", + "attribution_state", +) +TREND_TABLE = "trends" +TREND_COLUMNS: tuple[str, ...] = ( + "key", + "time_ms", + "bucket_end_ms", + "day", + "skill", + "recipe", + "step", + "harness", + "provider", + "model", + "measures", + "failure_share", + "eligible_sessions", + "untimed_sessions", + "future_sessions", + "retained_from_ms", + "window_start_ms", + "window_end_ms", +) +GAP_TABLE = "gaps" +PARITY_TABLE = "parity" +COVERAGE_COLUMNS: tuple[str, ...] = ( + "key", + "session_key", + "time_ms", + "harness", + "provider", + "model", + "level", + "skill", + "recipe", + "step", + "population", + "field", + "state", + "measure", + "state_counts", + "eligible_count", + "observation_count", + "question", + "prerequisite", + "reason", + "source", ) FACETS: tuple[FacetDef, ...] = ( FacetDef("harness", "harness", "harness", (), None, None), @@ -187,45 +284,45 @@ class DeckViewDef(NamedTuple): "Where does context build up?", "Tune a subagent's output, or insert one to filter", "What do you need to know?", + TURN_TABLE, + "views/context.js", None, - None, - 4652, ), DeckViewDef( "errors", "Where are the errors?", "Which step of which skill to fix", "What do you need to know?", + ERROR_TABLE, + "views/errors.js", None, - None, - 4652, ), DeckViewDef( "trend", "What's the trend?", "Whether a change held, and over what window", "What do you need to know?", + TREND_TABLE, + "views/trend.js", None, - None, - 4652, ), DeckViewDef( "gaps", "What can't we see?", "Which telemetry gap to close first", "What do you need to know?", + GAP_TABLE, + "views/gaps.js", None, - None, - 4652, ), DeckViewDef( "parity", "Every provider, same fields", "Target state for cross-pair comparison", "If telemetry were complete", + PARITY_TABLE, + "views/parity.js", None, - None, - 4652, ), ) diff --git a/src/autoskillit/report/deck/_trend.py b/src/autoskillit/report/deck/_trend.py new file mode 100644 index 0000000000..4dcc6e2902 --- /dev/null +++ b/src/autoskillit/report/deck/_trend.py @@ -0,0 +1,344 @@ +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Mapping, Sequence +from datetime import UTC, datetime +from typing import Any + +from autoskillit.core import ( + MeasureRecord, + TokenMeasure, + TokenMeasureState, +) + +from ._measure_helpers import ( + TOKEN_FIELDS, + _aggregate_fields, + _coverage_state, + _measure_records, + _outcome_summary, + _ratio_field, + _source_pair, + _state_counts_empty, +) +from ._registry import WINDOWS +from ._view_common import DAY_MS + + +def _trend_measure_summary(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + if not rows: + return { + "measures": { + field: { + "state": "no_observations", + "value": None, + "observed_sessions": 0, + "eligible_sessions": 0, + "state_counts": _state_counts_empty(), + } + for field in TOKEN_FIELDS + }, + "failure_share": { + "state": "no_observations", + "value": None, + "sample_size": 0, + "excluded_runs": 0, + "unknown_runs": 0, + "eligible_count": 0, + "observation_count": 0, + "state_counts": _state_counts_empty(), + }, + } + base_records = _measure_records(rows) + coverage = _aggregate_fields(base_records, TOKEN_FIELDS, eligible_count=len(rows)) + measures: dict[str, Any] = {} + for field in TOKEN_FIELDS: + known = [ + record + for record in base_records + if record.measures[field].state + in (TokenMeasureState.MEASURED, TokenMeasureState.MEASURED_ZERO) + ] + ratio_records = [ + MeasureRecord( + record.pair, + {field: record.measures[field], "observed_sessions": TokenMeasure.observed(1)}, + ) + for record in known + ] + average = _ratio_field(ratio_records, field, "observed_sessions", eligible_count=len(rows)) + state_counts = coverage[field]["state_counts"] + measures[field] = { + "state": ( + average["state"] + if average["value"] is not None + else _coverage_state(state_counts, len(rows)) + ), + "value": average["value"], + "observed_sessions": average["sample_size"], + "eligible_sessions": len(rows), + "state_counts": state_counts, + } + outcome = _outcome_summary(rows) + failure_share = { + **outcome["failure_rate"], + "state_counts": outcome.get("state_counts", _state_counts_empty()), + "eligible_sessions": len(rows), + "observed_sessions": outcome["observed_count"], + "unknown_sessions": outcome["unknown_count"], + } + return {"measures": measures, "failure_share": failure_share} + + +def _trend_group_key(row: Mapping[str, Any]) -> tuple[Any, ...]: + pair = _source_pair(row) + return ( + row.get("skill") if isinstance(row.get("skill"), str) else None, + row.get("recipe") if isinstance(row.get("recipe"), str) else None, + row.get("step") if isinstance(row.get("step"), str) else None, + pair.harness, + pair.provider, + row.get("model") if isinstance(row.get("model"), str) else None, + ) + + +def _interval_summary( + rows: Sequence[Mapping[str, Any]], *, start_ms: int, end_ms: int +) -> dict[str, Any]: + summary = _trend_measure_summary(rows) + times = [ + row["time_ms"] + for row in rows + if isinstance(row.get("time_ms"), int) and not isinstance(row.get("time_ms"), bool) + ] + return { + "start_ms": start_ms, + "end_ms": end_ms, + "covered_start_ms": min(times) if times else None, + "covered_end_ms": max(times) if times else None, + "eligible_sessions": len(rows), + **summary, + } + + +def _trend_delta( + earlier: Mapping[str, Any], later: Mapping[str, Any], field: str, *, unit: str +) -> dict[str, Any]: + left = ( + earlier.get("measures", {}).get(field) + if field != "failure_share" + else earlier.get("failure_share") + ) + right = ( + later.get("measures", {}).get(field) + if field != "failure_share" + else later.get("failure_share") + ) + left_value = left.get("value") if isinstance(left, Mapping) else None + right_value = right.get("value") if isinstance(right, Mapping) else None + left_samples = ( + ( + left.get("observed_sessions", 0) + if field != "failure_share" + else left.get("sample_size", 0) + ) + if isinstance(left, Mapping) + else 0 + ) + right_samples = ( + ( + right.get("observed_sessions", 0) + if field != "failure_share" + else right.get("sample_size", 0) + ) + if isinstance(right, Mapping) + else 0 + ) + if ( + isinstance(left_value, (int, float)) + and not isinstance(left_value, bool) + and isinstance(right_value, (int, float)) + and not isinstance(right_value, bool) + and left_samples > 0 + and right_samples > 0 + ): + delta = ( + (right_value - left_value) * 100 + if field == "failure_share" + else right_value - left_value + ) + return { + "state": ( + TokenMeasureState.MEASURED_ZERO.value + if delta == 0 + else TokenMeasureState.MEASURED.value + ), + "value": delta, + "unit": unit, + "reason": None, + "earlier_samples": left_samples, + "later_samples": right_samples, + } + reason = ( + "earlier interval has no observed values" + if not left_samples or left_value is None + else "later interval has no observed values" + ) + return { + "state": "no_observations", + "value": None, + "unit": unit, + "reason": reason, + "earlier_samples": left_samples, + "later_samples": right_samples, + } + + +def _trend_view( + all_sessions: Sequence[Mapping[str, Any]], + selected_sessions: Sequence[Mapping[str, Any]], + *, + levels: Sequence[str | None], + window: str, + generated_at_ms: int, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], dict[str, Any]]: + level_set = set(levels) + population = [row for row in all_sessions if row.get("level") in level_set] + future_count = sum( + isinstance(row.get("time_ms"), int) + and not isinstance(row.get("time_ms"), bool) + and row["time_ms"] > generated_at_ms + for row in population + ) + untimed_count = sum(row.get("time_ms") is None for row in population) + timed = [ + row + for row in population + if isinstance(row.get("time_ms"), int) + and not isinstance(row.get("time_ms"), bool) + and row["time_ms"] <= generated_at_ms + ] + days = next((item.days for item in WINDOWS if item.key == window), None) + start_ms = ( + generated_at_ms - days * DAY_MS + if days is not None + else min((row["time_ms"] for row in timed), default=generated_at_ms) + ) + end_ms = generated_at_ms + eligible = [row for row in timed if start_ms <= row["time_ms"] <= end_ms] + groups: dict[tuple[Any, ...], list[Mapping[str, Any]]] = defaultdict(list) + for row in eligible: + groups[_trend_group_key(row)].append(row) + + day_starts = list(range(start_ms // DAY_MS * DAY_MS, end_ms // DAY_MS * DAY_MS + 1, DAY_MS)) + rows_out: list[dict[str, Any]] = [] + comparisons: list[dict[str, Any]] = [] + midpoint = start_ms + (end_ms - start_ms) // 2 + for identity, group_rows in sorted( + groups.items(), key=lambda item: tuple(str(v) for v in item[0]) + ): + skill, recipe, step, harness, provider, model = identity + group_population = [row for row in population if _trend_group_key(row) == identity] + group_future = sum( + isinstance(row.get("time_ms"), int) + and not isinstance(row.get("time_ms"), bool) + and row["time_ms"] > generated_at_ms + for row in group_population + ) + group_untimed = sum(row.get("time_ms") is None for row in group_population) + group_retained = min( + ( + row["time_ms"] + for row in group_population + if isinstance(row.get("time_ms"), int) + and not isinstance(row.get("time_ms"), bool) + and row["time_ms"] <= generated_at_ms + ), + default=None, + ) + for day_start in day_starts: + bucket_start = max(day_start, start_ms) + bucket_end = min(day_start + DAY_MS, end_ms) + bucket = [ + row + for row in group_rows + if bucket_start <= row["time_ms"] < day_start + DAY_MS + and row["time_ms"] <= bucket_end + ] + summary = _trend_measure_summary(bucket) + rows_out.append( + { + "key": ( + f"{window}:{skill}:{recipe}:{step}:" + f"{harness}:{provider}:{model}:{day_start}" + ), + "time_ms": bucket_start, + "bucket_end_ms": bucket_end, + "day": datetime.fromtimestamp(day_start / 1000, UTC).date().isoformat(), + "skill": skill, + "recipe": recipe, + "step": step, + "harness": harness, + "provider": provider, + "model": model, + **summary, + "eligible_sessions": len(bucket), + "untimed_sessions": group_untimed, + "future_sessions": group_future, + "retained_from_ms": group_retained, + "window_start_ms": start_ms, + "window_end_ms": end_ms, + } + ) + earlier_rows = [row for row in group_rows if start_ms <= row["time_ms"] < midpoint] + later_rows = [row for row in group_rows if midpoint <= row["time_ms"] <= end_ms] + earlier = _interval_summary(earlier_rows, start_ms=start_ms, end_ms=midpoint) + later = _interval_summary(later_rows, start_ms=midpoint, end_ms=end_ms) + deltas = { + field: _trend_delta( + earlier, + later, + field, + unit="tokens/session", + ) + for field in TOKEN_FIELDS + } + deltas["failure_share"] = _trend_delta( + earlier, later, "failure_share", unit="percentage_points" + ) + comparisons.append( + { + "skill": skill, + "recipe": recipe, + "step": step, + "harness": harness, + "provider": provider, + "model": model, + "window": window, + "earlier": earlier, + "later": later, + "deltas": deltas, + "untimed_sessions": group_untimed, + "future_sessions": group_future, + "retained_from_ms": group_retained, + } + ) + rows_out.sort( + key=lambda row: ( + row["time_ms"], + row["harness"], + row["provider"], + row["skill"] or "", + row["step"] or "", + ) + ) + metadata = { + "window_start_ms": start_ms, + "window_end_ms": end_ms, + "retained_from_ms": min((row["time_ms"] for row in timed), default=None), + "untimed_sessions": untimed_count, + "future_sessions": future_count, + "eligible_sessions": len(eligible), + "selected_sessions": len(selected_sessions), + } + return rows_out, comparisons, metadata diff --git a/src/autoskillit/report/deck/_view_common.py b/src/autoskillit/report/deck/_view_common.py new file mode 100644 index 0000000000..66fef5671c --- /dev/null +++ b/src/autoskillit/report/deck/_view_common.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +from collections.abc import Iterable, Mapping, Sequence +from datetime import datetime +from itertools import combinations +from typing import Any + +from ._measure_helpers import _source_pair +from ._registry import WINDOWS + +DAY_MS = 86_400_000 +SKILL_LEVELS = ("skill", "orchestrator", None) + + +def encode_table(rows: Sequence[Mapping[str, Any]], columns: Sequence[str]) -> dict[str, Any]: + return {"columns": list(columns), "rows": [[row.get(c) for c in columns] for row in rows]} + + +def _level_value(value: object) -> str | None: + return value if isinstance(value, str) else None + + +def _canonical_children( + sessions: Sequence[Mapping[str, Any]], + subagents: Iterable[Mapping[str, Any]], +) -> list[dict[str, Any]]: + parents = {row.get("key"): row for row in sessions if isinstance(row.get("key"), str)} + children = [] + for raw in subagents: + row = dict(raw) + parent_key = row.get("parent_session_key") + parent = parents.get(parent_key) if isinstance(parent_key, str) else None + role = row.get("role") + if ( + row.get("actor_level") != "L0" + or not isinstance(row.get("child_id"), str) + or not row["child_id"] + or not isinstance(role, str) + or not role + or parent is None + or not isinstance(row.get("native_parent_session_id"), str) + or row["native_parent_session_id"] != parent.get("session_id") + ): + continue + children.append( + { + **row, + "harness": parent.get("harness") or "unknown", + "provider": row.get("provider") or "unknown", + "time_ms": parent.get("time_ms"), + "level": parent.get("level"), + "recipe": parent.get("recipe"), + "step": parent.get("step"), + "skill": row.get("skill") if isinstance(row.get("skill"), str) else None, + "_parent_provider": parent.get("provider") or "unknown", + } + ) + return children + + +def _level_groups( + rows: Sequence[Mapping[str, Any]], +) -> Iterable[tuple[list[str | None], list[Mapping[str, Any]]]]: + levels = sorted( + {_level_value(row.get("level")) for row in rows}, + key=lambda level: (level is None, level or ""), + ) + for size in range(1, len(levels) + 1): + for selected in combinations(levels, size): + selected_set = set(selected) + members = [row for row in rows if row.get("level") in selected_set] + if members: + yield list(selected), members + + +def _window_level_blocks( + rows: Sequence[Mapping[str, Any]], *, generated_at_ms: int +) -> list[tuple[str, list[str | None], list[Mapping[str, Any]]]]: + blocks: list[tuple[str, list[str | None], list[Mapping[str, Any]]]] = [] + for window in WINDOWS: + in_window = [ + row + for row in rows + if window.days is None + or ( + row.get("time_ms") is not None + and row["time_ms"] >= generated_at_ms - window.days * DAY_MS + ) + ] + blocks.extend( + (window.key, selected, members) for selected, members in _level_groups(in_window) + ) + return blocks + + +def _time_in_window(time_ms: object, window: str, *, generated_at_ms: int) -> bool: + if window == "all": + return time_ms is None or ( + isinstance(time_ms, int) + and not isinstance(time_ms, bool) + and time_ms <= generated_at_ms + ) + days = next((item.days for item in WINDOWS if item.key == window), None) + if days is None or isinstance(time_ms, bool) or not isinstance(time_ms, int): + return False + return generated_at_ms - days * DAY_MS <= time_ms <= generated_at_ms + + +def _population_blocks( + rows: Sequence[Mapping[str, Any]], *, generated_at_ms: int +) -> list[tuple[str, list[str | None], list[Mapping[str, Any]]]]: + blocks: list[tuple[str, list[str | None], list[Mapping[str, Any]]]] = [] + for window in WINDOWS: + in_window = [ + row + for row in rows + if _time_in_window(row.get("time_ms"), window.key, generated_at_ms=generated_at_ms) + ] + blocks.extend( + (window.key, selected, members) for selected, members in _level_groups(in_window) + ) + return blocks + + +def _timestamp_ms(value: object) -> int | None: + if not isinstance(value, str) or not value: + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + if parsed.utcoffset() is None: + return None + return int(parsed.timestamp() * 1000) + + +def _session_identity( + row: Mapping[str, Any], +) -> tuple[str, str, str | None, str | None, str | None]: + pair = _source_pair(row) + return ( + pair.harness, + pair.provider, + row.get("skill") if isinstance(row.get("skill"), str) else None, + row.get("recipe") if isinstance(row.get("recipe"), str) else None, + row.get("step") if isinstance(row.get("step"), str) else None, + ) + + +def _window_level_rows( + rows: Sequence[Mapping[str, Any]], + *, + window: str, + levels: Sequence[str | None], + generated_at_ms: int, +) -> list[Mapping[str, Any]]: + level_set = set(levels) + return [ + row + for row in rows + if row.get("level") in level_set + and _time_in_window(row.get("time_ms"), window, generated_at_ms=generated_at_ms) + ] + + +def _prepared_view_block( + blocks: Sequence[Mapping[str, Any]], *, window: str, levels: Sequence[str | None] +) -> Mapping[str, Any] | None: + wanted = tuple(levels) + return next( + ( + block + for block in blocks + if block.get("window") == window and tuple(block.get("levels", ())) == wanted + ), + None, + ) + + +def _latest_time(rows: Sequence[Mapping[str, Any]], key: str = "time_ms") -> int | None: + values = [ + value + for row in rows + if isinstance((value := row.get(key)), int) and not isinstance(value, bool) + ] + return max(values, default=None) diff --git a/tests/_test_filter.py b/tests/_test_filter.py index 8a49c6f20b..2167bbb945 100644 --- a/tests/_test_filter.py +++ b/tests/_test_filter.py @@ -1178,7 +1178,8 @@ class SourceMapRejection(enum.StrEnum): # file-level: the Docker pin guard compares CODEX_CLI_MIN_VERSION from # execution.backends with the Dockerfile's CODEX_VERSION pin. "infra/test_docker_image.py", - # file-level: the deck column drift guard imports execution.ReportSessionRow. + # file-level: the deck column drift guard imports + # execution._report_index_types.ReportSessionRow. "report/test_deck_registry.py", "_llm_triage", "smoke_utils", diff --git a/tests/cli/_sessions_helpers.py b/tests/cli/_sessions_helpers.py index cd6747b48d..1910100b7f 100644 --- a/tests/cli/_sessions_helpers.py +++ b/tests/cli/_sessions_helpers.py @@ -6,6 +6,8 @@ import pytest +from autoskillit.core import TOKEN_USAGE_SCHEMA_VERSION, TURN_USAGE_SCHEMA_VERSION + def _configure_log_root(monkeypatch: pytest.MonkeyPatch, log_root: Path) -> None: monkeypatch.setattr( @@ -14,7 +16,7 @@ def _configure_log_root(monkeypatch: pytest.MonkeyPatch, log_root: Path) -> None ) -def _seed_session(log_root: Path) -> None: +def _seed_session(log_root: Path, **fields: object) -> None: log_root.mkdir(parents=True, exist_ok=True) session = { "dir_name": "session-1", @@ -22,5 +24,34 @@ def _seed_session(log_root: Path) -> None: "backend": "claude-code", "provider_used": "anthropic", "timestamp": "2020-01-01T00:00:00Z", + **fields, } (log_root / "sessions.jsonl").write_text(json.dumps(session) + "\n", encoding="utf-8") + + +def _seed_turn_ledger(log_root: Path) -> None: + session_dir = log_root / "sessions" / "session-1" + session_dir.mkdir(parents=True, exist_ok=True) + descriptor = { + "schema_version": TOKEN_USAGE_SCHEMA_VERSION, + "turn_usage_file": "turn_usage.jsonl", + "turn_usage_count": 1, + "turn_usage_schema_version": TURN_USAGE_SCHEMA_VERSION, + } + turn = { + "backend": "claude-code", + "provider_used": "anthropic", + "message_id": "message-1", + "request_id": "request-1", + "timestamp": "2020-01-01T00:00:00Z", + "model": "claude-opus-4-1-20250805", + "input_tokens": {"state": "measured", "value": 75}, + "output_tokens": {"state": "measured_zero", "value": 0}, + "cache_read_tokens": {"state": "measured", "value": 50}, + "cache_write_tokens": {"state": "unavailable", "value": None}, + "peak_context": {"state": "unknown", "value": None}, + "context_window_tokens": 100, + "context_fraction": 0.5, + } + (session_dir / "token_usage.json").write_text(json.dumps(descriptor), encoding="utf-8") + (session_dir / "turn_usage.jsonl").write_text(json.dumps(turn) + "\n", encoding="utf-8") diff --git a/tests/cli/test_sessions_deck_cmd.py b/tests/cli/test_sessions_deck_cmd.py index c6a01cbb96..611d4af506 100644 --- a/tests/cli/test_sessions_deck_cmd.py +++ b/tests/cli/test_sessions_deck_cmd.py @@ -11,7 +11,7 @@ import pytest from autoskillit.core import ArtifactLease -from tests.cli._sessions_helpers import _configure_log_root, _seed_session +from tests.cli._sessions_helpers import _configure_log_root, _seed_session, _seed_turn_ledger pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium] @@ -30,6 +30,7 @@ def test_sessions_deck_passes_all_fact_collections( requests={"request": {"key": "request"}}, tools={"tool": {"key": "tool"}}, subagents={"child": {"key": "child", "actor_level": "L0"}}, + turns={"turn": {"key": "turn", "session_key": "parent"}}, ) received: dict[str, Any] = {} @@ -46,7 +47,7 @@ def capture_deck(session_rows: Any, **kwargs: Any) -> str: _sessions.sessions_deck(str(out)) assert received["session_rows"] == list(facts.sessions.values()) - for name in ("request", "tool", "subagent"): + for name in ("request", "tool", "subagent", "turn"): assert list(received[f"{name}_rows"]) == list(getattr(facts, f"{name}s").values()) assert out.read_text(encoding="utf-8") == "" @@ -59,14 +60,22 @@ def test_sessions_deck_refreshes_index_and_writes_session_data( from autoskillit.cli.ops._sessions import sessions_deck log_root = tmp_path / "logs" - _seed_session(log_root) + _seed_session( + log_root, + success=False, + skill_command="/autoskillit:implement", + session_type="skill", + recipe_name="implementation", + step_name="build", + ) + _seed_turn_ledger(log_root) _configure_log_root(monkeypatch, log_root) out = tmp_path / "deck.html" sessions_deck(str(out)) assert capsys.readouterr().out == ( - f"report index: walked 2 items, wrote 1 rows\ndeck: wrote {out} (1 session rows)\n" + f"report index: walked 2 items, wrote 2 rows\ndeck: wrote {out} (1 session rows)\n" ) html = out.read_text(encoding="utf-8") match = re.search(r']*id="deck-data"[^>]*>(.*?)', html, re.DOTALL) @@ -75,6 +84,18 @@ def test_sessions_deck_refreshes_index_and_writes_session_data( sessions = data["tables"]["sessions"] assert len(sessions["rows"]) == 1 assert sessions["rows"][0][sessions["columns"].index("harness")] == "claude-code" + turns = data["tables"]["turns"] + assert len(turns["rows"]) == 1 + turn = dict(zip(turns["columns"], turns["rows"][0], strict=True)) + assert turn["session_key"] == "session-1" + assert turn["model"] == "claude-opus-4-1-20250805" + assert turn["output_tokens"] == {"state": "measured_zero", "value": 0} + assert turn["cache_write_tokens"] == {"state": "unavailable", "value": None} + assert all( + view["status"] == "built" + for view in data["views"] + if view["id"] in {"context", "errors", "trend", "gaps", "parity"} + ) def test_sessions_deck_reports_lease_contention_without_writing_output( diff --git a/tests/cli/test_sessions_index_cmd.py b/tests/cli/test_sessions_index_cmd.py index fc0747b8dd..ba3761f8ef 100644 --- a/tests/cli/test_sessions_index_cmd.py +++ b/tests/cli/test_sessions_index_cmd.py @@ -8,7 +8,7 @@ from autoskillit.core import ArtifactLease from autoskillit.execution import REPORT_INDEX_SCHEMA_VERSION -from tests.cli._sessions_helpers import _configure_log_root, _seed_session +from tests.cli._sessions_helpers import _configure_log_root, _seed_session, _seed_turn_ledger pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium] @@ -30,7 +30,7 @@ def test_sessions_index_reports_without_creating_rows( index_dir = log_root / "report-index" assert out == ( f"report index v{REPORT_INDEX_SCHEMA_VERSION} at {index_dir}: " - "sessions=0 requests=0 tools=0 subagents=0\n" + "sessions=0 requests=0 tools=0 subagents=0 turns=0\n" ) assert not (index_dir / "rows.jsonl").exists() @@ -44,21 +44,22 @@ def test_sessions_index_updates_and_rebuilds( log_root = tmp_path / "logs" _seed_session(log_root) + _seed_turn_ledger(log_root) _configure_log_root(monkeypatch, log_root) sessions_index(update=True) index_dir = log_root / "report-index" assert capsys.readouterr().out == ( - "report index: walked 2 items, wrote 1 rows\n" + "report index: walked 2 items, wrote 2 rows\n" f"report index v{REPORT_INDEX_SCHEMA_VERSION} at {index_dir}: " - "sessions=1 requests=0 tools=0 subagents=0\n" + "sessions=1 requests=0 tools=0 subagents=0 turns=1\n" ) sessions_index(rebuild=True) assert capsys.readouterr().out == ( - "report index: walked 2 items, wrote 1 rows\n" + "report index: walked 2 items, wrote 2 rows\n" f"report index v{REPORT_INDEX_SCHEMA_VERSION} at {index_dir}: " - "sessions=1 requests=0 tools=0 subagents=0\n" + "sessions=1 requests=0 tools=0 subagents=0 turns=1\n" ) diff --git a/tests/contracts/test_package_gateways.py b/tests/contracts/test_package_gateways.py index a75911004e..d201b9e6b7 100644 --- a/tests/contracts/test_package_gateways.py +++ b/tests/contracts/test_package_gateways.py @@ -507,6 +507,7 @@ def test_measure_aggregation_ships_as_projected_hook_asset() -> None: { "anomaly_detection", "_native_child_projection", + "_native_parent_context", "linux_tracing", "otlp_sink", "report_walk", diff --git a/tests/execution/_report_index_fixtures.py b/tests/execution/_report_index_fixtures.py index 029c623b97..68d87effc0 100644 --- a/tests/execution/_report_index_fixtures.py +++ b/tests/execution/_report_index_fixtures.py @@ -7,8 +7,13 @@ from __future__ import annotations +import json from typing import Any +import tiktoken + +from autoskillit.core import TOKEN_USAGE_SCHEMA_VERSION, TURN_USAGE_SCHEMA_VERSION + CLAUDE_SCOPE: str = "com.anthropic.claude_code.events" @@ -65,3 +70,73 @@ def basic_session_row(dir_name: str, session_id: str, **fields: Any) -> dict[str **fields, } return session + + +def _jsonl(*records: dict[str, Any]) -> str: + return "".join(json.dumps(record, separators=(",", ":")) + "\n" for record in records) + + +def _encoding(name: str, *, merge_ab: bool = False) -> Any: + ranks = {bytes((value,)): value for value in range(256)} + if merge_ab: + ranks[b"ab"] = 256 + return tiktoken.Encoding( + name=name, + pat_str=r"[\s\S]+", + mergeable_ranks=ranks, + special_tokens={"<|endoftext|>": 257 if merge_ab else 256}, + ) + + +def _claude_call( + *, + child_id: str | None, + tool_id: str, + prompt: str, + model: str = "gpt-known", +) -> dict[str, Any]: + inputs: dict[str, str] = {"prompt": prompt} + if child_id is not None: + inputs["agent_id"] = child_id + return { + "type": "assistant", + "uuid": f"record-{tool_id}", + "timestamp": "2026-10-07T10:00:00Z", + "requestId": f"turn-{tool_id}", + "message": { + "id": f"message-{tool_id}", + "model": model, + "content": [{"type": "tool_use", "id": tool_id, "name": "Task", "input": inputs}], + }, + } + + +def _claude_result(*, tool_id: str, record_id: str, text: str) -> dict[str, Any]: + return { + "type": "user", + "uuid": record_id, + "timestamp": f"2026-10-07T10:00:0{record_id[-1]}Z", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": tool_id, + "content": [{"type": "text", "text": text}], + } + ] + }, + } + + +def _turn_usage_descriptor( + *, + count: int, + filename: str | None = "turn_usage.jsonl", + version: int = TURN_USAGE_SCHEMA_VERSION, +) -> dict[str, Any]: + return { + "schema_version": TOKEN_USAGE_SCHEMA_VERSION, + "turn_usage_file": filename, + "turn_usage_count": count, + "turn_usage_schema_version": version, + } diff --git a/tests/execution/test_parent_context_spans.py b/tests/execution/test_parent_context_spans.py new file mode 100644 index 0000000000..8238bd437e --- /dev/null +++ b/tests/execution/test_parent_context_spans.py @@ -0,0 +1,410 @@ +"""Focused tests for native parent-visible child text spans.""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest +import tiktoken + +from autoskillit.execution.evidence import _native_child_projection as projection +from autoskillit.execution.evidence import _native_parent_context as parent_context +from tests.execution._report_index_fixtures import _claude_call, _claude_result, _encoding, _jsonl + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] + + +def _spans( + transcript: str, + *, + child_id: str, + backend: str = "claude_code", + provider: str = "openai", +) -> list[dict[str, Any]]: + row = {"backend": backend.replace("_", "-"), "provider_used": provider} + return parent_context.parent_context_spans( + row, + child_id, + backend, + parent_context.parent_transcript_records(transcript, backend), + parent_complete=projection._complete_jsonl(transcript), + parent_transcript_reason=None, + tokenizer_version="0.14.0", + ) + + +@pytest.mark.parametrize( + ("phase", "event", "reason"), + [ + ( + "initialization", + "parent_context_tokenizer_unavailable", + "tokenizer_initialization_failed", + ), + ("encoding", "parent_context_text_encoding_failed", "tokenizer_encoding_failed"), + ], +) +def test_tokenizer_failures_keep_exception_diagnostics_and_unavailable_measure( + monkeypatch: pytest.MonkeyPatch, phase: str, event: str, reason: str +) -> None: + warnings: list[tuple[str, dict[str, Any]]] = [] + + def fail(_value: str) -> Any: + raise RuntimeError("tokenizer failure detail") + + encoding = SimpleNamespace(name="local-failing", encode_ordinary=fail) + monkeypatch.setattr( + tiktoken, "encoding_for_model", fail if phase == "initialization" else lambda _: encoding + ) + monkeypatch.setattr( + parent_context.logger, "warning", lambda name, **fields: warnings.append((name, fields)) + ) + transcript = _jsonl(_claude_call(child_id="child-1", tool_id="tool-1", prompt="text")) + + spans = _spans(transcript, child_id="child-1") + + assert spans[0]["measure"] == {"state": "unavailable", "value": None} + assert spans[0]["reason"] == reason + assert any( + name == event and fields["error"] == "RuntimeError" and fields["exc_info"] is True + for name, fields in warnings + ) + + +def test_claude_spans_count_exact_linked_text_deduplicate_replay_and_keep_deliveries( + monkeypatch: pytest.MonkeyPatch, +) -> None: + encoding = _encoding("local-byte-merge", merge_ab=True) + monkeypatch.setattr(tiktoken, "encoding_for_model", lambda _model: encoding) + prompt = "<|endoftext|>ab" + call = _claude_call(child_id="child-1", tool_id="tool-1", prompt=prompt) + transcript = _jsonl( + call, + call, + _claude_result(tool_id="tool-1", record_id="result-1", text="ab"), + _claude_result(tool_id="tool-1", record_id="result-2", text="<|endoftext|>"), + ) + + spans = _spans(transcript, child_id="child-1") + + assert [span["field"] for span in spans] == [ + "parent_prompt_tokens", + "subagent_return_tokens", + "subagent_return_tokens", + ] + assert spans[0]["measure"] == { + "state": "measured", + "value": len(encoding.encode_ordinary(prompt)), + } + assert spans[1]["measure"] == {"state": "measured", "value": 1} + assert spans[2]["measure"]["value"] == len(encoding.encode_ordinary("<|endoftext|>")) + assert spans[0]["invocation_id"] == "tool-1" + assert spans[0]["turn_id"] == "turn-tool-1" + assert spans[0]["model"] == "gpt-known" + assert spans[0]["encoding"] == "local-byte-merge" + assert spans[0]["source_id"] + assert len({span["source_id"] for span in spans[1:]}) == 2 + assert "<|endoftext|>ab" not in json.dumps(spans) + + +@pytest.mark.parametrize( + ("context", "call_metadata", "expected_model"), + [ + ({"turn_id": "context-turn", "model": "gpt-context"}, {"turn_id": "call-turn"}, None), + ( + {"turn_id": "call-turn", "model": "gpt-context"}, + {"turn_id": "call-turn", "model": "gpt-call"}, + None, + ), + (None, {"turn_id": "call-turn", "model": "gpt-explicit"}, "gpt-explicit"), + (None, {"model": "gpt-without-turn"}, None), + ], +) +def test_codex_model_requires_matching_context_or_explicit_call_turn( + monkeypatch: pytest.MonkeyPatch, + context: dict[str, str] | None, + call_metadata: dict[str, str], + expected_model: str | None, +) -> None: + encoding = _encoding("local-model-check") + monkeypatch.setattr(tiktoken, "encoding_for_model", lambda _model: encoding) + call_payload = { + "type": "function_call", + "call_id": "call-1", + "arguments": json.dumps({"agent_thread_id": "child-1", "message": "ab"}), + **call_metadata, + } + records = [] + if context is not None: + records.append({"type": "turn_context", "payload": context}) + records.extend( + [ + { + "type": "response_item", + "timestamp": "2026-10-07T10:00:00Z", + "payload": call_payload, + }, + { + "type": "response_item", + "timestamp": "2026-10-07T10:00:01Z", + "payload": {"type": "function_call_output", "call_id": "call-1", "output": "ab"}, + }, + ] + ) + + spans = _spans(_jsonl(*records), child_id="child-1", backend="codex") + + assert spans[0]["model"] == expected_model + if expected_model is None: + assert spans[0]["measure"] == {"state": "unavailable", "value": None} + assert spans[0]["reason"] == "parent_model_missing" + else: + assert spans[0]["measure"]["state"] == "measured" + assert spans[0]["encoding"] == "local-model-check" + + +def test_codex_uses_each_invocation_turn_model_and_pairs_call_id( + monkeypatch: pytest.MonkeyPatch, +) -> None: + encodings = { + "gpt-model-a": _encoding("local-a", merge_ab=True), + "gpt-model-b": _encoding("local-b"), + } + monkeypatch.setattr(tiktoken, "encoding_for_model", encodings.__getitem__) + records = [] + for turn, model, child_id, call_id in ( + ("turn-a", "gpt-model-a", "child-a", "call-a"), + ("turn-b", "gpt-model-b", "child-b", "call-b"), + ): + arguments = json.dumps( + {"agent_thread_id": child_id, "message": "ab"}, + separators=(",", ":"), + ) + records.extend( + [ + {"type": "turn_context", "payload": {"turn_id": turn, "model": model}}, + { + "type": "response_item", + "timestamp": "2026-10-07T10:00:00Z", + "payload": { + "type": "function_call", + "name": "spawn_agent", + "call_id": call_id, + "arguments": arguments, + }, + }, + { + "type": "response_item", + "timestamp": "2026-10-07T10:00:01Z", + "payload": { + "type": "function_call_output", + "call_id": call_id, + "output": "ab", + }, + }, + ] + ) + transcript = _jsonl(*records) + + first = _spans(transcript, child_id="child-a", backend="codex") + second = _spans(transcript, child_id="child-b", backend="codex") + + assert first[0]["model"] == "gpt-model-a" + assert first[0]["encoding"] == "local-a" + assert first[0]["measure"]["value"] == len(encodings["gpt-model-a"].encode_ordinary("ab")) + assert first[1]["measure"]["value"] == 1 + assert second[0]["model"] == "gpt-model-b" + assert second[0]["encoding"] == "local-b" + assert second[1]["measure"]["value"] == 2 + assert first[0]["turn_id"] == "turn-a" + assert second[0]["turn_id"] == "turn-b" + + +def test_codex_call_without_message_or_prompt_does_not_count_other_arguments( + monkeypatch: pytest.MonkeyPatch, +) -> None: + encoding = _encoding("local-no-prompt") + monkeypatch.setattr(tiktoken, "encoding_for_model", lambda _model: encoding) + transcript = _jsonl( + {"type": "turn_context", "payload": {"turn_id": "turn-1", "model": "gpt-known"}}, + { + "type": "response_item", + "timestamp": "2026-10-07T10:00:00Z", + "payload": { + "type": "function_call", + "call_id": "call-1", + "arguments": json.dumps( + {"agent_thread_id": "child-1", "description": "not the prompt"} + ), + }, + }, + { + "type": "response_item", + "timestamp": "2026-10-07T10:00:01Z", + "payload": {"type": "function_call_output", "call_id": "call-1", "output": "ab"}, + }, + ) + + spans = _spans(transcript, child_id="child-1", backend="codex") + + assert spans[0]["measure"] == {"state": "unavailable", "value": None} + assert spans[0]["reason"] == "parent_prompt_text_missing" + assert spans[1]["measure"] == {"state": "measured", "value": 2} + + +@pytest.mark.parametrize( + ("output", "expected_state", "expected_value", "expected_reason"), + [ + ( + json.dumps({"agent_id": "child-1", "status": "completed"}), + "unavailable", + None, + "parent_return_acknowledgement_only", + ), + ( + json.dumps({"agent_id": "child-1"}), + "unavailable", + None, + "parent_return_acknowledgement_only", + ), + (json.dumps({"agent_id": "child-1", "text": "ab"}), "measured", 2, None), + (json.dumps({"agent_id": "child-1", "response_text": "ab"}), "measured", 2, None), + ], +) +def test_codex_acknowledgement_is_not_return_text( + monkeypatch: pytest.MonkeyPatch, + output: str, + expected_state: str, + expected_value: int | None, + expected_reason: str | None, +) -> None: + encoding = _encoding("local-acknowledgement") + monkeypatch.setattr(tiktoken, "encoding_for_model", lambda _model: encoding) + transcript = _jsonl( + {"type": "turn_context", "payload": {"turn_id": "turn-1", "model": "gpt-known"}}, + { + "type": "response_item", + "timestamp": "2026-10-07T10:00:00Z", + "payload": { + "type": "function_call", + "call_id": "call-1", + "arguments": json.dumps({"agent_thread_id": "child-1", "message": "ab"}), + }, + }, + { + "type": "response_item", + "timestamp": "2026-10-07T10:00:01Z", + "payload": {"type": "function_call_output", "call_id": "call-1", "output": output}, + }, + ) + + spans = _spans(transcript, child_id="child-1", backend="codex") + returned = spans[1] + + assert returned["measure"]["state"] == expected_state + assert returned["measure"]["value"] == expected_value + assert returned["reason"] == expected_reason + + +def test_codex_json_child_text_without_ack_shape_remains_return_text( + monkeypatch: pytest.MonkeyPatch, +) -> None: + encoding = _encoding("local-json-child-text") + monkeypatch.setattr(tiktoken, "encoding_for_model", lambda _model: encoding) + output = json.dumps({"answer": "child-authored", "count": 1}) + transcript = _jsonl( + {"type": "turn_context", "payload": {"turn_id": "turn-1", "model": "gpt-known"}}, + { + "type": "response_item", + "timestamp": "2026-10-07T10:00:00Z", + "payload": { + "type": "function_call", + "call_id": "call-1", + "arguments": json.dumps({"agent_thread_id": "child-1", "message": "ab"}), + }, + }, + { + "type": "response_item", + "timestamp": "2026-10-07T10:00:01Z", + "payload": {"type": "function_call_output", "call_id": "call-1", "output": output}, + }, + ) + + spans = _spans(transcript, child_id="child-1", backend="codex") + + assert spans[1]["measure"] == { + "state": "measured", + "value": len(encoding.encode_ordinary(output)), + } + assert spans[1]["reason"] is None + + +@pytest.mark.parametrize( + ("provider", "expected_state", "expected_reason"), + [("openai", "measured_zero", None), ("anthropic", "unavailable", "unsupported_provider")], +) +def test_empty_prompt_is_zero_and_missing_return_or_unsupported_provider_stays_explicit( + monkeypatch: pytest.MonkeyPatch, + provider: str, + expected_state: str, + expected_reason: str | None, +) -> None: + encoding = _encoding("local-empty") + monkeypatch.setattr(tiktoken, "encoding_for_model", lambda _model: encoding) + spans = _spans( + _jsonl(_claude_call(child_id="child-1", tool_id="tool-1", prompt="")), + child_id="child-1", + provider=provider, + ) + + assert spans[0]["measure"]["state"] == expected_state + assert spans[0]["reason"] == expected_reason + assert spans[1]["measure"] == {"state": "unavailable", "value": None} + assert spans[1]["reason"] == "parent_return_missing" + + +def test_child_id_is_not_inferred_from_claude_tool_use_id() -> None: + transcript = _jsonl( + _claude_call(child_id=None, tool_id="child-1", prompt="do work"), + _claude_result(tool_id="child-1", record_id="result-1", text="done"), + ) + + spans = _spans(transcript, child_id="child-1") + + assert [span["measure"] for span in spans] == [ + {"state": "unavailable", "value": None}, + {"state": "unavailable", "value": None}, + ] + assert {span["reason"] for span in spans} == {"child_identity_unlinked"} + assert all(span["invocation_id"] is None for span in spans) + + +def test_native_child_dependency_tracks_parent_transcript_and_tokenizer_version( + tmp_path: Path, +) -> None: + row: dict[str, Any] = {"timestamp": "2026-10-07T10:00:00Z"} + transcript_path = tmp_path / "parent.jsonl" + transcript_path.write_text("first parent content\n", encoding="utf-8") + original_parent_fingerprint = projection._transcript_fingerprint( + transcript_path, transcript_path.read_text(encoding="utf-8") + ) + child: dict[str, Any] = { + "_parent_transcript_fingerprint": original_parent_fingerprint, + "_tokenizer_version": "0.14.0", + } + original = projection._child_dependency(row, child) + + transcript_path.write_text("replacement parent content\n", encoding="utf-8") + child["_parent_transcript_fingerprint"] = projection._transcript_fingerprint( + transcript_path, transcript_path.read_text(encoding="utf-8") + ) + changed_transcript = projection._child_dependency(row, child) + child["_parent_transcript_fingerprint"] = original_parent_fingerprint + child["_tokenizer_version"] = "0.14.1" + changed_tokenizer = projection._child_dependency(row, child) + + assert original != changed_transcript + assert original != changed_tokenizer diff --git a/tests/execution/test_report_index.py b/tests/execution/test_report_index.py index 5f6359ad18..ab2401dc5b 100644 --- a/tests/execution/test_report_index.py +++ b/tests/execution/test_report_index.py @@ -3,13 +3,18 @@ from __future__ import annotations import json +import os from pathlib import Path from typing import Any import pytest import autoskillit.execution.child_outcomes as child_snapshot -from autoskillit.core import ArtifactLease, ArtifactLeaseContention +import autoskillit.execution.evidence._native_child_projection as native_projection +from autoskillit.core import ( + ArtifactLease, + ArtifactLeaseContention, +) from autoskillit.execution import ( REPORT_INDEX_SCHEMA_VERSION, read_report_index, @@ -20,8 +25,15 @@ from autoskillit.execution.evidence.report_walk import WalkItem from tests.execution._report_index_fixtures import ( CLAUDE_SCOPE, + _claude_call, + _claude_result, + _encoding, + _turn_usage_descriptor, basic_session_row, ) +from tests.execution._report_index_fixtures import ( + _jsonl as _parent_jsonl, +) from tests.execution._report_index_fixtures import ( otlp_log_record as _log, ) @@ -70,6 +82,47 @@ def _basic_session(dir_name: str, session_id: str, **fields: Any) -> dict[str, A return basic_session_row(dir_name, session_id, **fields) +def _turn_usage_row( + *, backend: str, provider: str, model: str, value: int, timestamp: str | None +) -> dict[str, Any]: + return { + "backend": backend, + "provider_used": provider, + "message_id": f"message-{value}", + "request_id": f"request-{value}", + "timestamp": timestamp, + "model": model, + "input_tokens": {"state": "measured", "value": value}, + "output_tokens": {"state": "measured_zero", "value": 0}, + "cache_read_tokens": {"state": "unknown", "value": None}, + "cache_write_tokens": {"state": "unavailable", "value": None}, + "peak_context": {"state": "unknown", "value": None}, + "context_window_tokens": 100, + "context_fraction": None, + } + + +def _write_turn_ledger( + root: Path, + dir_name: str, + rows: list[dict[str, Any]], + *, + published: bool = True, +) -> None: + session_dir = root / "sessions" / dir_name + session_dir.mkdir(parents=True, exist_ok=True) + descriptor = _turn_usage_descriptor( + count=len(rows) if published else 0, + filename="turn_usage.jsonl" if published else None, + ) + (session_dir / "token_usage.json").write_bytes(_json_line(descriptor)) + sidecar = session_dir / "turn_usage.jsonl" + if published: + sidecar.write_bytes(b"".join(_json_line(row) for row in rows)) + else: + sidecar.unlink(missing_ok=True) + + def _write_native_child(log_root: Path, parent_id: str, child_id: str, *, model: str) -> Path: snapshot_path = child_snapshot.resolve_snapshot_path( log_root, backend="claude_code", parent_session_id=parent_id @@ -127,6 +180,83 @@ def _indexed_native_children(index_dir: Path) -> dict[str, dict[str, Any]]: } +def test_parent_context_spans_refresh_through_the_report_index( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import tiktoken + + encoding = _encoding("local-index-span", merge_ab=True) + monkeypatch.setattr(tiktoken, "encoding_for_model", lambda _model: encoding) + tokenizer_version = ["test-tokenizer-v1"] + monkeypatch.setattr(native_projection, "_tokenizer_version", lambda: tokenizer_version[0]) + + root = tmp_path / "logs" + parent_id = "parent-span-session" + child_id = "child-span-agent" + parent_transcript = root / f"{parent_id}.jsonl" + call = _claude_call(child_id=child_id, tool_id="tool-span-1", prompt="ab") + first_result = _claude_result(tool_id="tool-span-1", record_id="result-1", text="ab") + parent_transcript.parent.mkdir(parents=True, exist_ok=True) + parent_transcript.write_text(_parent_jsonl(call, first_result), encoding="utf-8") + _write_jsonl( + root / "sessions.jsonl", + [ + _basic_session( + "parent-attempt", + parent_id, + backend="claude-code", + provider_used="openai", + model_identifier="gpt-known", + claude_code_log=str(parent_transcript), + ) + ], + ) + _write_native_child(root, parent_id, child_id, model="gpt-known") + index_dir = tmp_path / "index" + + update_report_index(root, index_dir) + first = _indexed_native_children(index_dir)[child_id]["parent_context_spans"] + prompt_span = next(span for span in first if span["field"] == "parent_prompt_tokens") + first_return = next(span for span in first if span["field"] == "subagent_return_tokens") + old_return_source = first_return["source_id"] + assert prompt_span["measure"] == {"state": "measured", "value": 1} + assert first_return["measure"] == {"state": "measured", "value": 1} + assert prompt_span["timestamp"] + assert first_return["timestamp"] + assert prompt_span["invocation_id"] == "tool-span-1" + assert first_return["invocation_id"] == "tool-span-1" + assert prompt_span["turn_id"] == "turn-tool-span-1" + assert prompt_span["harness"] == "claude-code" + assert prompt_span["provider"] == "openai" + assert prompt_span["model"] == "gpt-known" + assert prompt_span["tokenizer_version"] == "test-tokenizer-v1" + assert prompt_span["encoding"] == "local-index-span" + assert prompt_span["source_id"] != old_return_source + + replacement_result = _claude_result(tool_id="tool-span-1", record_id="result-2", text="aa") + parent_transcript.write_text(_parent_jsonl(call, replacement_result), encoding="utf-8") + update_report_index(root, index_dir) + replaced = _indexed_native_children(index_dir)[child_id]["parent_context_spans"] + replaced_returns = [span for span in replaced if span["field"] == "subagent_return_tokens"] + assert len(replaced_returns) == 1 + assert replaced_returns[0]["source_id"] != old_return_source + assert replaced_returns[0]["measure"] == {"state": "measured", "value": 2} + + tokenizer_version[0] = "test-tokenizer-v2" + update_report_index(root, index_dir) + reencoded = _indexed_native_children(index_dir)[child_id]["parent_context_spans"] + assert {span["tokenizer_version"] for span in reencoded} == {"test-tokenizer-v2"} + assert next(span for span in reencoded if span["field"] == "subagent_return_tokens")[ + "measure" + ] == {"state": "measured", "value": 2} + + parent_transcript.unlink() + update_report_index(root, index_dir) + unavailable = _indexed_native_children(index_dir)[child_id]["parent_context_spans"] + assert {span["measure"]["state"] for span in unavailable} == {"unavailable"} + assert {span["reason"] for span in unavailable} == {"parent_transcript_missing"} + + @pytest.mark.parametrize("cache_write", [0, None]) def test_native_claude_input_is_inclusive_only_with_complete_components( tmp_path: Path, cache_write: int | None @@ -365,7 +495,7 @@ def test_resume_attempts_attribute_events_by_time(tmp_path: Path) -> None: } assert [by_id[name]["session_key"] for name in ("before", "between", "after")] == [ - "sid", + None, "sid", "sid_2020-01-01T00-00-01Z", ] @@ -374,6 +504,88 @@ def test_resume_attempts_attribute_events_by_time(tmp_path: Path) -> None: assert by_id["ambiguous"]["cache_write_tokens"]["state"] == "unknown" +def test_turn_index_tracks_live_and_archive_sidecar_replacements_and_deletions( + tmp_path: Path, +) -> None: + root = tmp_path / "logs" + live = _basic_session("live-attempt", "reused-session-id") + archived = _basic_session("archived-attempt", "reused-session-id") + _write_jsonl(root / "sessions.jsonl", [live]) + _write_jsonl(root / "sessions-archive.jsonl", [archived]) + initial_live = [ + _turn_usage_row( + backend="codex", + provider="openai", + model="gpt-old", + value=value, + timestamp=None, + ) + for value in (2, 3) + ] + archived_turn = _turn_usage_row( + backend="claude-code", + provider="anthropic", + model="claude-resolved", + value=7, + timestamp="2026-10-01T00:00:00Z", + ) + _write_turn_ledger(root, "live-attempt", initial_live) + _write_turn_ledger(root, "archived-attempt", [archived_turn]) + index_dir = tmp_path / "index" + + update_report_index(root, index_dir) + indexed = read_report_index(index_dir) + assert set(indexed.turns) == { + "archived-attempt:turn:0", + "live-attempt:turn:0", + "live-attempt:turn:1", + } + assert indexed.turns["live-attempt:turn:0"]["session_key"] == "live-attempt" + assert indexed.turns["archived-attempt:turn:0"]["session_key"] == "archived-attempt" + assert indexed.turns["live-attempt:turn:0"]["model"] == "gpt-old" + assert indexed.turns["archived-attempt:turn:0"]["model"] == "claude-resolved" + + descriptor_path = root / "sessions" / "live-attempt" / "token_usage.json" + descriptor_stat = descriptor_path.stat() + os.utime( + descriptor_path, + ns=(descriptor_stat.st_atime_ns, descriptor_stat.st_mtime_ns + 1_000_000), + ) + assert update_report_index(root, index_dir).rows_written > 0 + + replaced_live = [ + _turn_usage_row( + backend="codex", + provider="openai", + model="gpt-new", + value=value, + timestamp=None, + ) + for value in (20, 30) + ] + _write_turn_ledger(root, "live-attempt", replaced_live) + update_report_index(root, index_dir) + indexed = read_report_index(index_dir) + assert indexed.turns["live-attempt:turn:0"]["model"] == "gpt-new" + assert indexed.turns["live-attempt:turn:0"]["input_tokens"]["value"] == 20 + + _write_turn_ledger(root, "live-attempt", replaced_live[:1]) + update_report_index(root, index_dir) + indexed = read_report_index(index_dir) + assert "live-attempt:turn:0" in indexed.turns + assert "live-attempt:turn:1" not in indexed.turns + + archived_dir = root / "sessions" / "archived-attempt" + (archived_dir / "token_usage.json").unlink() + (archived_dir / "turn_usage.jsonl").unlink() + update_report_index(root, index_dir) + indexed = read_report_index(index_dir) + assert not any(turn["session_key"] == "archived-attempt" for turn in indexed.turns.values()) + assert indexed.sessions["archived-attempt"]["turn_usage_state"] == "unavailable" + assert indexed.sessions["archived-attempt"]["turn_usage_reason"] == "descriptor-missing" + assert update_report_index(root, index_dir).rows_written == 0 + + def test_rebuild_from_scratch_equals_incremental_build(tmp_path: Path) -> None: root = tmp_path / "logs" inc = tmp_path / "incremental" diff --git a/tests/execution/test_report_index_rows.py b/tests/execution/test_report_index_rows.py index 26848e1042..071140c946 100644 --- a/tests/execution/test_report_index_rows.py +++ b/tests/execution/test_report_index_rows.py @@ -10,12 +10,11 @@ import pytest from autoskillit.execution._report_index_rows import ( - REPORT_ROW_TYPES, - ReportSessionRow, normalize_report_row, resolve_token_measure, rows_for_walk_item, ) +from autoskillit.execution._report_index_types import REPORT_ROW_TYPES, ReportSessionRow from autoskillit.execution.evidence.report_walk import WalkItem from tests.execution._report_index_fixtures import otlp_log_record as _log @@ -56,6 +55,9 @@ def _session_item( turns: tuple[tuple[str, ...], ...] = (), available: bool = True, child_outcomes: tuple[dict[str, Any], ...] = (), + turn_usage_rows: tuple[dict[str, Any], ...] = (), + turn_usage_state: str = "unavailable", + turn_usage_reason: str | None = "descriptor-missing", ) -> WalkItem: return WalkItem( "session", @@ -71,6 +73,9 @@ def _session_item( "child_outcomes": child_outcomes, "transcripts_available": available, "transcript_unavailable_reasons": [], + "turn_usage_rows": list(turn_usage_rows), + "turn_usage_state": turn_usage_state, + "turn_usage_reason": turn_usage_reason, }, {}, ) @@ -151,6 +156,111 @@ def test_session_row_carries_facets_pair_and_resolved_measures() -> None: assert "private-turn-id" not in serialized +def test_turn_rows_keep_owner_and_ordinal_identity_with_measure_states() -> None: + turn = { + "backend": "codex", + "provider_used": "openai", + "message_id": "message-1", + "request_id": "request-1", + "timestamp": None, + "model": "gpt-resolved", + "input_tokens": {"state": "measured_zero", "value": 0}, + "output_tokens": {"state": "unknown", "value": None}, + "cache_read_tokens": {"state": "measured", "value": 12}, + "cache_write_tokens": {"state": "unavailable", "value": None}, + "context_window_tokens": 100, + "context_fraction": 0.12, + } + first, second = ( + rows_for_walk_item( + _session_item( + owner, + { + "session_id": "reused-id", + "backend": "codex", + "provider_used": "openai", + }, + turn_usage_rows=(turn,), + turn_usage_state="observed", + turn_usage_reason=None, + ) + ) + for owner in ("attempt-a", "attempt-b") + ) + + assert first[0]["turn_usage_state"] == "observed" + assert first[0]["turn_usage_reason"] is None + first_turn, second_turn = first[1], second[1] + assert first_turn["kind"] == "turn" + assert first_turn["source_id"] == first_turn["session_key"] == "attempt-a" + assert second_turn["source_id"] == second_turn["session_key"] == "attempt-b" + assert first_turn["key"] != second_turn["key"] + assert first_turn["ordinal"] == second_turn["ordinal"] == 0 + assert first_turn["session_id"] == "reused-id" + assert first_turn["time_ms"] is None + assert first_turn["request_id"] == "request-1" + assert first_turn["message_id"] == "message-1" + assert first_turn["harness"] == "codex" + assert first_turn["provider"] == "openai" + assert first_turn["model"] == "gpt-resolved" + assert first_turn["input_tokens"] == {"state": "measured_zero", "value": 0} + assert first_turn["output_tokens"] == {"state": "unknown", "value": None} + assert first_turn["context_window_tokens"] == 100 + assert first_turn["context_fraction"] == 0.12 + assert set(first_turn) == set(REPORT_ROW_TYPES["turn"].__annotations__) + + +def test_session_row_keeps_missing_turn_ledger_coverage() -> None: + (row,) = rows_for_walk_item( + _session_item( + "no-ledger", + { + "session_id": "sid", + "backend": "claude-code", + "provider_used": "anthropic", + }, + turn_usage_state="unavailable", + turn_usage_reason="ledger-not-published", + ) + ) + + assert row["turn_usage_state"] == "unavailable" + assert row["turn_usage_reason"] == "ledger-not-published" + + +def test_turn_measure_decoders_preserve_unknown_for_invalid_shapes() -> None: + turn = { + "backend": "codex", + "provider_used": "openai", + "message_id": None, + "request_id": None, + "timestamp": None, + "model": "gpt-resolved", + "input_tokens": {"state": "future-state", "value": 9}, + "output_tokens": {"state": "measured", "value": 4}, + "cache_read_tokens": {"state": "unknown", "value": None}, + "cache_write_tokens": {"state": "unavailable", "value": None}, + "context_window_tokens": None, + "context_fraction": None, + } + _, row = rows_for_walk_item( + _session_item( + "attempt", + {"session_id": "sid"}, + turn_usage_rows=(turn,), + turn_usage_state="observed", + turn_usage_reason=None, + ) + ) + + assert row["input_tokens"] == {"state": "unknown", "value": None} + invalid_serialized = normalize_report_row( + {**row, "input_tokens": {"state": "measured", "value": True}} + ) + assert invalid_serialized is not None + assert invalid_serialized["input_tokens"] == {"state": "unknown", "value": None} + + @pytest.mark.parametrize("session_type", ["skill", "orchestrator"]) def test_session_item_projects_verified_native_child_under_index_parent_key( session_type: str, @@ -386,6 +496,23 @@ def test_otlp_item_projects_request_tool_and_subagent_rows() -> None: assert positional_tool["key"] == f"{source_id}#3" assert subagent["key"] == f"{source_id}#4" assert subagent["model_swapped"] is False + spans = subagent["parent_context_spans"] + assert [span["field"] for span in spans] == [ + "parent_prompt_tokens", + "subagent_return_tokens", + ] + assert all(span["measure"] == {"state": "unavailable", "value": None} for span in spans) + assert all(span["reason"] == "parent_invocation_unverified" for span in spans) + assert all(span["source_id"] == f"{source_id}#4" for span in spans) + assert all(span["invocation_id"] is None and span["turn_id"] is None for span in spans) + assert all(span["harness"] == "claude-code" for span in spans) + assert all(span["provider"] is None and span["model"] is None for span in spans) + assert all( + span["timestamp"] is None + and span["tokenizer_version"] is None + and span["encoding"] is None + for span in spans + ) assert rows_for_walk_item(_otlp_item(source_id, _log("api_request", sid))) == [] assert ( rows_for_walk_item( diff --git a/tests/execution/test_report_index_schema.py b/tests/execution/test_report_index_schema.py index 74f771d174..4c1186b878 100644 --- a/tests/execution/test_report_index_schema.py +++ b/tests/execution/test_report_index_schema.py @@ -10,12 +10,12 @@ from autoskillit.execution._report_index_rows import ( _ROW_FIELDS, REPORT_INDEX_SCHEMA_VERSION, - REPORT_ROW_TYPES, ) +from autoskillit.execution._report_index_types import REPORT_ROW_TYPES pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] -_EXPECTED = (2, "e574d4e83971be001c1c92d29e807a421edbf705acbe5e72a03e7e513d43c4ba") +_EXPECTED = (3, "ce6665e9b25e6c0ea3d578271a782cec2b922899b055e9e81f7d74ad2360a6d2") # Locked snapshot of per-kind field name orderings that produced ``_EXPECTED``. # Bump ``_EXPECTED[0]`` and refresh this dict in lockstep when row fields change; # ``test_report_index_schema_version_matches_field_digest`` is what fails first @@ -71,6 +71,8 @@ "success", "time_ms", "tool_counts", + "turn_usage_reason", + "turn_usage_state", ], "subagent": [ "actor_level", @@ -85,6 +87,7 @@ "model", "model_swapped", "native_parent_session_id", + "parent_context_spans", "parent_session_key", "provider", "recipe", @@ -99,6 +102,31 @@ "transcript_state", "usage_state", ], + "turn": [ + "cache_read_tokens", + "cache_write_tokens", + "context_fraction", + "context_window_tokens", + "harness", + "input_tokens", + "key", + "kind", + "level", + "message_id", + "model", + "ordinal", + "output_tokens", + "provider", + "recipe", + "request_id", + "schema_version", + "session_id", + "session_key", + "skill", + "source_id", + "step", + "time_ms", + ], "tool": [ "agent_name", "duration_ms", diff --git a/tests/execution/test_report_walk.py b/tests/execution/test_report_walk.py index 51ee45b5af..a6325cdf20 100644 --- a/tests/execution/test_report_walk.py +++ b/tests/execution/test_report_walk.py @@ -11,7 +11,12 @@ import zstandard import autoskillit.execution.evidence._native_child_projection as native_child_projection -from autoskillit.core import iter_merged_assistant_turns +from autoskillit.core import ( + TOKEN_USAGE_SCHEMA_VERSION, + TURN_USAGE_SCHEMA_VERSION, + TokenMeasure, + iter_merged_assistant_turns, +) from autoskillit.execution.evidence.report_walk import ( SourceGapError, WalkItem, @@ -20,6 +25,7 @@ from autoskillit.execution.session_log.session_index import ( read_tolerant_session_index_rows, ) +from tests.execution._report_index_fixtures import _turn_usage_descriptor pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] @@ -65,6 +71,45 @@ def _session( return row +def _turn_usage_row( + *, + backend: str, + provider: str, + model: str, + timestamp: str | None, +) -> dict[str, Any]: + return { + "backend": backend, + "provider_used": provider, + "message_id": "message-1", + "request_id": "request-1", + "timestamp": timestamp, + "model": model, + "input_tokens": {"state": "measured_zero", "value": 0}, + "output_tokens": {"state": "unknown", "value": None}, + "cache_read_tokens": {"state": "measured", "value": 12}, + "cache_write_tokens": {"state": "unavailable", "value": None}, + "peak_context": {"state": "measured", "value": 12}, + "context_window_tokens": 100, + "context_fraction": 0.12, + } + + +def _write_turn_usage( + root: Path, + dir_name: str, + *, + descriptor: dict[str, Any] | None, + ledger: bytes | None, +) -> None: + session_dir = root / "sessions" / dir_name + session_dir.mkdir(parents=True, exist_ok=True) + if descriptor is not None: + (session_dir / "token_usage.json").write_text(json.dumps(descriptor), encoding="utf-8") + if ledger is not None: + (session_dir / "turn_usage.jsonl").write_bytes(ledger) + + def _consume( root: Path, store: dict[str, Any], @@ -138,6 +183,134 @@ def test_first_pass_emits_joinable_records_and_tolerates_old_index_rows( assert read_tolerant_session_index_rows(root / "sessions.jsonl") == [current, old_schema] +def test_turn_ledgers_are_admitted_for_live_and_archived_sessions( + tmp_path: Path, +) -> None: + root = tmp_path / "logs" + archived = _session("archived-attempt", "same-session-id", backend="claude-code") + live = _session("live-attempt", "same-session-id", backend="codex") + _write_jsonl(root / "sessions-archive.jsonl", [archived]) + _write_jsonl(root / "sessions.jsonl", [live]) + _write_turn_usage( + root, + "archived-attempt", + descriptor=_turn_usage_descriptor(count=1), + ledger=_json_line( + _turn_usage_row( + backend="claude-code", + provider="anthropic", + model="claude-resolved", + timestamp="2026-10-01T00:00:00Z", + ) + ), + ) + _write_turn_usage( + root, + "live-attempt", + descriptor=_turn_usage_descriptor(count=1), + ledger=_json_line( + { + **_turn_usage_row( + backend="codex", + provider="openai", + model="gpt-resolved", + timestamp=None, + ), + "model": None, + "context_window_tokens": None, + "context_fraction": None, + } + ), + ) + + records = { + item.source_id: item.record + for item in iter_report_walk(root) + if item.kind == "session" and item.record is not None + } + + for source_id, provider, model in ( + ("archived-attempt", "anthropic", "claude-resolved"), + ("live-attempt", "openai", None), + ): + assert records[source_id] is not None + assert records[source_id]["turn_usage_state"] == "observed" + assert records[source_id]["turn_usage_reason"] is None + (turn,) = records[source_id]["turn_usage_rows"] + assert turn["provider_used"] == provider + assert turn["model"] == model + assert turn["input_tokens"] == TokenMeasure.observed(0) + assert turn["output_tokens"] == TokenMeasure.unknown() + assert records["live-attempt"]["turn_usage_rows"][0]["timestamp"] is None + assert records["live-attempt"]["turn_usage_rows"][0]["context_window_tokens"] is None + assert records["live-attempt"]["turn_usage_rows"][0]["context_fraction"] is None + + +@pytest.mark.parametrize( + ("descriptor", "ledger", "reason"), + [ + (None, None, "descriptor-missing"), + ( + _turn_usage_descriptor(count=0, filename=None), + None, + "ledger-not-published", + ), + (_turn_usage_descriptor(count=0), None, "ledger-missing"), + ( + { + **_turn_usage_descriptor(count=1), + "schema_version": TOKEN_USAGE_SCHEMA_VERSION + 1, + }, + None, + "descriptor-unsupported-version", + ), + ( + _turn_usage_descriptor(count=1, version=TURN_USAGE_SCHEMA_VERSION + 1), + None, + "ledger-unsupported-version", + ), + ( + _turn_usage_descriptor(count=2), + _json_line( + _turn_usage_row( + backend="codex", + provider="openai", + model="gpt-resolved", + timestamp=None, + ) + ), + "ledger-count-mismatch", + ), + ( + _turn_usage_descriptor(count=2), + b'{"input_tokens":{"state":"measured","value":1}}\nnot-json', + "ledger-malformed", + ), + ( + _turn_usage_descriptor(count=1, filename="../turn_usage.jsonl"), + None, + "descriptor-invalid-ledger-file", + ), + ], +) +def test_invalid_or_absent_turn_ledgers_have_explicit_coverage( + tmp_path: Path, + descriptor: dict[str, Any] | None, + ledger: bytes | None, + reason: str, +) -> None: + root = tmp_path / "logs" + _write_jsonl(root / "sessions.jsonl", [_session("attempt", "sid")]) + _write_turn_usage(root, "attempt", descriptor=descriptor, ledger=ledger) + + (item,) = [candidate for candidate in iter_report_walk(root) if candidate.kind == "session"] + + assert item.record is not None + assert item.record["turn_usage_state"] == "unavailable" + assert item.record["turn_usage_reason"] == reason + assert item.record["turn_usage_rows"] == [] + + def test_session_walk_reads_child_outcomes_by_native_parent_id( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/report/test_deck_core_js.py b/tests/report/test_deck_core_js.py index 51324fbe4f..35467654ca 100644 --- a/tests/report/test_deck_core_js.py +++ b/tests/report/test_deck_core_js.py @@ -69,6 +69,56 @@ def test_href_for_keeps_cohort_params_and_applies_target_changes(deck_js: Any) - deck_js.call("DeckCore.hrefFor", route, {"view": "spend"}, COHORT_KEYS) == "#/spend?harness=codex&window=28d" ) + + +def test_href_for_preserves_exact_session_selection(deck_js: Any) -> None: + route = { + "view": "context", + "entity": None, + "params": { + "harness": ["codex"], + "provider": ["openai"], + "level": ["L2"], + "window": ["28d"], + "session": ["owner-key-1"], + }, + } + + href = deck_js.call("DeckCore.hrefFor", route, {"view": "gaps"}, COHORT_KEYS) + + assert href == ("#/gaps?harness=codex&provider=openai&level=L2&window=28d&session=owner-key-1") + assert deck_js.call("DeckCore.hrefFor", route, {"view": "errors"}, COHORT_KEYS) == ( + "#/errors?harness=codex&provider=openai&level=L2&window=28d" + ) + + +def test_href_for_keeps_explicit_identity_filters_across_evidence_views(deck_js: Any) -> None: + route = { + "view": "context", + "entity": None, + "params": { + "harness": ["codex"], + "window": ["7d"], + "skill": ["demo"], + "recipe": ["recipe-demo"], + "step": ["run"], + "model": ["gpt-test"], + "session": ["owner-key-1"], + }, + } + + href = deck_js.call("DeckCore.hrefFor", route, {"view": "parity"}, COHORT_KEYS) + decoded = deck_js.call("DeckCore.decodeRoute", href, "cohort") + + assert decoded["params"] == { + "harness": ["codex"], + "window": ["7d"], + "skill": ["demo"], + "recipe": ["recipe-demo"], + "step": ["run"], + "model": ["gpt-test"], + "session": ["owner-key-1"], + } assert ( deck_js.call( "DeckCore.hrefFor", @@ -268,6 +318,136 @@ def test_prepared_selection_returns_no_metrics_for_stale_scope(deck_js: Any) -> assert selected["roleRows"] == [] +def test_new_prepared_view_selects_actual_rows_and_exact_session_block(deck_js: Any) -> None: + chips = { + "window": [{"key": "7d", "match": "7d", "label": "7 days", "state": "live", "days": 7}], + "level": [{"key": "L2", "match": "orchestrator", "label": "L2", "state": "live"}], + "harness": [{"key": "codex", "match": "codex", "label": "Codex", "state": "live"}], + "provider": [{"key": "openai", "match": "openai", "label": "OpenAI", "state": "live"}], + } + selected_row = { + "key": "prepared-row", + "harness": "codex", + "provider": "openai", + "skill": "demo", + "recipe": "recipe-demo", + "step": "run", + "model": "gpt-test", + } + other_cohort_row = { + "key": "other-cohort", + "harness": "claude-code", + "provider": "anthropic", + } + session_row = {**selected_row, "key": "session-row"} + prepared = { + "skills": [ + { + "window": "7d", + "levels": ["orchestrator"], + "rows": [{"key": "fallback", "harness": "codex", "provider": "openai"}], + } + ], + "context": { + "blocks": [ + { + "window": "7d", + "levels": ["orchestrator"], + "rows": [ + selected_row, + other_cohort_row, + {**selected_row, "key": "other-step", "step": "review"}, + ], + "metrics": { + "parent_context": [ + selected_row, + other_cohort_row, + {**selected_row, "key": "other-step", "step": "review"}, + ] + }, + "sessions": { + "owner-key-1": { + "rows": [session_row], + "metrics": {"parent_context": [session_row]}, + } + }, + } + ] + }, + } + route = { + "view": "context", + "entity": None, + "params": { + "window": ["7d"], + "level": ["L2"], + "harness": ["codex"], + "provider": ["openai"], + "skill": ["demo"], + "recipe": ["recipe-demo"], + "step": ["run"], + "model": ["gpt-test"], + }, + } + + selected = deck_js.call("DeckCore.selectPrepared", prepared, "context", chips, route) + + assert selected["rows"] == [selected_row] + assert selected["metrics"]["parent_context"] == [selected_row] + assert selected["skillRows"] == [{"key": "fallback", "harness": "codex", "provider": "openai"}] + route["params"]["session"] = ["owner-key-1"] + session_selected = deck_js.call("DeckCore.selectPrepared", prepared, "context", chips, route) + assert session_selected["rows"] == [session_row] + assert session_selected["metrics"]["parent_context"] == [session_row] + assert session_selected["selection"]["session"] == "owner-key-1" + route["params"]["session"] = ["unknown-owner"] + missing_session = deck_js.call("DeckCore.selectPrepared", prepared, "context", chips, route) + assert missing_session["rows"] == [] + assert missing_session["metrics"] == [] + + route["params"]["session"] = ["owner-key-1"] + route["params"]["step"] = ["missing-step"] + missing_step = deck_js.call("DeckCore.selectPrepared", prepared, "context", chips, route) + assert missing_step["rows"] == [] + assert missing_step["metrics"]["parent_context"] == [] + + prepared["trend"] = { + "blocks": [ + { + "window": "7d", + "levels": ["orchestrator"], + "rows": [selected_row], + "metrics": {"comparisons": [selected_row]}, + } + ] + } + route["view"] = "trend" + route["params"]["session"] = ["owner-key-1"] + route["params"]["step"] = ["run"] + trend_selected = deck_js.call("DeckCore.selectPrepared", prepared, "trend", chips, route) + assert trend_selected["rows"] == [selected_row] + assert trend_selected["metrics"]["comparisons"] == [selected_row] + assert trend_selected["selection"]["session"] is None + + unattributed = { + "key": "unattributed-event", + "harness": None, + "provider": None, + "population": "unattributed_tool", + "attribution_state": "unattributed", + } + prepared["errors"] = { + "blocks": [ + {"window": "7d", "levels": ["orchestrator"], "rows": [unattributed], "metrics": {}} + ] + } + route["view"] = "errors" + for key in ("session", "skill", "recipe", "step", "model"): + route["params"].pop(key, None) + selected_errors = deck_js.call("DeckCore.selectPrepared", prepared, "errors", chips, route) + assert selected_errors["rows"] == [unattributed] + + @pytest.mark.parametrize( ("levels", "expected"), [ diff --git a/tests/report/test_deck_html.py b/tests/report/test_deck_html.py index 8f25fd71a5..bb9d0d077b 100644 --- a/tests/report/test_deck_html.py +++ b/tests/report/test_deck_html.py @@ -209,7 +209,7 @@ def test_deck_asset_directory_contains_only_registered_shell_assets() -> None: assert actual == {"deck.html", "deck.css", "core.js", "shell.js"} | view_assets -def test_built_wheel_contains_the_four_new_view_scripts(tmp_path: Path) -> None: +def test_built_wheel_contains_registered_view_scripts(tmp_path: Path) -> None: # REQ-108/REQ-158: isolate the plan's wheel-check directory per test worker. wheel_out = tmp_path / "wheel-check" result = subprocess.run( @@ -224,8 +224,9 @@ def test_built_wheel_contains_the_four_new_view_scripts(tmp_path: Path) -> None: assert len(wheels) == 1, f"Expected one built wheel, found {wheels}" with ZipFile(wheels[0]) as wheel: required = { - f"autoskillit/assets/deck/views/{view}.js" - for view in ("spend", "efficiency", "skill", "role") + f"autoskillit/assets/deck/{view.script}" + for view in DECK_VIEWS + if view.planned_issue is None and view.script is not None } missing = required - set(wheel.namelist()) assert not missing, f"Built wheel is missing view scripts: {sorted(missing)}" diff --git a/tests/report/test_deck_payload.py b/tests/report/test_deck_payload.py index 06d1f0970b..2bdb9f2703 100644 --- a/tests/report/test_deck_payload.py +++ b/tests/report/test_deck_payload.py @@ -15,15 +15,22 @@ aggregate_measures, measure_ratio, ) +from autoskillit.report.deck import _measure_helpers as deck_measure_helpers from autoskillit.report.deck import _payload as deck_payload_module +from autoskillit.report.deck import _view_common as deck_view_common from autoskillit.report.deck import build_deck_payload from autoskillit.report.deck._payload import encode_table from autoskillit.report.deck._registry import ( AVAILABILITY_VOCABULARY, DECK_VIEWS, + ERROR_TABLE, + GAP_TABLE, LANDING_VIEW, + PARITY_TABLE, SESSION_COLUMNS, SESSION_TABLE, + TREND_TABLE, + TURN_TABLE, ) from tests.report._fixtures import DECK_GENERATED_AT, subagent_row, token_measure from tests.report._fixtures import session_row as _row @@ -32,6 +39,28 @@ GEN_MS = 1_791_072_000_000 +@pytest.mark.parametrize( + ("function", "all_keys", "finite_keys"), + [ + ("_window_level_blocks", ["recent", "future", "untimed"], ["recent", "future"]), + ("_population_blocks", ["recent", "untimed"], ["recent"]), + ], +) +def test_level_grouping_preserves_each_window_time_policy( + function: str, all_keys: list[str], finite_keys: list[str] +) -> None: + rows = [ + {"key": "recent", "level": "skill", "time_ms": GEN_MS - 1}, + {"key": "future", "level": "skill", "time_ms": GEN_MS + 1}, + {"key": "untimed", "level": "skill", "time_ms": None}, + ] + blocks = getattr(deck_view_common, function)(rows, generated_at_ms=GEN_MS) + assert blocks + for window, levels, members in blocks: + assert levels == ["skill"] + assert [row["key"] for row in members] == (all_keys if window == "all" else finite_keys) + + def _payload(rows: list[dict[str, Any]]) -> dict[str, Any]: return build_deck_payload( rows, @@ -521,8 +550,8 @@ def definition_loader() -> tuple[SimpleNamespace, ...]: ) aggregate_calls: list[int] = [] ratio_calls: list[int] = [] - original_aggregate = getattr(deck_payload_module, "aggregate_measures", aggregate_measures) - original_ratio = getattr(deck_payload_module, "measure_ratio", measure_ratio) + original_aggregate = deck_measure_helpers.aggregate_measures + original_ratio = deck_measure_helpers.measure_ratio def aggregate_spy(records: Any, fields: Any, **kwargs: Any) -> Any: rows = list(records) @@ -534,8 +563,8 @@ def ratio_spy(records: Any, numerator: str, denominator: str, **kwargs: Any) -> ratio_calls.append(len(rows)) return original_ratio(rows, numerator, denominator, **kwargs) - monkeypatch.setattr(deck_payload_module, "aggregate_measures", aggregate_spy, raising=False) - monkeypatch.setattr(deck_payload_module, "measure_ratio", ratio_spy, raising=False) + monkeypatch.setattr(deck_measure_helpers, "aggregate_measures", aggregate_spy) + monkeypatch.setattr(deck_measure_helpers, "measure_ratio", ratio_spy) payload = build_deck_payload( sessions, request_rows=requests, @@ -672,7 +701,16 @@ def test_prepared_identity_tables_and_view_projections(prepared_metrics: SimpleN built_views = {view.view_id for view in DECK_VIEWS if view.planned_issue is None} assert set(prepared["view_chips"]) == built_views assert set(prepared["view_histories"]) == built_views - assert set(payload["tables"]) == {SESSION_TABLE, "skills", "roles"} + assert set(payload["tables"]) == { + SESSION_TABLE, + "skills", + "roles", + TURN_TABLE, + ERROR_TABLE, + TREND_TABLE, + GAP_TABLE, + PARITY_TABLE, + } def table_rows(table: dict[str, Any]) -> list[dict[str, Any]]: return [dict(zip(table["columns"], row, strict=True)) for row in table["rows"]] @@ -733,3 +771,373 @@ def test_observed_zero_ratio_can_be_flagged_with_complete_definitions( assert signal["definition_roles"] == [_AUDITOR] assert signal["review_eligible"] is True assert signal["review_reason"] is None + + +def _view_block(payload: dict[str, Any], view: str, window: str = "all") -> dict[str, Any]: + return next( + block + for block in payload["prepared"][view]["blocks"] + if block["window"] == window and block["levels"] == ["skill"] + ) + + +def _turn_fact( + session: dict[str, Any], + *, + time_ms: int, + model: str = "claude-opus-4-1-20250805", +) -> dict[str, Any]: + return { + "key": f"{session['key']}:turn:0", + "kind": "turn", + "session_key": session["key"], + "source_id": session["key"], + "session_id": session["session_id"], + "ordinal": 0, + "request_id": "req-1", + "message_id": "message-1", + "time_ms": time_ms, + "harness": session["harness"], + "provider": session["provider"], + "model": model, + "skill": session["skill"], + "recipe": session["recipe"], + "step": session["step"], + "level": session["level"], + "input_tokens": token_measure(75), + "output_tokens": token_measure(0), + "cache_read_tokens": token_measure(50), + "cache_write_tokens": token_measure(None, state="unavailable"), + "context_window_tokens": 100, + "context_fraction": 0.5, + } + + +def test_context_rows_keep_exact_turn_owner_model_states_and_ratio() -> None: + owner = _row( + "owner", + session_id="native-owner", + time_ms=GEN_MS - 1_000, + level="skill", + skill="demo", + recipe="recipe-demo", + step="run", + turn_usage_state="observed", + turn_usage_reason=None, + ) + turn = _turn_fact(owner, time_ms=GEN_MS - 900) + payload = build_deck_payload( + [owner], + request_rows=[ + { + "key": "native-owner:req-1", + "session_key": "owner", + "request_id": "req-1", + "model": "claude-opus-4-1-20250805", + } + ], + turn_rows=(row for row in [turn]), + generated_at=DECK_GENERATED_AT, + index_schema_version=7, + ) + + turns_table = payload["tables"][TURN_TABLE] + turn_row = dict(zip(turns_table["columns"], turns_table["rows"][0], strict=True)) + assert turn_row["session_key"] == "owner" + assert turn_row["model"] == "claude-opus-4-1-20250805" + assert turn_row["output_tokens"] == token_measure(0) + assert turn_row["cache_write_tokens"] == token_measure(None, state="unavailable") + + context = _view_block(payload, "context") + prepared_turn = next(row for row in context["rows"] if row["kind"] == "turn") + assert prepared_turn["session_key"] == "owner" + assert prepared_turn["context_fraction_percent"] == { + "state": "measured", + "value": 50.0, + "sample_size": 1, + "excluded_runs": 0, + "unknown_runs": 0, + } + assert prepared_turn["fraction_disagreement"] is False + assert prepared_turn["request_recorded"] is True + assert context["sessions"]["owner"]["rows"] == [prepared_turn] + + +def test_context_keeps_ordinal_order_across_an_untimed_missing_window_turn() -> None: + owner = _row( + "ordered-owner", + time_ms=GEN_MS - 1_000, + level="skill", + skill="demo", + recipe="recipe-demo", + step="run", + turn_usage_state="observed", + ) + turns = [ + _turn_fact(owner, time_ms=GEN_MS - 900), + _turn_fact(owner, time_ms=None), + _turn_fact(owner, time_ms=GEN_MS - 700), + ] + for ordinal, turn in enumerate(turns): + turn["ordinal"] = ordinal + turn["key"] = f"ordered-owner:turn:{ordinal}" + turn["context_window_tokens"] = None if ordinal == 1 else 100 + turn["context_fraction"] = None if ordinal == 1 else 0.5 + payload = build_deck_payload( + [owner], + turn_rows=turns, + generated_at=DECK_GENERATED_AT, + index_schema_version=7, + ) + + rows = _view_block(payload, "context")["rows"] + assert [row["ordinal"] for row in rows] == [0, 1, 2] + assert rows[1]["time_ms"] is None + assert rows[1]["context_fraction_percent"]["state"] == "unknown" + assert rows[1]["context_fraction_percent"]["value"] is None + + +def test_parent_span_provenance_is_json_safe_after_measure_aggregation() -> None: + owner = _row( + "span-owner", + session_id="native-span-owner", + time_ms=GEN_MS - 1_000, + level="skill", + skill="demo", + recipe="recipe-demo", + step="run", + ) + spans = [ + { + "field": "parent_prompt_tokens", + "measure": TokenMeasure.observed(6).to_dict(), + "invocation_id": "call-1", + "turn_id": "turn-1", + "source_id": "parent-prompt", + "timestamp": DECK_GENERATED_AT.isoformat(), + "harness": "claude-code", + "provider": "anthropic", + "model": "model-x", + "tokenizer_version": "test", + "encoding": "test", + "reason": None, + }, + { + "field": "subagent_return_tokens", + "measure": TokenMeasure.observed(4).to_dict(), + "invocation_id": "call-1", + "turn_id": "turn-1", + "source_id": "parent-return", + "timestamp": DECK_GENERATED_AT.isoformat(), + "harness": "claude-code", + "provider": "anthropic", + "model": "model-x", + "tokenizer_version": "test", + "encoding": "test", + "reason": None, + }, + ] + child = { + "key": "span-owner:child-1", + "child_id": "child-1", + "role": _AUDITOR, + "actor_level": "L0", + "parent_session_key": owner["key"], + "native_parent_session_id": owner["session_id"], + "provider": "anthropic", + "parent_context_spans": spans, + } + payload = build_deck_payload( + [owner], + subagent_rows=[child], + generated_at=DECK_GENERATED_AT, + index_schema_version=7, + ) + + encoded = json.dumps(payload, allow_nan=False) + context = _view_block(json.loads(encoded), "context") + model_row = next( + row for row in context["metrics"]["parent_context"] if row["model"] == "model-x" + ) + assert model_row["provenance"][0]["measure"] == {"state": "measured", "value": 6} + assert model_row["provenance"][1]["measure"] == {"state": "measured", "value": 4} + + +def test_errors_keep_session_and_tool_failures_separate_with_known_denominators() -> None: + common = { + "time_ms": GEN_MS - 1_000, + "level": "skill", + "skill": "demo", + "recipe": "recipe-demo", + "step": "run", + } + sessions = [ + _row("failed", **common, success=False, adjudication_reason="timeout"), + _row("passed", **common, success=True), + _row("unknown", **common, success=None), + ] + tools = [ + { + "key": "failed:tool-1", + "session_key": "failed", + "time_ms": GEN_MS - 900, + "success": False, + "error_type": "ToolError", + }, + { + "key": "unknown:tool-2", + "session_key": "unknown", + "time_ms": GEN_MS - 800, + "success": None, + "error_type": "ToolOutcomeUnknown", + }, + { + "key": "orphan:tool-3", + "session_key": None, + "time_ms": GEN_MS - 700, + "success": False, + "error_type": "OrphanError", + }, + ] + payload = build_deck_payload( + sessions, + tool_rows=tools, + generated_at=DECK_GENERATED_AT, + index_schema_version=7, + ) + rows = _view_block(payload, "errors")["rows"] + session_error = next(row for row in rows if row["population"] == "session_failure") + assert session_error["symptom"] == "timeout" + assert session_error["failures"]["value"] == 1 + assert session_error["failure_rate"]["value"] == 0.5 + assert session_error["eligible_count"] == 3 + assert session_error["observed_count"] == 2 + assert session_error["unknown_count"] == 1 + + tool_error = next(row for row in rows if row["symptom"] == "ToolError") + unknown_tool = next(row for row in rows if row["symptom"] == "ToolOutcomeUnknown") + orphan = next(row for row in rows if row["population"] == "unattributed_tool") + assert tool_error["population"] == "tool_failure" + assert tool_error["failures"]["value"] == 1 + assert unknown_tool["failures"]["state"] == "unknown" + assert orphan["harness"] == "unknown" and orphan["provider"] == "unknown" + assert orphan["attribution_state"] == "unattributed" + assert orphan["session_key"] is None + assert _chip(payload["chips"]["errors"], "harness", "unknown")["state"] == "live" + + +def test_trends_use_utc_days_and_disclose_future_untimed_and_interval_samples() -> None: + day_ms = 86_400_000 + sessions = [ + _row( + "earlier", + time_ms=GEN_MS - 5 * day_ms, + level="skill", + skill="demo", + recipe="recipe-demo", + step="run", + model="model-a", + success=True, + input_tokens=token_measure(10), + ), + _row( + "later", + time_ms=GEN_MS - day_ms, + level="skill", + skill="demo", + recipe="recipe-demo", + step="run", + model="model-a", + success=False, + input_tokens=token_measure(30), + ), + _row( + "future", + time_ms=GEN_MS + 1, + level="skill", + skill="demo", + recipe="recipe-demo", + step="run", + model="model-a", + success=False, + input_tokens=token_measure(999), + ), + _row( + "untimed", + level="skill", + skill="demo", + recipe="recipe-demo", + step="run", + model="model-a", + success=None, + input_tokens=token_measure(100), + ), + ] + payload = build_deck_payload( + sessions, + generated_at=DECK_GENERATED_AT, + index_schema_version=7, + ) + trend = _view_block(payload, "trend", "7d") + assert trend["metrics"]["future_sessions"] == 1 + assert trend["metrics"]["untimed_sessions"] == 1 + assert all(row["time_ms"] <= GEN_MS for row in trend["rows"]) + comparison = trend["metrics"]["comparisons"][0] + assert comparison["earlier"]["measures"]["input_tokens"]["value"] == 10.0 + assert comparison["later"]["measures"]["input_tokens"]["value"] == 30.0 + assert comparison["deltas"]["input_tokens"]["value"] == 20.0 + assert comparison["deltas"]["failure_share"]["value"] == 100.0 + assert any( + row["eligible_sessions"] == 0 + and row["measures"]["input_tokens"]["state"] == "no_observations" + for row in trend["rows"] + ) + + +def test_gap_and_parity_share_missing_evidence_states_and_session_selection() -> None: + owner = _row( + "empty-ledger", + time_ms=GEN_MS - 1_000, + level="skill", + skill="demo", + recipe="recipe-demo", + step="run", + turn_usage_state="unavailable", + turn_usage_reason="turn_usage_ledger_missing", + ) + payload = build_deck_payload( + [owner], + generated_at=DECK_GENERATED_AT, + index_schema_version=7, + ) + gaps = _view_block(payload, "gaps") + parity = _view_block(payload, "parity") + parity_by_key = {row["key"]: row for row in parity["rows"]} + assert all(parity_by_key[row["key"]]["state"] == row["state"] for row in gaps["rows"]) + turn_series = next( + row + for row in parity["rows"] + if row["field"] == "turn_series_coverage" and row["population"] == "turn" + ) + missing_tools = next( + row + for row in parity["rows"] + if row["field"] == "tool_errors" and row["population"] == "tool_result" + ) + prompt = next( + row + for row in parity["rows"] + if row["field"] == "parent_prompt_tokens" and row["population"] == "parent_context" + ) + assert turn_series["state"] == "unavailable" + assert "turn_usage_ledger_missing" in turn_series["reason"] + assert missing_tools["state"] == "no_observations" + assert prompt["state"] == "no_observations" + session_turn_gap = next( + row + for row in gaps["sessions"]["empty-ledger"]["rows"] + if row["field"] == "turn_series_coverage" and row["population"] == "turn" + ) + assert session_turn_gap["session_key"] == "empty-ledger" + assert session_turn_gap["state"] == "unavailable" + assert "turn_usage_ledger_missing" in session_turn_gap["reason"] diff --git a/tests/report/test_deck_registry.py b/tests/report/test_deck_registry.py index 7c208808ae..90ef6e7856 100644 --- a/tests/report/test_deck_registry.py +++ b/tests/report/test_deck_registry.py @@ -3,7 +3,7 @@ import pytest from autoskillit.core import SessionType, TokenMeasureState -from autoskillit.execution._report_index_rows import ReportSessionRow +from autoskillit.execution._report_index_types import ReportSessionRow from autoskillit.report.deck._registry import ( AVAILABILITY_VOCABULARY, DECK_VIEWS, @@ -94,13 +94,18 @@ def test_session_columns_follow_report_session_row_schema() -> None: assert len(SESSION_COLUMNS) == len(set(SESSION_COLUMNS)) -def test_reserved_measure_views_have_primary_tables_and_registered_scripts() -> None: +def test_built_views_have_primary_tables_and_registered_scripts() -> None: views = {view.view_id: view for view in DECK_VIEWS} expected = { "spend": ("skills", "views/spend.js"), "efficiency": ("skills", "views/efficiency.js"), "skill": ("skills", "views/skill.js"), "role": ("roles", "views/role.js"), + "context": ("turns", "views/context.js"), + "errors": ("errors", "views/errors.js"), + "trend": ("trends", "views/trend.js"), + "gaps": ("gaps", "views/gaps.js"), + "parity": ("parity", "views/parity.js"), } assert { diff --git a/tests/report/test_deck_shell_js.py b/tests/report/test_deck_shell_js.py index c625446f66..80b726fa29 100644 --- a/tests/report/test_deck_shell_js.py +++ b/tests/report/test_deck_shell_js.py @@ -1,5 +1,6 @@ """Browser-free contract tests for the observability deck's shell.""" +import json from typing import Any import pytest @@ -10,6 +11,54 @@ pytestmark = [pytest.mark.small] +_SHELL_DOM_HARNESS = r""" +function shellNode(tag) { + const node = {tag, attrs: {}, children: [], events: [], text: "", __shellNode: true}; + node.appendChild = child => { + if (!child || typeof child !== "object" || child.__shellNode !== true) { + throw new TypeError("appendChild requires a node"); + } + node.children.push(child); + return child; + }; + node.replaceChildren = (...children) => { node.children = children; node.text = ""; }; + node.setAttribute = (key, value) => { node.attrs[key] = String(value); }; + node.addEventListener = (event, callback) => { node.events.push([event, callback]); }; + Object.defineProperty(node, "textContent", { + get() { return node.text + node.children.map(child => + typeof child === "object" ? child.textContent : String(child)).join(""); }, + set(value) { node.text = String(value); node.children = []; } + }); + return node; +} +const shellNodes = Object.fromEntries(["deck-nav", "deck-cohort", "deck-view", "deck-foot"] + .map(id => [id, shellNode("div")])); +globalThis.document = { + createElement: shellNode, + createElementNS: (_namespace, tag) => shellNode(tag), + createTextNode: text => { + const node = shellNode("#text"); + node.textContent = String(text); + return node; + }, + getElementById: id => shellNodes[id] ?? null, + addEventListener() {} +}; +globalThis.location = {hash: ""}; +globalThis.window = {location, addEventListener() {}}; +globalThis.registerDeckProbe = viewId => { + DeckShell.registerView(viewId, ctx => { + globalThis.__selected = {rows: ctx.rows, metrics: ctx.metrics, session: ctx.selection.session}; + return ctx.el("p", {}, ["Selected: ", ctx.el("span", {}, + ctx.rows.map(row => row.key).join(",")), " selected."]); + }); + return true; +}; +globalThis.deckProbeResult = () => JSON.stringify(__selected); +globalThis.deckCohortText = () => shellNodes["deck-cohort"].textContent; +globalThis.deckViewText = () => shellNodes["deck-view"].textContent; +""" + def test_shell_registers_built_views_without_document(deck_asset: Any) -> None: with MiniRacer() as ctx: @@ -20,3 +69,78 @@ def test_shell_registers_built_views_without_document(deck_asset: Any) -> None: expected = [view.view_id for view in DECK_VIEWS if view.planned_issue is None] assert sorted(registered) == sorted(expected) + + +@pytest.mark.parametrize( + ("view_id", "headline"), + [ + ("context", "context record"), + ("errors", "symptom group"), + ("trend", "daily observation"), + ("gaps", "gap"), + ("parity", "coverage cell"), + ], +) +def test_shell_passes_selected_prepared_rows_to_renderer_and_headline( + deck_asset: Any, view_id: str, headline: str +) -> None: + selected_row = {"key": "prepared-population-row", "harness": "codex", "provider": "openai"} + chips = { + "window": [ + {"key": "all", "match": "all", "label": "all history", "state": "live", "days": None} + ], + "level": [{"key": "L2", "match": "orchestrator", "label": "L2", "state": "live"}], + "harness": [{"key": "codex", "match": "codex", "label": "Codex", "state": "live"}], + "provider": [{"key": "openai", "match": "openai", "label": "OpenAI", "state": "live"}], + } + session_rows = {"owner-key-1": {"rows": [selected_row], "metrics": {"session": True}}} + block = { + "window": "all", + "levels": ["orchestrator"], + "rows": [selected_row], + "metrics": {"selected": view_id}, + "sessions": session_rows, + } + payload = { + "tables": {"sessions": {"columns": ["session_key"], "rows": []}}, + "facets": [ + {"id": "window", "kind": "window", "label": "Window"}, + {"id": "level", "kind": "values", "label": "Level"}, + {"id": "harness", "kind": "values", "label": "Harness"}, + {"id": "provider", "kind": "values", "label": "Provider"}, + ], + "landing": "cohort", + "views": [ + { + "id": view_id, + "status": "built", + "group": "Evidence", + "question": view_id, + "decision": "inspect evidence", + } + ], + "chips": {view_id: chips}, + "prepared": {view_id: {"blocks": [block]}}, + "history": {"first_ms": None, "last_ms": None, "untimed": 0}, + "generated_at_ms": 0, + "index_schema_version": 3, + } + with MiniRacer() as ctx: + ctx.eval(deck_asset("core.js")) + ctx.eval(_SHELL_DOM_HARNESS) + ctx.eval(deck_asset("shell.js")) + ctx.call("registerDeckProbe", view_id) + ctx.eval( + 'location.hash = "#/" + "' + + view_id + + '?window=all&level=L2&harness=codex&provider=openai";' + ) + ctx.eval(f"DeckShell.boot({json.dumps(payload)});") + result = json.loads(ctx.call("deckProbeResult")) + cohort_text = ctx.call("deckCohortText") + view_text = ctx.call("deckViewText") + + assert result["rows"] == [selected_row] + assert result["metrics"] == {"selected": view_id} + assert f"1 {headline}" in cohort_text + assert view_text == "Selected: prepared-population-row selected." diff --git a/tests/report/test_deck_views_js.py b/tests/report/test_deck_views_js.py index e6da2c5274..dc06f2879a 100644 --- a/tests/report/test_deck_views_js.py +++ b/tests/report/test_deck_views_js.py @@ -1,13 +1,18 @@ """Browser-free renderer contracts for the observability deck's prepared views.""" +from datetime import UTC, datetime, timedelta from typing import Any import pytest from py_mini_racer import MiniRacer +from autoskillit.report.deck import build_deck_payload +from tests.report._fixtures import session_row, subagent_row, token_measure + pytestmark = [pytest.mark.small] VIEW_IDS = ("spend", "efficiency", "skill", "role") +NEW_VIEW_IDS = ("context", "errors", "trend", "gaps", "parity") _RENDERER_HARNESS = r""" globalThis.__deckViews = {}; @@ -19,12 +24,29 @@ function deckNode(tag, attrs = {}, children = null) { const node = {tag, attrs: {...attrs}, children: [], events: {}, hidden: attrs.hidden === true, - textContent: ""}; - node.appendChild = child => { node.children.push(child); return child; }; + textContent: "", __deckNode: true}; + node.appendChild = child => { + if (!child || typeof child !== "object" || child.__deckNode !== true) { + throw new TypeError("appendChild requires a node"); + } + node.children.push(child); + return child; + }; node.addEventListener = (name, callback) => { node.events[name] = callback; }; node.setAttribute = (name, value) => { node.attrs[name] = String(value); }; - if (Array.isArray(children)) node.children.push(...children); - else if (children !== null && children !== undefined) node.children.push(children); + if (Array.isArray(children)) { + for (const child of children) { + if (child === null || child === undefined) continue; + if (["string", "number", "boolean"].includes(typeof child)) { + node.appendChild(deckNode("#text", {}, String(child))); + } else { + node.appendChild(child); + } + } + } else if (children !== null && children !== undefined) { + if (typeof children === "object") node.appendChild(children); + else node.textContent = String(children); + } return node; } @@ -115,8 +137,16 @@ href: DeckCore.hrefFor(route, target, cohortKeys) }, text), sortableTable: deckTable, - availabilityCell: measure => deckNode("span", {"data-state": measure.state}, - measure.state === "measured" ? String(measure.value) : measure.state), + availabilityCell: measure => { + if (!["measured", "measured_zero", "unknown", "unavailable", "not_applicable"] + .includes(measure.state)) { + throw new Error("unknown availability state: " + measure.state); + } + return deckNode("span", {"data-state": measure.state}, + measure.state === "measured" ? String(measure.value) : + measure.state === "measured_zero" ? "0" : measure.state); + }, + svgElement: (tag, attrs, children) => deckNode(tag, attrs, children), barChart: (items, options) => deckNode("svg", {"aria-label": options.label}, JSON.stringify(items)) }; @@ -124,6 +154,12 @@ globalThis.__deckLastTree = __deckViews[id](context); return deckSnapshot(__deckLastTree); }, + renderPrepared(id, payload, route) { + const chips = payload.chips[id]; + const selected = DeckCore.selectPrepared(payload.prepared, id, chips, route); + return globalThis.DeckTest.render(id, {prepared: payload.prepared, rows: selected.rows, + metrics: selected.metrics, chips, selection: selected.selection, route}); + }, clickReview(index = 0) { const signals = deckFindAll(__deckLastTree, child => child.attrs && child.attrs["data-review-signal"] !== undefined); @@ -133,6 +169,18 @@ control.events.click(); return deckSnapshot(__deckLastTree).reviews[index]; }, + clickContextInput() { + const button = deckFind(__deckLastTree, node => + node.attrs && node.attrs.class === "context-input-toggle"); + const chart = deckFind(__deckLastTree, node => node.tag === "svg" && + node.attrs && node.attrs["aria-label"] === "Inclusive input tokens by turn"); + if (!button || !button.events.click || !chart) { + throw new Error("context input toggle is missing"); + } + const initial = {pressed: button.attrs["aria-pressed"], style: chart.attrs.style}; + button.events.click(); + return {initial, pressed: button.attrs["aria-pressed"], style: chart.attrs.style}; + }, snapshot() { return deckSnapshot(__deckLastTree); }, currentHash() { return location.hash; } }; @@ -143,7 +191,7 @@ def _load_renderers(deck_asset: Any) -> MiniRacer: context = MiniRacer() context.eval(deck_asset("core.js")) context.eval(_RENDERER_HARNESS) - for view_id in VIEW_IDS: + for view_id in (*VIEW_IDS, *NEW_VIEW_IDS): context.eval(deck_asset(f"views/{view_id}.js")) return context @@ -162,6 +210,7 @@ def _view_context(view_id: str, *, definitions_available: bool = True) -> dict[s } for role in roles } + ratio = { "state": "measured", "value": 2.5, @@ -239,6 +288,243 @@ def _view_context(view_id: str, *, definitions_available: bool = True) -> dict[s } +def _new_view_context(view_id: str) -> dict[str, Any]: + def measured(value: int | float) -> dict[str, Any]: + return {"state": "measured", "value": value} + + measured_zero = {"state": "measured_zero", "value": 0} + unknown = {"state": "unknown"} + unavailable = {"state": "unavailable"} + common = { + "harness": "codex", + "provider": "openai", + "skill": "summarizer", + "recipe": "daily", + "step": "draft", + "level": "orchestrator", + "session_key": "owner-key-1", + } + turn = { + **common, + "key": "turn-1", + "kind": "turn", + "ordinal": 1, + "time_ms": 1791235200000, + "model": "gpt-test", + "input_tokens": measured(75), + "output_tokens": measured_zero, + "cache_read_tokens": measured(50), + "cache_write_tokens": unavailable, + "context_window_tokens": 100, + "context_fraction_percent": measured(50), + "turn_usage_state": "observed", + "turn_usage_reason": None, + "fraction_disagreement": False, + } + coverage = { + **common, + "key": "coverage-1", + "kind": "coverage", + "turn_usage_state": "unavailable", + "turn_usage_reason": "turn ledger is missing", + "request_model": "gpt-test", + } + error = { + **common, + "key": "error-1", + "population": "session_failure", + "symptom": "unknown symptom", + "failures": {**measured(1), "state_counts": {"measured": 1}}, + "failure_rate": {**measured(0.25), "sample_size": 4}, + "eligible_count": 4, + "observed_count": 3, + "unknown_count": 1, + "tool_result_event_count": 2, + "tool_event_coverage": unknown, + "attribution_state": "owner indexed", + } + trend = { + **common, + "key": "trend-1", + "day": "2026-10-05", + "time_ms": 1791158400000, + "model": "gpt-test", + "measures": { + field: { + **measured(value), + "observed_sessions": 1, + "eligible_sessions": 2, + "state_counts": {"measured": 1, "unknown": 1}, + } + for field, value in { + "input_tokens": 75, + "output_tokens": 0, + "cache_read_tokens": 50, + "cache_write_tokens": 0, + }.items() + }, + "failure_share": {**measured_zero, "sample_size": 1}, + "eligible_sessions": 2, + "untimed_sessions": 1, + "future_sessions": 0, + "retained_from_ms": 1791158400000, + "window_start_ms": 1790553600000, + "window_end_ms": 1791763200000, + } + gap = { + **common, + "key": "gap-1", + "population": "session_turn", + "field": "context_window_tokens", + "state": "unavailable", + "measure": unavailable, + "state_counts": {"unavailable": 1}, + "eligible_count": 1, + "observation_count": 0, + "reason": "resolved model window is absent", + "question": "Which resolved window is needed?", + "prerequisite": "Resolve the turn model window", + "source": "turn usage ledger", + } + parity = [ + { + **gap, + "key": "parity-1", + "field": "input_tokens", + "state": "measured_zero", + "measure": measured_zero, + "state_counts": {"measured_zero": 1}, + "observation_count": 1, + }, + {**gap, "key": "parity-2", "field": "context_window_tokens"}, + ] + populations = { + "context": ( + [turn, coverage], + { + "parent_context": [ + { + **common, + "model": "gpt-test", + "prompt": { + **measured(12), + "state_counts": {"measured": 1}, + "eligible_count": 1, + "observation_count": 1, + }, + "returned": unavailable, + "comparison": { + "state": "no_observations", + "reason": "no matched invocation", + }, + "eligible_invocations": 1, + "matched_invocations": 0, + "provenance": ["parent transcript"], + "source_basis": "recorded text at parent model", + } + ] + }, + ), + "errors": ([error], {}), + "trend": ( + [trend], + { + "comparisons": [ + { + **common, + "model": "gpt-test", + "earlier": { + "start_ms": 1, + "end_ms": 2, + "covered_start_ms": 1, + "covered_end_ms": 2, + "measures": {"input_tokens": measured(50)}, + "failure_share": measured_zero, + }, + "later": { + "start_ms": 3, + "end_ms": 4, + "covered_start_ms": 3, + "covered_end_ms": 4, + "measures": {"input_tokens": measured(75)}, + "failure_share": measured_zero, + }, + "deltas": { + "input_tokens": { + **measured(25), + "unit": "tokens/session", + "earlier_samples": 1, + "later_samples": 1, + }, + "failure_share": { + **measured_zero, + "unit": "percentage_points", + "earlier_samples": 1, + "later_samples": 1, + }, + }, + } + ] + }, + ), + "gaps": ([gap], {}), + "parity": (parity, {}), + } + rows, metrics = populations[view_id] + params = { + "window": ["all"], + "level": ["L2"], + "harness": ["codex"], + "provider": ["openai"], + } + if view_id in {"context", "gaps", "parity"}: + params["session"] = ["owner-key-1"] + return { + "prepared": {}, + "rows": rows, + "metrics": metrics, + "chips": {}, + "selection": {}, + "route": {"view": view_id, "entity": None, "params": params}, + } + + +@pytest.mark.parametrize( + ("measure", "expected"), + [ + ({"state": "measured_zero", "value": 0}, {"primitive": "measured_zero"}), + ( + {"state": "mixed"}, + {"class": "coverage-state coverage-state--mixed", "text": "mixed coverage"}, + ), + ( + {"state": "no_observations"}, + {"class": "coverage-state coverage-state--no_observations", "text": "no observations"}, + ), + (None, {"class": "coverage-state coverage-state--unknown", "text": "unknown"}), + ], +) +def test_shared_coverage_cell_keeps_primitive_and_summary_states_distinct( + deck_asset: Any, measure: dict[str, Any] | None, expected: dict[str, str] +) -> None: + context = _load_renderers(deck_asset) + try: + context.eval(""" + globalThis.coverageCell = (measure, className) => DeckCore.coverageStateCell({ + availabilityCell: item => ({primitive: item.state}), + el: (tag, attrs, text) => ({class: attrs.class, text}) + }, measure, className); + """) + assert context.call("coverageCell", measure) == expected + if "class" in expected: + assert context.call("coverageCell", measure, "coverage-state") == { + **expected, + "class": "coverage-state", + } + finally: + context.close() + + def test_each_prepared_view_renders_its_population_and_cohort_links(deck_asset: Any) -> None: context = _load_renderers(deck_asset) try: @@ -268,6 +554,176 @@ def test_each_prepared_view_renders_its_population_and_cohort_links(deck_asset: context.close() +@pytest.mark.parametrize( + ("view_id", "marker"), + [ + ("context", "Cache-read proxy"), + ("errors", "unknown symptom"), + ("trend", "failure share"), + ("gaps", "Which resolved window is needed?"), + ("parity", "coverage parity"), + ], +) +def test_view_renderers_keep_prepared_populations_and_filter_links( + deck_asset: Any, view_id: str, marker: str +) -> None: + context = _load_renderers(deck_asset) + try: + rendered = context.call("DeckTest.render", view_id, _new_view_context(view_id)) + assert marker.lower() in rendered["text"].lower() + assert any( + "harness=codex" in href and "provider=openai" in href for href in rendered["links"] + ) + if view_id in {"context", "gaps", "parity"}: + assert any("session=owner-key-1" in href for href in rendered["links"]) + finally: + context.close() + + +def test_context_input_toggle_shows_inclusive_input_series(deck_asset: Any) -> None: + context = _load_renderers(deck_asset) + try: + context.call("DeckTest.render", "context", _new_view_context("context")) + input_toggle = context.call("DeckTest.clickContextInput") + assert input_toggle["initial"] == {"pressed": "false", "style": "display:none"} + assert input_toggle["pressed"] == "true" + assert input_toggle["style"] == "display:block" + finally: + context.close() + + +def test_python_prepared_payload_renders_all_new_views(deck_asset: Any) -> None: + generated_at = datetime(2026, 10, 7, tzinfo=UTC) + event_at = generated_at - timedelta(seconds=5) + event_ms = int(event_at.timestamp() * 1000) + owner = session_row( + "s1", + session_id="native-s1", + time_ms=event_ms, + harness="codex", + provider="openai", + level="skill", + skill="demo", + recipe="recipe-demo", + step="run", + success=False, + subtype="ToolError", + input_tokens=token_measure(75), + output_tokens=token_measure(0), + cache_read_tokens=token_measure(50), + cache_write_tokens=token_measure(None, state="unavailable"), + ) + turn = { + "key": "s1:turn:0", + "kind": "turn", + "session_key": "s1", + "session_id": "native-s1", + "source_id": "s1", + "ordinal": 0, + "time_ms": event_ms, + "harness": "codex", + "provider": "openai", + "model": "gpt-known", + "skill": "demo", + "recipe": "recipe-demo", + "step": "run", + "level": "skill", + "input_tokens": token_measure(75), + "output_tokens": token_measure(0), + "cache_read_tokens": token_measure(50), + "cache_write_tokens": token_measure(None, state="unavailable"), + "context_window_tokens": 100, + "context_fraction": 0.5, + "request_id": "req-1", + } + request = { + "key": "s1:req-1", + "kind": "request", + "session_key": "s1", + "request_id": "req-1", + "time_ms": event_ms, + "model": "gpt-known", + } + tool = { + "key": "s1:tool-1", + "kind": "tool", + "session_key": "s1", + "time_ms": event_ms, + "success": False, + "error_type": "ToolError", + } + child = subagent_row( + "child-1", + parent=owner, + role="reviewer", + skill="demo", + provider="openai", + input_tokens=11, + output_tokens=7, + cache_read_tokens=0, + cache_write_tokens=None, + tool_counts={"read_file": 1}, + ) + child["parent_context_spans"] = [ + { + "field": "parent_prompt_tokens", + "measure": {"state": "measured", "value": 13}, + "invocation_id": "invocation-1", + "turn_id": "turn-1", + "source_id": "parent-transcript-1", + "timestamp": event_at.isoformat(), + "harness": "codex", + "provider": "openai", + "model": "gpt-known", + "tokenizer_version": "0.14.0", + "encoding": "cl100k_base", + "reason": None, + }, + { + "field": "subagent_return_tokens", + "measure": {"state": "measured", "value": 9}, + "invocation_id": "invocation-1", + "turn_id": "turn-1", + "source_id": "parent-transcript-2", + "timestamp": event_at.isoformat(), + "harness": "codex", + "provider": "openai", + "model": "gpt-known", + "tokenizer_version": "0.14.0", + "encoding": "cl100k_base", + "reason": None, + }, + ] + payload = build_deck_payload( + [owner], + request_rows=[request], + tool_rows=[tool], + subagent_rows=[child], + turn_rows=[turn], + generated_at=generated_at, + index_schema_version=3, + ) + context = _load_renderers(deck_asset) + try: + for view_id, marker in { + "context": "Context occupancy and per-turn tokens", + "errors": "Failure symptoms by skill and step", + "trend": "failure share", + "gaps": "Live evidence gaps", + "parity": "Harness and provider coverage parity", + }.items(): + rendered = context.call( + "DeckTest.renderPrepared", + view_id, + payload, + {"view": view_id, "entity": None, "params": {}}, + ) + assert rendered["tag"] in {"div", "section"} + assert marker.lower() in rendered["text"].lower() + finally: + context.close() + + def test_manual_review_flags_require_definitions_and_reset_on_rerender( deck_asset: Any, ) -> None: diff --git a/uv.lock b/uv.lock index 831bdeddad..c47fd14ac8 100644 --- a/uv.lock +++ b/uv.lock @@ -153,6 +153,7 @@ dependencies = [ { name = "pyyaml" }, { name = "regex" }, { name = "structlog" }, + { name = "tiktoken" }, { name = "uvloop", marker = "sys_platform != 'win32'" }, { name = "zstandard" }, ] @@ -216,6 +217,7 @@ requires-dist = [ { name = "regex", specifier = ">=2026.1" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15.0" }, { name = "structlog", specifier = ">=25.0,<26.0" }, + { name = "tiktoken", specifier = ">=0.14.0" }, { name = "tree-sitter", marker = "extra == 'dev'", specifier = ">=0.24" }, { name = "tree-sitter-language-pack", marker = "extra == 'dev'", specifier = ">=0.10" }, { name = "tree-sitter-python", marker = "extra == 'dev'", specifier = ">=0.23" }, @@ -362,6 +364,153 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, ] +[[package]] +name = "charset-normalizer" +version = "3.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/1c/f41d4e74c28ab327ff3acd36053f7ea506c55872d7a90b0fa71aa3ab0c89/charset_normalizer-3.5.2.tar.gz", hash = "sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef", size = 172659, upload-time = "2026-09-30T04:39:23.398Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/67/6a0b94a7960d5e1b5eacd2fb529f3fccc47db4644f7f0a7cfdcfc3be578a/charset_normalizer-3.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3d21b8b13c7592db2ac5e544a6d83187b995257472b0c9e8351b6d507ae37ed6", size = 370642, upload-time = "2026-09-30T04:35:06.91Z" }, + { url = "https://files.pythonhosted.org/packages/fb/94/01009e13b94041599004edf32e56e382c24e570f60f79bab8efe45cfe1eb/charset_normalizer-3.5.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d760fe2a4d7c3b226cb9026d6a842868d52a7901bd98420e1baf14e80da85cf5", size = 259280, upload-time = "2026-09-30T04:35:08.448Z" }, + { url = "https://files.pythonhosted.org/packages/66/85/3b5358f60a13210f0b67d3755c168ef758701b021e655d88d4da28554467/charset_normalizer-3.5.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c9790464842f85f437dbbb54417eda1e0e6bfc52dd8d22d6fd1c994b73b2dc74", size = 245335, upload-time = "2026-09-30T04:35:10.104Z" }, + { url = "https://files.pythonhosted.org/packages/74/75/77c1c479b09ecd751d1e767b251ea5c14d4d50ff757bf404afab2692f600/charset_normalizer-3.5.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4685902cf26edf013ed7a3da0f426ebba7a00ebb9541386d835afbf002c11cab", size = 291707, upload-time = "2026-09-30T04:35:11.575Z" }, + { url = "https://files.pythonhosted.org/packages/0b/0d/363f78cacb70f58f15f4b083961bbd9d292f335d3f5c66fc4f1cfe69cb90/charset_normalizer-3.5.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4495c5002a7b28557e7e222e77e0b661183e432b7d6d2e788101e3f240e05b8c", size = 286747, upload-time = "2026-09-30T04:35:13.022Z" }, + { url = "https://files.pythonhosted.org/packages/e4/ed/cf505d3011ffceb12c2067a7a5d3cfe92b875d4d44bb0ff0d69375e2c184/charset_normalizer-3.5.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:211d5a3eb6af8f513b8d4ca19a8c1b7accab1b5f0d3175f9826b03c1a920dc1f", size = 269972, upload-time = "2026-09-30T04:35:14.606Z" }, + { url = "https://files.pythonhosted.org/packages/15/d8/f0a93a431d170e7ca681d4f6650fee3de934d18560e474e7267eb4b0f987/charset_normalizer-3.5.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ef4fcbf3327382cd4c9f540babd61248208af7b93eec4de397b4d5f58a09e288", size = 269339, upload-time = "2026-09-30T04:35:16.087Z" }, + { url = "https://files.pythonhosted.org/packages/86/bd/9b2bd1c5b7af02462c9752d33994834ff972a96b4c483eefde9e594488e2/charset_normalizer-3.5.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd16aabe4a02a297c23417aa17ac6299dbd8c49f673bcd645b4929b11f5a4400", size = 261683, upload-time = "2026-09-30T04:35:17.488Z" }, + { url = "https://files.pythonhosted.org/packages/76/a5/cac540ab0fd61f3fec88ad3dbb64509e71424593d73cfdfff5ab3e4db279/charset_normalizer-3.5.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:fb9e68df06293761f9fe66ade60a9bc6d0f5e42b8acf2939a9158af86ab0e5bd", size = 248143, upload-time = "2026-09-30T04:35:18.849Z" }, + { url = "https://files.pythonhosted.org/packages/71/7a/ff467301deef2089fad87f72df9e000a26a78fec7acbb18e1999371b8369/charset_normalizer-3.5.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:59f63901b0031c3136cf64704dcb21de0bbae62ce2c9529bc39d27665463de37", size = 292358, upload-time = "2026-09-30T04:35:20.326Z" }, + { url = "https://files.pythonhosted.org/packages/ad/77/22d7e785d1e210afc2e2f58600dd1799d17a35665faf84383f002826c5f8/charset_normalizer-3.5.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:304d5463e65a35d7bb0850550e0780395395f6fcf452f04db7d5ca7cecc425ac", size = 268766, upload-time = "2026-09-30T04:35:21.72Z" }, + { url = "https://files.pythonhosted.org/packages/ae/91/e8e946267f1c2d9e2bd651726e2fbd2addf02c4d36cea5069e32ca9d7bb5/charset_normalizer-3.5.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:9cf9b1a857e25c4baceeb3624e92a56df3668f398c4acba74e174d81fb4d1d3a", size = 287432, upload-time = "2026-09-30T04:35:23.273Z" }, + { url = "https://files.pythonhosted.org/packages/4e/88/7561d8a88d555e7df6623abe7c0070b4baf47549b9408783a2ae0a1a6cf7/charset_normalizer-3.5.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:114e4d0c92d618409ed82a99e22b5c5e768fe995f2973f78265f4524f49d4640", size = 272470, upload-time = "2026-09-30T04:35:24.655Z" }, + { url = "https://files.pythonhosted.org/packages/35/7e/578c702301ec036f01455f30744a08d2b42f6ab35b9b2d4bf8cae0ef2a80/charset_normalizer-3.5.2-cp311-cp311-win32.whl", hash = "sha256:2625388c6c754520c37abaf3b41eb34d1cc4a373f457898f08606c8e362b891d", size = 186688, upload-time = "2026-09-30T04:35:26.225Z" }, + { url = "https://files.pythonhosted.org/packages/e8/fc/fdf8cf52ff21cd5bf158f20978991cf985325842f74283eb6df26c8a39d8/charset_normalizer-3.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:87e50a3e7cb90af586b6c5faf23e302a970415ac73bd7bd90a515a04b427ef96", size = 214932, upload-time = "2026-09-30T04:35:27.796Z" }, + { url = "https://files.pythonhosted.org/packages/97/66/3e45a506d8110b632541faf9a9470185aa9878f1ed44020f31346c1c5e5b/charset_normalizer-3.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:254eb48b9fa5ee9898a3c445825a1f340fe53712a098904b39b0bddba8ea3cb1", size = 202828, upload-time = "2026-09-30T04:35:29.259Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c8/693809898870237d82785a03f3b2b58fe4c9f14669f84a7d4e623c92a59e/charset_normalizer-3.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491", size = 367780, upload-time = "2026-09-30T04:35:30.888Z" }, + { url = "https://files.pythonhosted.org/packages/c9/87/2fea8c13dc24b3ca9c6f803a5b2dfdeae73eb4f9e12c7885ed908ff0433c/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c", size = 246730, upload-time = "2026-09-30T04:35:32.286Z" }, + { url = "https://files.pythonhosted.org/packages/a8/9e/09efac30b937722f46d3110ba30b875b24b2e3a266ed746cc4e376a94d80/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0", size = 237707, upload-time = "2026-09-30T04:35:33.709Z" }, + { url = "https://files.pythonhosted.org/packages/9e/18/70d76670b13686237863a379928d60bd10e021f17d243ab3d7014c4a5f4e/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51", size = 273050, upload-time = "2026-09-30T04:35:35.138Z" }, + { url = "https://files.pythonhosted.org/packages/54/e2/77a8b09d5adc013ed07b95b01b8b8fa5441c4e810e83ee7e4aae2fa4d91a/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5", size = 270345, upload-time = "2026-09-30T04:35:36.502Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c5/38806a25ab5e65fc178f39affeda20858efafede2fce1ffc2556cfc9fe73/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649", size = 257601, upload-time = "2026-09-30T04:35:37.919Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8d/213565184708fdb263ae55e2c04ee1ff748129dd65d48ed0e3502da9c85a/charset_normalizer-3.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e", size = 252222, upload-time = "2026-09-30T04:35:39.544Z" }, + { url = "https://files.pythonhosted.org/packages/7e/24/76d2cefc25472531e4c5c7dfff68865eb1c39b78482f0fdc15b46f047830/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346", size = 248482, upload-time = "2026-09-30T04:35:41.088Z" }, + { url = "https://files.pythonhosted.org/packages/7d/dc/65a801b66ab4c197e22c433ab25e7ac24324ac6f45a2269aca42cce309bf/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1", size = 241206, upload-time = "2026-09-30T04:35:42.59Z" }, + { url = "https://files.pythonhosted.org/packages/a7/95/ca9b5eabde673002c6f1e7ada1b223916fe18f6d661da7aabd4d643718f1/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875", size = 273190, upload-time = "2026-09-30T04:35:44.347Z" }, + { url = "https://files.pythonhosted.org/packages/2d/8b/803b4d2a3f6e1740f63f1e87b04d14b42f3d4fdfe6ed7d4db2d34102b14f/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1", size = 253527, upload-time = "2026-09-30T04:35:45.915Z" }, + { url = "https://files.pythonhosted.org/packages/a9/55/93c0e5dbd085ae0471346026abbe7e0db9ea2d6fea74e51f0b5a46f233a7/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413", size = 271285, upload-time = "2026-09-30T04:35:47.49Z" }, + { url = "https://files.pythonhosted.org/packages/95/69/0dbd0e0b9b16cfa816cdfcb3e2e3854a1f680dc07fb1245ea125e7448060/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869", size = 260010, upload-time = "2026-09-30T04:35:48.996Z" }, + { url = "https://files.pythonhosted.org/packages/58/9d/e7b88e7b1bf403590c3b573277b5e1e488c68c7a6fbacca310a2c324e90c/charset_normalizer-3.5.2-cp312-cp312-win32.whl", hash = "sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e", size = 184125, upload-time = "2026-09-30T04:35:50.777Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e6/e6e083884cbcfd49c64865af05027fe7011be7b2d9179524f099a1b611f3/charset_normalizer-3.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc", size = 207486, upload-time = "2026-09-30T04:35:52.194Z" }, + { url = "https://files.pythonhosted.org/packages/c4/e3/017aea0911ada7405a825c7d937eb3a13009664e2f5b38e8c4bbf2abf894/charset_normalizer-3.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3", size = 196734, upload-time = "2026-09-30T04:35:53.636Z" }, + { url = "https://files.pythonhosted.org/packages/c5/34/68292d68512768591aaff07c59bb53ee31341c87759433a859c4641a50c2/charset_normalizer-3.5.2-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5", size = 225946, upload-time = "2026-09-30T04:35:55.313Z" }, + { url = "https://files.pythonhosted.org/packages/e3/80/bee0b01b90ccd5322ae1d0abb33fab1bd95b7c2eadaf02aeccf22e04ee83/charset_normalizer-3.5.2-cp313-cp313-android_24_x86_64.whl", hash = "sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e", size = 238619, upload-time = "2026-09-30T04:35:56.863Z" }, + { url = "https://files.pythonhosted.org/packages/78/6e/60ce52a85a7fd631ae8482ae6d74521014ca2f255892679484dc04d7ef56/charset_normalizer-3.5.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a", size = 206701, upload-time = "2026-09-30T04:35:58.639Z" }, + { url = "https://files.pythonhosted.org/packages/36/8c/71aafad23f971afc84c2b295bc0c560739ce1dac558aad9fec22e39f3639/charset_normalizer-3.5.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d", size = 210506, upload-time = "2026-09-30T04:36:00.147Z" }, + { url = "https://files.pythonhosted.org/packages/91/da/3c5a7798c046df7d2d68ad653cf5b6c5a8bfee225055a843c6f2f42aac1a/charset_normalizer-3.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055", size = 366626, upload-time = "2026-09-30T04:36:01.77Z" }, + { url = "https://files.pythonhosted.org/packages/e1/16/710ac3de2ee354e2bd1a9c94efe45a2d27b5c6ad39b2d6a905be2c094b6c/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858", size = 244624, upload-time = "2026-09-30T04:36:03.389Z" }, + { url = "https://files.pythonhosted.org/packages/d6/39/45c7439f5b63d24f7d5b2a1d760f34af7628782d7144b4cc8ded45c2d4bc/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234", size = 235731, upload-time = "2026-09-30T04:36:04.987Z" }, + { url = "https://files.pythonhosted.org/packages/4d/34/38f3154785ce92e9f56eb226f4d35bdfae6b008480dd055f58837a89c810/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21", size = 270352, upload-time = "2026-09-30T04:36:06.412Z" }, + { url = "https://files.pythonhosted.org/packages/04/f3/859f74e7babc977705026b30593b3be04049632a522fb7000f83c033d747/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718", size = 267393, upload-time = "2026-09-30T04:36:07.865Z" }, + { url = "https://files.pythonhosted.org/packages/4b/85/41d27f234b82e47c167a5f6c0f62501dc0c640585ff4aba79e08a390336a/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4", size = 254660, upload-time = "2026-09-30T04:36:09.248Z" }, + { url = "https://files.pythonhosted.org/packages/58/ca/5d1a997587febe5b26d8daffe363b5c1a091cece19828eec6502fd09c5ef/charset_normalizer-3.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3", size = 249996, upload-time = "2026-09-30T04:36:10.73Z" }, + { url = "https://files.pythonhosted.org/packages/b3/1f/d1e78246f7ed60c8c8d606b4ac27f66ce49cc3e95f24893ccbeba9f77302/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c", size = 247028, upload-time = "2026-09-30T04:36:12.294Z" }, + { url = "https://files.pythonhosted.org/packages/8e/37/eba316edd4f0c4d3a5d945924c4eeeae59abac4056aa815d8a4268f863a2/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429", size = 240562, upload-time = "2026-09-30T04:36:13.887Z" }, + { url = "https://files.pythonhosted.org/packages/c8/8e/aaa037d40ca9ef045977f1a661048b1aa33f223adfce3452fe9be9f79d14/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f", size = 271699, upload-time = "2026-09-30T04:36:15.41Z" }, + { url = "https://files.pythonhosted.org/packages/26/19/1c1c9f75974adf523b87f34b8a2adc5a435cd65916812bcbd0dfa45f9a29/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a", size = 252030, upload-time = "2026-09-30T04:36:16.839Z" }, + { url = "https://files.pythonhosted.org/packages/bc/90/0660ef18e18df0a4d2a1a0edff7dfbba42d4e50ef2425557a5bb7051f77b/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00", size = 268565, upload-time = "2026-09-30T04:36:18.468Z" }, + { url = "https://files.pythonhosted.org/packages/79/ba/57adc269824e8658f1a0f97a9e514c247445a9632b3419b97e0ba37f16dc/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d", size = 257627, upload-time = "2026-09-30T04:36:19.938Z" }, + { url = "https://files.pythonhosted.org/packages/9a/85/33abd4315c052d3d4f54c92b1ee49bfbc0dc7115a981e462a793b6d2ab87/charset_normalizer-3.5.2-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3", size = 143407, upload-time = "2026-09-30T04:36:21.376Z" }, + { url = "https://files.pythonhosted.org/packages/4f/de/6435e18d1aaa5d910b896d551411c96af1f42a0c56c29afc2016c61ccc2e/charset_normalizer-3.5.2-cp313-cp313-win32.whl", hash = "sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd", size = 183207, upload-time = "2026-09-30T04:36:22.776Z" }, + { url = "https://files.pythonhosted.org/packages/9c/76/b8ec57f4e9ee3253541abf95e4a462c0175fe8032dcd070f1f2421240942/charset_normalizer-3.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639", size = 206489, upload-time = "2026-09-30T04:36:24.306Z" }, + { url = "https://files.pythonhosted.org/packages/3e/60/c647c6ae47480221e875ea5d743ff94946f7416e3c69415ab772928e8d32/charset_normalizer-3.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3", size = 195501, upload-time = "2026-09-30T04:36:25.846Z" }, + { url = "https://files.pythonhosted.org/packages/58/ca/7aa91362a2f77ac8e9e28a9b902a74f7d0e11a851ef0d27a74308da8cd90/charset_normalizer-3.5.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187", size = 226010, upload-time = "2026-09-30T04:36:27.669Z" }, + { url = "https://files.pythonhosted.org/packages/a8/cf/ac8878d0322cf88a1aad4c7b147db32ca0bd806eb0060957b2e31486dbe6/charset_normalizer-3.5.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad", size = 239266, upload-time = "2026-09-30T04:36:29.434Z" }, + { url = "https://files.pythonhosted.org/packages/c9/6d/9a08d7e0b29b7208e2c6c01dc56c8e0520e7c7beadbbfb024b58fd69c8a5/charset_normalizer-3.5.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf", size = 206783, upload-time = "2026-09-30T04:36:30.872Z" }, + { url = "https://files.pythonhosted.org/packages/82/44/b0aa350280e6ff5a5492d17cf10460dd39d5ee848f872f7ba2df10607f60/charset_normalizer-3.5.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995", size = 210449, upload-time = "2026-09-30T04:36:32.625Z" }, + { url = "https://files.pythonhosted.org/packages/7c/8a/40db9aa9f5907bb0e6f8b6d64064bf8852fb33d4b813ff9414911df7647c/charset_normalizer-3.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424", size = 367815, upload-time = "2026-09-30T04:36:34.197Z" }, + { url = "https://files.pythonhosted.org/packages/7f/72/9c5e7707b57c8ddfa9ddf7b0b1d009d7fbab9e9e887d5b721060f37e307d/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13", size = 247095, upload-time = "2026-09-30T04:36:35.803Z" }, + { url = "https://files.pythonhosted.org/packages/83/09/71e453691e927de4ddf792770cfaab3f49d494e222f66ea5e404bbd5e39c/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d", size = 233942, upload-time = "2026-09-30T04:36:37.407Z" }, + { url = "https://files.pythonhosted.org/packages/9f/86/85c84e4da8b27dd409577d9437926ff581c5f9d3c66038dc68c1a526de51/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4", size = 273762, upload-time = "2026-09-30T04:36:38.904Z" }, + { url = "https://files.pythonhosted.org/packages/92/08/564955a4b5f2ccb410ab480bbe8c6a18063ff27f2d35458731c4a5335df9/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438", size = 269514, upload-time = "2026-09-30T04:36:40.469Z" }, + { url = "https://files.pythonhosted.org/packages/18/24/bad3ac4271589df29cf5ce2f5ae490518a5739358052bd0d61209e6fea54/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a", size = 255056, upload-time = "2026-09-30T04:36:42.02Z" }, + { url = "https://files.pythonhosted.org/packages/d6/3e/350d89ad49916b86554d6f5f2d03ec1152148f87e5ff735106c6a03b1a36/charset_normalizer-3.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56", size = 251311, upload-time = "2026-09-30T04:36:43.577Z" }, + { url = "https://files.pythonhosted.org/packages/56/5b/4970a2d154df502e133402906dd04e3ae7cada7b3011283c88d0479a2585/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd", size = 250190, upload-time = "2026-09-30T04:36:45.185Z" }, + { url = "https://files.pythonhosted.org/packages/88/8c/f1a91bddc8fb47c2889e29ea7ea49a194eb0d9868675d786806519c00d76/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204", size = 237098, upload-time = "2026-09-30T04:36:46.689Z" }, + { url = "https://files.pythonhosted.org/packages/24/0e/bb5dace3cc7e79068425386a6589c19b5a2ab5fefc2a46abea6919683332/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7", size = 274228, upload-time = "2026-09-30T04:36:48.31Z" }, + { url = "https://files.pythonhosted.org/packages/9d/79/b849ad523017ea9f5a45581bbebed91439e0cf42fd2860a6f64e358eb5a6/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd", size = 252401, upload-time = "2026-09-30T04:36:50.091Z" }, + { url = "https://files.pythonhosted.org/packages/89/8c/75469d690cf47200bce8f6cad7655724fc23148e147abfc5ce78b5f65863/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc", size = 271846, upload-time = "2026-09-30T04:36:51.719Z" }, + { url = "https://files.pythonhosted.org/packages/26/cd/6d52d3c7437cdcf2e310ce9f28f282e733d4ef60ed19105d1819c356255f/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874", size = 258151, upload-time = "2026-09-30T04:36:53.234Z" }, + { url = "https://files.pythonhosted.org/packages/f7/4c/070b38bdb5f49a70199fce923ec0726a49536a63ab262abbfcaaf351110b/charset_normalizer-3.5.2-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655", size = 143570, upload-time = "2026-09-30T04:36:54.816Z" }, + { url = "https://files.pythonhosted.org/packages/81/84/9ebfc8ed6c8c4fcd8e726ff6bf220cc8deb3966e31dce9be8dd8aa017e64/charset_normalizer-3.5.2-cp314-cp314-win32.whl", hash = "sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0", size = 186412, upload-time = "2026-09-30T04:36:56.643Z" }, + { url = "https://files.pythonhosted.org/packages/d1/78/5ed86f743d4bc350db307e7636419a0a5ee1d91806d30c7f667bd5c80dae/charset_normalizer-3.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c", size = 210965, upload-time = "2026-09-30T04:36:58.205Z" }, + { url = "https://files.pythonhosted.org/packages/53/94/a3a7698e9b1a395e1eb99ccd9a324be9347973bff4e72db2a06496d7cd27/charset_normalizer-3.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253", size = 200406, upload-time = "2026-09-30T04:36:59.764Z" }, + { url = "https://files.pythonhosted.org/packages/c1/48/c5dd00d5ef7791f02666de250a5bb6071e29b7e133cf4b835800b6d3bc27/charset_normalizer-3.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709", size = 396871, upload-time = "2026-09-30T04:37:01.543Z" }, + { url = "https://files.pythonhosted.org/packages/12/c8/8379554b42e8368161d898476686947a0fdbd3e8865170d7909dcabfdee8/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084", size = 249711, upload-time = "2026-09-30T04:37:03.111Z" }, + { url = "https://files.pythonhosted.org/packages/4a/eb/2ddb1035d17320caa9f41682935123a9a250277b261c3efc86b2d2a21343/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb", size = 231146, upload-time = "2026-09-30T04:37:04.721Z" }, + { url = "https://files.pythonhosted.org/packages/4a/24/2ecb4bde104322cd7859d6594fcfa74649f8d90b3221c9feecbef149875b/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f", size = 271552, upload-time = "2026-09-30T04:37:06.295Z" }, + { url = "https://files.pythonhosted.org/packages/3f/98/9d5f6ebc3aee9fef5d30b4aff11fb2ab7a1222b4064f8ef2c7c87cde217a/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09", size = 269810, upload-time = "2026-09-30T04:37:07.905Z" }, + { url = "https://files.pythonhosted.org/packages/09/e1/a3b06a10461b1b7628853c934c644e03bc28e42767116afb52f19a56519b/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80", size = 259845, upload-time = "2026-09-30T04:37:09.554Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d3/6f561f74a296cf27d61775a1dc665ad13f3bff6a798810ca05907f37a7c4/charset_normalizer-3.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c", size = 252825, upload-time = "2026-09-30T04:37:11.274Z" }, + { url = "https://files.pythonhosted.org/packages/26/9f/69e13ca3b18f43e0eafcd34c04a45b732ae22a43b54a5fc9e119103356eb/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f", size = 252659, upload-time = "2026-09-30T04:37:12.941Z" }, + { url = "https://files.pythonhosted.org/packages/73/a9/ace29806a0dae18939919c76ba526472d83214afa101105fabff2cf30625/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03", size = 238819, upload-time = "2026-09-30T04:37:14.659Z" }, + { url = "https://files.pythonhosted.org/packages/f8/c1/6116d52a2e3311ec80f21f5fb5e17b27405f10b9608af8f6e69516841a1b/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604", size = 272973, upload-time = "2026-09-30T04:37:16.346Z" }, + { url = "https://files.pythonhosted.org/packages/19/aa/9955c7e93bba10a9c7e8f7a5031b7ced66f3a1883a55c00712b8d5850ff3/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8", size = 253692, upload-time = "2026-09-30T04:37:18.212Z" }, + { url = "https://files.pythonhosted.org/packages/bb/33/2a6ae7fdc1b10cb581cef91addd8cdfc5f40d50abb5702309369d5834579/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93", size = 268757, upload-time = "2026-09-30T04:37:19.877Z" }, + { url = "https://files.pythonhosted.org/packages/a2/22/80992720a0282cd39bba1db35868e6b9c22f41281160143a836544bc1d8a/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915", size = 262220, upload-time = "2026-09-30T04:37:21.583Z" }, + { url = "https://files.pythonhosted.org/packages/92/9f/181fd07e1bffea1d95cd80c84ac537354f50699c22cfc4d3c02b6fc16208/charset_normalizer-3.5.2-cp314-cp314t-win32.whl", hash = "sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5", size = 197797, upload-time = "2026-09-30T04:37:23.235Z" }, + { url = "https://files.pythonhosted.org/packages/49/1c/25d8415ec1c4f2f41f1680435e4c87cfb378ff2f677d950946f2a45d0632/charset_normalizer-3.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc", size = 222765, upload-time = "2026-09-30T04:37:24.891Z" }, + { url = "https://files.pythonhosted.org/packages/3e/b4/46b48f013dadfc0d0d33b375438e31bdf5a989dc68389c6bf627054d4df9/charset_normalizer-3.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105", size = 210985, upload-time = "2026-09-30T04:37:26.634Z" }, + { url = "https://files.pythonhosted.org/packages/ca/e9/34e597dee616d0b8ee4b34d29399e85c2204ade174157a48505d42baa4ff/charset_normalizer-3.5.2-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26", size = 367676, upload-time = "2026-09-30T04:37:28.329Z" }, + { url = "https://files.pythonhosted.org/packages/60/9f/a5d1c91c0263745e2cd344c5a4415d787c575501ab1d449f1148ac6b495d/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364", size = 248414, upload-time = "2026-09-30T04:37:30.167Z" }, + { url = "https://files.pythonhosted.org/packages/26/79/e697f77464748a3ee3cf490c83d592459400d4898380d66c38366b03080c/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253", size = 244775, upload-time = "2026-09-30T04:37:31.964Z" }, + { url = "https://files.pythonhosted.org/packages/ca/87/3d42a42e18ea066e2513936fd678a00696e77878b5ae04528976abdbcb83/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0", size = 273877, upload-time = "2026-09-30T04:37:33.661Z" }, + { url = "https://files.pythonhosted.org/packages/c3/76/8a28136f3938ba9836f84280ce0c4d61ed1cf15a036b2034900c62634162/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc", size = 269455, upload-time = "2026-09-30T04:37:35.573Z" }, + { url = "https://files.pythonhosted.org/packages/a0/a1/4fbf5d0f0f1b2a080474c1cf9a2f12c4c6531bb0e8ba591055e846d2b4e9/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229", size = 257336, upload-time = "2026-09-30T04:37:37.397Z" }, + { url = "https://files.pythonhosted.org/packages/ba/a2/8b50aa320adb880ad579518e6f718f24944804b42a88b83d267d5d444125/charset_normalizer-3.5.2-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5", size = 257584, upload-time = "2026-09-30T04:37:39.522Z" }, + { url = "https://files.pythonhosted.org/packages/a5/57/50e3fed84e175f40349bd0da7a4fce94c87f0378f52d74f511d89e0bdc20/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98", size = 251854, upload-time = "2026-09-30T04:37:41.23Z" }, + { url = "https://files.pythonhosted.org/packages/d6/54/f7fbb3493c9f49091213b9c2d6dd65800696f1ce1a3f196a4205f50417b1/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3", size = 248767, upload-time = "2026-09-30T04:37:42.883Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/b3a6385acc5a1e45b39ae9c90bfb9cf838a09b9dd37ef2740ab4c6b4a2eb/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2", size = 274479, upload-time = "2026-09-30T04:37:44.658Z" }, + { url = "https://files.pythonhosted.org/packages/89/44/809913e2cfd279e635a9294fdbbfb1b1dc62a8189d473d561f649fce98d8/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf", size = 258310, upload-time = "2026-09-30T04:37:46.529Z" }, + { url = "https://files.pythonhosted.org/packages/af/a2/f28400ab13359d91bd39179df8e149376b9bf36588e739a3a4f9de2b84b2/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95", size = 271100, upload-time = "2026-09-30T04:37:48.399Z" }, + { url = "https://files.pythonhosted.org/packages/e9/89/9bab37955edf0adb3b66f8a3a6617d9f2f487e0d56f295a6a286cb640aa6/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d", size = 259936, upload-time = "2026-09-30T04:37:50.023Z" }, + { url = "https://files.pythonhosted.org/packages/23/b5/4459e08d45a679f903d50fea08bc52cfa728cca4d7bd02c757b5e5abda2e/charset_normalizer-3.5.2-cp315-cp315-win32.whl", hash = "sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847", size = 186301, upload-time = "2026-09-30T04:37:51.722Z" }, + { url = "https://files.pythonhosted.org/packages/98/e8/55d5fd3935b4bce6da4fe0df61898e8c82653e317e677bd58aceb9c60f13/charset_normalizer-3.5.2-cp315-cp315-win_amd64.whl", hash = "sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8", size = 210858, upload-time = "2026-09-30T04:37:53.427Z" }, + { url = "https://files.pythonhosted.org/packages/a9/5b/974423c2fd8e524c7a7f64318c1e02240ef954912fa2b4d70344107b9c68/charset_normalizer-3.5.2-cp315-cp315-win_arm64.whl", hash = "sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a", size = 200309, upload-time = "2026-09-30T04:37:55.015Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f9/00ee0195db1013d8f7c416fd770fbeb560bb46eb2e36b054d05cb56f6cfa/charset_normalizer-3.5.2-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1", size = 395343, upload-time = "2026-09-30T04:37:56.743Z" }, + { url = "https://files.pythonhosted.org/packages/04/3a/c00b50e94c964cf934c7899cd47c97952fc11dad71cc5884b3c61795b09b/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b", size = 250114, upload-time = "2026-09-30T04:37:58.607Z" }, + { url = "https://files.pythonhosted.org/packages/50/27/d102dc880bbcffd0479ab64dfc1fb96777a854355a55e2bda72a71efadcb/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f", size = 239997, upload-time = "2026-09-30T04:38:00.511Z" }, + { url = "https://files.pythonhosted.org/packages/a5/4a/bf7ef45794dd293fab5f98a9309817977fbb845b9998f171b8cc5d8437a3/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3", size = 272313, upload-time = "2026-09-30T04:38:02.509Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ee/008a2837737991474c5754bb3191010007663860979701990982a502cbaf/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e", size = 269986, upload-time = "2026-09-30T04:38:04.435Z" }, + { url = "https://files.pythonhosted.org/packages/93/ad/bd74a283940dc910c5b14f8e4f80a248082bc9c0fcbe1f54530cb6d9cc5e/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9", size = 260202, upload-time = "2026-09-30T04:38:06.549Z" }, + { url = "https://files.pythonhosted.org/packages/8a/7b/ed341c66f69f688723501fac752be3d63c7159ca0d0d4174fc611e5710bb/charset_normalizer-3.5.2-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a", size = 259083, upload-time = "2026-09-30T04:38:08.311Z" }, + { url = "https://files.pythonhosted.org/packages/cc/9d/e41588b777965e5031a43128a1e96173ebb35ac75fc53ec3b517e7c21cd4/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115", size = 253081, upload-time = "2026-09-30T04:38:10.402Z" }, + { url = "https://files.pythonhosted.org/packages/81/35/b761eb6d8c1eb218b9b42b9b4d5ac902afdc399fb6dac6f9a9aac7bda589/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c", size = 248712, upload-time = "2026-09-30T04:38:12.317Z" }, + { url = "https://files.pythonhosted.org/packages/4d/2c/147169a041b747759f37405c0a97157e8e92de967968373101ff14915cba/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d", size = 274176, upload-time = "2026-09-30T04:38:14.138Z" }, + { url = "https://files.pythonhosted.org/packages/f0/2d/0ff8db0d373ba8538db686db11cd7e8912031490b9e4f383b41912e8d594/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d", size = 259413, upload-time = "2026-09-30T04:38:15.841Z" }, + { url = "https://files.pythonhosted.org/packages/8a/8e/b4a085fb47c9d3a7e43576a4784fdd8fe23f907514a972de8086edaf7a48/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4", size = 269434, upload-time = "2026-09-30T04:38:17.626Z" }, + { url = "https://files.pythonhosted.org/packages/83/1c/d8d8d7322a7c3eecdf3237a4a419cf41d2eaad8e006ce7dfdd9d4c8fa2eb/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b", size = 262716, upload-time = "2026-09-30T04:38:19.214Z" }, + { url = "https://files.pythonhosted.org/packages/a0/16/0e4c6ba9b44e97a2da150e52d331e8f9c968b21b358fbffa6c856cebcd89/charset_normalizer-3.5.2-cp315-cp315t-win32.whl", hash = "sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800", size = 197159, upload-time = "2026-09-30T04:38:21.037Z" }, + { url = "https://files.pythonhosted.org/packages/be/33/e90bc2b1374f7f36ef106f56620de5a783907e19ca857efe2277e31cac3e/charset_normalizer-3.5.2-cp315-cp315t-win_amd64.whl", hash = "sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21", size = 221322, upload-time = "2026-09-30T04:38:22.886Z" }, + { url = "https://files.pythonhosted.org/packages/66/89/dfa6dcb08c200b7830ab56439e8c1890f2971d51aafbb3937894a2e7fcfc/charset_normalizer-3.5.2-cp315-cp315t-win_arm64.whl", hash = "sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58", size = 209192, upload-time = "2026-09-30T04:38:24.648Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ab/176fbfd5b64939c55d652366aa5b9ef1d767af207a3aa6ebeb0d226c484d/charset_normalizer-3.5.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd", size = 331815, upload-time = "2026-09-30T04:38:26.216Z" }, + { url = "https://files.pythonhosted.org/packages/7e/84/371eac6b30bdbcbf2d632a1a01809103459216fcaae61b8b8d922c1bfb8a/charset_normalizer-3.5.2-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7", size = 253276, upload-time = "2026-09-30T04:38:28.032Z" }, + { url = "https://files.pythonhosted.org/packages/43/6f/c4fbae58febff71709c51bc7e18fdfa55341dc382704740f9f0cbf03817b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f", size = 241239, upload-time = "2026-09-30T04:38:29.732Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/458c3f42164a07d0c5210798e9e704b39e540a6793b05aba67f3a35243a9/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93", size = 231121, upload-time = "2026-09-30T04:38:31.462Z" }, + { url = "https://files.pythonhosted.org/packages/09/54/ab9e89367076f6331bb6c65c4bf14a5361fa5191cb6561bf534f18504e1b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade", size = 260350, upload-time = "2026-09-30T04:38:33.239Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c1/061431ecc688d9d76602502cb57cc01e691e682c18f1beb45f9673b5bbd2/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0", size = 255430, upload-time = "2026-09-30T04:38:34.865Z" }, + { url = "https://files.pythonhosted.org/packages/8d/1f/20c8949f0676f7ab811abdeb7f4d7f1cbc6e61ff20bef08b44edeb092bc8/charset_normalizer-3.5.2-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26", size = 250612, upload-time = "2026-09-30T04:38:36.649Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9e/46f2fa4c431fc98c4ae76a8cb5bdca54e0341e3cfc3fcfd8e82740250818/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011", size = 242083, upload-time = "2026-09-30T04:38:38.26Z" }, + { url = "https://files.pythonhosted.org/packages/bd/39/559be29a0c0f086e0bba6922babd38916cc5e0b58ced4de13ee01ea05508/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621", size = 232738, upload-time = "2026-09-30T04:38:39.81Z" }, + { url = "https://files.pythonhosted.org/packages/ff/6c/387b0e4f756a282831c1d9fc6aeb6c51ca4507ca202767c8de15ce9b12e2/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4", size = 260703, upload-time = "2026-09-30T04:38:41.346Z" }, + { url = "https://files.pythonhosted.org/packages/96/92/1fdf015f09ef449f50d3ac4b67c90887c9c318b727daa95cc4f866e6521d/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e", size = 247622, upload-time = "2026-09-30T04:38:42.937Z" }, + { url = "https://files.pythonhosted.org/packages/dc/3c/8e7b8a5671ad5d433669fb2a76f1a0164df2d9b1718b0206bc2a16d840cc/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c", size = 257500, upload-time = "2026-09-30T04:38:44.604Z" }, + { url = "https://files.pythonhosted.org/packages/b4/f0/45b579df5cabc1d5d53ea1cc35e8437d3ca768c0acccc7041517cb6fbb32/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0", size = 255100, upload-time = "2026-09-30T04:38:46.289Z" }, + { url = "https://files.pythonhosted.org/packages/31/68/fdec18a343f5fb3f310588dd478b09ac4799e0b187dbade3a8cd776f03ef/charset_normalizer-3.5.2-cp37-abi3-win32.whl", hash = "sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf", size = 174499, upload-time = "2026-09-30T04:38:47.999Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8a/b618149cc5207943a0242068d7a27897f56a62947b5a039085f2a22029f8/charset_normalizer-3.5.2-cp37-abi3-win_amd64.whl", hash = "sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036", size = 200092, upload-time = "2026-09-30T04:38:49.707Z" }, + { url = "https://files.pythonhosted.org/packages/03/cf/4c66866fa9e2b1c78e3c911516d1de497a677b7ac60f1eceda74ce777ca3/charset_normalizer-3.5.2-cp37-abi3-win_arm64.whl", hash = "sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e", size = 294363, upload-time = "2026-09-30T04:38:51.312Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ad/d07d7862a62ffa6d79d68074d14823243dd235a77c45262acbf6adeb28bf/charset_normalizer-3.5.2-py3-none-any.whl", hash = "sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685", size = 68872, upload-time = "2026-09-30T04:39:21.828Z" }, +] + [[package]] name = "click" version = "8.4.1" @@ -2009,6 +2158,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/80/35b4c33c804a165a7f55289afda3ea9e3eb6d15800341a2d66455c0f1f30/regex-2026.5.9-cp314-cp314t-win_arm64.whl", hash = "sha256:5e41809d2683fcde7d5a8c87a6567ba1fb1ce0de9f31bff578de00a4b2d76daa", size = 275713, upload-time = "2026-05-09T23:15:16.98Z" }, ] +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + [[package]] name = "rich" version = "15.0.0" @@ -2254,6 +2418,67 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a8/45/a132b9074aa18e799b891b91ad72133c98d8042c70f6240e4c5f9dabee2f/structlog-25.5.0-py3-none-any.whl", hash = "sha256:a8453e9b9e636ec59bd9e79bbd4a72f025981b3ba0f5837aebf48f02f37a7f9f", size = 72510, upload-time = "2025-10-27T08:28:21.535Z" }, ] +[[package]] +name = "tiktoken" +version = "0.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "regex" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/62/167a842aa0429d45f5e797354fd4343a96f6043d67d0513c675c7b8d36e6/tiktoken-0.14.0.tar.gz", hash = "sha256:231dec90efcdccf1b565a1416107736f1e09b1a08fe736ef9d6363e626d03874", size = 38898, upload-time = "2026-08-17T19:49:49.514Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/c5/9d848b7f408241171e1f843deb8bfa626086452bc9c78beee500829583e3/tiktoken-0.14.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:c2edf09b381fafbc014ae8e018ed25087abb9a3dafa8465a0ea63c6558c47a79", size = 1094971, upload-time = "2026-08-17T19:48:40.347Z" }, + { url = "https://files.pythonhosted.org/packages/2d/a9/d94302340304328961d6f0c35ca4e60617fbb57a5cf667e2ed1692cb9e57/tiktoken-0.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cd8ca1305c1c902fe42c486165f2e4808d9997625c98ffb05b9e0366d99d3948", size = 1042916, upload-time = "2026-08-17T19:48:41.541Z" }, + { url = "https://files.pythonhosted.org/packages/c8/b6/31da98ee871383509cae2ba96a9ddef1965e3c4f8cb6dc7bcda3379398db/tiktoken-0.14.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:1f83081065ee5833d35b49e9180f3d8d15622a603dd1c435da0da6cc12b3662f", size = 1188650, upload-time = "2026-08-17T19:48:42.729Z" }, + { url = "https://files.pythonhosted.org/packages/24/65/8c5dddd7cb67f6571d154a58d7c6e2f07da54bf84c49b6a1839965b7c35e/tiktoken-0.14.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:f5e7665f6624e052e5e7f6a36919ab69279decdc976d7b16b4fa15e1897d0513", size = 1206378, upload-time = "2026-08-17T19:48:44.013Z" }, + { url = "https://files.pythonhosted.org/packages/d1/04/522ec59d30dd9a2f3ab837011cd4fc5d1178dc4a2fa07c9fa4b90af6ba9d/tiktoken-0.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:144a3fc369f92b7d548995217c5d6e84038d3572157a0f6f34080d65291d0f78", size = 1253694, upload-time = "2026-08-17T19:48:45.597Z" }, + { url = "https://files.pythonhosted.org/packages/69/84/9019e272bad188a1c61ecf44f25a9ba2368744644e3ac1f3d6516f3c9e80/tiktoken-0.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:151d37a150c8f3dfc5f4345597b10e101876bd1bd13494e0185af6b508758d2e", size = 1317873, upload-time = "2026-08-17T19:48:46.792Z" }, + { url = "https://files.pythonhosted.org/packages/24/7f/fff1217240343c0c11b5938b98aeae0e3a266cacfac25f86f91cdcd748f0/tiktoken-0.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:c77d4a3e1deb2707819df92046b89aad1ac81d27e07616b797cbff3f62c037da", size = 944395, upload-time = "2026-08-17T19:48:48.028Z" }, + { url = "https://files.pythonhosted.org/packages/8c/da/e273746b9d24a63c776bc60fba914351573ad9c575b52601eb5e60632564/tiktoken-0.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8e947aefe98ef74cce94923f90e48c98fe34eb1ec0a6bfdfadfc5a96359bfc36", size = 1094408, upload-time = "2026-08-17T19:48:49.269Z" }, + { url = "https://files.pythonhosted.org/packages/69/9f/fe6b1aca23331aa5271df5a4bd07bf68a7059254d47faee1b8272592a777/tiktoken-0.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d6cebe67765569df3dafac8474e4eccf5c19d24140492567a5e58a11445732a4", size = 1038499, upload-time = "2026-08-17T19:48:50.666Z" }, + { url = "https://files.pythonhosted.org/packages/0b/35/e9f47647c9e163bd1de30fe1a491669b7248cfc67b7404c35c009a701e1a/tiktoken-0.14.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:7db45b98e94adf4173a5cd7422b150999a7ee11ff847783a14f6e1b80cc38cb6", size = 1186355, upload-time = "2026-08-17T19:48:51.93Z" }, + { url = "https://files.pythonhosted.org/packages/51/11/9976ad86980a00cdef05e730a0127a2578a1bc6d11644d8d47246de2eb26/tiktoken-0.14.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:7896eea257fe497a2b7134474d909156c6744ce8da35bce88011a960e008aa0d", size = 1204197, upload-time = "2026-08-17T19:48:53.18Z" }, + { url = "https://files.pythonhosted.org/packages/d4/9c/7035b0bcfaa68d1ee4803fc5be5214ad865669b05bd20e7105ae8a18afc6/tiktoken-0.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b950248272f1b303dc32986396e2dccfa10cf6d1e83ec8f0bba1776660305482", size = 1250635, upload-time = "2026-08-17T19:48:54.392Z" }, + { url = "https://files.pythonhosted.org/packages/bc/1d/69cabf18bed7f4366da076735816abce0d4db3fae491ae338a6612128777/tiktoken-0.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3de75343041a1c57333b1e707ac8a9769738241d7d6a55d39e12cf84548337c6", size = 1316085, upload-time = "2026-08-17T19:48:55.525Z" }, + { url = "https://files.pythonhosted.org/packages/bd/bd/a2e884fb1402cba5be08836590320012b2d8ada0e2eef9911a64df4bcd2d/tiktoken-0.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:087538c080e5ff421abd3a0785ed63c5111d06af98e6cd0d374dbe5969147ca3", size = 941208, upload-time = "2026-08-17T19:48:56.938Z" }, + { url = "https://files.pythonhosted.org/packages/50/53/ee1453623bf65f019328721ccb6587846d2c5b7b82f34e73ca09101f072e/tiktoken-0.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9c5fe393aab56469f04e432ff851216d3def3436cf5f07e442a240164bf500f", size = 1094198, upload-time = "2026-08-17T19:48:57.955Z" }, + { url = "https://files.pythonhosted.org/packages/ad/5f/6448cfe278c3664ba9ec5b5ac08344341f7dc3d42888476e215a14eda2be/tiktoken-0.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cbe2cc3bba939bcdaf103e03df9d5039d33887080b315624be28ec69059e5f94", size = 1038820, upload-time = "2026-08-17T19:48:59.015Z" }, + { url = "https://files.pythonhosted.org/packages/69/3b/d67eac1bcce9dee3abe23aff5e3ded3116bbebaf67b80a0811c06d3806fc/tiktoken-0.14.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:2157f52e4b4d7ac5ecc7457b3716834706e7ef9a46f5144029bfeb7cf71f4e06", size = 1186175, upload-time = "2026-08-17T19:49:00.068Z" }, + { url = "https://files.pythonhosted.org/packages/37/62/cae690d9783146b0f81f564ada0f8f611de68178c0c9c7e1e969f0516b48/tiktoken-0.14.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:26e60f6a956ee171ab728b37b8439905d7ea1db435c30f9822f291e9861c861d", size = 1203884, upload-time = "2026-08-17T19:49:01.163Z" }, + { url = "https://files.pythonhosted.org/packages/b9/1e/633e30237b94e383cf814145499079f3bb9cdd4aeafc1bc42e01b0f810a6/tiktoken-0.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:380873f330b741c4435574f37edb20813d04603ace2d53e0a63560e1fec83010", size = 1250980, upload-time = "2026-08-17T19:49:02.274Z" }, + { url = "https://files.pythonhosted.org/packages/cb/56/4c12f07b812f84206f38d723eb1ebfdd34bad9309b5dbc0bee6bbcff4cbf/tiktoken-0.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3fd7c14b1cb45b486c39fc9b3443bb341f3e2fc7e6f31247f3435a5836651632", size = 1315434, upload-time = "2026-08-17T19:49:03.434Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e0/c65603f0c44811def666d3fbf611bf2af3b5e1ef613e06c19411419830b3/tiktoken-0.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:90a762670c7f968184723769a06ed51f5cf5ce5dcd1e30164f25c72d85c2d1f1", size = 940883, upload-time = "2026-08-17T19:49:04.583Z" }, + { url = "https://files.pythonhosted.org/packages/59/b0/1cf129f4af8fc513931f931023def596b7c4bfc77026513cd9d851da9e88/tiktoken-0.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:e067f4cbcc5d036e8aff7fe7a6b530a8f4de2e4616ad9005a24a1879e24e6450", size = 1096273, upload-time = "2026-08-17T19:49:05.807Z" }, + { url = "https://files.pythonhosted.org/packages/62/85/2ae74575e321148484147e10b53c3b1717c59ebaa9edb4fe18b1f5c055f8/tiktoken-0.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f2af4a336ea56d6c14f27741a0e1d8294a35dd0b038bcf990d232ebb54eb994b", size = 1040269, upload-time = "2026-08-17T19:49:06.943Z" }, + { url = "https://files.pythonhosted.org/packages/89/29/92a1120a12e4bcf2d5464350d1a91b68a433d63ce656bb7f806c27aec09c/tiktoken-0.14.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f702e0aeeb6506e57687e881c59e844ebe8f0a6a097ddafe20e3ab25f387be4e", size = 1186101, upload-time = "2026-08-17T19:49:08.102Z" }, + { url = "https://files.pythonhosted.org/packages/5b/7d/144af98dc5ad68108451a82e2f5a17f80e2663f5115058b8dfd215c1ad02/tiktoken-0.14.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:e3442bbb2f0c588cec876061e37ae67b455b9df9978b003c8fe30e45f2ef5b42", size = 1204457, upload-time = "2026-08-17T19:49:09.28Z" }, + { url = "https://files.pythonhosted.org/packages/e6/1f/be7cb06ab2108f612f3e92e7b76cf391e192db0db37a984616f0cc32aafc/tiktoken-0.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:979c1524f753b662b0f3cd261b135afe6659cce33caaa7a5ea00dd1756b3055c", size = 1251716, upload-time = "2026-08-17T19:49:10.509Z" }, + { url = "https://files.pythonhosted.org/packages/ab/6b/81f158d0f90adb826cd704069c2129a046cb784a2a09861009519fc41cf4/tiktoken-0.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2cc19ac87b41c9493c9778ff5847f0c8bbcf5bd0ec6b87ce06c1c802adc8a771", size = 1315432, upload-time = "2026-08-17T19:49:11.844Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ec/f5fa35ec13f07279fdcaf3cc9c04bbb154ea591d23978651f2b672593e8a/tiktoken-0.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:eceeff0c62419bc78d4b6e70a4762a4d25df3ae8f2d5946e3853ce93e7a57098", size = 988046, upload-time = "2026-08-17T19:49:13.282Z" }, + { url = "https://files.pythonhosted.org/packages/68/c9/7756717408d3d0dfea3f046c9466144b28afde39ff69d5808f2475dcd7f5/tiktoken-0.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:6eb94895c45f26bb8f5546e5fd8a069efcf6e3f108ea9d5cbe3bf6f7f3983438", size = 1096261, upload-time = "2026-08-17T19:49:14.351Z" }, + { url = "https://files.pythonhosted.org/packages/79/29/46ad8061f57bd9f8b2ea0aa82bf574e0f2aa040b0857a1582adba9957899/tiktoken-0.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:86951a971c53979ec857bd8c4a32dc227ab0fd33f6c12a3bd62d3fbf5f0bfcaa", size = 1040183, upload-time = "2026-08-17T19:49:15.707Z" }, + { url = "https://files.pythonhosted.org/packages/5a/7c/3184d17b868456f17b60b1a75f5ec0405618a43aa753336df341d8f11781/tiktoken-0.14.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:e2eca764c53490f8930dbce329e0769f11108d87d908282a80c5c130e26e7037", size = 1186719, upload-time = "2026-08-17T19:49:16.84Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e8/46de4400d5bf859f640feee85bd7e32235f68ddf25db53c63be78e581e3a/tiktoken-0.14.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:26cc4b4840fa0e9f4b72ed489883e12f57e00d1021ca794720e3c29a12f0edef", size = 1204660, upload-time = "2026-08-17T19:49:17.987Z" }, + { url = "https://files.pythonhosted.org/packages/29/ce/af8964c38bc8226dd8950305b7a255fa33345d5572f78af7275a313d28e0/tiktoken-0.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2fc834fbe3f6a0736905c36ab709537e6840dbd63b982dc9e0216ae7d305ba1a", size = 1250932, upload-time = "2026-08-17T19:49:19.28Z" }, + { url = "https://files.pythonhosted.org/packages/1d/4b/323631116fc986d9cc5bbeb2b8223c7c85e61a8bb94ea5ab4951023b149b/tiktoken-0.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ca4db6ff5c5bf600f9b7761a0070ed44dfe5797a76bd432fb978bc480ef40c58", size = 1315190, upload-time = "2026-08-17T19:49:20.467Z" }, + { url = "https://files.pythonhosted.org/packages/18/8b/ba48a73729c9270989b36f37ab2ed5525e52690d715097c9fa791aaa5d05/tiktoken-0.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:7aab286a020660a039097912a088236b985d18a3090d73f136c4413d29d37ca0", size = 987717, upload-time = "2026-08-17T19:49:21.704Z" }, + { url = "https://files.pythonhosted.org/packages/1d/10/b73b7e319179e0f60b32475f783b044f9cece872c53b6662664e9084b0d0/tiktoken-0.14.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:14b47e3674f2624803a8acc8fb367b7e24fc53055f9df3296482fe9a3a34a232", size = 1096280, upload-time = "2026-08-17T19:49:22.779Z" }, + { url = "https://files.pythonhosted.org/packages/c2/6b/09999a9bf1d559670d1680e8f8e419ac0e2c5f6aac82e9bfdf70f260b30a/tiktoken-0.14.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:19d643d701fdaa70e5b9c7f8f96abcaffe77ca5e482a3a1a7dde46feb4284695", size = 1040433, upload-time = "2026-08-17T19:49:23.998Z" }, + { url = "https://files.pythonhosted.org/packages/cd/7b/8537be0836f3df99b2a636b44399bfa43cd757f2b8b4097dacb794cf24a7/tiktoken-0.14.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:e4ddf863b59347deaa92302dcd90e5eb003cdc9be06ec2b692c38d1bdd9efd49", size = 1186989, upload-time = "2026-08-17T19:49:25.021Z" }, + { url = "https://files.pythonhosted.org/packages/7c/9d/f9c56d7a943a4468abf9ef37661bb9b8e0cd3aa8aa87368c7146cc3f3222/tiktoken-0.14.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:60c47ca69ddda0dea8256fffd12e1b86f4b59734a20e4a70c61f63cc5f021df4", size = 1204615, upload-time = "2026-08-17T19:49:26.37Z" }, + { url = "https://files.pythonhosted.org/packages/4b/d2/98a38579db25c4a8a84e31dd95d9072ec5f21f7e70de591da0412e29b25b/tiktoken-0.14.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:728303a072163130c5b477b1f20d6211895569c1d5302c24ffc93a3009160871", size = 1251828, upload-time = "2026-08-17T19:49:27.423Z" }, + { url = "https://files.pythonhosted.org/packages/0c/83/467be424746c039c5493c0f4102feab16b9b48eb6f5c089b2a2438e3cde2/tiktoken-0.14.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:3c5349c9f916283bba32bec8af69b763e4faa304dc004d0eaaea66a3cf004c1f", size = 1316260, upload-time = "2026-08-17T19:49:29.101Z" }, + { url = "https://files.pythonhosted.org/packages/02/ee/ddf46ca78e371f5890e96b6e7d089a85b3536432be219851eb0481786ca8/tiktoken-0.14.0-cp315-cp315-win_amd64.whl", hash = "sha256:1b6e4adcfd285c44502aed51df98aaaca4f0fea028165dbf8a9e857b9f98d8ea", size = 988230, upload-time = "2026-08-17T19:49:30.246Z" }, + { url = "https://files.pythonhosted.org/packages/2a/00/5162e90c851a28da18ed382d34898b79a8022548e5619a64e14c03ce7c3d/tiktoken-0.14.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:11d8211b290855d2721334ff17dd9b3a17bfb26872be01f25d73612ef7ece890", size = 1096186, upload-time = "2026-08-17T19:49:31.656Z" }, + { url = "https://files.pythonhosted.org/packages/65/97/a5a7bfccf25b1bb65e82bae8edff11ac3c9c041c374b7b4a823d60c38133/tiktoken-0.14.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:d0781223705199b289faa59601bb9c2441712d4c600dd13c43d8fd6a33d22cd5", size = 1039947, upload-time = "2026-08-17T19:49:32.848Z" }, + { url = "https://files.pythonhosted.org/packages/fb/ba/ef427fc638f1439181c5e12dd26b70e881861f89c007aa7e5b36300f8342/tiktoken-0.14.0-cp315-cp315t-manylinux_2_28_aarch64.whl", hash = "sha256:2ea70afba6b9eddbf22c165142e5f0a2ad7aa36a452873c48b57bb2aeb8492ae", size = 1186997, upload-time = "2026-08-17T19:49:34.121Z" }, + { url = "https://files.pythonhosted.org/packages/3e/88/2f3f85a968cdc514152129af0a060ebcccb067005a2f29b0d5ef3c838514/tiktoken-0.14.0-cp315-cp315t-manylinux_2_28_x86_64.whl", hash = "sha256:78571efc311c30b73f31eb949a921d6dac39a5d9dc42d1cfa8f8db157b3447b1", size = 1205211, upload-time = "2026-08-17T19:49:35.284Z" }, + { url = "https://files.pythonhosted.org/packages/4e/f6/80760e98a08e6649d2d68afb6035af713121dfb615acce8c4f73810ec438/tiktoken-0.14.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:86f66c85e796f5d05d5c4a60ec1d40cbfebc47a32464053528c797163fa9ab89", size = 1251479, upload-time = "2026-08-17T19:49:36.419Z" }, + { url = "https://files.pythonhosted.org/packages/c5/84/50966fb6918a0fb9b32721277e5342bf729a2d74350074d662fbedf9772e/tiktoken-0.14.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:149d97453c4c98c04b081d64a85e635921269b532710d6faf81e9e82b790e7d3", size = 1316673, upload-time = "2026-08-17T19:49:37.756Z" }, + { url = "https://files.pythonhosted.org/packages/35/5e/9b01afd037bfa22a0033963fa091e0f75b6fb15cd85bffb42ff86e697323/tiktoken-0.14.0-cp315-cp315t-win_amd64.whl", hash = "sha256:561e7580f84a79859af1ef6f676968e9030fcc3fe195700b15235bca64f009c9", size = 987929, upload-time = "2026-08-17T19:49:38.947Z" }, +] + [[package]] name = "tomli" version = "2.4.1" @@ -2416,6 +2641,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/25/2c87754f3a9e692315f7b811244090e68f362979fc8886b3fbd2985a1d8c/uncalled_for-0.3.2-py3-none-any.whl", hash = "sha256:0ff60b142c7d1f8070bde9d42afaa70aedc77dcc10998c227687e9c15713418e", size = 11444, upload-time = "2026-05-06T13:38:24.025Z" }, ] +[[package]] +name = "urllib3" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, +] + [[package]] name = "uvicorn" version = "0.48.0"