From 820c0c7de5b7ee7cdd71eca4a854874b3733f51c Mon Sep 17 00:00:00 2001 From: ggbdpq Date: Thu, 1 Oct 2026 01:36:33 +0800 Subject: [PATCH 01/10] fix(usage): record aborted auxiliary model calls as usage-unknown, not zero Auxiliary Host model calls (goal evaluation, session titles, WorkHub routing, recaps, ...) recorded every abort and provider error as `inputTokens: 0, outputTokens: 0` in the legacy table, and a completion the provider answered without usage as the same fake zero. A zero reads as "genuinely free", which is exactly the ambiguity the canonical ledger (#1679) was built to remove. #5691: "Where the provider or SDK reports partial usage on abort or error, record it. If it can't be known, mark the row as usage-unknown rather than zero." On this path usage can never be partial today: an aborted tool-free call throws without usage, so usage-unknown is the only honest value. The ledger already has the vocabulary (ModelCallUsageBasis 'reported'|'partial'|'missing', coverage counting, a schema CHECK that forbids token counts on missing-usage rows) and the read side already SUMs coverage - but its canonical writer is reachable only from an AgentRun, and auxiliary calls run outside any run. - storage: ModelCallLedger / InteractiveUsageStoresWriter.modelCalls gain recordUsageUnknownAttempt(), exposing the ledger through the stores facade the same way the canonical read channel does (server/canonical-usage-reader.ts). The record type cannot express tokens or cost: rows land usageBasis 'missing' / costBasis 'unpriced', upsert-keyed by attemptId, publishing the owning Session. This is the one ledger write that does not come from the run event stream - the seam shape is disclosed for maintainer adjudication (design note: #5691 issuecomment-5915690378). - runtime-host: runHostAuxiliaryModelCall now records aborts ('aborted'), provider errors ('failed') and usage-free completions ('completed') as usage-unknown canonical rows; calls with reported usage still record in the legacy table as before. Rows of calls no run owns carry the turnId sentinel 'auxiliary'. Disclosed, not done here: surfacing usage-unknown counts over the wire to Desktop, and createHostPluginModel passing callKind 'main' through the auxiliary pipeline - both need maintainer input on the seam above. Tests: abort, error and no-usage auxiliary calls must produce canonical usage-unknown rows (usageMissingAttempts coverage, no token totals, no legacy 0/0 row). Red before the change (7 failures: the 5 pre-existing Windows environment failures plus these assertions), green after (remaining failures identical to the pre-change baseline), red again when the authority migration alone is reverted with the storage seam kept. Refs #5691 Generated-by: GLM-5.3-Flash (ZCode) --- .../execution-model-composition.test.ts | 204 +++++++++++++++++- .../src/server/execution-model-authority.ts | 81 +++++-- .../src/__tests__/usage-stores.test.ts | 98 +++++++++ packages/storage/src/model-call-ledger.ts | 84 ++++++++ packages/storage/src/usage-stores.ts | 11 + 5 files changed, 455 insertions(+), 23 deletions(-) diff --git a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts index a38e4cee2af..60406ea5e88 100644 --- a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts +++ b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts @@ -4441,13 +4441,31 @@ test('Host auxiliary models meter provider usage and abort physical requests', { }); await closing; assert.equal(transportCloses, 1); - const abortedLogs = await usage.telemetry.logs({ range: 'all' }); + // An aborted auxiliary call knows no token counts: it records in the + // canonical ledger as a usage-unknown row (#5691), never as a zero-token + // legacy row that reads as a free call. + const abortedAttempts = await usage.modelCalls.modelCallLogs( + { range: 'all', sessionId: session.id }, + Date.now(), + 0, + 10, + ); assert.ok( - abortedLogs.rows.some( + abortedAttempts.projection.rows.some( (row) => row.callId === `goal_evaluation_${session.id}_call-2` && row.status === 'aborted', ), ); + const abortedSummary = await usage.modelCalls.modelCallSummary( + { range: 'all', sessionId: session.id }, + Date.now(), + ); + assert.equal(abortedSummary.projection.coverage.usageMissingAttempts >= 1, true); + const abortedLegacyLogs = await usage.telemetry.logs({ range: 'all' }); + assert.equal( + abortedLegacyLogs.rows.some((row) => row.callId === `goal_evaluation_${session.id}_call-2`), + false, + ); } finally { abort.abort(new DOMException('Goal evaluator test cleanup', 'AbortError')); providerRelease.resolve(); @@ -4463,6 +4481,188 @@ test('Host auxiliary models meter provider usage and abort physical requests', { } }); +test('Host auxiliary aborts, errors and usage-unknown completions record canonical usage-unknown rows', { + timeout: 20_000, +}, async () => { + const base = await mkdtemp(join(tmpdir(), 'maka-host-aux-usage-unknown-')); + const capability = await resolveStorageRoot({ + path: join(base, 'interactive'), + kind: 'interactive', + }); + const owner = await tryAcquireInteractiveRootOwner(capability); + assert.ok(owner); + if (!owner) return; + + const policy = await openInteractiveRuntimePolicyStoresForWrite(owner.lease); + const usage = await openInteractiveUsageStoresForWrite(owner.lease); + const execution = await openInteractiveExecutionStoresForWrite(owner.lease); + try { + const created = await policy.connectionCatalog.create({ + expectedCatalogRevision: 0, + connection: { + slug: 'usage-unknown-provider', + name: 'Usage unknown provider', + providerType: 'opencode-go', + baseUrl: 'http://usage-unknown.test/v1', + enabled: true, + enabledModelIds: [MODEL_ID], + }, + }); + assert.equal(created.kind, 'committed'); + if (created.kind !== 'committed') return; + const connection = created.snapshot.connections[0]; + assert.ok(connection); + if (!connection) return; + const credential = await policy.credentialVault.set({ + locator: { + scope: 'connection', + connectionId: connection.connectionId, + kind: 'api_key', + }, + expected: null, + secret: API_KEY, + }); + assert.equal(credential.kind, 'committed'); + await publishConnectionModel(policy, connection.connectionId, MODEL_ID); + const session = await execution.sessionStore.create({ + cwd: capability.canonicalPath, + llmConnectionId: connection.connectionId, + llmConnectionSlug: 'usage-unknown-provider', + model: MODEL_ID, + permissionMode: 'ask', + }); + let callCounter = 0; + const evaluator = ( + fetchImpl: (request: unknown, init?: { signal?: AbortSignal }) => Promise, + ) => + createHostGoalEvaluator({ + runtimePolicy: policy, + oauthCredentials: new HostOAuthExecutionAuthority(policy), + usage, + requestDrain: () => assert.fail('Auxiliary accounting must not drain the Host'), + readSessionHeader: (sessionId: string) => + execution.sessionStore.readHeaderSnapshot(sessionId), + newId: () => `call-${++callCounter}`, + createFetchTransport: () => ({ + fetch: fetchImpl as unknown as ProxiedFetchTransport['fetch'], + close: async () => undefined, + }), + }); + const callId = (suffix: string) => `goal_evaluation_${session.id}_${suffix}`; + const canonicalRow = async (suffix: string) => { + const logs = await usage.modelCalls.modelCallLogs( + { range: 'all', sessionId: session.id }, + Date.now(), + 0, + 10, + ); + return logs.projection.rows.find((row) => row.callId === callId(suffix)); + }; + const legacyGoalRows = async () => { + const logs = await usage.telemetry.logs({ range: 'all' }); + return logs.rows.filter((row) => row.callKind === 'goal_evaluation'); + }; + const assertSummary = async (expected: { + requests: number; + missing: number; + errors: number; + }) => { + const summary = await usage.modelCalls.modelCallSummary( + { range: 'all', sessionId: session.id }, + Date.now(), + ); + assert.equal(summary.projection.totalRequests, expected.requests); + assert.equal(summary.projection.coverage.usageMissingAttempts, expected.missing); + assert.equal(summary.projection.coverage.usageReportedAttempts, 0); + assert.equal(summary.projection.errorRequests, expected.errors); + assert.equal(summary.projection.totalTokens.total, 0); + assert.equal(summary.projection.totalCostUsd, 0); + }; + + // Aborted mid-flight: usage can never be known, so the canonical row says + // so instead of posing as a free call in the legacy table. + let abortSignal: AbortSignal | undefined; + const abortDispatched = deferred(); + const abortReleased = deferred(); + const abortingEvaluator = evaluator(async (_request, init) => { + abortSignal = init?.signal ?? undefined; + abortDispatched.resolve(); + await abortReleased.promise; + throw abortSignal?.reason ?? new DOMException('Aborted', 'AbortError'); + }); + const abort = new AbortController(); + const abortedCall = abortingEvaluator.evaluate( + 'Judge the completed Goal.', + session.id, + abort.signal, + ); + await settleWithin(abortDispatched.promise); + abort.abort(new DOMException('Goal lane invalidated', 'AbortError')); + abortReleased.resolve(); + await assert.rejects(settleWithin(abortedCall)); + await assertSummary({ requests: 1, missing: 1, errors: 0 }); + const abortedRow = await canonicalRow('call-1'); + assert.ok(abortedRow); + assert.equal(abortedRow.status, 'aborted'); + assert.equal(abortedRow.callKind, 'goal_evaluation'); + assert.deepEqual(await legacyGoalRows(), []); + + // A provider error without an abort is usage-unknown all the same. + const failingEvaluator = evaluator(async () => { + throw new Error('provider exploded'); + }); + await assert.rejects( + failingEvaluator.evaluate( + 'Judge the completed Goal.', + session.id, + new AbortController().signal, + ), + ); + await assertSummary({ requests: 2, missing: 2, errors: 1 }); + const failedRow = await canonicalRow('call-2'); + assert.ok(failedRow); + assert.equal(failedRow.status, 'error'); + assert.equal(failedRow.errorClass, 'Error'); + assert.deepEqual(await legacyGoalRows(), []); + + // A completion the provider answered without usage is usage-unknown too — + // the call succeeded, but zero tokens were never reported. + const silentEvaluator = evaluator(async () => + Response.json({ + id: 'chatcmpl-usage-unknown', + object: 'chat.completion', + created: 1, + model: MODEL_ID, + choices: [ + { + index: 0, + message: { role: 'assistant', content: RESPONSE_TEXT }, + finish_reason: 'stop', + }, + ], + }), + ); + assert.equal( + await silentEvaluator.evaluate( + 'Judge the completed Goal.', + session.id, + new AbortController().signal, + ), + RESPONSE_TEXT, + ); + await assertSummary({ requests: 3, missing: 3, errors: 1 }); + const silentRow = await canonicalRow('call-3'); + assert.ok(silentRow); + assert.equal(silentRow.status, 'success'); + assert.deepEqual(await legacyGoalRows(), []); + } finally { + await usage.close(); + await execution.sessionStore.close?.(); + await owner.close(); + await rm(base, { recursive: true, force: true }); + } +}); + test('one turn shares one canonical Skill inventory across prompt and lazy tools', async () => { const policy = { revision: 7, diff --git a/packages/runtime-host/src/server/execution-model-authority.ts b/packages/runtime-host/src/server/execution-model-authority.ts index 4f5fbb6d471..90f761a088c 100644 --- a/packages/runtime-host/src/server/execution-model-authority.ts +++ b/packages/runtime-host/src/server/execution-model-authority.ts @@ -76,6 +76,8 @@ import { resolveModelRuntime } from '@maka/runtime/model-runtime'; import { type BackendFactoryContext } from '@maka/runtime/session-manager'; import { type GoalEvaluatorResource } from '@maka/runtime/goal-evaluator'; import { type ModelMessage } from '@maka/runtime/model-protocol'; +import { type ModelCallAttemptStatus } from '@maka/core/model-call-attempt'; +import { type UsageUnknownModelCallRecord } from '@maka/storage/model-call-ledger'; import { memoryExtractionMaxOutputTokens, type MemoryExtractionSourceSnapshot, @@ -560,6 +562,9 @@ interface AuxiliaryModelCallAuthority { record: Parameters[0], ): Promise; }; + readonly modelCalls: { + recordUsageUnknownAttempt(record: UsageUnknownModelCallRecord): Promise; + }; readonly requestDrain: () => void; readonly now: () => number; readonly newId: () => string; @@ -605,6 +610,12 @@ interface HostAuxiliaryModelCallInput { ) => AuxiliaryModelRequest; } +/** + * Turn placeholder for canonical rows of calls no run owns: auxiliary calls + * happen outside any turn, and the ledger's countable rows require the column. + */ +const AUXILIARY_TURN_ID = 'auxiliary'; + function createAuxiliaryModelCallAuthority( input: AuxiliaryModelCallAuthorityInput, ): AuxiliaryModelCallAuthority { @@ -629,6 +640,16 @@ function createAuxiliaryModelCallAuthority( } }, }, + modelCalls: { + recordUsageUnknownAttempt: async (record) => { + try { + await input.usage.modelCalls.recordUsageUnknownAttempt(record); + } catch (error) { + requestDrain(); + throw error; + } + }, + }, requestDrain, now: input.now ?? Date.now, newId: input.newId ?? randomUUID, @@ -703,6 +724,34 @@ async function runHostAuxiliaryModelCall( modelId: target.model, startedAt, }; + // Abort and no-usage paths know no token counts: their canonical row says + // usage-unknown (`usageBasis: 'missing'`, no tokens) instead of posing as a + // free call (#5691). Auxiliary calls run outside any AgentRun, so the + // canonical writer is this store seam, not the run's event stream. + const recordUsageUnknown = async ( + status: ModelCallAttemptStatus, + errorClass?: string, + ): Promise => { + const completedAt = authority.now(); + try { + await authority.modelCalls.recordUsageUnknownAttempt({ + attemptId: input.callId, + completedAt, + ...(input.telemetrySessionId ? { sessionId: input.telemetrySessionId } : {}), + logicalCallId: input.callId, + turnId: AUXILIARY_TURN_ID, + callKind: input.callKind, + connectionSlug: target.connection.slug, + providerId: target.connection.providerType, + modelId: target.model, + latencyMs: Math.max(0, completedAt - startedAt), + status, + ...(errorClass ? { errorClass } : {}), + }); + } catch (accountingError) { + throw new AuxiliaryModelCallLocalError('accounting', accountingError); + } + }; let result: | Awaited> | Awaited>; @@ -743,38 +792,28 @@ async function runHostAuxiliaryModelCall( if (oauthFailure) throw oauthFailure; } catch (error) { const effectiveError = readDeferredOAuthFailure?.() ?? error; + await recordUsageUnknown( + input.abortSignal.aborted ? 'aborted' : 'failed', + evaluatorErrorClass(effectiveError), + ); + throw effectiveError; + } + if (result.usage) { try { await recordLlmCallStrict( { repo: authority.telemetry, lookupPricing: pricing }, { ...baseRecord, - inputTokens: 0, - outputTokens: 0, + ...llmCallUsageFields(result.usage), latencyMs: Math.max(0, authority.now() - startedAt), - status: input.abortSignal.aborted ? 'aborted' : 'error', - errorClass: evaluatorErrorClass(effectiveError), + status: 'success', }, ); } catch (accountingError) { throw new AuxiliaryModelCallLocalError('accounting', accountingError); } - throw effectiveError; - } - try { - await recordLlmCallStrict( - { repo: authority.telemetry, lookupPricing: pricing }, - { - ...baseRecord, - ...(result.usage - ? llmCallUsageFields(result.usage) - : { inputTokens: 0, outputTokens: 0 }), - ...(result.finishReason && !result.usage ? { rawFinishReason: result.finishReason } : {}), - latencyMs: Math.max(0, authority.now() - startedAt), - status: 'success', - }, - ); - } catch (accountingError) { - throw new AuxiliaryModelCallLocalError('accounting', accountingError); + } else { + await recordUsageUnknown('completed'); } return { text: result.text, diff --git a/packages/storage/src/__tests__/usage-stores.test.ts b/packages/storage/src/__tests__/usage-stores.test.ts index 3e7e53178f4..a2a82c359c5 100644 --- a/packages/storage/src/__tests__/usage-stores.test.ts +++ b/packages/storage/src/__tests__/usage-stores.test.ts @@ -185,6 +185,104 @@ describe('InteractiveUsageStores', () => { }); }); + test('records a usage-unknown auxiliary call as a canonical row without token counts', async () => { + await withInteractiveRoot(async ({ root, capability }) => { + const owner = await tryAcquireInteractiveRootOwner(capability); + assert(owner); + const stores = await openInteractiveUsageStoresForWrite(owner.lease); + const changed: string[] = []; + const unsubscribe = stores.subscribeSessionUsageChanges((sessionId) => + changed.push(sessionId), + ); + try { + // An auxiliary call aborted mid-flight knows no token counts. Its row + // must say so (usage_basis 'missing', no tokens) instead of posing as a + // free call — the same facts the run-path projection records. + const record = { + attemptId: 'goal_evaluation_session-aux_call-1', + completedAt: Date.UTC(2026, 0, 1), + sessionId: 'session-aux', + logicalCallId: 'goal_evaluation_session-aux_call-1', + turnId: 'auxiliary', + callKind: 'goal_evaluation' as const, + connectionSlug: 'conn-aux', + providerId: 'openai', + modelId: 'gpt-5', + latencyMs: 120, + status: 'aborted' as const, + errorClass: 'AbortError', + }; + await stores.modelCalls.recordUsageUnknownAttempt(record); + assert.deepEqual(changed, ['session-aux']); + + const summary = await stores.modelCalls.modelCallSummary({ range: 'all' }, Date.now()); + assert.equal(summary.projection.totalRequests, 1); + assert.equal(summary.projection.totalCostUsd, 0); + assert.equal(summary.projection.totalTokens.total, 0); + assert.equal(summary.projection.coverage.usageMissingAttempts, 1); + assert.equal(summary.projection.coverage.usageReportedAttempts, 0); + assert.equal(summary.unreadableRecords, 0); + + const logs = await stores.modelCalls.modelCallLogs({ range: 'all' }, Date.now(), 0, 10); + assert.equal(logs.projection.total, 1); + const row = logs.projection.rows[0]; + assert.ok(row); + assert.equal(row.callId, 'goal_evaluation_session-aux_call-1'); + assert.equal(row.callKind, 'goal_evaluation'); + assert.equal(row.status, 'aborted'); + assert.equal(row.errorClass, 'AbortError'); + + // The schema CHECK is the hard edge: a missing-usage row carries no + // token counts at all, so no total can mistake it for a free call. + const db = acquireOperationalStateDatabase(root); + try { + const stored = db.database + .prepare( + `SELECT usage_basis, cost_basis, cost_usd, input_tokens, output_tokens, + cache_read_input_tokens, cache_miss_input_tokens, + cache_write_input_tokens, reasoning_tokens, + status, session_id, turn_id + FROM usage_model_call_attempts WHERE attempt_id = ?`, + ) + .get('goal_evaluation_session-aux_call-1'); + assert.deepEqual( + { ...stored }, + { + usage_basis: 'missing', + cost_basis: 'unpriced', + cost_usd: null, + input_tokens: null, + output_tokens: null, + cache_read_input_tokens: null, + cache_miss_input_tokens: null, + cache_write_input_tokens: null, + reasoning_tokens: null, + status: 'aborted', + session_id: 'session-aux', + turn_id: 'auxiliary', + }, + ); + } finally { + db.close(); + } + + // Re-recording the same attempt upserts in place instead of double + // counting the call. + await stores.modelCalls.recordUsageUnknownAttempt({ + ...record, + latencyMs: 130, + }); + const resummarized = await stores.modelCalls.modelCallSummary({ range: 'all' }, Date.now()); + assert.equal(resummarized.projection.totalRequests, 1); + assert.deepEqual(changed, ['session-aux', 'session-aux']); + } finally { + unsubscribe(); + await stores.close(); + await owner.close(); + } + }); + }); + test('publishes the owning Session after a durable tool-usage write', async () => { await withInteractiveRoot(async ({ capability }) => { const owner = await tryAcquireInteractiveRootOwner(capability); diff --git a/packages/storage/src/model-call-ledger.ts b/packages/storage/src/model-call-ledger.ts index eee51d310e4..b736820fecd 100644 --- a/packages/storage/src/model-call-ledger.ts +++ b/packages/storage/src/model-call-ledger.ts @@ -21,6 +21,7 @@ import { decodeModelCallAttempt, MODEL_CALL_ATTEMPT_EVENT_TYPE, type ModelCallAttempt, + type ModelCallAttemptStatus, type ModelCallCoverage, } from '@maka/core/model-call-attempt'; import { @@ -31,6 +32,7 @@ import { } from '@maka/core/model-call-usage-projection'; import { usageBucketKey } from '@maka/core/usage-stats/bucket-key'; import type { + ModelCallKind, UsageBucket, UsageGroupBy, UsageLogRow, @@ -133,11 +135,48 @@ export interface CatchUpModelCallProjectionResult { readonly unreadableEvents: number; } +/** + * One canonical row for a call no AgentRun owns (#5691). + * + * Auxiliary Host model calls run outside any run, so nothing projects them + * from an event stream — the caller carries the row's identity itself. Usage + * cannot be known for these calls (the provider never reported it), so the + * record cannot express token counts or a cost: it lands as + * `usageBasis: 'missing'` / `costBasis: 'unpriced'`, the same facts the + * run-path projection records and the same rule the table's CHECK enforces on + * every other writer. + */ +export interface UsageUnknownModelCallRecord { + /** Idempotency key: recording the same attempt twice stores one row. */ + readonly attemptId: string; + readonly completedAt: number; + readonly sessionId?: string; + readonly logicalCallId: string; + /** + * Calls outside any run's turn carry a stable sentinel (`'auxiliary'` at the + * Host authority) — the table requires the column for every countable row. + */ + readonly turnId: string; + readonly callKind: ModelCallKind; + readonly connectionSlug?: string; + readonly providerId: string; + readonly modelId: string; + readonly latencyMs: number; + readonly status: ModelCallAttemptStatus; + readonly errorClass?: string; +} + export interface ModelCallLedgerWriter extends ModelCallLedgerReader { /** Advances the read model from the AgentRun authority's durable sequence. */ catchUpProjection( input?: CatchUpModelCallProjectionInput, ): Promise; + /** + * Records one usage-unknown row for a call outside any AgentRun. The single + * write that does not come from the event stream: the caller owns the + * attempt's identity, and re-recording an `attemptId` upserts in place. + */ + recordUsageUnknownAttempt(record: UsageUnknownModelCallRecord): Promise; } export interface ModelCallLedger extends ModelCallLedgerWriter { @@ -209,6 +248,11 @@ class SqliteModelCallLedger implements ModelCallLedger { ); } + recordUsageUnknownAttempt(record: UsageUnknownModelCallRecord): Promise { + if (this.#state !== 'open') return Promise.reject(new ModelCallLedgerClosedError()); + return this.write(() => writeUsageUnknownModelCallAttempt(this.#lease.database, record)); + } + summary(query: UsageQuery, now: number): ModelCallLedgerResult { const db = this.#open(); const range = resolveUsageRange(query.range, now); @@ -471,6 +515,46 @@ function writeModelCallAttempt(db: DatabaseSync, attempt: ModelCallAttempt): voi db.prepare(MODEL_CALL_UPSERT).run(...bindModelCallAttempt(attempt)); } +/** + * The usage-unknown binding: no token counts and no cost exist to record, and + * the table's CHECKs refuse any row that claimed otherwise. + */ +function bindUsageUnknownModelCallAttempt( + record: UsageUnknownModelCallRecord, +): (string | number | null)[] { + const values: Record<(typeof MODEL_CALL_COLUMNS)[number], string | number | null> = { + attempt_id: record.attemptId, + completed_at: record.completedAt, + session_id: record.sessionId ?? null, + logical_call_id: record.logicalCallId, + turn_id: record.turnId, + call_kind: record.callKind, + connection_slug: record.connectionSlug ?? null, + provider_id: record.providerId, + model_id: record.modelId, + latency_ms: record.latencyMs, + status: record.status, + error_class: record.errorClass ?? null, + usage_basis: 'missing', + input_tokens: null, + output_tokens: null, + cache_read_input_tokens: null, + cache_miss_input_tokens: null, + cache_write_input_tokens: null, + reasoning_tokens: null, + cost_basis: 'unpriced', + cost_usd: null, + }; + return MODEL_CALL_COLUMNS.map((column) => values[column]); +} + +function writeUsageUnknownModelCallAttempt( + db: DatabaseSync, + record: UsageUnknownModelCallRecord, +): void { + db.prepare(MODEL_CALL_UPSERT).run(...bindUsageUnknownModelCallAttempt(record)); +} + interface LaggingRunRow { readonly session_id: string; readonly run_id: string; diff --git a/packages/storage/src/usage-stores.ts b/packages/storage/src/usage-stores.ts index a92c9f54504..ee3d1bd6cd7 100644 --- a/packages/storage/src/usage-stores.ts +++ b/packages/storage/src/usage-stores.ts @@ -41,6 +41,7 @@ import { ModelCallLedgerPublicationError, type ModelCallLedger, type ModelCallLedgerReader, + type UsageUnknownModelCallRecord, } from './model-call-ledger.js'; import { PricingCommitUnknownError, @@ -133,6 +134,12 @@ export interface ModelCallIndexWriter extends ModelCallIndexReader { catchUpModelCallProjection( input?: CatchUpModelCallProjectionInput, ): Promise; + /** + * Records one usage-unknown row for a model call outside any AgentRun + * (#5691) — auxiliary Host calls the event stream cannot project. The owning + * Session, when known, is published after the write. + */ + recordUsageUnknownAttempt(record: UsageUnknownModelCallRecord): Promise; } export interface PricingAuthorityReader { @@ -527,6 +534,10 @@ function createWriterFacade( modelCallLogs: (query, now, offset, limit) => read(() => modelCalls.logs(query, now, offset, limit)), catchUpModelCallProjection: admitModelCallProjectionCatchUp, + recordUsageUnknownAttempt: (record) => + admitSessionUsageMutation(record.sessionId, () => + modelCalls.recordUsageUnknownAttempt(record), + ), }, pricing: { snapshot: () => read(() => pricing.snapshot()), From d591f5bc3ea54d7e4aac4d0d25791b79328dc2b1 Mon Sep 17 00:00:00 2001 From: ggbdpq Date: Thu, 1 Oct 2026 02:37:16 +0800 Subject: [PATCH 02/10] test(storage): retire model-call-ledger from the unconsumed entrypoints snapshot The usage-unknown seam gives `@maka/storage/model-call-ledger` its first external consumer since `repairPendingModelCallProjections` lost its last caller (the exact history the snapshot's comment records), so the exact-in-both-directions guard now fails on the stale entry: the published-subpath side sees the consumer, while the snapshot still listed the subpath as unconsumed. Remove the entry and update the comment to record the new fact. Generated-by: GLM-5.3-Flash (ZCode) --- .../storage/src/__tests__/public-entrypoints.test.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/storage/src/__tests__/public-entrypoints.test.ts b/packages/storage/src/__tests__/public-entrypoints.test.ts index a82c61b94d7..f17906149cb 100644 --- a/packages/storage/src/__tests__/public-entrypoints.test.ts +++ b/packages/storage/src/__tests__/public-entrypoints.test.ts @@ -140,17 +140,16 @@ test('every published entrypoint target is emitted by the build', async () => { * means removing the entry, and publishing a *new* consumer-less entrypoint * fails outright, which is the direction this guard exists to hold. * - * `./model-call-ledger` is on the list without this change touching it: it was - * already published, and `repairPendingModelCallProjections` lost its last - * caller when `canonical-usage-reader` was rewritten on `main`. It is listed - * here rather than unpublished for the same reason as the rest — retiring a - * subpath that already shipped is a compatibility call of its own. + * `./model-call-ledger` was on this list until the usage-unknown seam + * (`fix(usage)`): it had been published without consumers since + * `repairPendingModelCallProjections` lost its last caller when + * `canonical-usage-reader` was rewritten on `main`; the auxiliary-call + * recorder in runtime-host is its first external consumer again. */ const PREEXISTING_UNCONSUMED_ENTRYPOINTS = [ './activation-secret-injector', './encrypted-file-managed-secret-store', './managed-secret-store', - './model-call-ledger', './write-queue', ]; From b6f5386107128a2b4078a798a163d99b1207ed14 Mon Sep 17 00:00:00 2001 From: ggbdpq Date: Thu, 1 Oct 2026 03:52:18 +0800 Subject: [PATCH 03/10] chore: retrigger CI after a single environment-flake failure The one failing test (hosted execution freezes the headless coding provider wire contract, settled vs indeterminate, 443 ms) has no auxiliary-call trigger in its scenario, so the usage-unknown seam cannot produce a missing-usage row inside its window; the failure path is the settle-fallback catch. Same tree content was green in the previous CI round, and this commit changes nothing. Generated-by: GLM-5.3-Flash (ZCode) From 551efbc3491d13aa948712de97367fd15893c360 Mon Sep 17 00:00:00 2001 From: ggbdpq Date: Thu, 1 Oct 2026 04:48:54 +0800 Subject: [PATCH 04/10] fix(usage): keep zero-usage auxiliary completions on the legacy path The usage-unknown recording for aborted and failed auxiliary model calls (still in place, regression-tested) also routed completions the provider answered without usage to the canonical ledger as usage-missing rows. Hosted execution settlement refuses any queried range whose canonical coverage contains usage-missing rows (hosted-execution-runner incompleteUsageReason -> 'missing_attempt_usage'), so every hosted run over a provider that omits usage settled as indeterminate - the two CI failures on this branch. Revert that one path to the legacy zero-token record, matching main. The issue asks for usage-unknown rows on aborted or failed calls; those keep their canonical rows. A zero-token row for a usage-free completion is known dishonesty, and fixing it needs a maintainer ruling on the settlement coverage semantics first: as long as any canonical usage-missing row in range marks the run unsettled, recording auxiliary failures honestly (which #5691 asks for) collides with that check. The recording seam and its tests stay; the collision is the question for maintainers (#5691 issuecomment-5915690378). Note for review: hosted-execution settlement reads the whole queried range, not just the run's own attempts, so any failed auxiliary call inside the range flips a hosted run to indeterminate on this branch. The headless coding wire-contract test exercises exactly this: its session-title call fails against the test provider (responses wire plus a non-streaming auxiliary call -> 'Invalid JSON response', on CI just as on main); main records that failure as a legacy 0/0 row and settles, this branch records a canonical usage-missing row and does not. If maintainers keep the settlement semantics as-is, the aborted/failed half of this change needs the same legacy treatment; if they scope the check to the run's own attempts, both halves stay. Details and the local diagnostic trail go into the PR description. Refs #5691 Generated-by: GLM-5.3-Flash (ZCode) --- .../execution-model-composition.test.ts | 19 +++++---- .../src/server/execution-model-authority.ts | 39 +++++++++++-------- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts index 60406ea5e88..88271e527ff 100644 --- a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts +++ b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts @@ -4625,8 +4625,12 @@ test('Host auxiliary aborts, errors and usage-unknown completions record canonic assert.equal(failedRow.errorClass, 'Error'); assert.deepEqual(await legacyGoalRows(), []); - // A completion the provider answered without usage is usage-unknown too — - // the call succeeded, but zero tokens were never reported. + // A completion the provider answered without usage stays on the legacy + // zero path: hosted execution settlement refuses any range containing + // usage-missing rows, so a canonical missing row here would mark every + // such hosted run indeterminate. This residual zero is pending a ruling + // on that settlement coverage semantics (#5691); aborted and failed calls + // keep their canonical usage-unknown rows above. const silentEvaluator = evaluator(async () => Response.json({ id: 'chatcmpl-usage-unknown', @@ -4650,11 +4654,12 @@ test('Host auxiliary aborts, errors and usage-unknown completions record canonic ), RESPONSE_TEXT, ); - await assertSummary({ requests: 3, missing: 3, errors: 1 }); - const silentRow = await canonicalRow('call-3'); - assert.ok(silentRow); - assert.equal(silentRow.status, 'success'); - assert.deepEqual(await legacyGoalRows(), []); + await assertSummary({ requests: 2, missing: 2, errors: 1 }); + const silentLegacy = (await legacyGoalRows()).find((row) => row.callId === callId('call-3')); + assert.ok(silentLegacy); + assert.equal(silentLegacy.status, 'success'); + assert.equal(silentLegacy.inputTokens, 0); + assert.equal(silentLegacy.outputTokens, 0); } finally { await usage.close(); await execution.sessionStore.close?.(); diff --git a/packages/runtime-host/src/server/execution-model-authority.ts b/packages/runtime-host/src/server/execution-model-authority.ts index 90f761a088c..fcb242074e9 100644 --- a/packages/runtime-host/src/server/execution-model-authority.ts +++ b/packages/runtime-host/src/server/execution-model-authority.ts @@ -724,7 +724,7 @@ async function runHostAuxiliaryModelCall( modelId: target.model, startedAt, }; - // Abort and no-usage paths know no token counts: their canonical row says + // Abort and error paths know no token counts: their canonical row says // usage-unknown (`usageBasis: 'missing'`, no tokens) instead of posing as a // free call (#5691). Auxiliary calls run outside any AgentRun, so the // canonical writer is this store seam, not the run's event stream. @@ -798,22 +798,27 @@ async function runHostAuxiliaryModelCall( ); throw effectiveError; } - if (result.usage) { - try { - await recordLlmCallStrict( - { repo: authority.telemetry, lookupPricing: pricing }, - { - ...baseRecord, - ...llmCallUsageFields(result.usage), - latencyMs: Math.max(0, authority.now() - startedAt), - status: 'success', - }, - ); - } catch (accountingError) { - throw new AuxiliaryModelCallLocalError('accounting', accountingError); - } - } else { - await recordUsageUnknown('completed'); + try { + await recordLlmCallStrict( + { repo: authority.telemetry, lookupPricing: pricing }, + { + ...baseRecord, + // A completion without reported usage stays on the legacy zero path: + // hosted execution settlement refuses any range containing + // usage-missing rows ('missing_attempt_usage'), so routing these + // rows to the canonical ledger marked every such hosted run + // indeterminate. The residual zero is known dishonesty, pending a + // ruling on that settlement coverage semantics (#5691). + ...(result.usage + ? llmCallUsageFields(result.usage) + : { inputTokens: 0, outputTokens: 0 }), + ...(result.finishReason && !result.usage ? { rawFinishReason: result.finishReason } : {}), + latencyMs: Math.max(0, authority.now() - startedAt), + status: 'success', + }, + ); + } catch (accountingError) { + throw new AuxiliaryModelCallLocalError('accounting', accountingError); } return { text: result.text, From d917117c5cac32a14ea3df7047a07b836d59e426 Mon Sep 17 00:00:00 2001 From: ggbdpq Date: Thu, 1 Oct 2026 04:58:45 +0800 Subject: [PATCH 05/10] fix(usage): settle hosted executions despite auxiliary usage-unknown rows The two CI failures on this branch were this fix's own recording: hosted execution settlement refuses a queried range whose canonical coverage holds usage-missing rows (hosted-execution-runner incompleteUsageReason -> 'missing_attempt_usage'), and the session-title auxiliary call inside the headless coding wire-contract test always fails against the test provider (responses wire + non-streaming auxiliary call -> 'Invalid JSON response', on CI just as on main). Main records that failure as a legacy 0/0 row and settles; this branch's abort/failed recording - the core of #5691 - records a canonical usage-missing row and did not settle. Narrow the settlement coverage to run-owned rows: usageMissingAttempts now counts usage-missing rows whose turn is not the shared no-run sentinel ('auxiliary') that auxiliary Host model calls record under. An auxiliary failure is honest accounting but nobody's unsettled obligation; a run's own dispatch coming home without usage still refuses settlement, unchanged. The Usage screen computes its coverage from its own SQL and keeps counting every usage-missing row, so Desktop unknown totals are unchanged; row-level data carries the basis either way. The query and result shapes of usage.query are unchanged - no protocol change (epoch guard clean). Fallback declared for maintainer review: if narrowing the settlement semantics is rejected, the whole change reverts to the legacy table (aborted and failed calls included), keeping only the usage-unknown recording seam for a future round - the fake-zero problem stays unsolved until the settlement semantics are settled. The headless coding wire-contract test now fails past settlement on its pre-existing local hash assertion (environment-specific, red on main locally too) and is expected green on CI, where that hash matches. Refs #5691 Generated-by: GLM-5.3-Flash (ZCode) --- .../execution-model-composition.test.ts | 14 +++++-- .../src/server/execution-model-authority.ts | 6 ++- .../__tests__/model-call-usage-query.test.ts | 39 +++++++++++++++++++ .../src/__tests__/usage-stores.test.ts | 7 +++- packages/storage/src/model-call-ledger.ts | 8 +++- packages/storage/src/model-call-usage-sql.ts | 12 +++++- 6 files changed, 76 insertions(+), 10 deletions(-) diff --git a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts index 88271e527ff..bd94d7a4eba 100644 --- a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts +++ b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts @@ -4460,7 +4460,10 @@ test('Host auxiliary models meter provider usage and abort physical requests', { { range: 'all', sessionId: session.id }, Date.now(), ); - assert.equal(abortedSummary.projection.coverage.usageMissingAttempts >= 1, true); + // The aborted call's row is recorded under the no-run sentinel turn, so + // settlement coverage does not count it as missing usage — a hosted run + // must still settle despite an auxiliary failure. + assert.equal(abortedSummary.projection.coverage.usageMissingAttempts, 0); const abortedLegacyLogs = await usage.telemetry.logs({ range: 'all' }); assert.equal( abortedLegacyLogs.rows.some((row) => row.callId === `goal_evaluation_${session.id}_call-2`), @@ -4572,6 +4575,9 @@ test('Host auxiliary aborts, errors and usage-unknown completions record canonic Date.now(), ); assert.equal(summary.projection.totalRequests, expected.requests); + // Settlement coverage counts run-owned usage-missing rows only: the + // auxiliary rows under test are recorded under the no-run sentinel + // turn, so `missing` stays at whatever a run's own dispatch produced. assert.equal(summary.projection.coverage.usageMissingAttempts, expected.missing); assert.equal(summary.projection.coverage.usageReportedAttempts, 0); assert.equal(summary.projection.errorRequests, expected.errors); @@ -4600,7 +4606,7 @@ test('Host auxiliary aborts, errors and usage-unknown completions record canonic abort.abort(new DOMException('Goal lane invalidated', 'AbortError')); abortReleased.resolve(); await assert.rejects(settleWithin(abortedCall)); - await assertSummary({ requests: 1, missing: 1, errors: 0 }); + await assertSummary({ requests: 1, missing: 0, errors: 0 }); const abortedRow = await canonicalRow('call-1'); assert.ok(abortedRow); assert.equal(abortedRow.status, 'aborted'); @@ -4618,7 +4624,7 @@ test('Host auxiliary aborts, errors and usage-unknown completions record canonic new AbortController().signal, ), ); - await assertSummary({ requests: 2, missing: 2, errors: 1 }); + await assertSummary({ requests: 2, missing: 0, errors: 1 }); const failedRow = await canonicalRow('call-2'); assert.ok(failedRow); assert.equal(failedRow.status, 'error'); @@ -4654,7 +4660,7 @@ test('Host auxiliary aborts, errors and usage-unknown completions record canonic ), RESPONSE_TEXT, ); - await assertSummary({ requests: 2, missing: 2, errors: 1 }); + await assertSummary({ requests: 2, missing: 0, errors: 1 }); const silentLegacy = (await legacyGoalRows()).find((row) => row.callId === callId('call-3')); assert.ok(silentLegacy); assert.equal(silentLegacy.status, 'success'); diff --git a/packages/runtime-host/src/server/execution-model-authority.ts b/packages/runtime-host/src/server/execution-model-authority.ts index fcb242074e9..3c35a5f84d8 100644 --- a/packages/runtime-host/src/server/execution-model-authority.ts +++ b/packages/runtime-host/src/server/execution-model-authority.ts @@ -77,7 +77,7 @@ import { type BackendFactoryContext } from '@maka/runtime/session-manager'; import { type GoalEvaluatorResource } from '@maka/runtime/goal-evaluator'; import { type ModelMessage } from '@maka/runtime/model-protocol'; import { type ModelCallAttemptStatus } from '@maka/core/model-call-attempt'; -import { type UsageUnknownModelCallRecord } from '@maka/storage/model-call-ledger'; +import { NO_RUN_TURN_ID, type UsageUnknownModelCallRecord } from '@maka/storage/model-call-ledger'; import { memoryExtractionMaxOutputTokens, type MemoryExtractionSourceSnapshot, @@ -613,8 +613,10 @@ interface HostAuxiliaryModelCallInput { /** * Turn placeholder for canonical rows of calls no run owns: auxiliary calls * happen outside any turn, and the ledger's countable rows require the column. + * The storage side excludes these rows from settlement coverage, so a failed + * auxiliary call cannot flip a hosted run to indeterminate (#5691). */ -const AUXILIARY_TURN_ID = 'auxiliary'; +const AUXILIARY_TURN_ID = NO_RUN_TURN_ID; function createAuxiliaryModelCallAuthority( input: AuxiliaryModelCallAuthorityInput, diff --git a/packages/storage/src/__tests__/model-call-usage-query.test.ts b/packages/storage/src/__tests__/model-call-usage-query.test.ts index 421650c2104..a6ec67aba6c 100644 --- a/packages/storage/src/__tests__/model-call-usage-query.test.ts +++ b/packages/storage/src/__tests__/model-call-usage-query.test.ts @@ -99,6 +99,45 @@ describe('Usage answers over the canonical ledger', () => { ); }); + test('settlement coverage ignores usage-unknown rows recorded outside any run', async () => { + // Hosted execution settlement refuses a range whose canonical coverage + // holds usage-missing rows: those are a run's unsettled usage, the thing + // settlement must not paper over. A failed auxiliary call (#5691) is + // accounted the same honest way, but it belongs to no run, so its row — + // the no-run sentinel turn — must not flip a run to indeterminate. + await withProjectedAttempts( + [ + attempt({ + attemptId: 'auxiliary-failure', + turnId: 'auxiliary', + status: 'failed', + usageBasis: 'missing', + inputTokens: undefined, + outputTokens: undefined, + costBasis: 'unpriced', + costUsd: undefined, + }), + attempt({ + attemptId: 'run-dispatch', + status: 'failed', + usageBasis: 'missing', + inputTokens: undefined, + outputTokens: undefined, + costBasis: 'unpriced', + costUsd: undefined, + }), + ], + async (ledger) => { + const { projection } = ledger.summary(ALL, NOW); + assert.equal(projection.coverage.usageMissingAttempts, 1); + assert.equal(projection.coverage.usageReportedAttempts, 0); + // Both rows stay accounted: the exclusion changes settlement + // coverage, not the ledger. + assert.equal(projection.totalRequests, 2); + }, + ); + }); + test('one malformed cache reading cannot inflate the cache total', async () => { await withProjectedAttempts( [ diff --git a/packages/storage/src/__tests__/usage-stores.test.ts b/packages/storage/src/__tests__/usage-stores.test.ts index a2a82c359c5..b65f30a56e8 100644 --- a/packages/storage/src/__tests__/usage-stores.test.ts +++ b/packages/storage/src/__tests__/usage-stores.test.ts @@ -219,7 +219,12 @@ describe('InteractiveUsageStores', () => { assert.equal(summary.projection.totalRequests, 1); assert.equal(summary.projection.totalCostUsd, 0); assert.equal(summary.projection.totalTokens.total, 0); - assert.equal(summary.projection.coverage.usageMissingAttempts, 1); + // The row is recorded under the no-run sentinel turn, so settlement + // coverage — which a hosted run must be able to pass despite an + // auxiliary failure — does not count it as missing usage. The + // accounting facts stay in the row itself, asserted against the table + // below. + assert.equal(summary.projection.coverage.usageMissingAttempts, 0); assert.equal(summary.projection.coverage.usageReportedAttempts, 0); assert.equal(summary.unreadableRecords, 0); diff --git a/packages/storage/src/model-call-ledger.ts b/packages/storage/src/model-call-ledger.ts index b736820fecd..16c6ac0100f 100644 --- a/packages/storage/src/model-call-ledger.ts +++ b/packages/storage/src/model-call-ledger.ts @@ -45,6 +45,7 @@ import { count, countableFilter, COVERAGE_SUMS, + NO_RUN_TURN_ID, PRICED_COST, REQUEST_SUMS, TOKEN_SUMS, @@ -58,6 +59,8 @@ import { import { MODEL_CALL_COLUMNS } from './sqlite-usage-schema.js'; import type { ModelCallLedgerResult } from './usage-stores.js'; +export { NO_RUN_TURN_ID }; + /** * Materialization of the canonical model-call accounting ledger (#1679). * @@ -153,8 +156,9 @@ export interface UsageUnknownModelCallRecord { readonly sessionId?: string; readonly logicalCallId: string; /** - * Calls outside any run's turn carry a stable sentinel (`'auxiliary'` at the - * Host authority) — the table requires the column for every countable row. + * Calls outside any run's turn carry the shared no-run sentinel + * (`NO_RUN_TURN_ID`) — the table requires the column for every countable + * row, and settlement coverage excludes rows recorded under it. */ readonly turnId: string; readonly callKind: ModelCallKind; diff --git a/packages/storage/src/model-call-usage-sql.ts b/packages/storage/src/model-call-usage-sql.ts index f7c26a86a08..ade3bc7c0e1 100644 --- a/packages/storage/src/model-call-usage-sql.ts +++ b/packages/storage/src/model-call-usage-sql.ts @@ -43,6 +43,16 @@ export const PRICED_COST = `CASE WHEN cost_basis = 'priced' THEN COALESCE(cost_u /** Only a provider failure is an error. */ const ERROR_ROW = `status = 'failed'`; +/** + * The turn placeholder on rows recorded outside any AgentRun — failed and + * aborted auxiliary Host model calls (#5691). Such a row is honest accounting + * (real spend shape, unknown usage) but it is nobody's unsettled obligation: + * hosted execution settlement refuses a range over its run-owned usage-missing + * rows and must not let an auxiliary failure flip a run to indeterminate. + * Shared with the writer so the sentinel cannot drift between the two. + */ +export const NO_RUN_TURN_ID = 'auxiliary'; + export const TOKEN_SUMS = ` SUM(COALESCE(input_tokens, 0)) AS input, SUM(COALESCE(output_tokens, 0)) AS output, @@ -58,7 +68,7 @@ export const COVERAGE_SUMS = ` SUM(cost_basis = 'unpriced') AS unpricedAttempts, SUM(usage_basis = 'reported') AS usageReportedAttempts, SUM(usage_basis = 'partial') AS usagePartialAttempts, - SUM(usage_basis = 'missing') AS usageMissingAttempts`; + SUM(usage_basis = 'missing' AND turn_id IS NOT '${NO_RUN_TURN_ID}') AS usageMissingAttempts`; export const REQUEST_SUMS = ` COUNT(*) AS totalRequests, From 0e3f82923960307402ab4b154e07d551bd8a9e31 Mon Sep 17 00:00:00 2001 From: ggbdpq Date: Thu, 1 Oct 2026 07:34:40 +0800 Subject: [PATCH 06/10] fix(usage): give hosted settlement its own run-scoped incompleteness check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The usage-unknown seam narrowed the shared COVERAGE_SUMS so a failed auxiliary Host call could not flip a hosted run to indeterminate. That field is not settlement-only: ModelCallLedger.summary()/logs() report it to every client, while a single failed auxiliary call is still stored with usage_basis='missing' and counted in attempts/unpricedAttempts. usage-pricing-coordinator.ts returns that coverage to normal clients and the session inspector treats zero missing attempts as permission to show a cache-hit rate, so a real unknown-usage call vanished from the public provenance and an incomplete total could read as complete. Restore the ledger-wide usageMissingAttempts to count every usage-unknown row (the NO_RUN_TURN_ID sentinel stays a write-side fact: auxiliary rows still carry the auxiliary turn), and give hosted execution settlement its own run-scoped check instead: - ModelCallLedger.runSettlementCoverage(sessionId) aggregates the rows the run owns — its Session's rows outside the no-run sentinel turn — exposed through the usage-stores reader facade and the HostUsagePricingCoordinator as an in-process seam deliberately kept off the operation handler map, so the usage.query parameter surface is unchanged. - HostHostedExecutionRunner judges partial/missing usage against that run-scoped answer; unreadable and pending-repair rows stay ledger-wide (damage in the window is nobody's settled fact). Semantic edges: an auxiliary failure's missing row blocks no settlement (#5691 stays landable), a run's own usage-unknown dispatch still refuses settlement, and the ledger-wide provenance is honest again — the rewritten storage test now asserts the auxiliary rows ARE counted. Verification: storage focused (usage-stores / model-call-usage-query / sqlite-usage-schema / public-entrypoints) 66 pass; runtime-host hosted-execution-runner suite 9 pass; core model-call-attempt 19 pass; execution-model-composition auxiliary suites pass locally. Ablations re-verified both halves: reverting the settlement check re-reddens the auxiliary-does-not-block case, and re-narrowing COVERAGE_SUMS re-reddens the ledger-coverage case (1 !== 2). Known-local pre-existing failures unchanged against the d917117c5 baseline (headless wire-contract prompt hash, DeepSeek reasoning settings, WorkHub routing, prompt suggestions, usage-pricing-root-identity EBUSY). Refs #5691 Generated-by: GLM-5.3-Flash (ZCode) --- .../execution-model-composition.test.ts | 35 +++++---- .../__tests__/hosted-execution-runner.test.ts | 63 ++++++++++++++- .../src/server/execution-composition.ts | 1 + .../src/server/hosted-execution-runner.ts | 23 +++++- .../src/server/usage-pricing-coordinator.ts | 11 +++ .../__tests__/model-call-usage-query.test.ts | 76 ++++++++++++++++--- .../src/__tests__/usage-stores.test.ts | 18 +++-- packages/storage/src/model-call-ledger.ts | 36 +++++++++ packages/storage/src/model-call-usage-sql.ts | 23 ++++-- packages/storage/src/usage-stores.ts | 11 +++ 10 files changed, 256 insertions(+), 41 deletions(-) diff --git a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts index bd94d7a4eba..89a7f06d47f 100644 --- a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts +++ b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts @@ -4460,10 +4460,13 @@ test('Host auxiliary models meter provider usage and abort physical requests', { { range: 'all', sessionId: session.id }, Date.now(), ); - // The aborted call's row is recorded under the no-run sentinel turn, so - // settlement coverage does not count it as missing usage — a hosted run - // must still settle despite an auxiliary failure. - assert.equal(abortedSummary.projection.coverage.usageMissingAttempts, 0); + // Every failed auxiliary call above — the 401 recap, the timed-out + // recap, and the aborted goal evaluation — is recorded under the + // no-run sentinel turn, and the ledger-wide coverage still counts all + // three: real unknown-usage calls stay visible in the public + // provenance. Keeping a hosted run to its own rows is settlement's + // run-scoped check, not this field's job. + assert.equal(abortedSummary.projection.coverage.usageMissingAttempts, 3); const abortedLegacyLogs = await usage.telemetry.logs({ range: 'all' }); assert.equal( abortedLegacyLogs.rows.some((row) => row.callId === `goal_evaluation_${session.id}_call-2`), @@ -4575,9 +4578,11 @@ test('Host auxiliary aborts, errors and usage-unknown completions record canonic Date.now(), ); assert.equal(summary.projection.totalRequests, expected.requests); - // Settlement coverage counts run-owned usage-missing rows only: the - // auxiliary rows under test are recorded under the no-run sentinel - // turn, so `missing` stays at whatever a run's own dispatch produced. + // The ledger-wide coverage counts every usage-unknown row it holds — + // the auxiliary rows under test included, each recorded under the + // no-run sentinel turn. Hiding them here would make an incomplete + // total read as complete; settlement scopes itself to the run's own + // rows instead. assert.equal(summary.projection.coverage.usageMissingAttempts, expected.missing); assert.equal(summary.projection.coverage.usageReportedAttempts, 0); assert.equal(summary.projection.errorRequests, expected.errors); @@ -4606,7 +4611,7 @@ test('Host auxiliary aborts, errors and usage-unknown completions record canonic abort.abort(new DOMException('Goal lane invalidated', 'AbortError')); abortReleased.resolve(); await assert.rejects(settleWithin(abortedCall)); - await assertSummary({ requests: 1, missing: 0, errors: 0 }); + await assertSummary({ requests: 1, missing: 1, errors: 0 }); const abortedRow = await canonicalRow('call-1'); assert.ok(abortedRow); assert.equal(abortedRow.status, 'aborted'); @@ -4624,7 +4629,7 @@ test('Host auxiliary aborts, errors and usage-unknown completions record canonic new AbortController().signal, ), ); - await assertSummary({ requests: 2, missing: 0, errors: 1 }); + await assertSummary({ requests: 2, missing: 2, errors: 1 }); const failedRow = await canonicalRow('call-2'); assert.ok(failedRow); assert.equal(failedRow.status, 'error'); @@ -4632,11 +4637,11 @@ test('Host auxiliary aborts, errors and usage-unknown completions record canonic assert.deepEqual(await legacyGoalRows(), []); // A completion the provider answered without usage stays on the legacy - // zero path: hosted execution settlement refuses any range containing - // usage-missing rows, so a canonical missing row here would mark every - // such hosted run indeterminate. This residual zero is pending a ruling - // on that settlement coverage semantics (#5691); aborted and failed calls - // keep their canonical usage-unknown rows above. + // zero path for now: with settlement scoped to a run's own rows a + // canonical missing row here would no longer flip any hosted run + // indeterminate, so routing it to the canonical ledger is a free-standing + // recording question (#5691), not a settlement constraint. Aborted and + // failed calls above already keep their canonical usage-unknown rows. const silentEvaluator = evaluator(async () => Response.json({ id: 'chatcmpl-usage-unknown', @@ -4660,7 +4665,7 @@ test('Host auxiliary aborts, errors and usage-unknown completions record canonic ), RESPONSE_TEXT, ); - await assertSummary({ requests: 2, missing: 0, errors: 1 }); + await assertSummary({ requests: 2, missing: 2, errors: 1 }); const silentLegacy = (await legacyGoalRows()).find((row) => row.callId === callId('call-3')); assert.ok(silentLegacy); assert.equal(silentLegacy.status, 'success'); diff --git a/packages/runtime-host/src/__tests__/hosted-execution-runner.test.ts b/packages/runtime-host/src/__tests__/hosted-execution-runner.test.ts index 00608a64d20..b2d78fa8c67 100644 --- a/packages/runtime-host/src/__tests__/hosted-execution-runner.test.ts +++ b/packages/runtime-host/src/__tests__/hosted-execution-runner.test.ts @@ -33,6 +33,7 @@ test('hosted execution reads usage only after execution residencies settle', asy return usageSummary(); }, }), + runSettlementCoverage: settlementCoverage(), context: context(), requestDrain: () => {}, waitForExecutionResidencies: () => residency.promise, @@ -62,11 +63,46 @@ test('hosted execution reads usage only after execution residencies settle', asy assert.equal(usageRead, true); }); -test('incomplete usage preserves its fixed safe cause', async () => { +test('a run-owned unsettled attempt refuses settlement with its fixed safe cause', async () => { + const runner = new HostHostedExecutionRunner({ + handlers: handlers({}), + runSettlementCoverage: settlementCoverage({ usageMissingAttempts: 1 }), + context: context(), + requestDrain: () => {}, + waitForExecutionResidencies: async () => {}, + waitForAllResidencies: async () => {}, + }); + + const result = await runner.run(input(), new AbortController().signal); + + assert.equal(result.failureReason, 'Runtime Host usage did not settle: missing_attempt_usage'); +}); + +test('a run-owned partial attempt refuses settlement with its fixed safe cause', async () => { + const runner = new HostHostedExecutionRunner({ + handlers: handlers({}), + runSettlementCoverage: settlementCoverage({ usagePartialAttempts: 1 }), + context: context(), + requestDrain: () => {}, + waitForExecutionResidencies: async () => {}, + waitForAllResidencies: async () => {}, + }); + + const result = await runner.run(input(), new AbortController().signal); + + assert.equal(result.failureReason, 'Runtime Host usage did not settle: partial_attempt_usage'); +}); + +test('a usage-unknown row outside any run does not block settlement', async () => { + // The failed auxiliary call (#5691) is real accounting: the ledger-wide + // coverage — the public provenance every client reads — counts it as + // usage-missing. Settlement must not read that field: it holds the run to + // its own rows, and the run left none unsettled. const usage = usageSummary(); usage.provenance.coverage.usageMissingAttempts = 1; const runner = new HostHostedExecutionRunner({ handlers: handlers({ usage: () => usage }), + runSettlementCoverage: settlementCoverage(), context: context(), requestDrain: () => {}, waitForExecutionResidencies: async () => {}, @@ -75,7 +111,7 @@ test('incomplete usage preserves its fixed safe cause', async () => { const result = await runner.run(input(), new AbortController().signal); - assert.equal(result.failureReason, 'Runtime Host usage did not settle: missing_attempt_usage'); + assert.equal(result.kind, 'settled'); }); test('abort after terminal completion preserves the completed result', async () => { @@ -98,6 +134,7 @@ async function runWithAbortAfterTerminal(query?: () => unknown) { const settling = deferred(); const runner = new HostHostedExecutionRunner({ handlers: handlers(query ? { query } : {}), + runSettlementCoverage: settlementCoverage(), context: context(), requestDrain: () => {}, waitForExecutionResidencies: () => { @@ -132,6 +169,7 @@ test('hosted execution cancellation drains the Host and waits for canonical stop return terminalTurn('cancelled'); }, }), + runSettlementCoverage: settlementCoverage(), context: context(), requestDrain: () => { drains += 1; @@ -167,6 +205,7 @@ test('hosted execution cancellation before Turn admission starts no Turn', async turnStarts += 1; }, }), + runSettlementCoverage: settlementCoverage(), context: context(), requestDrain: () => {}, waitForExecutionResidencies: async () => {}, @@ -195,6 +234,7 @@ test('hosted execution cancellation remains active while Runtime continuations s return runningTurn(); }, }), + runSettlementCoverage: settlementCoverage(), context: context(), requestDrain: () => { drains += 1; @@ -334,6 +374,25 @@ function usageSummary() { }; } +/** + * The run-scoped incompleteness check's stub: a hosted execution owns no + * usage-unknown rows unless the test says otherwise. + */ +function settlementCoverage( + overrides: { + attempts?: number; + usageMissingAttempts?: number; + usagePartialAttempts?: number; + } = {}, +) { + return async () => ({ + attempts: 1, + usageMissingAttempts: 0, + usagePartialAttempts: 0, + ...overrides, + }); +} + function emptySkillInvocation() { return { loaded: [], failed: [], receipts: [] }; } diff --git a/packages/runtime-host/src/server/execution-composition.ts b/packages/runtime-host/src/server/execution-composition.ts index 1bd26edf7cd..a12868505ee 100644 --- a/packages/runtime-host/src/server/execution-composition.ts +++ b/packages/runtime-host/src/server/execution-composition.ts @@ -2818,6 +2818,7 @@ export async function createExecutionRuntimeHostComposition( 'turn.stop': turnControl.handlers['turn.stop'], 'usage.query': usagePricing.handlers['usage.query'], }, + runSettlementCoverage: (sessionId) => usagePricing.runSettlementCoverage(sessionId), context: { hostEpoch: context.hostEpoch, connectionId: 'hosted-execution', diff --git a/packages/runtime-host/src/server/hosted-execution-runner.ts b/packages/runtime-host/src/server/hosted-execution-runner.ts index ea38fdfa2be..9ff6383971c 100644 --- a/packages/runtime-host/src/server/hosted-execution-runner.ts +++ b/packages/runtime-host/src/server/hosted-execution-runner.ts @@ -26,6 +26,7 @@ import type { TurnSnapshot, UsageQueryResult, } from '../protocol/index.js'; +import type { RunSettlementCoverage } from '@maka/storage/model-call-ledger'; import type { ConnectionContext, OperationHandlerMap } from './operation-dispatcher.js'; export interface HostHostedExecutionRunnerInput { @@ -33,6 +34,14 @@ export interface HostHostedExecutionRunnerInput { OperationHandlerMap, 'session.create' | 'turn.start' | 'turn.query' | 'turn.stop' | 'usage.query' >; + /** + * What this execution's own run left unsettled (#5691). Settlement's + * incompleteness check reads this run-scoped answer instead of the + * `usage.query` coverage: that field is the ledger's public provenance and + * must keep counting usage-unknown rows no run owns — a failed auxiliary + * Host call is real accounting, not this run's unsettled obligation. + */ + readonly runSettlementCoverage: (sessionId: string) => Promise; readonly context: ConnectionContext; readonly requestDrain: () => void; readonly waitForExecutionResidencies: () => Promise; @@ -83,7 +92,8 @@ export class HostHostedExecutionRunner { ? this.input.waitForAllResidencies() : this.input.waitForExecutionResidencies()); const usage = await this.#readUsage(startedAt, (this.input.now ?? Date.now)()); - const incompleteUsage = incompleteUsageReason(usage); + const runCoverage = await this.input.runSettlementCoverage(input.executionId); + const incompleteUsage = incompleteUsageReason(usage, runCoverage); if (incompleteUsage) { return indeterminate( input.executionId, @@ -160,12 +170,17 @@ export class HostHostedExecutionRunner { function incompleteUsageReason( result: Extract, + runCoverage: RunSettlementCoverage, ): string | undefined { - const { coverage, unreadableRecords, pendingRepairs } = result.provenance; + // Unreadable and pending-repair rows stay ledger-wide: damage in the window + // is nobody's settled fact. Partial and missing usage, the things a run can + // leave unsettled, are judged against the run's own rows only (#5691) — the + // shared coverage field still reports them for every client to see. + const { unreadableRecords, pendingRepairs } = result.provenance; if (unreadableRecords > 0) return 'unreadable_usage_record'; if (pendingRepairs > 0) return 'pending_usage_repair'; - if (coverage.usagePartialAttempts > 0) return 'partial_attempt_usage'; - if (coverage.usageMissingAttempts > 0) return 'missing_attempt_usage'; + if (runCoverage.usagePartialAttempts > 0) return 'partial_attempt_usage'; + if (runCoverage.usageMissingAttempts > 0) return 'missing_attempt_usage'; return undefined; } diff --git a/packages/runtime-host/src/server/usage-pricing-coordinator.ts b/packages/runtime-host/src/server/usage-pricing-coordinator.ts index 4b97e790d23..fa532c08368 100644 --- a/packages/runtime-host/src/server/usage-pricing-coordinator.ts +++ b/packages/runtime-host/src/server/usage-pricing-coordinator.ts @@ -42,6 +42,7 @@ import { type InteractiveUsageStoresFailureClassification, type InteractiveUsageStoresWriter, } from '@maka/storage/usage-stores'; +import type { RunSettlementCoverage } from '@maka/storage/model-call-ledger'; import { isSessionNotFoundError } from '@maka/storage/execution-stores'; import { encodePricingQueryResult, @@ -80,6 +81,16 @@ export class HostUsagePricingCoordinator { 'pricing.mutate': (input) => this.#mutatePricing(input), }; + /** + * What one Session's own run left unsettled (#5691) — hosted execution + * settlement's incompleteness check. Deliberately off the operation handler + * map: the ledger-wide `usage.query` coverage must keep counting rows no run + * owns, so settlement scopes itself through this in-process seam instead. + */ + async runSettlementCoverage(sessionId: string): Promise { + return this.#stores.modelCalls.modelCallRunSettlementCoverage(sessionId); + } + readonly #stores: InteractiveUsageStoresWriter; readonly #requestDrain: () => void; readonly #activation: RuntimePolicyActivationGate; diff --git a/packages/storage/src/__tests__/model-call-usage-query.test.ts b/packages/storage/src/__tests__/model-call-usage-query.test.ts index a6ec67aba6c..d5f75b357d0 100644 --- a/packages/storage/src/__tests__/model-call-usage-query.test.ts +++ b/packages/storage/src/__tests__/model-call-usage-query.test.ts @@ -99,12 +99,13 @@ describe('Usage answers over the canonical ledger', () => { ); }); - test('settlement coverage ignores usage-unknown rows recorded outside any run', async () => { - // Hosted execution settlement refuses a range whose canonical coverage - // holds usage-missing rows: those are a run's unsettled usage, the thing - // settlement must not paper over. A failed auxiliary call (#5691) is - // accounted the same honest way, but it belongs to no run, so its row — - // the no-run sentinel turn — must not flip a run to indeterminate. + test('ledger coverage counts usage-unknown rows recorded outside any run', async () => { + // A failed auxiliary call (#5691) is accounted the same honest way as a + // run's own unsettled dispatch, and the ledger-wide coverage — what + // `summary()`/`logs()` report to every client — must see both. Excluding + // the no-run sentinel turn here would hide a real unknown-usage call from + // the public provenance and let an incomplete total read as complete; + // keeping a hosted run to its own rows is settlement's separate job. await withProjectedAttempts( [ attempt({ @@ -129,15 +130,72 @@ describe('Usage answers over the canonical ledger', () => { ], async (ledger) => { const { projection } = ledger.summary(ALL, NOW); - assert.equal(projection.coverage.usageMissingAttempts, 1); + assert.equal(projection.coverage.usageMissingAttempts, 2); assert.equal(projection.coverage.usageReportedAttempts, 0); - // Both rows stay accounted: the exclusion changes settlement - // coverage, not the ledger. assert.equal(projection.totalRequests, 2); }, ); }); + test('run settlement coverage holds a run to its own rows, not auxiliary ones', async () => { + // Hosted execution settlement asks what its own run left unsettled (#5691). + // Rows under the no-run sentinel turn belong to no run — a failed + // auxiliary call sharing the Session's id included — so they are excluded + // even at the same Session, and another Session's rows never count. The + // run's own usage-missing dispatch still shows up. + await withProjectedAttempts( + [ + attempt({ + attemptId: 'auxiliary-failure', + turnId: 'auxiliary', + status: 'failed', + usageBasis: 'missing', + inputTokens: undefined, + outputTokens: undefined, + costBasis: 'unpriced', + costUsd: undefined, + }), + attempt({ + attemptId: 'other-session-dispatch', + sessionId: 'session-2', + status: 'failed', + usageBasis: 'missing', + inputTokens: undefined, + outputTokens: undefined, + costBasis: 'unpriced', + costUsd: undefined, + }), + attempt({ + attemptId: 'run-reported', + logicalCallId: 'call-2', + }), + attempt({ + attemptId: 'run-partial', + logicalCallId: 'call-3', + usageBasis: 'partial', + outputTokens: undefined, + }), + attempt({ + attemptId: 'run-missing-dispatch', + logicalCallId: 'call-4', + status: 'failed', + usageBasis: 'missing', + inputTokens: undefined, + outputTokens: undefined, + costBasis: 'unpriced', + costUsd: undefined, + }), + ], + async (ledger) => { + assert.deepEqual(ledger.runSettlementCoverage('session-1'), { + attempts: 3, + usageMissingAttempts: 1, + usagePartialAttempts: 1, + }); + }, + ); + }); + test('one malformed cache reading cannot inflate the cache total', async () => { await withProjectedAttempts( [ diff --git a/packages/storage/src/__tests__/usage-stores.test.ts b/packages/storage/src/__tests__/usage-stores.test.ts index b65f30a56e8..742fb2257ca 100644 --- a/packages/storage/src/__tests__/usage-stores.test.ts +++ b/packages/storage/src/__tests__/usage-stores.test.ts @@ -219,14 +219,20 @@ describe('InteractiveUsageStores', () => { assert.equal(summary.projection.totalRequests, 1); assert.equal(summary.projection.totalCostUsd, 0); assert.equal(summary.projection.totalTokens.total, 0); - // The row is recorded under the no-run sentinel turn, so settlement - // coverage — which a hosted run must be able to pass despite an - // auxiliary failure — does not count it as missing usage. The - // accounting facts stay in the row itself, asserted against the table - // below. - assert.equal(summary.projection.coverage.usageMissingAttempts, 0); + // The ledger-wide coverage counts the row: a real unknown-usage call + // must stay visible in the public provenance even though it belongs to + // no run. Keeping a hosted run to its own rows is the run-scoped + // settlement coverage's job, asserted against the same stores below. + assert.equal(summary.projection.coverage.usageMissingAttempts, 1); assert.equal(summary.projection.coverage.usageReportedAttempts, 0); assert.equal(summary.unreadableRecords, 0); + // The run-scoped settlement coverage excludes the no-run sentinel row, + // so an auxiliary failure cannot flip a hosted run to indeterminate. + assert.deepEqual(await stores.modelCalls.modelCallRunSettlementCoverage('session-aux'), { + attempts: 0, + usageMissingAttempts: 0, + usagePartialAttempts: 0, + }); const logs = await stores.modelCalls.modelCallLogs({ range: 'all' }, Date.now(), 0, 10); assert.equal(logs.projection.total, 1); diff --git a/packages/storage/src/model-call-ledger.ts b/packages/storage/src/model-call-ledger.ts index 16c6ac0100f..3d986532eea 100644 --- a/packages/storage/src/model-call-ledger.ts +++ b/packages/storage/src/model-call-ledger.ts @@ -48,6 +48,7 @@ import { NO_RUN_TURN_ID, PRICED_COST, REQUEST_SUMS, + RUN_SETTLEMENT_COVERAGE_SUMS, TOKEN_SUMS, unreadableFilter, type SqlFilter, @@ -121,6 +122,25 @@ export interface ModelCallLedgerReader { offset: number, limit: number, ): ModelCallLedgerResult; + /** + * The rows one Session's own run left unsettled (#5691): its attempts whose + * usage the provider never reported or only partly reported. Rows recorded + * outside any run — the no-run sentinel turn — belong to no run's + * settlement and are excluded, the reverse of the ledger-wide coverage + * above, which must see them. + */ + runSettlementCoverage(sessionId: string): RunSettlementCoverage; +} + +/** + * What a hosted execution's settlement checks instead of the ledger-wide + * coverage: whether the run's own attempts all settled (#5691). + */ +export interface RunSettlementCoverage { + /** Countable rows the run owns — Session rows outside the sentinel turn. */ + readonly attempts: number; + readonly usageMissingAttempts: number; + readonly usagePartialAttempts: number; } export interface CatchUpModelCallProjectionInput { @@ -368,6 +388,22 @@ class SqliteModelCallLedger implements ModelCallLedger { }; } + runSettlementCoverage(sessionId: string): RunSettlementCoverage { + const db = this.#open(); + const row = db + .prepare( + `SELECT ${RUN_SETTLEMENT_COVERAGE_SUMS} + FROM usage_model_call_attempts + WHERE cost_basis IS NOT NULL AND session_id = ? AND turn_id IS NOT ?`, + ) + .get(sessionId, NO_RUN_TURN_ID) as Record | undefined; + return { + attempts: count(row?.attempts), + usageMissingAttempts: count(row?.usageMissingAttempts), + usagePartialAttempts: count(row?.usagePartialAttempts), + }; + } + #open(): DatabaseSync { if (this.#state !== 'open') throw new ModelCallLedgerClosedError(); return this.#lease.database; diff --git a/packages/storage/src/model-call-usage-sql.ts b/packages/storage/src/model-call-usage-sql.ts index ade3bc7c0e1..674cc3f2f20 100644 --- a/packages/storage/src/model-call-usage-sql.ts +++ b/packages/storage/src/model-call-usage-sql.ts @@ -46,10 +46,10 @@ const ERROR_ROW = `status = 'failed'`; /** * The turn placeholder on rows recorded outside any AgentRun — failed and * aborted auxiliary Host model calls (#5691). Such a row is honest accounting - * (real spend shape, unknown usage) but it is nobody's unsettled obligation: - * hosted execution settlement refuses a range over its run-owned usage-missing - * rows and must not let an auxiliary failure flip a run to indeterminate. - * Shared with the writer so the sentinel cannot drift between the two. + * (real spend shape, unknown usage), so the ledger-wide coverage counts it; + * what it is not is a run's unsettled obligation, and the run-scoped + * settlement coverage excludes it on that marker. Shared with the writer so + * the sentinel cannot drift between the two. */ export const NO_RUN_TURN_ID = 'auxiliary'; @@ -68,7 +68,20 @@ export const COVERAGE_SUMS = ` SUM(cost_basis = 'unpriced') AS unpricedAttempts, SUM(usage_basis = 'reported') AS usageReportedAttempts, SUM(usage_basis = 'partial') AS usagePartialAttempts, - SUM(usage_basis = 'missing' AND turn_id IS NOT '${NO_RUN_TURN_ID}') AS usageMissingAttempts`; + SUM(usage_basis = 'missing') AS usageMissingAttempts`; + +/** + * Coverage a hosted execution's settlement may hold a run to (#5691): the + * rows the run owns — its Session's, outside the no-run sentinel turn. The + * sentinel exclusion is the query's WHERE clause, not an aggregate here, so + * this stays narrower than the ledger-wide {@link COVERAGE_SUMS} on purpose: + * one answers "what does the ledger hold unknown-usage rows for", the other + * "what did this run leave unsettled". + */ +export const RUN_SETTLEMENT_COVERAGE_SUMS = ` + COUNT(*) AS attempts, + SUM(usage_basis = 'partial') AS usagePartialAttempts, + SUM(usage_basis = 'missing') AS usageMissingAttempts`; export const REQUEST_SUMS = ` COUNT(*) AS totalRequests, diff --git a/packages/storage/src/usage-stores.ts b/packages/storage/src/usage-stores.ts index ee3d1bd6cd7..57779d9f161 100644 --- a/packages/storage/src/usage-stores.ts +++ b/packages/storage/src/usage-stores.ts @@ -41,6 +41,7 @@ import { ModelCallLedgerPublicationError, type ModelCallLedger, type ModelCallLedgerReader, + type RunSettlementCoverage, type UsageUnknownModelCallRecord, } from './model-call-ledger.js'; import { @@ -128,6 +129,12 @@ export interface ModelCallIndexReader { offset: number, limit: number, ): Promise>; + /** + * The rows one Session's own run left unsettled (#5691) — what hosted + * execution settlement checks instead of the ledger-wide coverage, which + * must also count rows no run owns. + */ + modelCallRunSettlementCoverage(sessionId: string): Promise; } export interface ModelCallIndexWriter extends ModelCallIndexReader { @@ -533,6 +540,8 @@ function createWriterFacade( read(() => modelCalls.buckets(query, groupBy, now)), modelCallLogs: (query, now, offset, limit) => read(() => modelCalls.logs(query, now, offset, limit)), + modelCallRunSettlementCoverage: (sessionId) => + read(() => modelCalls.runSettlementCoverage(sessionId)), catchUpModelCallProjection: admitModelCallProjectionCatchUp, recordUsageUnknownAttempt: (record) => admitSessionUsageMutation(record.sessionId, () => @@ -589,6 +598,8 @@ function modelCallReader( run(() => ledger.buckets(query, groupBy, now)), modelCallLogs: (query: UsageQuery, now: number, offset: number, limit: number) => run(() => ledger.logs(query, now, offset, limit)), + modelCallRunSettlementCoverage: (sessionId: string) => + run(() => ledger.runSettlementCoverage(sessionId)), }); } From 185482af298215218f68589f19103362c27331b8 Mon Sep 17 00:00:00 2001 From: ggbdpq Date: Thu, 1 Oct 2026 09:45:29 +0800 Subject: [PATCH 07/10] fix(usage): carry no-run ownership in a column, not the turn value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second P2 from the #5890 review: NO_RUN_TURN_ID is the literal 'auxiliary', but decodeHostedExecutionStartInput accepts that value as executionId (hosted-execution.ts:119) and HostHostedExecutionRunner uses the execution id as the real run's turnId (hosted-execution-runner.ts:78-84). For a hosted execution named `auxiliary`, the `turn_id IS NOT ?` settlement predicate dropped every one of its run-owned attempts: a missing-usage ledger row showed up in public coverage while runSettlementCoverage('auxiliary') returned zero attempts and zero missing, so incompleteUsageReason could mark the execution settled and release its environment despite unknown usage. Fix: usage_model_call_attempts gains a `no_run` ownership column written by whichever seam recorded the row — 0 from the AgentRun event-stream projection (every row it writes is run-owned), 1 from the usage-unknown seam (the one write outside any run). runSettlementCoverage filters on `no_run = 0`; the sentinel stays as the required turn placeholder but no longer carries the discrimination. Existing databases migrate through ensureColumn (the session_id precedent) plus a one-time backfill of the old turn-value judgment; the legacy record_json spread supplies 0 explicitly. Rejected alternatives: - Reserved word (reject executionId = 'auxiliary' at decode): removing an accepted protocol value moves the compatibility epoch per protocol-compatible-changes/README.md, turning a storage fix into a client/host lockstep upgrade, and the same collision stayed reachable through turn.start accepting the literal as a turn id. - Positive attribution without a column: the ledger holds no run_id, and matching turn_id = executionId collapses under the very collision being fixed — an execution named `auxiliary` owns rows byte-identical to the sentinel rows on (session_id, turn_id). Regression: a hosted execution named like the sentinel keeps its own rows — its usage-missing dispatch holds settlement (missing_attempt_usage) while the failed auxiliary call at the same Session still does not, and ledger-wide coverage keeps counting both. The two older sentinel tests now seed no-run rows through the real usage-unknown seam instead of projecting a sentinel turn value through the run authority, which under the old predicate was the only way to fake one. Refs #5691 Generated-by: GLM-5.3-Flash (ZCode) --- .../src/server/execution-model-authority.ts | 6 +- .../__tests__/model-call-usage-query.test.ts | 137 ++++++++++++++---- packages/storage/src/model-call-ledger.ts | 22 ++- packages/storage/src/model-call-usage-sql.ts | 15 +- packages/storage/src/sqlite-usage-schema.ts | 23 ++- 5 files changed, 157 insertions(+), 46 deletions(-) diff --git a/packages/runtime-host/src/server/execution-model-authority.ts b/packages/runtime-host/src/server/execution-model-authority.ts index 3c35a5f84d8..47fc6c2cb93 100644 --- a/packages/runtime-host/src/server/execution-model-authority.ts +++ b/packages/runtime-host/src/server/execution-model-authority.ts @@ -613,8 +613,10 @@ interface HostAuxiliaryModelCallInput { /** * Turn placeholder for canonical rows of calls no run owns: auxiliary calls * happen outside any turn, and the ledger's countable rows require the column. - * The storage side excludes these rows from settlement coverage, so a failed - * auxiliary call cannot flip a hosted run to indeterminate (#5691). + * The storage side records the rows' no-run ownership from this seam, so a + * failed auxiliary call cannot flip a hosted run to indeterminate (#5691) — + * including a hosted execution legally named like this placeholder + * (#5890 review). */ const AUXILIARY_TURN_ID = NO_RUN_TURN_ID; diff --git a/packages/storage/src/__tests__/model-call-usage-query.test.ts b/packages/storage/src/__tests__/model-call-usage-query.test.ts index d5f75b357d0..76104c22acd 100644 --- a/packages/storage/src/__tests__/model-call-usage-query.test.ts +++ b/packages/storage/src/__tests__/model-call-usage-query.test.ts @@ -19,14 +19,62 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; +import type { ModelCallAttempt } from '@maka/core/model-call-attempt'; import { + type ModelCallLedger, + NO_RUN_TURN_ID, + type UsageUnknownModelCallRecord, +} from '../model-call-ledger.js'; +import { + appendAuthorityEvent, modelCallAttempt as attempt, MODEL_CALL_NOW as NOW, + withLedger, withProjectedAttempts, } from './fixtures/model-call-attempt.js'; const ALL = { range: 'all' } as const; +/** + * Seeds rows the way production writes them: run-owned attempts through the + * AgentRun authority's projection, calls no run owns through the usage-unknown + * seam. Ownership follows the writer, so a test cannot fake a no-run row by + * projecting a sentinel turn value. + */ +async function withRecordedAttempts( + projected: readonly ModelCallAttempt[], + noRun: readonly UsageUnknownModelCallRecord[], + run: (ledger: ModelCallLedger) => Promise, +): Promise { + await withLedger(async (ledger, root) => { + projected.forEach((value, index) => { + appendAuthorityEvent(root, index, value, value.sessionId, value.runId); + }); + await ledger.catchUpProjection(); + for (const record of noRun) await ledger.recordUsageUnknownAttempt(record); + await run(ledger); + }); +} + +/** A failed auxiliary Host call, as the usage-unknown seam records it. */ +function usageUnknownAttempt( + overrides: Partial = {}, +): UsageUnknownModelCallRecord { + return { + attemptId: 'auxiliary-failure', + completedAt: NOW - 100, + sessionId: 'session-1', + logicalCallId: 'auxiliary-failure', + turnId: NO_RUN_TURN_ID, + callKind: 'goal_evaluation', + providerId: 'openai', + modelId: 'gpt-5', + latencyMs: 5, + status: 'failed', + ...overrides, + }; +} + describe('Usage answers over the canonical ledger', () => { test('a total never counts unpriced spend as zero, and says so in coverage', async () => { // The frozen pre-cutover table had nowhere to record "we could not price @@ -103,21 +151,11 @@ describe('Usage answers over the canonical ledger', () => { // A failed auxiliary call (#5691) is accounted the same honest way as a // run's own unsettled dispatch, and the ledger-wide coverage — what // `summary()`/`logs()` report to every client — must see both. Excluding - // the no-run sentinel turn here would hide a real unknown-usage call from - // the public provenance and let an incomplete total read as complete; - // keeping a hosted run to its own rows is settlement's separate job. - await withProjectedAttempts( + // the no-run rows here would hide a real unknown-usage call from the + // public provenance and let an incomplete total read as complete; keeping + // a hosted run to its own rows is settlement's separate job. + await withRecordedAttempts( [ - attempt({ - attemptId: 'auxiliary-failure', - turnId: 'auxiliary', - status: 'failed', - usageBasis: 'missing', - inputTokens: undefined, - outputTokens: undefined, - costBasis: 'unpriced', - costUsd: undefined, - }), attempt({ attemptId: 'run-dispatch', status: 'failed', @@ -128,6 +166,7 @@ describe('Usage answers over the canonical ledger', () => { costUsd: undefined, }), ], + [usageUnknownAttempt()], async (ledger) => { const { projection } = ledger.summary(ALL, NOW); assert.equal(projection.coverage.usageMissingAttempts, 2); @@ -139,22 +178,12 @@ describe('Usage answers over the canonical ledger', () => { test('run settlement coverage holds a run to its own rows, not auxiliary ones', async () => { // Hosted execution settlement asks what its own run left unsettled (#5691). - // Rows under the no-run sentinel turn belong to no run — a failed - // auxiliary call sharing the Session's id included — so they are excluded - // even at the same Session, and another Session's rows never count. The - // run's own usage-missing dispatch still shows up. - await withProjectedAttempts( + // Rows the usage-unknown seam recorded — a failed auxiliary call sharing + // the Session's id included — belong to no run and are excluded even at + // the same Session, and another Session's rows never count. The run's own + // usage-missing dispatch still shows up. + await withRecordedAttempts( [ - attempt({ - attemptId: 'auxiliary-failure', - turnId: 'auxiliary', - status: 'failed', - usageBasis: 'missing', - inputTokens: undefined, - outputTokens: undefined, - costBasis: 'unpriced', - costUsd: undefined, - }), attempt({ attemptId: 'other-session-dispatch', sessionId: 'session-2', @@ -186,6 +215,7 @@ describe('Usage answers over the canonical ledger', () => { costUsd: undefined, }), ], + [usageUnknownAttempt()], async (ledger) => { assert.deepEqual(ledger.runSettlementCoverage('session-1'), { attempts: 3, @@ -196,6 +226,55 @@ describe('Usage answers over the canonical ledger', () => { ); }); + test('run settlement coverage holds a hosted execution named like the sentinel to its own rows', async () => { + // The execution id is a client-chosen entity id and the runner reuses it as + // the root Turn id, so a hosted execution named `auxiliary` owns rows under + // `turn_id = 'auxiliary'` — on (session_id, turn_id) indistinguishable from + // no-run sentinel rows (#5890 review). Settlement must read ownership from + // how a row was recorded, not from the turn value, or this execution's + // missing-usage dispatch disappears from its own settlement and + // `incompleteUsageReason` releases the environment with unknown usage. + await withRecordedAttempts( + [ + attempt({ + attemptId: 'run-reported', + sessionId: 'auxiliary', + runId: 'run-auxiliary', + turnId: 'auxiliary', + logicalCallId: 'call-reported', + }), + attempt({ + attemptId: 'run-missing-dispatch', + sessionId: 'auxiliary', + runId: 'run-auxiliary', + turnId: 'auxiliary', + logicalCallId: 'call-missing', + status: 'failed', + usageBasis: 'missing', + inputTokens: undefined, + outputTokens: undefined, + costBasis: 'unpriced', + costUsd: undefined, + }), + ], + [usageUnknownAttempt({ sessionId: 'auxiliary' })], + async (ledger) => { + // The run-owned missing dispatch is this execution's unsettled + // obligation; the failed auxiliary call is nobody's. Public coverage + // keeps counting both. + assert.deepEqual(ledger.runSettlementCoverage('auxiliary'), { + attempts: 2, + usageMissingAttempts: 1, + usagePartialAttempts: 0, + }); + assert.equal( + ledger.summary({ range: 'all' }, NOW).projection.coverage.usageMissingAttempts, + 2, + ); + }, + ); + }); + test('one malformed cache reading cannot inflate the cache total', async () => { await withProjectedAttempts( [ diff --git a/packages/storage/src/model-call-ledger.ts b/packages/storage/src/model-call-ledger.ts index 3d986532eea..cabfefa425c 100644 --- a/packages/storage/src/model-call-ledger.ts +++ b/packages/storage/src/model-call-ledger.ts @@ -125,9 +125,9 @@ export interface ModelCallLedgerReader { /** * The rows one Session's own run left unsettled (#5691): its attempts whose * usage the provider never reported or only partly reported. Rows recorded - * outside any run — the no-run sentinel turn — belong to no run's - * settlement and are excluded, the reverse of the ledger-wide coverage - * above, which must see them. + * outside any run — by the usage-unknown seam, which the ledger marks + * `no_run` — belong to no run's settlement and are excluded, the reverse of + * the ledger-wide coverage above, which must see them. */ runSettlementCoverage(sessionId: string): RunSettlementCoverage; } @@ -137,7 +137,7 @@ export interface ModelCallLedgerReader { * coverage: whether the run's own attempts all settled (#5691). */ export interface RunSettlementCoverage { - /** Countable rows the run owns — Session rows outside the sentinel turn. */ + /** Countable rows the run owns — its Session's rows outside the `no_run` mark. */ readonly attempts: number; readonly usageMissingAttempts: number; readonly usagePartialAttempts: number; @@ -178,7 +178,9 @@ export interface UsageUnknownModelCallRecord { /** * Calls outside any run's turn carry the shared no-run sentinel * (`NO_RUN_TURN_ID`) — the table requires the column for every countable - * row, and settlement coverage excludes rows recorded under it. + * row. The value is a placeholder, not the discriminator: the ledger + * records the row's no-run ownership from this seam itself, so an + * execution legally named like the sentinel keeps its own rows. */ readonly turnId: string; readonly callKind: ModelCallKind; @@ -394,9 +396,9 @@ class SqliteModelCallLedger implements ModelCallLedger { .prepare( `SELECT ${RUN_SETTLEMENT_COVERAGE_SUMS} FROM usage_model_call_attempts - WHERE cost_basis IS NOT NULL AND session_id = ? AND turn_id IS NOT ?`, + WHERE cost_basis IS NOT NULL AND session_id = ? AND no_run = 0`, ) - .get(sessionId, NO_RUN_TURN_ID) as Record | undefined; + .get(sessionId) as Record | undefined; return { attempts: count(row?.attempts), usageMissingAttempts: count(row?.usageMissingAttempts), @@ -547,6 +549,9 @@ function bindModelCallAttempt(attempt: ModelCallAttempt): (string | number | nul reasoning_tokens: attempt.reasoningTokens ?? null, cost_basis: attempt.costBasis, cost_usd: attempt.costUsd ?? null, + // The projection's only source is the AgentRun authority: every row it + // writes is run-owned, whatever turn the run ran under. + no_run: 0, }; return MODEL_CALL_COLUMNS.map((column) => values[column]); } @@ -584,6 +589,9 @@ function bindUsageUnknownModelCallAttempt( reasoning_tokens: null, cost_basis: 'unpriced', cost_usd: null, + // The one write outside any run's event stream: the ledger marks the row + // no-run here rather than inferring ownership from the turn value. + no_run: 1, }; return MODEL_CALL_COLUMNS.map((column) => values[column]); } diff --git a/packages/storage/src/model-call-usage-sql.ts b/packages/storage/src/model-call-usage-sql.ts index 674cc3f2f20..15606a15599 100644 --- a/packages/storage/src/model-call-usage-sql.ts +++ b/packages/storage/src/model-call-usage-sql.ts @@ -46,10 +46,12 @@ const ERROR_ROW = `status = 'failed'`; /** * The turn placeholder on rows recorded outside any AgentRun — failed and * aborted auxiliary Host model calls (#5691). Such a row is honest accounting - * (real spend shape, unknown usage), so the ledger-wide coverage counts it; - * what it is not is a run's unsettled obligation, and the run-scoped - * settlement coverage excludes it on that marker. Shared with the writer so - * the sentinel cannot drift between the two. + * (real spend shape, unknown usage), so the ledger-wide coverage counts it. + * The value is a placeholder for the table's required turn column, not the + * settlement discriminator: rows carry a `no_run` column written by whichever + * seam recorded them, because a hosted execution legally named `auxiliary` + * owns rows under this same turn value (#5890 review). Shared with the writer + * so the placeholder cannot drift. */ export const NO_RUN_TURN_ID = 'auxiliary'; @@ -72,8 +74,9 @@ export const COVERAGE_SUMS = ` /** * Coverage a hosted execution's settlement may hold a run to (#5691): the - * rows the run owns — its Session's, outside the no-run sentinel turn. The - * sentinel exclusion is the query's WHERE clause, not an aggregate here, so + * rows the run owns — the ones its Session's event-stream projection + * recorded, outside the `no_run` mark the usage-unknown seam writes. The + * ownership exclusion is the query's WHERE clause, not an aggregate here, so * this stays narrower than the ledger-wide {@link COVERAGE_SUMS} on purpose: * one answers "what does the ledger hold unknown-usage rows for", the other * "what did this run leave unsettled". diff --git a/packages/storage/src/sqlite-usage-schema.ts b/packages/storage/src/sqlite-usage-schema.ts index cf796d75c7b..e249e049335 100644 --- a/packages/storage/src/sqlite-usage-schema.ts +++ b/packages/storage/src/sqlite-usage-schema.ts @@ -19,7 +19,9 @@ import type { DatabaseSync } from 'node:sqlite'; -export const SQLITE_USAGE_SCHEMA_VERSION = 9; +import { NO_RUN_TURN_ID } from './model-call-usage-sql.js'; + +export const SQLITE_USAGE_SCHEMA_VERSION = 10; /** * The canonical ledger's columns, in the order every statement binds them. @@ -49,6 +51,7 @@ export const MODEL_CALL_COLUMNS = [ 'reasoning_tokens', 'cost_basis', 'cost_usd', + 'no_run', ] as const; /** @@ -140,6 +143,11 @@ const MODEL_CALL_TABLE = ` reasoning_tokens INTEGER, cost_basis TEXT, cost_usd REAL, + -- Which writer recorded the row: the run's event-stream projection (0) or + -- the usage-unknown seam for calls no AgentRun owns (1). Settlement reads + -- this, not the turn value — a hosted execution legally named like the + -- no-run sentinel owns rows under the sentinel turn too (#5890 review). + no_run INTEGER NOT NULL DEFAULT 0, CHECK (${MODEL_CALL_REQUIRED_COLUMNS.map( (column) => `(${column} IS NULL) = (cost_basis IS NULL)`, ).join(' AND ')}), @@ -221,6 +229,14 @@ export function migrateSqliteUsageDatabase(db: DatabaseSync): void { // through, and the conversion below reads one. ensureColumn(db, 'usage_model_call_attempts', 'session_id', 'TEXT'); spreadModelCallRecordJson(db); + // Rows recorded before the ownership column existed were already + // discriminated by the no-run sentinel turn; carry that judgment over once + // so a database from an earlier build keeps its no-run rows excluded when + // the column takes over the discrimination. + ensureColumn(db, 'usage_model_call_attempts', 'no_run', 'INTEGER NOT NULL DEFAULT 0'); + db.prepare( + `UPDATE usage_model_call_attempts SET no_run = 1 WHERE turn_id = ? AND no_run = 0`, + ).run(NO_RUN_TURN_ID); db.exec(` CREATE INDEX IF NOT EXISTS usage_model_call_attempts_completed_at ON usage_model_call_attempts(completed_at DESC, attempt_id); @@ -364,7 +380,10 @@ function spreadModelCallRecordJson(db: DatabaseSync): void { attempt_id, completed_at, session_id, - ${pricing} + ${pricing}, + -- Pre-column rows all predate the usage-unknown seam: every one of them + -- was recorded by the run's event-stream projection. + 0 AS no_run FROM classified; DROP TABLE usage_model_call_attempts_blob; From c05ad5cc9c47bbb45acebaf4da1140709f0a3b12 Mon Sep 17 00:00:00 2001 From: ggbdpq Date: Thu, 1 Oct 2026 18:50:11 +0800 Subject: [PATCH 08/10] fix(usage): drop the no_run backfill that reused the collision-prone turn predicate The schema-10 migration backfilled `no_run = 1` from `turn_id = 'auxiliary'` - the same collision the column exists to remove. A hosted execution legally named `auxiliary` projects its run-owned missing-usage rows with that turn value, and the operational migration re-runs every scope's migrator, so a close/reopen (or an unrelated scope's migration) flipped those rows to `no_run = 1`: settlement then reported zero attempts while public coverage still reported the missing attempt, releasing the environment despite unknown usage. The backfill served databases from this PR's own intermediate builds - recording auxiliary attempts is what introduced the rows, so no released database can hold them. Drop the UPDATE and keep the column recorder-owned; the settlement-coverage regression in model-call-usage-query.test.ts covers the legal-name case. Refs #5691 Generated-by: GLM-5.3-Flash (ZCode) --- packages/storage/src/sqlite-usage-schema.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/storage/src/sqlite-usage-schema.ts b/packages/storage/src/sqlite-usage-schema.ts index e249e049335..19fe2036c00 100644 --- a/packages/storage/src/sqlite-usage-schema.ts +++ b/packages/storage/src/sqlite-usage-schema.ts @@ -19,7 +19,6 @@ import type { DatabaseSync } from 'node:sqlite'; -import { NO_RUN_TURN_ID } from './model-call-usage-sql.js'; export const SQLITE_USAGE_SCHEMA_VERSION = 10; @@ -229,14 +228,12 @@ export function migrateSqliteUsageDatabase(db: DatabaseSync): void { // through, and the conversion below reads one. ensureColumn(db, 'usage_model_call_attempts', 'session_id', 'TEXT'); spreadModelCallRecordJson(db); - // Rows recorded before the ownership column existed were already - // discriminated by the no-run sentinel turn; carry that judgment over once - // so a database from an earlier build keeps its no-run rows excluded when - // the column takes over the discrimination. + // no_run is written by the recorder, never inferred from the turn value: + // a backfill keyed on the auxiliary turn string would flip a legally named + // execution's own run-owned rows (see the settlement-coverage regression in + // model-call-usage-query.test.ts). Databases from before this column cannot + // hold auxiliary rows - recording them is what introduced the column. ensureColumn(db, 'usage_model_call_attempts', 'no_run', 'INTEGER NOT NULL DEFAULT 0'); - db.prepare( - `UPDATE usage_model_call_attempts SET no_run = 1 WHERE turn_id = ? AND no_run = 0`, - ).run(NO_RUN_TURN_ID); db.exec(` CREATE INDEX IF NOT EXISTS usage_model_call_attempts_completed_at ON usage_model_call_attempts(completed_at DESC, attempt_id); From 6929b9624b74cb1c073caa459f4e93df2e6471a8 Mon Sep 17 00:00:00 2001 From: ggbdpq Date: Thu, 1 Oct 2026 18:56:16 +0800 Subject: [PATCH 09/10] chore(usage): drop the stray blank line left by the backfill removal Generated-by: GLM-5.3-Flash (ZCode) --- packages/storage/src/sqlite-usage-schema.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/storage/src/sqlite-usage-schema.ts b/packages/storage/src/sqlite-usage-schema.ts index 19fe2036c00..c25b37bb226 100644 --- a/packages/storage/src/sqlite-usage-schema.ts +++ b/packages/storage/src/sqlite-usage-schema.ts @@ -19,7 +19,6 @@ import type { DatabaseSync } from 'node:sqlite'; - export const SQLITE_USAGE_SCHEMA_VERSION = 10; /** From 5993466f44940c9e2689653428eeef10073c287d Mon Sep 17 00:00:00 2001 From: ggbdpq Date: Fri, 9 Oct 2026 01:21:25 +0800 Subject: [PATCH 10/10] fix(usage): scope run settlement coverage to the settlement window The completeness check guarded window-wide totals with run-owned rows: #readUsage reads a usage.query summary over the whole window with no session predicate, while runSettlementCoverage filtered session_id = executionId AND no_run = 0. A hosted execution that delegates to a subagent therefore settled as complete when only a child Session's model call was missing or partial usage - undercounted tokens and, for partial rows, a non-null undercounted costUsd, where the old scope returned indeterminate: missing_attempt_usage. Scope the coverage to the same window #readUsage reads: pass startedAt and the settled-at instant through instead of the execution id, keeping no_run = 0 so usage-unknown sentinel rows stay excluded. The window comparison mirrors countableFilter exactly (completed_at >= from AND completed_at <= to). Drop the computed-but-unread attempts field, pin the schema 9 -> 10 upgrade path (column restored in place with its recorder-owned default, idempotent on rerun), and note in the PR that a schema-10 database opened by an older build is refused. Addresses the P2 and P3s from liugddx's review. Generated-by: GLM-5.3-Flash (ZCode) --- .../__tests__/hosted-execution-runner.test.ts | 9 +- .../src/server/execution-composition.ts | 2 +- .../src/server/hosted-execution-runner.ts | 18 ++-- .../src/server/usage-pricing-coordinator.ts | 13 +-- .../__tests__/model-call-usage-query.test.ts | 82 ++++++++++++++----- .../src/__tests__/sqlite-usage-schema.test.ts | 44 ++++++++++ .../src/__tests__/usage-stores.test.ts | 8 +- packages/storage/src/model-call-ledger.ts | 27 +++--- packages/storage/src/model-call-usage-sql.ts | 1 - packages/storage/src/usage-stores.ts | 12 +-- 10 files changed, 149 insertions(+), 67 deletions(-) diff --git a/packages/runtime-host/src/__tests__/hosted-execution-runner.test.ts b/packages/runtime-host/src/__tests__/hosted-execution-runner.test.ts index b2d78fa8c67..40624dc78c1 100644 --- a/packages/runtime-host/src/__tests__/hosted-execution-runner.test.ts +++ b/packages/runtime-host/src/__tests__/hosted-execution-runner.test.ts @@ -379,14 +379,9 @@ function usageSummary() { * usage-unknown rows unless the test says otherwise. */ function settlementCoverage( - overrides: { - attempts?: number; - usageMissingAttempts?: number; - usagePartialAttempts?: number; - } = {}, + overrides: { usageMissingAttempts?: number; usagePartialAttempts?: number } = {}, ) { - return async () => ({ - attempts: 1, + return async (_from: number, _to: number) => ({ usageMissingAttempts: 0, usagePartialAttempts: 0, ...overrides, diff --git a/packages/runtime-host/src/server/execution-composition.ts b/packages/runtime-host/src/server/execution-composition.ts index a12868505ee..15bc1215424 100644 --- a/packages/runtime-host/src/server/execution-composition.ts +++ b/packages/runtime-host/src/server/execution-composition.ts @@ -2818,7 +2818,7 @@ export async function createExecutionRuntimeHostComposition( 'turn.stop': turnControl.handlers['turn.stop'], 'usage.query': usagePricing.handlers['usage.query'], }, - runSettlementCoverage: (sessionId) => usagePricing.runSettlementCoverage(sessionId), + runSettlementCoverage: (from, to) => usagePricing.runSettlementCoverage(from, to), context: { hostEpoch: context.hostEpoch, connectionId: 'hosted-execution', diff --git a/packages/runtime-host/src/server/hosted-execution-runner.ts b/packages/runtime-host/src/server/hosted-execution-runner.ts index 9ff6383971c..243a700e4b9 100644 --- a/packages/runtime-host/src/server/hosted-execution-runner.ts +++ b/packages/runtime-host/src/server/hosted-execution-runner.ts @@ -35,13 +35,13 @@ export interface HostHostedExecutionRunnerInput { 'session.create' | 'turn.start' | 'turn.query' | 'turn.stop' | 'usage.query' >; /** - * What this execution's own run left unsettled (#5691). Settlement's - * incompleteness check reads this run-scoped answer instead of the + * What the settlement window left unsettled (#5890). Settlement's + * incompleteness check reads this window-scoped answer instead of the * `usage.query` coverage: that field is the ledger's public provenance and * must keep counting usage-unknown rows no run owns — a failed auxiliary * Host call is real accounting, not this run's unsettled obligation. */ - readonly runSettlementCoverage: (sessionId: string) => Promise; + readonly runSettlementCoverage: (from: number, to: number) => Promise; readonly context: ConnectionContext; readonly requestDrain: () => void; readonly waitForExecutionResidencies: () => Promise; @@ -91,8 +91,9 @@ export class HostHostedExecutionRunner { await (signal.aborted ? this.input.waitForAllResidencies() : this.input.waitForExecutionResidencies()); - const usage = await this.#readUsage(startedAt, (this.input.now ?? Date.now)()); - const runCoverage = await this.input.runSettlementCoverage(input.executionId); + const settledAt = (this.input.now ?? Date.now)(); + const usage = await this.#readUsage(startedAt, settledAt); + const runCoverage = await this.input.runSettlementCoverage(startedAt, settledAt); const incompleteUsage = incompleteUsageReason(usage, runCoverage); if (incompleteUsage) { return indeterminate( @@ -173,9 +174,10 @@ function incompleteUsageReason( runCoverage: RunSettlementCoverage, ): string | undefined { // Unreadable and pending-repair rows stay ledger-wide: damage in the window - // is nobody's settled fact. Partial and missing usage, the things a run can - // leave unsettled, are judged against the run's own rows only (#5691) — the - // shared coverage field still reports them for every client to see. + // is nobody's settled fact. Partial and missing usage are judged against the + // same window as the totals this check guards (#5890), so a delegated child + // Session cannot settle underneath them — the `no_run` sentinel rows stay + // excluded, and the shared coverage field still reports everything. const { unreadableRecords, pendingRepairs } = result.provenance; if (unreadableRecords > 0) return 'unreadable_usage_record'; if (pendingRepairs > 0) return 'pending_usage_repair'; diff --git a/packages/runtime-host/src/server/usage-pricing-coordinator.ts b/packages/runtime-host/src/server/usage-pricing-coordinator.ts index fa532c08368..b5dcebcafe2 100644 --- a/packages/runtime-host/src/server/usage-pricing-coordinator.ts +++ b/packages/runtime-host/src/server/usage-pricing-coordinator.ts @@ -82,13 +82,14 @@ export class HostUsagePricingCoordinator { }; /** - * What one Session's own run left unsettled (#5691) — hosted execution - * settlement's incompleteness check. Deliberately off the operation handler - * map: the ledger-wide `usage.query` coverage must keep counting rows no run - * owns, so settlement scopes itself through this in-process seam instead. + * What the settlement window itself left unsettled (#5890) — hosted + * execution settlement's incompleteness check reads the same window as the + * totals it guards, minus the `no_run` sentinel rows. Deliberately off the + * operation handler map: the ledger-wide `usage.query` coverage keeps a + * different shape, so settlement scopes itself through this in-process seam. */ - async runSettlementCoverage(sessionId: string): Promise { - return this.#stores.modelCalls.modelCallRunSettlementCoverage(sessionId); + async runSettlementCoverage(from: number, to: number): Promise { + return this.#stores.modelCalls.modelCallRunSettlementCoverage(from, to); } readonly #stores: InteractiveUsageStoresWriter; diff --git a/packages/storage/src/__tests__/model-call-usage-query.test.ts b/packages/storage/src/__tests__/model-call-usage-query.test.ts index 76104c22acd..1afb45fa492 100644 --- a/packages/storage/src/__tests__/model-call-usage-query.test.ts +++ b/packages/storage/src/__tests__/model-call-usage-query.test.ts @@ -176,12 +176,12 @@ describe('Usage answers over the canonical ledger', () => { ); }); - test('run settlement coverage holds a run to its own rows, not auxiliary ones', async () => { - // Hosted execution settlement asks what its own run left unsettled (#5691). - // Rows the usage-unknown seam recorded — a failed auxiliary call sharing - // the Session's id included — belong to no run and are excluded even at - // the same Session, and another Session's rows never count. The run's own - // usage-missing dispatch still shows up. + test('run settlement coverage guards every session in the window, not the auxiliary sentinel', async () => { + // Settlement guards the window-wide totals, so it must count every + // Session's unsettled dispatch in that window (#5890): the pre-window + // session filter here would have let a delegated child's missing usage + // settle as complete. The `no_run` sentinel — the failed auxiliary call + // the usage-unknown seam recorded — belongs to no run and stays out. await withRecordedAttempts( [ attempt({ @@ -217,23 +217,24 @@ describe('Usage answers over the canonical ledger', () => { ], [usageUnknownAttempt()], async (ledger) => { - assert.deepEqual(ledger.runSettlementCoverage('session-1'), { - attempts: 3, - usageMissingAttempts: 1, + // The run's own missing dispatch and the other session's (a delegated + // child's) both count; the `no_run` sentinel does not. + assert.deepEqual(ledger.runSettlementCoverage(NOW - 1_000, NOW), { + usageMissingAttempts: 2, usagePartialAttempts: 1, }); }, ); }); - test('run settlement coverage holds a hosted execution named like the sentinel to its own rows', async () => { - // The execution id is a client-chosen entity id and the runner reuses it as - // the root Turn id, so a hosted execution named `auxiliary` owns rows under - // `turn_id = 'auxiliary'` — on (session_id, turn_id) indistinguishable from - // no-run sentinel rows (#5890 review). Settlement must read ownership from - // how a row was recorded, not from the turn value, or this execution's - // missing-usage dispatch disappears from its own settlement and - // `incompleteUsageReason` releases the environment with unknown usage. + test('run settlement coverage excludes the no-run sentinel, not an execution named like it', async () => { + // The exclusion must key on how a row was recorded (`no_run`), never on + // the turn value: a hosted execution whose id is the client-chosen + // `auxiliary` owns rows under `turn_id = 'auxiliary'`, and with window + // scoping (#5890) those must keep counting while the genuine sentinel row + // next to them stays out — or this execution's missing-usage dispatch + // disappears and `incompleteUsageReason` releases the environment with + // unknown usage. await withRecordedAttempts( [ attempt({ @@ -260,10 +261,10 @@ describe('Usage answers over the canonical ledger', () => { [usageUnknownAttempt({ sessionId: 'auxiliary' })], async (ledger) => { // The run-owned missing dispatch is this execution's unsettled - // obligation; the failed auxiliary call is nobody's. Public coverage - // keeps counting both. - assert.deepEqual(ledger.runSettlementCoverage('auxiliary'), { - attempts: 2, + // obligation and counts inside the window; the failed auxiliary call + // is nobody's (`no_run`) and stays excluded. Public coverage keeps + // counting both. + assert.deepEqual(ledger.runSettlementCoverage(NOW - 1_000, NOW), { usageMissingAttempts: 1, usagePartialAttempts: 0, }); @@ -275,6 +276,45 @@ describe('Usage answers over the canonical ledger', () => { ); }); + test('run settlement coverage reads the window, not one session (#5890 review)', async () => { + // Settlement guards the window-wide totals that #readUsage reads, so its + // coverage must read the same window: a child Session's missing dispatch + // inside the window is exactly the undercount that would otherwise settle + // as complete, and a window-external row is nobody's obligation here. + await withRecordedAttempts( + [ + attempt({ + attemptId: 'outside-window-missing', + completedAt: NOW - 5_000, + status: 'failed', + usageBasis: 'missing', + inputTokens: undefined, + outputTokens: undefined, + costBasis: 'unpriced', + costUsd: undefined, + }), + attempt({ attemptId: 'root-ok' }), + attempt({ + attemptId: 'child-missing', + sessionId: 'session-1-child', + status: 'failed', + usageBasis: 'missing', + inputTokens: undefined, + outputTokens: undefined, + costBasis: 'unpriced', + costUsd: undefined, + }), + ], + [], + async (ledger) => { + assert.deepEqual(ledger.runSettlementCoverage(NOW - 1_000, NOW), { + usageMissingAttempts: 1, + usagePartialAttempts: 0, + }); + }, + ); + }); + test('one malformed cache reading cannot inflate the cache total', async () => { await withProjectedAttempts( [ diff --git a/packages/storage/src/__tests__/sqlite-usage-schema.test.ts b/packages/storage/src/__tests__/sqlite-usage-schema.test.ts index f84590b3b5b..2ac5213bbe8 100644 --- a/packages/storage/src/__tests__/sqlite-usage-schema.test.ts +++ b/packages/storage/src/__tests__/sqlite-usage-schema.test.ts @@ -21,6 +21,15 @@ import assert from 'node:assert/strict'; import { DatabaseSync } from 'node:sqlite'; import { test } from 'node:test'; import { migrateSqliteUsageDatabase } from '../sqlite-usage-schema.js'; +import { + assertCurrentOperationalTargetSchema, + ensureOperationalSchemaRegistry, +} from '../operational-target-schema.js'; +import { migrateSqliteArtifactDatabase } from '../sqlite-artifact-schema.js'; +import { migrateSqliteCoreExecutionDatabase } from '../sqlite-core-execution-schema.js'; +import { migrateSqliteRuntimeDatabase } from '../sqlite-runtime-schema.js'; +import { migrateSqliteSessionMetadataDatabase } from '../sqlite-session-metadata-schema.js'; +import { migrateSqliteWorkflowDatabase } from '../sqlite-workflow-schema.js'; import { MODEL_CALL_NOW as NOW, modelCallAttempt as attempt, @@ -331,3 +340,38 @@ test('Usage title revision ignores unrelated metadata and covers every activity database.close(); } }); + +test('a schema-9 usage database upgrades to the target shape and stays idempotent', () => { + const database = new DatabaseSync(':memory:'); + try { + // Build the full operational target, then regress usage to its schema-9 + // shape. The #5890 target adds `no_run` as the usage delta, so a real + // schema-9 database is exactly this table minus that column. + migrateSqliteRuntimeDatabase(database); + migrateSqliteSessionMetadataDatabase(database); + migrateSqliteCoreExecutionDatabase(database); + migrateSqliteWorkflowDatabase(database); + migrateSqliteUsageDatabase(database); + migrateSqliteArtifactDatabase(database); + ensureOperationalSchemaRegistry(database); + database.exec('ALTER TABLE usage_model_call_attempts DROP COLUMN no_run'); + + migrateSqliteUsageDatabase(database); + + // The upgrade restores the recorder-owned default in place — the DDL + // signature matches a fresh create only because ADD COLUMN appends the + // column the same way — and a rerun changes nothing. + assertCurrentOperationalTargetSchema(database); + const restored = database + .prepare( + "SELECT \"notnull\" AS not_null, dflt_value FROM pragma_table_info('usage_model_call_attempts') WHERE name = 'no_run'", + ) + .get() as { not_null: number; dflt_value: string }; + assert.equal(restored.not_null, 1); + assert.equal(restored.dflt_value, '0'); + migrateSqliteUsageDatabase(database); + assertCurrentOperationalTargetSchema(database); + } finally { + database.close(); + } +}); diff --git a/packages/storage/src/__tests__/usage-stores.test.ts b/packages/storage/src/__tests__/usage-stores.test.ts index 742fb2257ca..50de4e4addf 100644 --- a/packages/storage/src/__tests__/usage-stores.test.ts +++ b/packages/storage/src/__tests__/usage-stores.test.ts @@ -226,10 +226,10 @@ describe('InteractiveUsageStores', () => { assert.equal(summary.projection.coverage.usageMissingAttempts, 1); assert.equal(summary.projection.coverage.usageReportedAttempts, 0); assert.equal(summary.unreadableRecords, 0); - // The run-scoped settlement coverage excludes the no-run sentinel row, - // so an auxiliary failure cannot flip a hosted run to indeterminate. - assert.deepEqual(await stores.modelCalls.modelCallRunSettlementCoverage('session-aux'), { - attempts: 0, + // The settlement coverage excludes the no-run sentinel row — its + // usage_basis never enters any run's window scope — so an auxiliary + // failure cannot flip a hosted run to indeterminate. + assert.deepEqual(await stores.modelCalls.modelCallRunSettlementCoverage(0, Date.now()), { usageMissingAttempts: 0, usagePartialAttempts: 0, }); diff --git a/packages/storage/src/model-call-ledger.ts b/packages/storage/src/model-call-ledger.ts index cabfefa425c..69ee75d124f 100644 --- a/packages/storage/src/model-call-ledger.ts +++ b/packages/storage/src/model-call-ledger.ts @@ -123,22 +123,23 @@ export interface ModelCallLedgerReader { limit: number, ): ModelCallLedgerResult; /** - * The rows one Session's own run left unsettled (#5691): its attempts whose - * usage the provider never reported or only partly reported. Rows recorded - * outside any run — by the usage-unknown seam, which the ledger marks - * `no_run` — belong to no run's settlement and are excluded, the reverse of - * the ledger-wide coverage above, which must see them. + * What the settlement window itself left unsettled (#5890): the attempts + * between `from` and `to` inclusive whose usage the provider never reported + * or only partly reported, across every Session — the same scope as the + * window-wide totals this check guards. Rows recorded outside any run — by + * the usage-unknown seam, which the ledger marks `no_run` — belong to no + * run's settlement and are excluded, the reverse of the ledger-wide + * coverage above, which must see them. */ - runSettlementCoverage(sessionId: string): RunSettlementCoverage; + runSettlementCoverage(from: number, to: number): RunSettlementCoverage; } /** * What a hosted execution's settlement checks instead of the ledger-wide - * coverage: whether the run's own attempts all settled (#5691). + * coverage: whether everything in the window settled (#5890, widening #5691's + * run-owned scope so delegated Sessions cannot settle underneath it). */ export interface RunSettlementCoverage { - /** Countable rows the run owns — its Session's rows outside the `no_run` mark. */ - readonly attempts: number; readonly usageMissingAttempts: number; readonly usagePartialAttempts: number; } @@ -390,17 +391,17 @@ class SqliteModelCallLedger implements ModelCallLedger { }; } - runSettlementCoverage(sessionId: string): RunSettlementCoverage { + runSettlementCoverage(from: number, to: number): RunSettlementCoverage { const db = this.#open(); const row = db .prepare( `SELECT ${RUN_SETTLEMENT_COVERAGE_SUMS} FROM usage_model_call_attempts - WHERE cost_basis IS NOT NULL AND session_id = ? AND no_run = 0`, + WHERE cost_basis IS NOT NULL AND no_run = 0 + AND completed_at >= ? AND completed_at <= ?`, ) - .get(sessionId) as Record | undefined; + .get(from, to) as Record | undefined; return { - attempts: count(row?.attempts), usageMissingAttempts: count(row?.usageMissingAttempts), usagePartialAttempts: count(row?.usagePartialAttempts), }; diff --git a/packages/storage/src/model-call-usage-sql.ts b/packages/storage/src/model-call-usage-sql.ts index 15606a15599..3f3f547366d 100644 --- a/packages/storage/src/model-call-usage-sql.ts +++ b/packages/storage/src/model-call-usage-sql.ts @@ -82,7 +82,6 @@ export const COVERAGE_SUMS = ` * "what did this run leave unsettled". */ export const RUN_SETTLEMENT_COVERAGE_SUMS = ` - COUNT(*) AS attempts, SUM(usage_basis = 'partial') AS usagePartialAttempts, SUM(usage_basis = 'missing') AS usageMissingAttempts`; diff --git a/packages/storage/src/usage-stores.ts b/packages/storage/src/usage-stores.ts index 57779d9f161..f0bf1b747bf 100644 --- a/packages/storage/src/usage-stores.ts +++ b/packages/storage/src/usage-stores.ts @@ -130,11 +130,11 @@ export interface ModelCallIndexReader { limit: number, ): Promise>; /** - * The rows one Session's own run left unsettled (#5691) — what hosted + * What the settlement window itself left unsettled (#5890) — what hosted * execution settlement checks instead of the ledger-wide coverage, which * must also count rows no run owns. */ - modelCallRunSettlementCoverage(sessionId: string): Promise; + modelCallRunSettlementCoverage(from: number, to: number): Promise; } export interface ModelCallIndexWriter extends ModelCallIndexReader { @@ -540,8 +540,8 @@ function createWriterFacade( read(() => modelCalls.buckets(query, groupBy, now)), modelCallLogs: (query, now, offset, limit) => read(() => modelCalls.logs(query, now, offset, limit)), - modelCallRunSettlementCoverage: (sessionId) => - read(() => modelCalls.runSettlementCoverage(sessionId)), + modelCallRunSettlementCoverage: (from, to) => + read(() => modelCalls.runSettlementCoverage(from, to)), catchUpModelCallProjection: admitModelCallProjectionCatchUp, recordUsageUnknownAttempt: (record) => admitSessionUsageMutation(record.sessionId, () => @@ -598,8 +598,8 @@ function modelCallReader( run(() => ledger.buckets(query, groupBy, now)), modelCallLogs: (query: UsageQuery, now: number, offset: number, limit: number) => run(() => ledger.logs(query, now, offset, limit)), - modelCallRunSettlementCoverage: (sessionId: string) => - run(() => ledger.runSettlementCoverage(sessionId)), + modelCallRunSettlementCoverage: (from: number, to: number) => + run(() => ledger.runSettlementCoverage(from, to)), }); }