diff --git a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts index a38e4cee2af..89a7f06d47f 100644 --- a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts +++ b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts @@ -4441,13 +4441,37 @@ test('Host auxiliary models meter provider usage and abort physical requests', { }); await closing; assert.equal(transportCloses, 1); - const abortedLogs = await usage.telemetry.logs({ range: 'all' }); + // An aborted auxiliary call knows no token counts: it records in the + // canonical ledger as a usage-unknown row (#5691), never as a zero-token + // legacy row that reads as a free call. + const abortedAttempts = await usage.modelCalls.modelCallLogs( + { range: 'all', sessionId: session.id }, + Date.now(), + 0, + 10, + ); assert.ok( - abortedLogs.rows.some( + abortedAttempts.projection.rows.some( (row) => row.callId === `goal_evaluation_${session.id}_call-2` && row.status === 'aborted', ), ); + const abortedSummary = await usage.modelCalls.modelCallSummary( + { range: 'all', sessionId: session.id }, + Date.now(), + ); + // Every failed auxiliary call above — the 401 recap, the timed-out + // recap, and the aborted goal evaluation — is recorded under the + // no-run sentinel turn, and the ledger-wide coverage still counts all + // three: real unknown-usage calls stay visible in the public + // provenance. Keeping a hosted run to its own rows is settlement's + // run-scoped check, not this field's job. + assert.equal(abortedSummary.projection.coverage.usageMissingAttempts, 3); + const abortedLegacyLogs = await usage.telemetry.logs({ range: 'all' }); + assert.equal( + abortedLegacyLogs.rows.some((row) => row.callId === `goal_evaluation_${session.id}_call-2`), + false, + ); } finally { abort.abort(new DOMException('Goal evaluator test cleanup', 'AbortError')); providerRelease.resolve(); @@ -4463,6 +4487,198 @@ test('Host auxiliary models meter provider usage and abort physical requests', { } }); +test('Host auxiliary aborts, errors and usage-unknown completions record canonical usage-unknown rows', { + timeout: 20_000, +}, async () => { + const base = await mkdtemp(join(tmpdir(), 'maka-host-aux-usage-unknown-')); + const capability = await resolveStorageRoot({ + path: join(base, 'interactive'), + kind: 'interactive', + }); + const owner = await tryAcquireInteractiveRootOwner(capability); + assert.ok(owner); + if (!owner) return; + + const policy = await openInteractiveRuntimePolicyStoresForWrite(owner.lease); + const usage = await openInteractiveUsageStoresForWrite(owner.lease); + const execution = await openInteractiveExecutionStoresForWrite(owner.lease); + try { + const created = await policy.connectionCatalog.create({ + expectedCatalogRevision: 0, + connection: { + slug: 'usage-unknown-provider', + name: 'Usage unknown provider', + providerType: 'opencode-go', + baseUrl: 'http://usage-unknown.test/v1', + enabled: true, + enabledModelIds: [MODEL_ID], + }, + }); + assert.equal(created.kind, 'committed'); + if (created.kind !== 'committed') return; + const connection = created.snapshot.connections[0]; + assert.ok(connection); + if (!connection) return; + const credential = await policy.credentialVault.set({ + locator: { + scope: 'connection', + connectionId: connection.connectionId, + kind: 'api_key', + }, + expected: null, + secret: API_KEY, + }); + assert.equal(credential.kind, 'committed'); + await publishConnectionModel(policy, connection.connectionId, MODEL_ID); + const session = await execution.sessionStore.create({ + cwd: capability.canonicalPath, + llmConnectionId: connection.connectionId, + llmConnectionSlug: 'usage-unknown-provider', + model: MODEL_ID, + permissionMode: 'ask', + }); + let callCounter = 0; + const evaluator = ( + fetchImpl: (request: unknown, init?: { signal?: AbortSignal }) => Promise, + ) => + createHostGoalEvaluator({ + runtimePolicy: policy, + oauthCredentials: new HostOAuthExecutionAuthority(policy), + usage, + requestDrain: () => assert.fail('Auxiliary accounting must not drain the Host'), + readSessionHeader: (sessionId: string) => + execution.sessionStore.readHeaderSnapshot(sessionId), + newId: () => `call-${++callCounter}`, + createFetchTransport: () => ({ + fetch: fetchImpl as unknown as ProxiedFetchTransport['fetch'], + close: async () => undefined, + }), + }); + const callId = (suffix: string) => `goal_evaluation_${session.id}_${suffix}`; + const canonicalRow = async (suffix: string) => { + const logs = await usage.modelCalls.modelCallLogs( + { range: 'all', sessionId: session.id }, + Date.now(), + 0, + 10, + ); + return logs.projection.rows.find((row) => row.callId === callId(suffix)); + }; + const legacyGoalRows = async () => { + const logs = await usage.telemetry.logs({ range: 'all' }); + return logs.rows.filter((row) => row.callKind === 'goal_evaluation'); + }; + const assertSummary = async (expected: { + requests: number; + missing: number; + errors: number; + }) => { + const summary = await usage.modelCalls.modelCallSummary( + { range: 'all', sessionId: session.id }, + Date.now(), + ); + assert.equal(summary.projection.totalRequests, expected.requests); + // The ledger-wide coverage counts every usage-unknown row it holds — + // the auxiliary rows under test included, each recorded under the + // no-run sentinel turn. Hiding them here would make an incomplete + // total read as complete; settlement scopes itself to the run's own + // rows instead. + assert.equal(summary.projection.coverage.usageMissingAttempts, expected.missing); + assert.equal(summary.projection.coverage.usageReportedAttempts, 0); + assert.equal(summary.projection.errorRequests, expected.errors); + assert.equal(summary.projection.totalTokens.total, 0); + assert.equal(summary.projection.totalCostUsd, 0); + }; + + // Aborted mid-flight: usage can never be known, so the canonical row says + // so instead of posing as a free call in the legacy table. + let abortSignal: AbortSignal | undefined; + const abortDispatched = deferred(); + const abortReleased = deferred(); + const abortingEvaluator = evaluator(async (_request, init) => { + abortSignal = init?.signal ?? undefined; + abortDispatched.resolve(); + await abortReleased.promise; + throw abortSignal?.reason ?? new DOMException('Aborted', 'AbortError'); + }); + const abort = new AbortController(); + const abortedCall = abortingEvaluator.evaluate( + 'Judge the completed Goal.', + session.id, + abort.signal, + ); + await settleWithin(abortDispatched.promise); + abort.abort(new DOMException('Goal lane invalidated', 'AbortError')); + abortReleased.resolve(); + await assert.rejects(settleWithin(abortedCall)); + await assertSummary({ requests: 1, missing: 1, errors: 0 }); + const abortedRow = await canonicalRow('call-1'); + assert.ok(abortedRow); + assert.equal(abortedRow.status, 'aborted'); + assert.equal(abortedRow.callKind, 'goal_evaluation'); + assert.deepEqual(await legacyGoalRows(), []); + + // A provider error without an abort is usage-unknown all the same. + const failingEvaluator = evaluator(async () => { + throw new Error('provider exploded'); + }); + await assert.rejects( + failingEvaluator.evaluate( + 'Judge the completed Goal.', + session.id, + new AbortController().signal, + ), + ); + await assertSummary({ requests: 2, missing: 2, errors: 1 }); + const failedRow = await canonicalRow('call-2'); + assert.ok(failedRow); + assert.equal(failedRow.status, 'error'); + assert.equal(failedRow.errorClass, 'Error'); + assert.deepEqual(await legacyGoalRows(), []); + + // A completion the provider answered without usage stays on the legacy + // zero path for now: with settlement scoped to a run's own rows a + // canonical missing row here would no longer flip any hosted run + // indeterminate, so routing it to the canonical ledger is a free-standing + // recording question (#5691), not a settlement constraint. Aborted and + // failed calls above already keep their canonical usage-unknown rows. + const silentEvaluator = evaluator(async () => + Response.json({ + id: 'chatcmpl-usage-unknown', + object: 'chat.completion', + created: 1, + model: MODEL_ID, + choices: [ + { + index: 0, + message: { role: 'assistant', content: RESPONSE_TEXT }, + finish_reason: 'stop', + }, + ], + }), + ); + assert.equal( + await silentEvaluator.evaluate( + 'Judge the completed Goal.', + session.id, + new AbortController().signal, + ), + RESPONSE_TEXT, + ); + await assertSummary({ requests: 2, missing: 2, errors: 1 }); + const silentLegacy = (await legacyGoalRows()).find((row) => row.callId === callId('call-3')); + assert.ok(silentLegacy); + assert.equal(silentLegacy.status, 'success'); + assert.equal(silentLegacy.inputTokens, 0); + assert.equal(silentLegacy.outputTokens, 0); + } finally { + await usage.close(); + await execution.sessionStore.close?.(); + await owner.close(); + await rm(base, { recursive: true, force: true }); + } +}); + test('one turn shares one canonical Skill inventory across prompt and lazy tools', async () => { const policy = { revision: 7, diff --git a/packages/runtime-host/src/__tests__/hosted-execution-runner.test.ts b/packages/runtime-host/src/__tests__/hosted-execution-runner.test.ts index 00608a64d20..40624dc78c1 100644 --- a/packages/runtime-host/src/__tests__/hosted-execution-runner.test.ts +++ b/packages/runtime-host/src/__tests__/hosted-execution-runner.test.ts @@ -33,6 +33,7 @@ test('hosted execution reads usage only after execution residencies settle', asy return usageSummary(); }, }), + runSettlementCoverage: settlementCoverage(), context: context(), requestDrain: () => {}, waitForExecutionResidencies: () => residency.promise, @@ -62,11 +63,46 @@ test('hosted execution reads usage only after execution residencies settle', asy assert.equal(usageRead, true); }); -test('incomplete usage preserves its fixed safe cause', async () => { +test('a run-owned unsettled attempt refuses settlement with its fixed safe cause', async () => { + const runner = new HostHostedExecutionRunner({ + handlers: handlers({}), + runSettlementCoverage: settlementCoverage({ usageMissingAttempts: 1 }), + context: context(), + requestDrain: () => {}, + waitForExecutionResidencies: async () => {}, + waitForAllResidencies: async () => {}, + }); + + const result = await runner.run(input(), new AbortController().signal); + + assert.equal(result.failureReason, 'Runtime Host usage did not settle: missing_attempt_usage'); +}); + +test('a run-owned partial attempt refuses settlement with its fixed safe cause', async () => { + const runner = new HostHostedExecutionRunner({ + handlers: handlers({}), + runSettlementCoverage: settlementCoverage({ usagePartialAttempts: 1 }), + context: context(), + requestDrain: () => {}, + waitForExecutionResidencies: async () => {}, + waitForAllResidencies: async () => {}, + }); + + const result = await runner.run(input(), new AbortController().signal); + + assert.equal(result.failureReason, 'Runtime Host usage did not settle: partial_attempt_usage'); +}); + +test('a usage-unknown row outside any run does not block settlement', async () => { + // The failed auxiliary call (#5691) is real accounting: the ledger-wide + // coverage — the public provenance every client reads — counts it as + // usage-missing. Settlement must not read that field: it holds the run to + // its own rows, and the run left none unsettled. const usage = usageSummary(); usage.provenance.coverage.usageMissingAttempts = 1; const runner = new HostHostedExecutionRunner({ handlers: handlers({ usage: () => usage }), + runSettlementCoverage: settlementCoverage(), context: context(), requestDrain: () => {}, waitForExecutionResidencies: async () => {}, @@ -75,7 +111,7 @@ test('incomplete usage preserves its fixed safe cause', async () => { const result = await runner.run(input(), new AbortController().signal); - assert.equal(result.failureReason, 'Runtime Host usage did not settle: missing_attempt_usage'); + assert.equal(result.kind, 'settled'); }); test('abort after terminal completion preserves the completed result', async () => { @@ -98,6 +134,7 @@ async function runWithAbortAfterTerminal(query?: () => unknown) { const settling = deferred(); const runner = new HostHostedExecutionRunner({ handlers: handlers(query ? { query } : {}), + runSettlementCoverage: settlementCoverage(), context: context(), requestDrain: () => {}, waitForExecutionResidencies: () => { @@ -132,6 +169,7 @@ test('hosted execution cancellation drains the Host and waits for canonical stop return terminalTurn('cancelled'); }, }), + runSettlementCoverage: settlementCoverage(), context: context(), requestDrain: () => { drains += 1; @@ -167,6 +205,7 @@ test('hosted execution cancellation before Turn admission starts no Turn', async turnStarts += 1; }, }), + runSettlementCoverage: settlementCoverage(), context: context(), requestDrain: () => {}, waitForExecutionResidencies: async () => {}, @@ -195,6 +234,7 @@ test('hosted execution cancellation remains active while Runtime continuations s return runningTurn(); }, }), + runSettlementCoverage: settlementCoverage(), context: context(), requestDrain: () => { drains += 1; @@ -334,6 +374,20 @@ function usageSummary() { }; } +/** + * The run-scoped incompleteness check's stub: a hosted execution owns no + * usage-unknown rows unless the test says otherwise. + */ +function settlementCoverage( + overrides: { usageMissingAttempts?: number; usagePartialAttempts?: number } = {}, +) { + return async (_from: number, _to: number) => ({ + usageMissingAttempts: 0, + usagePartialAttempts: 0, + ...overrides, + }); +} + function emptySkillInvocation() { return { loaded: [], failed: [], receipts: [] }; } diff --git a/packages/runtime-host/src/server/execution-composition.ts b/packages/runtime-host/src/server/execution-composition.ts index 1bd26edf7cd..15bc1215424 100644 --- a/packages/runtime-host/src/server/execution-composition.ts +++ b/packages/runtime-host/src/server/execution-composition.ts @@ -2818,6 +2818,7 @@ export async function createExecutionRuntimeHostComposition( 'turn.stop': turnControl.handlers['turn.stop'], 'usage.query': usagePricing.handlers['usage.query'], }, + runSettlementCoverage: (from, to) => usagePricing.runSettlementCoverage(from, to), context: { hostEpoch: context.hostEpoch, connectionId: 'hosted-execution', diff --git a/packages/runtime-host/src/server/execution-model-authority.ts b/packages/runtime-host/src/server/execution-model-authority.ts index 4f5fbb6d471..47fc6c2cb93 100644 --- a/packages/runtime-host/src/server/execution-model-authority.ts +++ b/packages/runtime-host/src/server/execution-model-authority.ts @@ -76,6 +76,8 @@ import { resolveModelRuntime } from '@maka/runtime/model-runtime'; import { type BackendFactoryContext } from '@maka/runtime/session-manager'; import { type GoalEvaluatorResource } from '@maka/runtime/goal-evaluator'; import { type ModelMessage } from '@maka/runtime/model-protocol'; +import { type ModelCallAttemptStatus } from '@maka/core/model-call-attempt'; +import { NO_RUN_TURN_ID, type UsageUnknownModelCallRecord } from '@maka/storage/model-call-ledger'; import { memoryExtractionMaxOutputTokens, type MemoryExtractionSourceSnapshot, @@ -560,6 +562,9 @@ interface AuxiliaryModelCallAuthority { record: Parameters[0], ): Promise; }; + readonly modelCalls: { + recordUsageUnknownAttempt(record: UsageUnknownModelCallRecord): Promise; + }; readonly requestDrain: () => void; readonly now: () => number; readonly newId: () => string; @@ -605,6 +610,16 @@ interface HostAuxiliaryModelCallInput { ) => AuxiliaryModelRequest; } +/** + * Turn placeholder for canonical rows of calls no run owns: auxiliary calls + * happen outside any turn, and the ledger's countable rows require the column. + * The storage side records the rows' no-run ownership from this seam, so a + * failed auxiliary call cannot flip a hosted run to indeterminate (#5691) — + * including a hosted execution legally named like this placeholder + * (#5890 review). + */ +const AUXILIARY_TURN_ID = NO_RUN_TURN_ID; + function createAuxiliaryModelCallAuthority( input: AuxiliaryModelCallAuthorityInput, ): AuxiliaryModelCallAuthority { @@ -629,6 +644,16 @@ function createAuxiliaryModelCallAuthority( } }, }, + modelCalls: { + recordUsageUnknownAttempt: async (record) => { + try { + await input.usage.modelCalls.recordUsageUnknownAttempt(record); + } catch (error) { + requestDrain(); + throw error; + } + }, + }, requestDrain, now: input.now ?? Date.now, newId: input.newId ?? randomUUID, @@ -703,6 +728,34 @@ async function runHostAuxiliaryModelCall( modelId: target.model, startedAt, }; + // Abort and error paths know no token counts: their canonical row says + // usage-unknown (`usageBasis: 'missing'`, no tokens) instead of posing as a + // free call (#5691). Auxiliary calls run outside any AgentRun, so the + // canonical writer is this store seam, not the run's event stream. + const recordUsageUnknown = async ( + status: ModelCallAttemptStatus, + errorClass?: string, + ): Promise => { + const completedAt = authority.now(); + try { + await authority.modelCalls.recordUsageUnknownAttempt({ + attemptId: input.callId, + completedAt, + ...(input.telemetrySessionId ? { sessionId: input.telemetrySessionId } : {}), + logicalCallId: input.callId, + turnId: AUXILIARY_TURN_ID, + callKind: input.callKind, + connectionSlug: target.connection.slug, + providerId: target.connection.providerType, + modelId: target.model, + latencyMs: Math.max(0, completedAt - startedAt), + status, + ...(errorClass ? { errorClass } : {}), + }); + } catch (accountingError) { + throw new AuxiliaryModelCallLocalError('accounting', accountingError); + } + }; let result: | Awaited> | Awaited>; @@ -743,21 +796,10 @@ async function runHostAuxiliaryModelCall( if (oauthFailure) throw oauthFailure; } catch (error) { const effectiveError = readDeferredOAuthFailure?.() ?? error; - try { - await recordLlmCallStrict( - { repo: authority.telemetry, lookupPricing: pricing }, - { - ...baseRecord, - inputTokens: 0, - outputTokens: 0, - latencyMs: Math.max(0, authority.now() - startedAt), - status: input.abortSignal.aborted ? 'aborted' : 'error', - errorClass: evaluatorErrorClass(effectiveError), - }, - ); - } catch (accountingError) { - throw new AuxiliaryModelCallLocalError('accounting', accountingError); - } + await recordUsageUnknown( + input.abortSignal.aborted ? 'aborted' : 'failed', + evaluatorErrorClass(effectiveError), + ); throw effectiveError; } try { @@ -765,6 +807,12 @@ async function runHostAuxiliaryModelCall( { repo: authority.telemetry, lookupPricing: pricing }, { ...baseRecord, + // A completion without reported usage stays on the legacy zero path: + // hosted execution settlement refuses any range containing + // usage-missing rows ('missing_attempt_usage'), so routing these + // rows to the canonical ledger marked every such hosted run + // indeterminate. The residual zero is known dishonesty, pending a + // ruling on that settlement coverage semantics (#5691). ...(result.usage ? llmCallUsageFields(result.usage) : { inputTokens: 0, outputTokens: 0 }), diff --git a/packages/runtime-host/src/server/hosted-execution-runner.ts b/packages/runtime-host/src/server/hosted-execution-runner.ts index ea38fdfa2be..243a700e4b9 100644 --- a/packages/runtime-host/src/server/hosted-execution-runner.ts +++ b/packages/runtime-host/src/server/hosted-execution-runner.ts @@ -26,6 +26,7 @@ import type { TurnSnapshot, UsageQueryResult, } from '../protocol/index.js'; +import type { RunSettlementCoverage } from '@maka/storage/model-call-ledger'; import type { ConnectionContext, OperationHandlerMap } from './operation-dispatcher.js'; export interface HostHostedExecutionRunnerInput { @@ -33,6 +34,14 @@ export interface HostHostedExecutionRunnerInput { OperationHandlerMap, 'session.create' | 'turn.start' | 'turn.query' | 'turn.stop' | 'usage.query' >; + /** + * What the settlement window left unsettled (#5890). Settlement's + * incompleteness check reads this window-scoped answer instead of the + * `usage.query` coverage: that field is the ledger's public provenance and + * must keep counting usage-unknown rows no run owns — a failed auxiliary + * Host call is real accounting, not this run's unsettled obligation. + */ + readonly runSettlementCoverage: (from: number, to: number) => Promise; readonly context: ConnectionContext; readonly requestDrain: () => void; readonly waitForExecutionResidencies: () => Promise; @@ -82,8 +91,10 @@ export class HostHostedExecutionRunner { await (signal.aborted ? this.input.waitForAllResidencies() : this.input.waitForExecutionResidencies()); - const usage = await this.#readUsage(startedAt, (this.input.now ?? Date.now)()); - const incompleteUsage = incompleteUsageReason(usage); + const settledAt = (this.input.now ?? Date.now)(); + const usage = await this.#readUsage(startedAt, settledAt); + const runCoverage = await this.input.runSettlementCoverage(startedAt, settledAt); + const incompleteUsage = incompleteUsageReason(usage, runCoverage); if (incompleteUsage) { return indeterminate( input.executionId, @@ -160,12 +171,18 @@ export class HostHostedExecutionRunner { function incompleteUsageReason( result: Extract, + runCoverage: RunSettlementCoverage, ): string | undefined { - const { coverage, unreadableRecords, pendingRepairs } = result.provenance; + // Unreadable and pending-repair rows stay ledger-wide: damage in the window + // is nobody's settled fact. Partial and missing usage are judged against the + // same window as the totals this check guards (#5890), so a delegated child + // Session cannot settle underneath them — the `no_run` sentinel rows stay + // excluded, and the shared coverage field still reports everything. + const { unreadableRecords, pendingRepairs } = result.provenance; if (unreadableRecords > 0) return 'unreadable_usage_record'; if (pendingRepairs > 0) return 'pending_usage_repair'; - if (coverage.usagePartialAttempts > 0) return 'partial_attempt_usage'; - if (coverage.usageMissingAttempts > 0) return 'missing_attempt_usage'; + if (runCoverage.usagePartialAttempts > 0) return 'partial_attempt_usage'; + if (runCoverage.usageMissingAttempts > 0) return 'missing_attempt_usage'; return undefined; } diff --git a/packages/runtime-host/src/server/usage-pricing-coordinator.ts b/packages/runtime-host/src/server/usage-pricing-coordinator.ts index 4b97e790d23..b5dcebcafe2 100644 --- a/packages/runtime-host/src/server/usage-pricing-coordinator.ts +++ b/packages/runtime-host/src/server/usage-pricing-coordinator.ts @@ -42,6 +42,7 @@ import { type InteractiveUsageStoresFailureClassification, type InteractiveUsageStoresWriter, } from '@maka/storage/usage-stores'; +import type { RunSettlementCoverage } from '@maka/storage/model-call-ledger'; import { isSessionNotFoundError } from '@maka/storage/execution-stores'; import { encodePricingQueryResult, @@ -80,6 +81,17 @@ export class HostUsagePricingCoordinator { 'pricing.mutate': (input) => this.#mutatePricing(input), }; + /** + * What the settlement window itself left unsettled (#5890) — hosted + * execution settlement's incompleteness check reads the same window as the + * totals it guards, minus the `no_run` sentinel rows. Deliberately off the + * operation handler map: the ledger-wide `usage.query` coverage keeps a + * different shape, so settlement scopes itself through this in-process seam. + */ + async runSettlementCoverage(from: number, to: number): Promise { + return this.#stores.modelCalls.modelCallRunSettlementCoverage(from, to); + } + readonly #stores: InteractiveUsageStoresWriter; readonly #requestDrain: () => void; readonly #activation: RuntimePolicyActivationGate; diff --git a/packages/storage/src/__tests__/model-call-usage-query.test.ts b/packages/storage/src/__tests__/model-call-usage-query.test.ts index 421650c2104..1afb45fa492 100644 --- a/packages/storage/src/__tests__/model-call-usage-query.test.ts +++ b/packages/storage/src/__tests__/model-call-usage-query.test.ts @@ -19,14 +19,62 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; +import type { ModelCallAttempt } from '@maka/core/model-call-attempt'; import { + type ModelCallLedger, + NO_RUN_TURN_ID, + type UsageUnknownModelCallRecord, +} from '../model-call-ledger.js'; +import { + appendAuthorityEvent, modelCallAttempt as attempt, MODEL_CALL_NOW as NOW, + withLedger, withProjectedAttempts, } from './fixtures/model-call-attempt.js'; const ALL = { range: 'all' } as const; +/** + * Seeds rows the way production writes them: run-owned attempts through the + * AgentRun authority's projection, calls no run owns through the usage-unknown + * seam. Ownership follows the writer, so a test cannot fake a no-run row by + * projecting a sentinel turn value. + */ +async function withRecordedAttempts( + projected: readonly ModelCallAttempt[], + noRun: readonly UsageUnknownModelCallRecord[], + run: (ledger: ModelCallLedger) => Promise, +): Promise { + await withLedger(async (ledger, root) => { + projected.forEach((value, index) => { + appendAuthorityEvent(root, index, value, value.sessionId, value.runId); + }); + await ledger.catchUpProjection(); + for (const record of noRun) await ledger.recordUsageUnknownAttempt(record); + await run(ledger); + }); +} + +/** A failed auxiliary Host call, as the usage-unknown seam records it. */ +function usageUnknownAttempt( + overrides: Partial = {}, +): UsageUnknownModelCallRecord { + return { + attemptId: 'auxiliary-failure', + completedAt: NOW - 100, + sessionId: 'session-1', + logicalCallId: 'auxiliary-failure', + turnId: NO_RUN_TURN_ID, + callKind: 'goal_evaluation', + providerId: 'openai', + modelId: 'gpt-5', + latencyMs: 5, + status: 'failed', + ...overrides, + }; +} + describe('Usage answers over the canonical ledger', () => { test('a total never counts unpriced spend as zero, and says so in coverage', async () => { // The frozen pre-cutover table had nowhere to record "we could not price @@ -99,6 +147,174 @@ describe('Usage answers over the canonical ledger', () => { ); }); + test('ledger coverage counts usage-unknown rows recorded outside any run', async () => { + // A failed auxiliary call (#5691) is accounted the same honest way as a + // run's own unsettled dispatch, and the ledger-wide coverage — what + // `summary()`/`logs()` report to every client — must see both. Excluding + // the no-run rows here would hide a real unknown-usage call from the + // public provenance and let an incomplete total read as complete; keeping + // a hosted run to its own rows is settlement's separate job. + await withRecordedAttempts( + [ + attempt({ + attemptId: 'run-dispatch', + status: 'failed', + usageBasis: 'missing', + inputTokens: undefined, + outputTokens: undefined, + costBasis: 'unpriced', + costUsd: undefined, + }), + ], + [usageUnknownAttempt()], + async (ledger) => { + const { projection } = ledger.summary(ALL, NOW); + assert.equal(projection.coverage.usageMissingAttempts, 2); + assert.equal(projection.coverage.usageReportedAttempts, 0); + assert.equal(projection.totalRequests, 2); + }, + ); + }); + + test('run settlement coverage guards every session in the window, not the auxiliary sentinel', async () => { + // Settlement guards the window-wide totals, so it must count every + // Session's unsettled dispatch in that window (#5890): the pre-window + // session filter here would have let a delegated child's missing usage + // settle as complete. The `no_run` sentinel — the failed auxiliary call + // the usage-unknown seam recorded — belongs to no run and stays out. + await withRecordedAttempts( + [ + attempt({ + attemptId: 'other-session-dispatch', + sessionId: 'session-2', + status: 'failed', + usageBasis: 'missing', + inputTokens: undefined, + outputTokens: undefined, + costBasis: 'unpriced', + costUsd: undefined, + }), + attempt({ + attemptId: 'run-reported', + logicalCallId: 'call-2', + }), + attempt({ + attemptId: 'run-partial', + logicalCallId: 'call-3', + usageBasis: 'partial', + outputTokens: undefined, + }), + attempt({ + attemptId: 'run-missing-dispatch', + logicalCallId: 'call-4', + status: 'failed', + usageBasis: 'missing', + inputTokens: undefined, + outputTokens: undefined, + costBasis: 'unpriced', + costUsd: undefined, + }), + ], + [usageUnknownAttempt()], + async (ledger) => { + // The run's own missing dispatch and the other session's (a delegated + // child's) both count; the `no_run` sentinel does not. + assert.deepEqual(ledger.runSettlementCoverage(NOW - 1_000, NOW), { + usageMissingAttempts: 2, + usagePartialAttempts: 1, + }); + }, + ); + }); + + test('run settlement coverage excludes the no-run sentinel, not an execution named like it', async () => { + // The exclusion must key on how a row was recorded (`no_run`), never on + // the turn value: a hosted execution whose id is the client-chosen + // `auxiliary` owns rows under `turn_id = 'auxiliary'`, and with window + // scoping (#5890) those must keep counting while the genuine sentinel row + // next to them stays out — or this execution's missing-usage dispatch + // disappears and `incompleteUsageReason` releases the environment with + // unknown usage. + await withRecordedAttempts( + [ + attempt({ + attemptId: 'run-reported', + sessionId: 'auxiliary', + runId: 'run-auxiliary', + turnId: 'auxiliary', + logicalCallId: 'call-reported', + }), + attempt({ + attemptId: 'run-missing-dispatch', + sessionId: 'auxiliary', + runId: 'run-auxiliary', + turnId: 'auxiliary', + logicalCallId: 'call-missing', + status: 'failed', + usageBasis: 'missing', + inputTokens: undefined, + outputTokens: undefined, + costBasis: 'unpriced', + costUsd: undefined, + }), + ], + [usageUnknownAttempt({ sessionId: 'auxiliary' })], + async (ledger) => { + // The run-owned missing dispatch is this execution's unsettled + // obligation and counts inside the window; the failed auxiliary call + // is nobody's (`no_run`) and stays excluded. Public coverage keeps + // counting both. + assert.deepEqual(ledger.runSettlementCoverage(NOW - 1_000, NOW), { + usageMissingAttempts: 1, + usagePartialAttempts: 0, + }); + assert.equal( + ledger.summary({ range: 'all' }, NOW).projection.coverage.usageMissingAttempts, + 2, + ); + }, + ); + }); + + test('run settlement coverage reads the window, not one session (#5890 review)', async () => { + // Settlement guards the window-wide totals that #readUsage reads, so its + // coverage must read the same window: a child Session's missing dispatch + // inside the window is exactly the undercount that would otherwise settle + // as complete, and a window-external row is nobody's obligation here. + await withRecordedAttempts( + [ + attempt({ + attemptId: 'outside-window-missing', + completedAt: NOW - 5_000, + status: 'failed', + usageBasis: 'missing', + inputTokens: undefined, + outputTokens: undefined, + costBasis: 'unpriced', + costUsd: undefined, + }), + attempt({ attemptId: 'root-ok' }), + attempt({ + attemptId: 'child-missing', + sessionId: 'session-1-child', + status: 'failed', + usageBasis: 'missing', + inputTokens: undefined, + outputTokens: undefined, + costBasis: 'unpriced', + costUsd: undefined, + }), + ], + [], + async (ledger) => { + assert.deepEqual(ledger.runSettlementCoverage(NOW - 1_000, NOW), { + usageMissingAttempts: 1, + usagePartialAttempts: 0, + }); + }, + ); + }); + test('one malformed cache reading cannot inflate the cache total', async () => { await withProjectedAttempts( [ diff --git a/packages/storage/src/__tests__/public-entrypoints.test.ts b/packages/storage/src/__tests__/public-entrypoints.test.ts index a82c61b94d7..f17906149cb 100644 --- a/packages/storage/src/__tests__/public-entrypoints.test.ts +++ b/packages/storage/src/__tests__/public-entrypoints.test.ts @@ -140,17 +140,16 @@ test('every published entrypoint target is emitted by the build', async () => { * means removing the entry, and publishing a *new* consumer-less entrypoint * fails outright, which is the direction this guard exists to hold. * - * `./model-call-ledger` is on the list without this change touching it: it was - * already published, and `repairPendingModelCallProjections` lost its last - * caller when `canonical-usage-reader` was rewritten on `main`. It is listed - * here rather than unpublished for the same reason as the rest — retiring a - * subpath that already shipped is a compatibility call of its own. + * `./model-call-ledger` was on this list until the usage-unknown seam + * (`fix(usage)`): it had been published without consumers since + * `repairPendingModelCallProjections` lost its last caller when + * `canonical-usage-reader` was rewritten on `main`; the auxiliary-call + * recorder in runtime-host is its first external consumer again. */ const PREEXISTING_UNCONSUMED_ENTRYPOINTS = [ './activation-secret-injector', './encrypted-file-managed-secret-store', './managed-secret-store', - './model-call-ledger', './write-queue', ]; diff --git a/packages/storage/src/__tests__/sqlite-usage-schema.test.ts b/packages/storage/src/__tests__/sqlite-usage-schema.test.ts index f84590b3b5b..2ac5213bbe8 100644 --- a/packages/storage/src/__tests__/sqlite-usage-schema.test.ts +++ b/packages/storage/src/__tests__/sqlite-usage-schema.test.ts @@ -21,6 +21,15 @@ import assert from 'node:assert/strict'; import { DatabaseSync } from 'node:sqlite'; import { test } from 'node:test'; import { migrateSqliteUsageDatabase } from '../sqlite-usage-schema.js'; +import { + assertCurrentOperationalTargetSchema, + ensureOperationalSchemaRegistry, +} from '../operational-target-schema.js'; +import { migrateSqliteArtifactDatabase } from '../sqlite-artifact-schema.js'; +import { migrateSqliteCoreExecutionDatabase } from '../sqlite-core-execution-schema.js'; +import { migrateSqliteRuntimeDatabase } from '../sqlite-runtime-schema.js'; +import { migrateSqliteSessionMetadataDatabase } from '../sqlite-session-metadata-schema.js'; +import { migrateSqliteWorkflowDatabase } from '../sqlite-workflow-schema.js'; import { MODEL_CALL_NOW as NOW, modelCallAttempt as attempt, @@ -331,3 +340,38 @@ test('Usage title revision ignores unrelated metadata and covers every activity database.close(); } }); + +test('a schema-9 usage database upgrades to the target shape and stays idempotent', () => { + const database = new DatabaseSync(':memory:'); + try { + // Build the full operational target, then regress usage to its schema-9 + // shape. The #5890 target adds `no_run` as the usage delta, so a real + // schema-9 database is exactly this table minus that column. + migrateSqliteRuntimeDatabase(database); + migrateSqliteSessionMetadataDatabase(database); + migrateSqliteCoreExecutionDatabase(database); + migrateSqliteWorkflowDatabase(database); + migrateSqliteUsageDatabase(database); + migrateSqliteArtifactDatabase(database); + ensureOperationalSchemaRegistry(database); + database.exec('ALTER TABLE usage_model_call_attempts DROP COLUMN no_run'); + + migrateSqliteUsageDatabase(database); + + // The upgrade restores the recorder-owned default in place — the DDL + // signature matches a fresh create only because ADD COLUMN appends the + // column the same way — and a rerun changes nothing. + assertCurrentOperationalTargetSchema(database); + const restored = database + .prepare( + "SELECT \"notnull\" AS not_null, dflt_value FROM pragma_table_info('usage_model_call_attempts') WHERE name = 'no_run'", + ) + .get() as { not_null: number; dflt_value: string }; + assert.equal(restored.not_null, 1); + assert.equal(restored.dflt_value, '0'); + migrateSqliteUsageDatabase(database); + assertCurrentOperationalTargetSchema(database); + } finally { + database.close(); + } +}); diff --git a/packages/storage/src/__tests__/usage-stores.test.ts b/packages/storage/src/__tests__/usage-stores.test.ts index 3e7e53178f4..50de4e4addf 100644 --- a/packages/storage/src/__tests__/usage-stores.test.ts +++ b/packages/storage/src/__tests__/usage-stores.test.ts @@ -185,6 +185,115 @@ describe('InteractiveUsageStores', () => { }); }); + test('records a usage-unknown auxiliary call as a canonical row without token counts', async () => { + await withInteractiveRoot(async ({ root, capability }) => { + const owner = await tryAcquireInteractiveRootOwner(capability); + assert(owner); + const stores = await openInteractiveUsageStoresForWrite(owner.lease); + const changed: string[] = []; + const unsubscribe = stores.subscribeSessionUsageChanges((sessionId) => + changed.push(sessionId), + ); + try { + // An auxiliary call aborted mid-flight knows no token counts. Its row + // must say so (usage_basis 'missing', no tokens) instead of posing as a + // free call — the same facts the run-path projection records. + const record = { + attemptId: 'goal_evaluation_session-aux_call-1', + completedAt: Date.UTC(2026, 0, 1), + sessionId: 'session-aux', + logicalCallId: 'goal_evaluation_session-aux_call-1', + turnId: 'auxiliary', + callKind: 'goal_evaluation' as const, + connectionSlug: 'conn-aux', + providerId: 'openai', + modelId: 'gpt-5', + latencyMs: 120, + status: 'aborted' as const, + errorClass: 'AbortError', + }; + await stores.modelCalls.recordUsageUnknownAttempt(record); + assert.deepEqual(changed, ['session-aux']); + + const summary = await stores.modelCalls.modelCallSummary({ range: 'all' }, Date.now()); + assert.equal(summary.projection.totalRequests, 1); + assert.equal(summary.projection.totalCostUsd, 0); + assert.equal(summary.projection.totalTokens.total, 0); + // The ledger-wide coverage counts the row: a real unknown-usage call + // must stay visible in the public provenance even though it belongs to + // no run. Keeping a hosted run to its own rows is the run-scoped + // settlement coverage's job, asserted against the same stores below. + assert.equal(summary.projection.coverage.usageMissingAttempts, 1); + assert.equal(summary.projection.coverage.usageReportedAttempts, 0); + assert.equal(summary.unreadableRecords, 0); + // The settlement coverage excludes the no-run sentinel row — its + // usage_basis never enters any run's window scope — so an auxiliary + // failure cannot flip a hosted run to indeterminate. + assert.deepEqual(await stores.modelCalls.modelCallRunSettlementCoverage(0, Date.now()), { + usageMissingAttempts: 0, + usagePartialAttempts: 0, + }); + + const logs = await stores.modelCalls.modelCallLogs({ range: 'all' }, Date.now(), 0, 10); + assert.equal(logs.projection.total, 1); + const row = logs.projection.rows[0]; + assert.ok(row); + assert.equal(row.callId, 'goal_evaluation_session-aux_call-1'); + assert.equal(row.callKind, 'goal_evaluation'); + assert.equal(row.status, 'aborted'); + assert.equal(row.errorClass, 'AbortError'); + + // The schema CHECK is the hard edge: a missing-usage row carries no + // token counts at all, so no total can mistake it for a free call. + const db = acquireOperationalStateDatabase(root); + try { + const stored = db.database + .prepare( + `SELECT usage_basis, cost_basis, cost_usd, input_tokens, output_tokens, + cache_read_input_tokens, cache_miss_input_tokens, + cache_write_input_tokens, reasoning_tokens, + status, session_id, turn_id + FROM usage_model_call_attempts WHERE attempt_id = ?`, + ) + .get('goal_evaluation_session-aux_call-1'); + assert.deepEqual( + { ...stored }, + { + usage_basis: 'missing', + cost_basis: 'unpriced', + cost_usd: null, + input_tokens: null, + output_tokens: null, + cache_read_input_tokens: null, + cache_miss_input_tokens: null, + cache_write_input_tokens: null, + reasoning_tokens: null, + status: 'aborted', + session_id: 'session-aux', + turn_id: 'auxiliary', + }, + ); + } finally { + db.close(); + } + + // Re-recording the same attempt upserts in place instead of double + // counting the call. + await stores.modelCalls.recordUsageUnknownAttempt({ + ...record, + latencyMs: 130, + }); + const resummarized = await stores.modelCalls.modelCallSummary({ range: 'all' }, Date.now()); + assert.equal(resummarized.projection.totalRequests, 1); + assert.deepEqual(changed, ['session-aux', 'session-aux']); + } finally { + unsubscribe(); + await stores.close(); + await owner.close(); + } + }); + }); + test('publishes the owning Session after a durable tool-usage write', async () => { await withInteractiveRoot(async ({ capability }) => { const owner = await tryAcquireInteractiveRootOwner(capability); diff --git a/packages/storage/src/model-call-ledger.ts b/packages/storage/src/model-call-ledger.ts index eee51d310e4..69ee75d124f 100644 --- a/packages/storage/src/model-call-ledger.ts +++ b/packages/storage/src/model-call-ledger.ts @@ -21,6 +21,7 @@ import { decodeModelCallAttempt, MODEL_CALL_ATTEMPT_EVENT_TYPE, type ModelCallAttempt, + type ModelCallAttemptStatus, type ModelCallCoverage, } from '@maka/core/model-call-attempt'; import { @@ -31,6 +32,7 @@ import { } from '@maka/core/model-call-usage-projection'; import { usageBucketKey } from '@maka/core/usage-stats/bucket-key'; import type { + ModelCallKind, UsageBucket, UsageGroupBy, UsageLogRow, @@ -43,8 +45,10 @@ import { count, countableFilter, COVERAGE_SUMS, + NO_RUN_TURN_ID, PRICED_COST, REQUEST_SUMS, + RUN_SETTLEMENT_COVERAGE_SUMS, TOKEN_SUMS, unreadableFilter, type SqlFilter, @@ -56,6 +60,8 @@ import { import { MODEL_CALL_COLUMNS } from './sqlite-usage-schema.js'; import type { ModelCallLedgerResult } from './usage-stores.js'; +export { NO_RUN_TURN_ID }; + /** * Materialization of the canonical model-call accounting ledger (#1679). * @@ -116,6 +122,26 @@ export interface ModelCallLedgerReader { offset: number, limit: number, ): ModelCallLedgerResult; + /** + * What the settlement window itself left unsettled (#5890): the attempts + * between `from` and `to` inclusive whose usage the provider never reported + * or only partly reported, across every Session — the same scope as the + * window-wide totals this check guards. Rows recorded outside any run — by + * the usage-unknown seam, which the ledger marks `no_run` — belong to no + * run's settlement and are excluded, the reverse of the ledger-wide + * coverage above, which must see them. + */ + runSettlementCoverage(from: number, to: number): RunSettlementCoverage; +} + +/** + * What a hosted execution's settlement checks instead of the ledger-wide + * coverage: whether everything in the window settled (#5890, widening #5691's + * run-owned scope so delegated Sessions cannot settle underneath it). + */ +export interface RunSettlementCoverage { + readonly usageMissingAttempts: number; + readonly usagePartialAttempts: number; } export interface CatchUpModelCallProjectionInput { @@ -133,11 +159,51 @@ export interface CatchUpModelCallProjectionResult { readonly unreadableEvents: number; } +/** + * One canonical row for a call no AgentRun owns (#5691). + * + * Auxiliary Host model calls run outside any run, so nothing projects them + * from an event stream — the caller carries the row's identity itself. Usage + * cannot be known for these calls (the provider never reported it), so the + * record cannot express token counts or a cost: it lands as + * `usageBasis: 'missing'` / `costBasis: 'unpriced'`, the same facts the + * run-path projection records and the same rule the table's CHECK enforces on + * every other writer. + */ +export interface UsageUnknownModelCallRecord { + /** Idempotency key: recording the same attempt twice stores one row. */ + readonly attemptId: string; + readonly completedAt: number; + readonly sessionId?: string; + readonly logicalCallId: string; + /** + * Calls outside any run's turn carry the shared no-run sentinel + * (`NO_RUN_TURN_ID`) — the table requires the column for every countable + * row. The value is a placeholder, not the discriminator: the ledger + * records the row's no-run ownership from this seam itself, so an + * execution legally named like the sentinel keeps its own rows. + */ + readonly turnId: string; + readonly callKind: ModelCallKind; + readonly connectionSlug?: string; + readonly providerId: string; + readonly modelId: string; + readonly latencyMs: number; + readonly status: ModelCallAttemptStatus; + readonly errorClass?: string; +} + export interface ModelCallLedgerWriter extends ModelCallLedgerReader { /** Advances the read model from the AgentRun authority's durable sequence. */ catchUpProjection( input?: CatchUpModelCallProjectionInput, ): Promise; + /** + * Records one usage-unknown row for a call outside any AgentRun. The single + * write that does not come from the event stream: the caller owns the + * attempt's identity, and re-recording an `attemptId` upserts in place. + */ + recordUsageUnknownAttempt(record: UsageUnknownModelCallRecord): Promise; } export interface ModelCallLedger extends ModelCallLedgerWriter { @@ -209,6 +275,11 @@ class SqliteModelCallLedger implements ModelCallLedger { ); } + recordUsageUnknownAttempt(record: UsageUnknownModelCallRecord): Promise { + if (this.#state !== 'open') return Promise.reject(new ModelCallLedgerClosedError()); + return this.write(() => writeUsageUnknownModelCallAttempt(this.#lease.database, record)); + } + summary(query: UsageQuery, now: number): ModelCallLedgerResult { const db = this.#open(); const range = resolveUsageRange(query.range, now); @@ -320,6 +391,22 @@ class SqliteModelCallLedger implements ModelCallLedger { }; } + runSettlementCoverage(from: number, to: number): RunSettlementCoverage { + const db = this.#open(); + const row = db + .prepare( + `SELECT ${RUN_SETTLEMENT_COVERAGE_SUMS} + FROM usage_model_call_attempts + WHERE cost_basis IS NOT NULL AND no_run = 0 + AND completed_at >= ? AND completed_at <= ?`, + ) + .get(from, to) as Record | undefined; + return { + usageMissingAttempts: count(row?.usageMissingAttempts), + usagePartialAttempts: count(row?.usagePartialAttempts), + }; + } + #open(): DatabaseSync { if (this.#state !== 'open') throw new ModelCallLedgerClosedError(); return this.#lease.database; @@ -463,6 +550,9 @@ function bindModelCallAttempt(attempt: ModelCallAttempt): (string | number | nul reasoning_tokens: attempt.reasoningTokens ?? null, cost_basis: attempt.costBasis, cost_usd: attempt.costUsd ?? null, + // The projection's only source is the AgentRun authority: every row it + // writes is run-owned, whatever turn the run ran under. + no_run: 0, }; return MODEL_CALL_COLUMNS.map((column) => values[column]); } @@ -471,6 +561,49 @@ function writeModelCallAttempt(db: DatabaseSync, attempt: ModelCallAttempt): voi db.prepare(MODEL_CALL_UPSERT).run(...bindModelCallAttempt(attempt)); } +/** + * The usage-unknown binding: no token counts and no cost exist to record, and + * the table's CHECKs refuse any row that claimed otherwise. + */ +function bindUsageUnknownModelCallAttempt( + record: UsageUnknownModelCallRecord, +): (string | number | null)[] { + const values: Record<(typeof MODEL_CALL_COLUMNS)[number], string | number | null> = { + attempt_id: record.attemptId, + completed_at: record.completedAt, + session_id: record.sessionId ?? null, + logical_call_id: record.logicalCallId, + turn_id: record.turnId, + call_kind: record.callKind, + connection_slug: record.connectionSlug ?? null, + provider_id: record.providerId, + model_id: record.modelId, + latency_ms: record.latencyMs, + status: record.status, + error_class: record.errorClass ?? null, + usage_basis: 'missing', + input_tokens: null, + output_tokens: null, + cache_read_input_tokens: null, + cache_miss_input_tokens: null, + cache_write_input_tokens: null, + reasoning_tokens: null, + cost_basis: 'unpriced', + cost_usd: null, + // The one write outside any run's event stream: the ledger marks the row + // no-run here rather than inferring ownership from the turn value. + no_run: 1, + }; + return MODEL_CALL_COLUMNS.map((column) => values[column]); +} + +function writeUsageUnknownModelCallAttempt( + db: DatabaseSync, + record: UsageUnknownModelCallRecord, +): void { + db.prepare(MODEL_CALL_UPSERT).run(...bindUsageUnknownModelCallAttempt(record)); +} + interface LaggingRunRow { readonly session_id: string; readonly run_id: string; diff --git a/packages/storage/src/model-call-usage-sql.ts b/packages/storage/src/model-call-usage-sql.ts index f7c26a86a08..3f3f547366d 100644 --- a/packages/storage/src/model-call-usage-sql.ts +++ b/packages/storage/src/model-call-usage-sql.ts @@ -43,6 +43,18 @@ export const PRICED_COST = `CASE WHEN cost_basis = 'priced' THEN COALESCE(cost_u /** Only a provider failure is an error. */ const ERROR_ROW = `status = 'failed'`; +/** + * The turn placeholder on rows recorded outside any AgentRun — failed and + * aborted auxiliary Host model calls (#5691). Such a row is honest accounting + * (real spend shape, unknown usage), so the ledger-wide coverage counts it. + * The value is a placeholder for the table's required turn column, not the + * settlement discriminator: rows carry a `no_run` column written by whichever + * seam recorded them, because a hosted execution legally named `auxiliary` + * owns rows under this same turn value (#5890 review). Shared with the writer + * so the placeholder cannot drift. + */ +export const NO_RUN_TURN_ID = 'auxiliary'; + export const TOKEN_SUMS = ` SUM(COALESCE(input_tokens, 0)) AS input, SUM(COALESCE(output_tokens, 0)) AS output, @@ -60,6 +72,19 @@ export const COVERAGE_SUMS = ` SUM(usage_basis = 'partial') AS usagePartialAttempts, SUM(usage_basis = 'missing') AS usageMissingAttempts`; +/** + * Coverage a hosted execution's settlement may hold a run to (#5691): the + * rows the run owns — the ones its Session's event-stream projection + * recorded, outside the `no_run` mark the usage-unknown seam writes. The + * ownership exclusion is the query's WHERE clause, not an aggregate here, so + * this stays narrower than the ledger-wide {@link COVERAGE_SUMS} on purpose: + * one answers "what does the ledger hold unknown-usage rows for", the other + * "what did this run leave unsettled". + */ +export const RUN_SETTLEMENT_COVERAGE_SUMS = ` + SUM(usage_basis = 'partial') AS usagePartialAttempts, + SUM(usage_basis = 'missing') AS usageMissingAttempts`; + export const REQUEST_SUMS = ` COUNT(*) AS totalRequests, SUM(${PRICED_COST}) AS totalCostUsd, diff --git a/packages/storage/src/sqlite-usage-schema.ts b/packages/storage/src/sqlite-usage-schema.ts index cf796d75c7b..c25b37bb226 100644 --- a/packages/storage/src/sqlite-usage-schema.ts +++ b/packages/storage/src/sqlite-usage-schema.ts @@ -19,7 +19,7 @@ import type { DatabaseSync } from 'node:sqlite'; -export const SQLITE_USAGE_SCHEMA_VERSION = 9; +export const SQLITE_USAGE_SCHEMA_VERSION = 10; /** * The canonical ledger's columns, in the order every statement binds them. @@ -49,6 +49,7 @@ export const MODEL_CALL_COLUMNS = [ 'reasoning_tokens', 'cost_basis', 'cost_usd', + 'no_run', ] as const; /** @@ -140,6 +141,11 @@ const MODEL_CALL_TABLE = ` reasoning_tokens INTEGER, cost_basis TEXT, cost_usd REAL, + -- Which writer recorded the row: the run's event-stream projection (0) or + -- the usage-unknown seam for calls no AgentRun owns (1). Settlement reads + -- this, not the turn value — a hosted execution legally named like the + -- no-run sentinel owns rows under the sentinel turn too (#5890 review). + no_run INTEGER NOT NULL DEFAULT 0, CHECK (${MODEL_CALL_REQUIRED_COLUMNS.map( (column) => `(${column} IS NULL) = (cost_basis IS NULL)`, ).join(' AND ')}), @@ -221,6 +227,12 @@ export function migrateSqliteUsageDatabase(db: DatabaseSync): void { // through, and the conversion below reads one. ensureColumn(db, 'usage_model_call_attempts', 'session_id', 'TEXT'); spreadModelCallRecordJson(db); + // no_run is written by the recorder, never inferred from the turn value: + // a backfill keyed on the auxiliary turn string would flip a legally named + // execution's own run-owned rows (see the settlement-coverage regression in + // model-call-usage-query.test.ts). Databases from before this column cannot + // hold auxiliary rows - recording them is what introduced the column. + ensureColumn(db, 'usage_model_call_attempts', 'no_run', 'INTEGER NOT NULL DEFAULT 0'); db.exec(` CREATE INDEX IF NOT EXISTS usage_model_call_attempts_completed_at ON usage_model_call_attempts(completed_at DESC, attempt_id); @@ -364,7 +376,10 @@ function spreadModelCallRecordJson(db: DatabaseSync): void { attempt_id, completed_at, session_id, - ${pricing} + ${pricing}, + -- Pre-column rows all predate the usage-unknown seam: every one of them + -- was recorded by the run's event-stream projection. + 0 AS no_run FROM classified; DROP TABLE usage_model_call_attempts_blob; diff --git a/packages/storage/src/usage-stores.ts b/packages/storage/src/usage-stores.ts index a92c9f54504..f0bf1b747bf 100644 --- a/packages/storage/src/usage-stores.ts +++ b/packages/storage/src/usage-stores.ts @@ -41,6 +41,8 @@ import { ModelCallLedgerPublicationError, type ModelCallLedger, type ModelCallLedgerReader, + type RunSettlementCoverage, + type UsageUnknownModelCallRecord, } from './model-call-ledger.js'; import { PricingCommitUnknownError, @@ -127,12 +129,24 @@ export interface ModelCallIndexReader { offset: number, limit: number, ): Promise>; + /** + * What the settlement window itself left unsettled (#5890) — what hosted + * execution settlement checks instead of the ledger-wide coverage, which + * must also count rows no run owns. + */ + modelCallRunSettlementCoverage(from: number, to: number): Promise; } export interface ModelCallIndexWriter extends ModelCallIndexReader { catchUpModelCallProjection( input?: CatchUpModelCallProjectionInput, ): Promise; + /** + * Records one usage-unknown row for a model call outside any AgentRun + * (#5691) — auxiliary Host calls the event stream cannot project. The owning + * Session, when known, is published after the write. + */ + recordUsageUnknownAttempt(record: UsageUnknownModelCallRecord): Promise; } export interface PricingAuthorityReader { @@ -526,7 +540,13 @@ function createWriterFacade( read(() => modelCalls.buckets(query, groupBy, now)), modelCallLogs: (query, now, offset, limit) => read(() => modelCalls.logs(query, now, offset, limit)), + modelCallRunSettlementCoverage: (from, to) => + read(() => modelCalls.runSettlementCoverage(from, to)), catchUpModelCallProjection: admitModelCallProjectionCatchUp, + recordUsageUnknownAttempt: (record) => + admitSessionUsageMutation(record.sessionId, () => + modelCalls.recordUsageUnknownAttempt(record), + ), }, pricing: { snapshot: () => read(() => pricing.snapshot()), @@ -578,6 +598,8 @@ function modelCallReader( run(() => ledger.buckets(query, groupBy, now)), modelCallLogs: (query: UsageQuery, now: number, offset: number, limit: number) => run(() => ledger.logs(query, now, offset, limit)), + modelCallRunSettlementCoverage: (from: number, to: number) => + run(() => ledger.runSettlementCoverage(from, to)), }); }