From c7d97e274e3af8ca711098ae6bf9c20dae854638 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Wed, 30 Sep 2026 04:05:35 +0000 Subject: [PATCH 1/7] fix(spend): keep OAuth logs out of pool identity --- src/server/messages-native.ts | 1 + src/server/request-log.ts | 2 ++ src/server/responses/request-spend.ts | 6 ++++-- structure/transports/responses-spend.md | 4 ++++ .../responses/responses-spend-ledger-wiring.test.ts | 12 ++++++++++++ 5 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/server/messages-native.ts b/src/server/messages-native.ts index b593af2d1b4..6e82bf2544e 100644 --- a/src/server/messages-native.ts +++ b/src/server/messages-native.ts @@ -290,6 +290,7 @@ export async function handleNativeMessages(options: HandleNativeMessagesOptions) logCtx.inboundProtocol = "messages"; logCtx.model = route.modelId; logCtx.provider = route.providerName; + logCtx.spendPoolId = route.providerName; logCtx.providerAdapter = route.provider.adapter; logCtx.requestedModel = requestedModel; if (route.routeReason === "model-alias" || route.modelId !== requestedModel) logCtx.requestedAlias = requestedModel; diff --git a/src/server/request-log.ts b/src/server/request-log.ts index f7986a07e0e..f984671f11f 100644 --- a/src/server/request-log.ts +++ b/src/server/request-log.ts @@ -199,6 +199,8 @@ export interface RequestLogContext { spendOutputCeilingTokens?: number; /** Pre-send input estimate reserved for spend only; unlike usageLogInputTokens it never enters usage. */ spendInputEstimateTokens?: number; + /** Canonical provider-pool spend identity; independent of mutable, account-specific log labels. */ + spendPoolId?: string; /** Settles this request's durable spend entries from `addFinalRequestLog`. */ spendTracker?: RequestSpendSettlement; attempts?: PersistedUsageAttempt[]; diff --git a/src/server/responses/request-spend.ts b/src/server/responses/request-spend.ts index 8485b3d79dd..f18d031c36b 100644 --- a/src/server/responses/request-spend.ts +++ b/src/server/responses/request-spend.ts @@ -43,7 +43,7 @@ export interface RequestSpendTracker extends RequestSendObserver, RequestSpendSe export function createRequestSpendTracker( logCtx: Pick< RequestLogContext, - "provider" | "accountLogLabel" | "usageLogInputTokens" | "spendOutputCeilingTokens" | "spendInputEstimateTokens" + "provider" | "accountLogLabel" | "usageLogInputTokens" | "spendOutputCeilingTokens" | "spendInputEstimateTokens" | "spendPoolId" > & Partial>, rootId: string | undefined, injected?: SpendReservationLedger, @@ -88,7 +88,9 @@ export function createRequestSpendTracker( // Already the privacy-safe label the request log uses, and the ledger aliases it // again on the way to disk. A raw credential never reaches either. ...(logCtx.accountLogLabel !== undefined ? { identityId: logCtx.accountLogLabel } : {}), - ...(logCtx.provider !== undefined ? { poolId: logCtx.provider } : {}), + ...((logCtx.spendPoolId ?? logCtx.provider) !== undefined + ? { poolId: logCtx.spendPoolId ?? logCtx.provider } + : {}), }, inputTokens: logCtx.spendInputEstimateTokens ?? logCtx.usageLogInputTokens ?? 0, outputCeilingTokens: logCtx.spendOutputCeilingTokens ?? 0, diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index 6bfc03a8938..735e8084e48 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -102,6 +102,10 @@ so one ledger entry per increment is one entry per send, and a dispatch path add forget to book. The previous attempt at this wiring shipped the whole reserve/dispatch/settle vocabulary with no caller at all (#4707), which is the failure mode this shape rules out. +Provider-pool accounting uses the canonical routed provider identity, not the mutable display label +that may identify an OAuth account in request logs. Native Messages therefore shares the same pool +ceiling as Responses even when its Anthropic log label includes an account ordinal. + A booking is confirmed dispatched only once a LATER send exists, because that later send proves the earlier one left. The newest booking stays open, so a reservation the budget hands back during this process's lifetime can still be released for free. diff --git a/tests/responses/responses-spend-ledger-wiring.test.ts b/tests/responses/responses-spend-ledger-wiring.test.ts index e1c973fc055..2aafa48cd02 100644 --- a/tests/responses/responses-spend-ledger-wiring.test.ts +++ b/tests/responses/responses-spend-ledger-wiring.test.ts @@ -88,6 +88,18 @@ describe("the request path books every physical send on the durable ledger", () expect(root?.unresolved).toBe(1000); }); + test("a canonical pool identity is independent of an account-specific display label", () => { + const ledger = createSpendReservationLedger({ journal: memoryJournal() }); + const tracker = createRequestSpendTracker(logContext({ + provider: "anthropic-p123abc", + spendPoolId: "anthropic", + }), undefined, ledger); + + expect(tracker.charge()).toBe(true); + expect(ledger.snapshot("pool", "anthropic")?.reserved).toBe(500); + expect(ledger.snapshot("pool", "anthropic-p123abc")).toBeUndefined(); + }); + test("a reservation the budget hands back releases its tokens instead of booking spend", () => { const ledger = createSpendReservationLedger({ journal: memoryJournal() }); const tracker = createRequestSpendTracker(logContext(), "root-c", ledger); From 44b3e14f8b46a9e49e0cbb2a77df5157b9e8e3c3 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Thu, 1 Oct 2026 04:15:25 -0700 Subject: [PATCH 2/7] fix(spend): preserve explicit historical pool continuity Aggregate verified salted aliases without moving original balances; fail closed on unresolved pool history before dispatch. Preserve compatibility metadata in v1 checkpoints and document the supported rollback boundary. Add synthetic cross-version, retention, durability, and rootless preflight coverage. --- .../docs/reference/configuration/server.md | 56 ++ scripts/test-layout/layout.json | 1 + src/config/diagnostics.ts | 5 + src/lib/spend-pool-continuity.ts | 98 +++ src/lib/spend-reservation-ledger.ts | 109 +++- src/lib/workflow-budget.ts | 17 +- src/server/index.ts | 6 +- src/server/index/spend-ledger-lifecycle.ts | 6 +- src/server/responses/request-send-budget.ts | 10 +- src/server/responses/request-spend.ts | 9 +- src/server/workflow-refusal.ts | 12 + src/types/config.ts | 2 + structure/config.md | 2 +- structure/runtime.md | 2 +- structure/transports/responses-spend.md | 41 ++ tests/config/config-spend-ceilings.test.ts | 23 +- tests/fixtures/spend-ledger-f7a50dc3.ts.txt | 586 ++++++++++++++++++ tests/fixtures/test-layout-expected.json | 1 + tests/helpers/legacy-spend-ledger.ts | 16 + tests/lib/spend-pool-continuity.test.ts | 249 ++++++++ 20 files changed, 1216 insertions(+), 35 deletions(-) create mode 100644 src/lib/spend-pool-continuity.ts create mode 100644 tests/fixtures/spend-ledger-f7a50dc3.ts.txt create mode 100644 tests/helpers/legacy-spend-ledger.ts create mode 100644 tests/lib/spend-pool-continuity.test.ts diff --git a/docs-site/src/content/docs/reference/configuration/server.md b/docs-site/src/content/docs/reference/configuration/server.md index e97fab4cdca..ca72348a5fb 100644 --- a/docs-site/src/content/docs/reference/configuration/server.md +++ b/docs-site/src/content/docs/reference/configuration/server.md @@ -40,6 +40,7 @@ runs helper features around provider requests. | `codexProviderDisplayName?` | `string` | `"OpenCodex Proxy"` | Label Codex shows for the injected `opencodex` provider, written as its `name` field in `config.toml` and the reference profile. Presentation only: routing resolves through the provider id `opencodex`, so a rename never moves `model_provider = "opencodex"` or the `[model_providers.opencodex]` header and cannot orphan threads already tagged with that id. Codex refuses to load a provider with no name, so there is no way to omit the field — choose a neutral label instead. A blank, over-128-character, or control-character value is ignored and the default label is written. | | `resetCreditAutoRedeem?` | `{ enabled?: boolean; leadTimeMinutes?: number }` | off | Opt-in: redeem the main Codex account's soonest-expiring reset credit `leadTimeMinutes` (1–60, default 10) before it expires. Every attempt re-reads the upstream credit list first and skips when the credit is gone (for example, redeemed by hand); the `redeem_request_id` is journaled in `$OPENCODEX_HOME/reset-credit-auto-redeem.json` before the call so a crash replays the same idempotent request instead of spending a second credit. Servers sharing this configuration directory coordinate reservations and settlements so one process does not replace another's request record. Logs carry a hashed account key only. | | `syncResumeHistory?` | `boolean` | `true` | Reversible Codex App history compatibility. Original metadata is backed up and restored by `ocx stop` / `ocx restore`. | +| `spendPoolAliases?` | `Record` | unset | Explicit historical salted pool-alias to canonical provider mappings. Keep this key at the top level, outside `spend`. See [Historical pool continuity](#historical-pool-continuity). | | `shadowCallIntercept?` | `{ enabled?: boolean; model?: string; sourceModels?: string[] }` | off | Redirect recognized Codex helper/shadow calls to a chosen model while preserving the request's configured reasoning effort. The default source prefixes are `gpt-6-luna` and `gpt-5.6-luna`; older clients through 0.144.x used `gpt-5.4-mini`, which `sourceModels` can restore. | | `memoryModels?` | `{ extract?: { model: string; reasoningEffort?: string }; consolidation?: { model: string; reasoningEffort?: string } }` | off | Route Codex's two memory phases to a chosen model, with an optional reasoning effort per phase. See [Memory routing](#memory-routing). | | `webSearchSidecar?` | `OcxWebSearchSidecarConfig` | on when usable | Web-search sidecar options. | @@ -892,3 +893,58 @@ WebSocket control paths. See the canonical guide for [supported steering routes and settings](/guides/codex-integration/#steering-continuation-settings), [typed result and approval continuations](/guides/codex-integration/#rich-tool-results-and-explicit-approvals-after-response-completion), and [confirmation deadlines and retained context](/guides/codex-integration/#steering-confirmation-deadlines-and-retained-context). + + +## Historical pool continuity + +Provider-pool ceilings use the canonical routed provider, while request logs keep account-specific +display labels. Older journals may hold spend under those display labels. A journal stores salted +aliases, not the original provider/account names, so OpenCodex does not guess a mapping from a +current account roster, a label prefix, or a shortened account ID. + +If `spend.pool.maxTokens` is configured and positive historical pool balances remain unidentified, +inference admission returns local HTTP 429 with +`x-opencodex-local-refusal: workflow_pool_history_unresolved` before contacting a provider. +This can temporarily block otherwise valid requests, including requests without a workflow root. +After routing, preflight also refuses a canonical pool that is already exhausted by its combined +balances. This check does not turn post-reported passthrough sends into atomic reservations: +a crossing send, concurrent admissions or retries reported afterwards retain their existing limits. +Observe-only installs remain observe-only. Root and identity ceilings remain in force independently. + +To resolve it, an operator must verify which canonical provider each historical salted pool alias +belongs to, using their own retained evidence. Add a top-level `spendPoolAliases` object in +`config.json`: each key is the exact 32-character lowercase hexadecimal pool alias from that same +installation's journal; each value is its verified canonical provider ID. An old alias that already +represents the canonical provider still needs an explicit entry when it lacks identity metadata. +A positive alias that cannot be identified stays blocked. Do not infer a match from similar names, +account deletion, or a short-label collision, and do not share the journal or salt publicly. + +Keep `spendPoolAliases` outside `spend`: older versions reject unknown keys inside `spend` and can +disable the entire section. Invalid top-level mappings are rejected on configuration writes; +malformed `spendPoolAliases` hand edits retain existing ceilings and fail pool admission closed. Correct the mapping +and restart through the ordinary configuration workflow. Empty/removing mappings does not erase +links already recorded durably. A previously redirected alias cannot be reassigned to a different +group; verified canonical renames can join groups without splitting existing spend. + +Each original balance is counted once. Settled usage, in-flight reservations and unresolved usage +all count; unknown usage is never treated as a refund. Original reservation targets are retained. +New requests continue to record the canonical provider pool, and checkpoints retain salted identity +evidence without adding raw account/provider names. Unknown positive history and active or exhausted +groups are protected from cleanup; the existing dormant, under-limit retention rule still applies. +Identity evidence remains bounded; if its capacity is exhausted, pool admission stops rather than +forgetting it. No automatic tool reconstructs unverifiable history. + +### Safe rollback requirements + +A supported rollback must retain or backport both canonical request attribution and the +continuity-aware ledger reader/writer, with the same journal, salt and verified mappings. +Keep the latest journal: restoring a pre-upgrade copy would omit later spend. Do not remove records, +change the salt, raise ceilings, or disable enforcement to make a downgrade appear compatible. + +An unmodified older binary is **not a supported rollback**. It can read the v1 raw balances but +does not enforce the cross-alias provider total, may create a new account-label pool, and may discard +optional identity metadata when compacting. There is no automatic downgrade barrier. If such an old +writer has run, the new reader refuses unidentified positive history until the operator explicitly +verifies all remaining mappings again. Storage or journal-integrity denials are separate from an +alias problem and keep `workflow_spend_undurable`; unsafe-file/ownership failures may instead +propagate as storage errors, without admitting the request. diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 402eede4dc3..b5def6cb76c 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1632,6 +1632,7 @@ "spend-ledger-lifecycle.test.ts": "server", "spend-ledger-owner-startup.test.ts": "server", "spend-ledger-owner.test.ts": "lib", + "spend-pool-continuity.test.ts": "lib", "spend-reservation-ledger.test.ts": "lib", "sponsor-presets.test.ts": "providers", "sse-client-frame-bounds.test.ts": "responses", diff --git a/src/config/diagnostics.ts b/src/config/diagnostics.ts index dd559ed6097..bdbf180c909 100644 --- a/src/config/diagnostics.ts +++ b/src/config/diagnostics.ts @@ -1,3 +1,4 @@ +import { spendPoolAliasesError } from "../lib/spend-pool-continuity"; import { createHash } from "node:crypto"; import { lstatSync, readFileSync } from "node:fs"; import { join } from "node:path"; @@ -134,6 +135,9 @@ function validFileConfigDiagnostics(config: OcxConfig, rawParsed: unknown): Conf if (codexPoolWarning) warnings.push(codexPoolWarning); const spendWarning = malformedSpendWarning(rawParsed); if (spendWarning) warnings.push(spendWarning); + if (spendPoolAliasesError(rawConfigRecord(rawParsed)?.spendPoolAliases)) { + warnings.push("spendPoolAliases invalid: configured pool admission is refused until the mapping is corrected"); + } const plaintextWarning = malformedPlaintextV2AgentMessagesWarning(rawParsed); if (plaintextWarning) warnings.push(plaintextWarning); if (syncDisabledReason) { @@ -642,6 +646,7 @@ export function validateConfigCandidate(value: unknown): { ok: true; config: Ocx ?? agentTaskRecoveryError(value) ?? quotaResetNotifyError(value) ?? catalogAutoRefreshError(value) + ?? (spendPoolAliasesError(rawConfigRecord(value)?.spendPoolAliases) ? "schema_invalid: spendPoolAliases: invalid pool identity mapping" : null) ?? spendError(value) ?? codexPoolError(value) ?? googleAntigravityStaticCatalogVersionError(value) diff --git a/src/lib/spend-pool-continuity.ts b/src/lib/spend-pool-continuity.ts new file mode 100644 index 00000000000..991f8fe4541 --- /dev/null +++ b/src/lib/spend-pool-continuity.ts @@ -0,0 +1,98 @@ +/** Explicit, salted pool identity evidence. Never infer ownership from a label prefix. */ +export interface PoolBinding { readonly alias: string; readonly canonical: string } +export interface PoolContinuityRecord { + readonly v: 1; + readonly kind: "pool-continuity"; + readonly at: number; + readonly bindings: readonly PoolBinding[]; +} + +const isAlias = (value: unknown): value is string => + typeof value === "string" && /^[0-9a-f]{32}$/.test(value); + +/** Kept outside `spend`: older strict spend schemas must keep all configured ceilings. */ +export function spendPoolAliasesError(value: unknown): string | undefined { + if (value === undefined) return undefined; + if (!value || typeof value !== "object" || Array.isArray(value)) return "spendPoolAliases must be an object"; + if (Object.keys(value).length > 4_096) return "spendPoolAliases has too many entries"; + for (const [alias, provider] of Object.entries(value)) { + if (!isAlias(alias) || typeof provider !== "string" || !provider.trim() || provider.length > 256) { + return "spendPoolAliases requires exact 32-character salted pool aliases and nonempty provider IDs"; + } + } + return undefined; +} + +export function parsePoolContinuityRecord(value: unknown): PoolContinuityRecord | undefined { + if (!value || typeof value !== "object") return undefined; + const record = value as Record; + if (record.v !== 1 || record.kind !== "pool-continuity" + || typeof record.at !== "number" || !Number.isFinite(record.at) || record.at < 0 + || !Array.isArray(record.bindings) || record.bindings.length > 16_384) return undefined; + const bindings: PoolBinding[] = []; + const seen = new Set(); + for (const entry of record.bindings) { + if (!entry || typeof entry !== "object") return undefined; + const { alias, canonical } = entry as Partial; + if (!isAlias(alias) || !isAlias(canonical) || seen.has(alias)) return undefined; + seen.add(alias); + bindings.push({ alias, canonical }); + } + return { v: 1, kind: "pool-continuity", at: record.at, bindings }; +} + +/** A directed union: links may merge proven groups, never split or erase accounted spend. */ +export function createPoolContinuity() { + let bindings = new Map(); + const resolve = (alias: string, map = bindings): string => { + const seen = new Set(); + while (map.has(alias) && map.get(alias) !== alias) { + if (seen.has(alias)) throw new Error("cyclic pool identity evidence"); + seen.add(alias); + alias = map.get(alias)!; + } + return alias; + }; + const link = (map: Map, alias: string, canonical: string): boolean => { + const existing = map.get(alias); + const target = resolve(canonical, map); + if (existing !== undefined && existing !== alias && resolve(alias, map) !== target) return false; + // An explicit rename can join a formerly canonical alias to its successor. The old + // name still resolves to this same group, so neither a reload nor an old caller splits it. + if (target !== alias || existing === undefined) map.set(alias, target); + return true; + }; + return { + resolve: (alias: string): string => resolve(alias), + known: (alias: string): boolean => bindings.has(alias), + size: (): number => bindings.size, + record: (at: number): PoolContinuityRecord => ({ + v: 1, kind: "pool-continuity", at, + bindings: [...bindings].map(([alias, canonical]) => ({ alias, canonical })), + }), + restore(record: PoolContinuityRecord): boolean { + const next = new Map(bindings); + try { + for (const { alias, canonical } of record.bindings) if (!link(next, alias, canonical)) return false; + for (const alias of next.keys()) resolve(alias, next); + } catch { return false; } + bindings = next; + return true; + }, + prepare(config: unknown, requested: string | undefined, historical: ReadonlySet, + hash: (provider: string) => string, at: number, capacity: number): PoolContinuityRecord | false | undefined { + if (spendPoolAliasesError(config)) return false; + const next = new Map(bindings); + // Sort for deterministic multi-hop mappings regardless of JSON property order. Link + // conflicts fail closed; repeat entries and already-joined destinations are idempotent. + for (const [alias, provider] of Object.entries(config ?? {}).sort(([a], [b]) => a.localeCompare(b))) { + if (!link(next, alias, hash(provider as string))) return false; + } + if (requested !== undefined && !next.has(requested) && !historical.has(requested)) next.set(requested, requested); + if (next.size > Math.min(16_384, capacity)) return false; + if (next.size === bindings.size && [...next].every(([key, value]) => bindings.get(key) === value)) return undefined; + return { v: 1, kind: "pool-continuity", at, + bindings: [...next].map(([alias, canonical]) => ({ alias, canonical })) }; + }, + }; +} diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index 1a7ff2b4c64..04a5a313339 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -62,6 +62,7 @@ import { getConfigDir } from "../config/paths"; // Type-only, so it is erased before this module has a runtime import graph at all. The // config SHAPE is what this file needs; the config loader is what the note above keeps out. import type { OcxSpendConfig, OcxSpendScopeConfig } from "../types/config"; +import { createPoolContinuity, parsePoolContinuityRecord, type PoolContinuityRecord } from "./spend-pool-continuity"; import { assertNotRealHomeUnderTest } from "./test-home-guard"; // Windows chmod does not remove inherited ACEs; this is the repository's icacls path. import { hardenSecretPath } from "./windows-secret-acl"; @@ -93,6 +94,8 @@ export interface SpendScopeLimit { } export interface SpendReservationPolicy { + /** Exact historical salted pool alias -> canonical provider. Malformed input fails closed. */ + readonly poolAliases?: unknown; readonly root: SpendScopeLimit; readonly identity: SpendScopeLimit; readonly pool: SpendScopeLimit; @@ -181,6 +184,7 @@ export interface SpendReservationRequest { * to buy an unlimited number of physical sends while the scope totals never moved. */ export type SpendDenial = + | { readonly reason: "pool-history-unresolved" } | { readonly reason: "spend-limit-exceeded"; readonly scope: SpendScope; @@ -258,6 +262,7 @@ type JournalRecord = v: 1; kind: "checkpoint"; at: number; + poolContinuity?: PoolContinuityRecord; scopes: { scope: SpendScope; alias: string; settled: number; unresolved: number; seenAt: number }[]; sends: { send: string; status: ReservationStatus; targets: ScopeRef[]; tokens: number; at: number; resolvedAt: number }[]; }; @@ -350,7 +355,9 @@ export function parseSpendJournalRecord(line: string): JournalRecord | undefined if (!isCountable(e.tokens) || !isCountable(e.at) || !isCountable(e.resolvedAt)) return undefined; sends.push({ send: e.send, status: e.status, targets, tokens: e.tokens, at: e.at, resolvedAt: e.resolvedAt }); } - return { v: 1, kind: "checkpoint", at, scopes, sends }; + const poolContinuity = record.poolContinuity === undefined ? undefined : parsePoolContinuityRecord(record.poolContinuity); + if (record.poolContinuity !== undefined && !poolContinuity) return undefined; + return { v: 1, kind: "checkpoint", at, scopes, sends, ...(poolContinuity ? { poolContinuity } : {}) }; } default: return undefined; @@ -574,6 +581,8 @@ export interface ScopeSpendSnapshot { export interface SpendReservationLedger { reserve(request: SpendReservationRequest): SpendReservationDecision; + /** Pre-dispatch guard, including transports which report their sends after dispatch. */ + checkPoolContinuity(): SpendDenial | undefined; /** * The send left for upstream. Until this is called the reservation may be abandoned for * free; after it, a missing usage frame becomes unresolved spend. Returns false when the @@ -667,6 +676,7 @@ export function createSpendReservationLedger(options: { const compactAfterRecords = (): number => policy.compactAfterRecords ?? DEFAULT_COMPACT_AFTER_RECORDS; const scopes = new Map(); const reservations = new Map(); + const poolContinuity = createPoolContinuity(); let persistFailures = 0; let corruptRecords = 0; let recordsOnDisk = 0; @@ -690,6 +700,26 @@ export function createSpendReservationLedger(options: { return state; }; + const historicalPools = (): Set => new Set([...scopes].filter(([key, state]) => + key.startsWith("pool\0") && state.settled + state.reserved + state.unresolved > 0, + ).map(([key]) => key.slice(5))); + const unknownPoolHistory = (): boolean => [...historicalPools()].some(alias => !poolContinuity.known(alias)); + const poolState = (alias: string): ScopeState | undefined => { + const group = poolContinuity.resolve(alias); + let total: ScopeState | undefined; + for (const [key, state] of scopes) { + if (!key.startsWith("pool\0") || poolContinuity.resolve(key.slice(5)) !== group) continue; + total ??= { settled: 0, reserved: 0, unresolved: 0, lastSeenAt: 0 }; + total.settled += state.settled; + total.reserved += state.reserved; + total.unresolved += state.unresolved; + total.lastSeenAt = Math.max(total.lastSeenAt, state.lastSeenAt); + } + return total; + }; + const stateFor = (scope: SpendScope, alias: string): ScopeState | undefined => + scope === "pool" ? poolState(alias) : scopes.get(scopeKey(scope, alias)); + const limitFor = (scope: SpendScope): number | undefined => policy[scope].maxTokens; const isExhausted = (scope: SpendScope, state: ScopeState): boolean => { @@ -777,6 +807,7 @@ export function createSpendReservationLedger(options: { }; const applyCheckpoint = (record: Extract): void => { + if (record.poolContinuity && !poolContinuity.restore(record.poolContinuity)) corruptRecords += 1; scopes.clear(); reservations.clear(); for (const entry of record.scopes) { @@ -814,12 +845,13 @@ export function createSpendReservationLedger(options: { const line = lines[index] as string; const record = parseSpendJournalRecord(line); if (!record) { - // A rejected FINAL line is a torn tail write -- the process died between the write - // and its newline -- and is dropped quietly, because that record never completed and - // therefore never authorised anything. A rejected line ANYWHERE ELSE is different: - // the records after it did complete, so skipping it silently undercounts a scope and - // hands back budget. It is counted, and a configured limit refuses on it below. - if (index < lines.length - 1) corruptRecords += 1; + // A malformed final JSON line can be an incomplete tail write. Rejected records + // elsewhere cannot be skipped: later complete writes may have authorized spend. + // Only malformed JSON can be a torn tail. A complete unknown/invalid final + // record is evidence of an unsupported format, not permission to forget accounting. + let completeJson = false; + try { JSON.parse(line); completeJson = true; } catch { /* torn tail */ } + if (completeJson || index < lines.length - 1) corruptRecords += 1; continue; } switch (record.kind) { @@ -870,9 +902,12 @@ export function createSpendReservationLedger(options: { for (const [key, state] of scopes) { const separator = key.indexOf("\0"); const scope = key.slice(0, separator) as SpendScope; - if (state.reserved > 0) continue; - if (isExhausted(scope, state)) continue; - if (!force && state.lastSeenAt >= cutoff) continue; + const effective = scope === "pool" ? poolState(key.slice(separator + 1))! : state; + if (effective.reserved > 0) continue; + if (scope === "pool" && state.settled + state.unresolved > 0 + && !poolContinuity.known(key.slice(separator + 1))) continue; + if (isExhausted(scope, effective)) continue; + if (!force && effective.lastSeenAt >= cutoff) continue; candidates.push({ key, scope, alias: key.slice(separator + 1), seenAt: state.lastSeenAt }); } if (force) { @@ -915,10 +950,7 @@ export function createSpendReservationLedger(options: { * Bounded maps are not enough on their own: the file behind them is what replay reads, and * an uncompacted file grows forever on unique root and send ids. */ - const compact = (at: number): void => { - const rewrite = journal?.rewrite; - if (!journal || !rewrite || recordsOnDisk < compactAfterRecords()) return; - const checkpoint: JournalRecord = { + const checkpointRecord = (at: number, evidence = poolContinuity.record(at)): Extract => ({ v: 1, kind: "checkpoint", at, @@ -940,9 +972,15 @@ export function createSpendReservationLedger(options: { at: reservation.at, resolvedAt: reservation.resolvedAt, })), - }; + ...(evidence.bindings.length > 0 ? { poolContinuity: evidence } : {}), + }); + + const compact = (at: number): void => { + const rewrite = journal?.rewrite; + // A checkpoint must never erase evidence of replay corruption. + if (!journal || !rewrite || corruptRecords > 0 || recordsOnDisk < compactAfterRecords()) return; try { - rewrite.call(journal, [JSON.stringify(checkpoint)]); + rewrite.call(journal, [JSON.stringify(checkpointRecord(at))]); recordsOnDisk = 1; } catch (error) { if (error instanceof SpendLedgerOwnerError) throw error; @@ -952,6 +990,19 @@ export function createSpendReservationLedger(options: { } }; + const preparePoolContinuity = (at: number, requested?: string): SpendDenial | undefined => { + const evidence = poolContinuity.prepare(policy.poolAliases, requested, historicalPools(), + provider => aliasFor("pool", provider), at, maxTrackedScopes()); + if (evidence === false) return { reason: "pool-history-unresolved" }; + if (evidence) { + // A v1 checkpoint atomically carries the unchanged balances AND salted identity + // evidence. Older parsers can still read its balances; no unknown-record fence. + if (!append(checkpointRecord(at, evidence))) return { reason: "reserve-not-durable", sendId: "" }; + if (!poolContinuity.restore(evidence)) return { reason: "pool-history-unresolved" }; + } + return unknownPoolHistory() ? { reason: "pool-history-unresolved" } : undefined; + }; + /** The denial when tracking cannot fit this request, or undefined when it can. */ const makeRoom = (refs: readonly ScopeRef[], at: number): SpendDenial | undefined => { evictSends(at, false); @@ -977,6 +1028,13 @@ export function createSpendReservationLedger(options: { get degraded() { assertOwnedAccounting?.(); return persistFailures > 0 || corruptRecords > 0; }, get policy() { assertOwnedAccounting?.(); return policy; }, + checkPoolContinuity(): SpendDenial | undefined { + assertOwnedAccounting?.(); + if (policy.pool.maxTokens === undefined) return undefined; + if (corruptRecords > 0) return { reason: "journal-corrupt", corruptRecords }; + return preparePoolContinuity(now()); + }, + reserve(request: SpendReservationRequest): SpendReservationDecision { assertOwnedAccounting?.(); const tokens = sanitizeTokens(request.inputTokens) + sanitizeTokens(request.outputCeilingTokens); @@ -996,6 +1054,11 @@ export function createSpendReservationLedger(options: { if (enforced && corruptRecords > 0) { return { reserved: false, denial: { reason: "journal-corrupt", corruptRecords } }; } + const poolRef = refs.find(ref => ref.scope === "pool"); + const continuityDenial = preparePoolContinuity(at, poolRef?.alias); + if (continuityDenial && policy.pool.maxTokens !== undefined && request.alreadySent !== true) { + return { reserved: false, denial: continuityDenial }; + } const capacity = makeRoom(refs, at); if (capacity) return { reserved: false, denial: capacity }; @@ -1007,7 +1070,7 @@ export function createSpendReservationLedger(options: { // it is to let the total go OVER the ceiling so the next request can be refused. const limit = request.alreadySent === true ? undefined : limitFor(ref.scope); if (limit === undefined) continue; - const state = scopes.get(scopeKey(ref.scope, ref.alias)); + const state = stateFor(ref.scope, ref.alias); const projected = (state ? state.settled + state.reserved + state.unresolved : 0) + tokens; if (projected > limit) { const scopeId = ref.scope === "root" @@ -1088,7 +1151,7 @@ export function createSpendReservationLedger(options: { // Reading accounting from a handle whose ownership has ended is as wrong as writing it: // the figures describe a journal this process no longer owns. assertOwnedAccounting?.(); - const state = scopes.get(scopeKey(scope, aliasFor(scope, scopeId))); + const state = stateFor(scope, aliasFor(scope, scopeId)); if (!state) return undefined; return { settled: state.settled, @@ -1100,7 +1163,7 @@ export function createSpendReservationLedger(options: { exhausted(scope: SpendScope, scopeId: string): boolean { assertOwnedAccounting?.(); - const state = scopes.get(scopeKey(scope, aliasFor(scope, scopeId))); + const state = stateFor(scope, aliasFor(scope, scopeId)); return state !== undefined && isExhausted(scope, state); }, @@ -1120,6 +1183,11 @@ export function createSpendReservationLedger(options: { }; } +/** Does no I/O for installations without a pool ceiling. */ +export function sharedPoolContinuityDenial(): SpendDenial | undefined { + return sharedPolicy.pool.maxTokens === undefined ? undefined : sharedSpendLedger().checkPoolContinuity(); +} + let sharedLedger: SpendReservationLedger | undefined; /** * The operator policy in effect. Held beside the ledger rather than inside it because the @@ -1155,9 +1223,10 @@ const spendScopeLimitFromConfig = (scope: OcxSpendScopeConfig | undefined): Spen * ceiling would start refusing real traffic on the first upgrade that ran this code, against * a number nobody chose. There is deliberately no default figure here at all. */ -export function spendPolicyFromConfig(spend: OcxSpendConfig | undefined): SpendReservationPolicy { +export function spendPolicyFromConfig(spend: OcxSpendConfig | undefined, poolAliases?: unknown): SpendReservationPolicy { const retentionDays = spend?.retentionDays; return { + ...(poolAliases !== undefined ? { poolAliases } : {}), root: spendScopeLimitFromConfig(spend?.root), identity: spendScopeLimitFromConfig(spend?.identity), pool: spendScopeLimitFromConfig(spend?.pool), diff --git a/src/lib/workflow-budget.ts b/src/lib/workflow-budget.ts index ae12fee0753..41b72265b36 100644 --- a/src/lib/workflow-budget.ts +++ b/src/lib/workflow-budget.ts @@ -196,7 +196,8 @@ export type WorkflowDenial = /** This send id was already reserved once; a repeat buys no second dispatch. */ | "workflow-send-replayed" /** The reservation could not be made durable, and a configured ceiling requires it. */ - | "workflow-spend-undurable"; + | "workflow-spend-undurable" + | "workflow-pool-history-unresolved"; /** * The sentence an operator reads, plus the machine-readable name of the ceiling that fired. @@ -265,6 +266,8 @@ export function workflowDenialSummary( message: "This proxy refused the request locally: this send was already reserved once, and" + " a repeat buys no second dispatch.", }; + case "workflow-pool-history-unresolved": + return { code: "workflow_pool_history_unresolved", message: "Historical provider-pool spend needs verified alias mappings before dispatch. No provider was contacted." }; case "workflow-spend-undurable": return { code: "workflow_spend_undurable", @@ -582,6 +585,7 @@ export function admitWorkflowTurn( // an operator reading a 429 needs to know which of the three happened. const reason: WorkflowDenial = denial.reason === "duplicate-send-id" ? "workflow-send-replayed" + : denial.reason === "pool-history-unresolved" ? "workflow-pool-history-unresolved" : denial.reason === "reserve-not-durable" || denial.reason === "journal-corrupt" ? "workflow-spend-undurable" : denial.reason === "tracking-capacity-exhausted" @@ -718,7 +722,7 @@ function spentRootCeiling( } /** - * The token ceiling a root has already spent, or undefined when it has room or has none. + * A root or routed pool ceiling already spent, or undefined when neither is exhausted. * * The count-side twin of {@link workflowSendCeilingReached}, and the responses path calls both * at the same seam for the same reason: a refusal decided before dispatch can be reported as @@ -731,10 +735,15 @@ function spentRootCeiling( export function workflowSpendCeilingReached( rootId: string | undefined, spendLedger?: SpendReservationLedger, + poolId?: string, ): WorkflowSpendDenialDetail | undefined { - if (!rootId) return undefined; + if (!rootId && !poolId) return undefined; const ledger = spendLedger ?? (spendCeilingsConfigured() ? sharedSpendLedger() : undefined); - return ledger ? spentRootCeiling(rootId, ledger) : undefined; + if (!ledger) return undefined; + const root = rootId ? spentRootCeiling(rootId, ledger) : undefined; + if (root) return root; + const limit = ledger.policy.pool.maxTokens; + return poolId && limit !== undefined && ledger.exhausted("pool", poolId) ? { scope: "pool", limit } : undefined; } export interface WorkflowBudgetSnapshot { diff --git a/src/server/index.ts b/src/server/index.ts index bc827af06fe..78bb9fc8498 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -302,7 +302,7 @@ function startServerWithSpendLedgerOwner(port: number | undefined, deps: StartSe configureAppOwnedMemoryBudget(resolveAppOwnedMemoryBudgetBytes(config.appOwnedMemoryBudgetMb)); enforceAppOwnedMemoryBudget(); // Observe-only mode still journals physical sends, so every server owns before configuring. - spendLedgerLifecycle.configure(config.spend); + spendLedgerLifecycle.configure(config.spend, config.spendPoolAliases); // After ownership: a second server on the same home is refused above, so the process running // this line is the only one appending to usage.jsonl and the only one that may compact it. setUsageLedgerRetention(config.usageLedgerMaxBytes); @@ -507,7 +507,9 @@ function startServerWithSpendLedgerOwner(port: number | undefined, deps: StartSe const lease = tryAdmitTurn(sessionLaneIdFromRequest(req.headers)); if (!lease) return serverBusyResponse(req, "active turns", policy); // Root, lane, and the refusal that follows from them, all live in ./workflow-refusal. - const workflow = admitHttpWorkflowTurn(req.headers); + let workflow: ReturnType; + try { workflow = admitHttpWorkflowTurn(req.headers); } + catch (error) { lease.release(); throw error; } if (workflow && !workflow.admitted) { lease.release(); // withCors, because without Access-Control-Allow-Origin the exposed refusal header is diff --git a/src/server/index/spend-ledger-lifecycle.ts b/src/server/index/spend-ledger-lifecycle.ts index ddcc70a6a3b..a87f469cdbb 100644 --- a/src/server/index/spend-ledger-lifecycle.ts +++ b/src/server/index/spend-ledger-lifecycle.ts @@ -25,7 +25,7 @@ export function waitForFailedStartRollback(error: unknown): Promise { } export interface SpendLedgerServerLifecycle { - configure(spend: OcxSpendConfig | undefined): void; + configure(spend: OcxSpendConfig | undefined, poolAliases?: unknown): void; track }>(server: T): T; release(): void; releaseAfterFailedStart(): Promise; @@ -45,8 +45,8 @@ export function acquireSpendLedgerServerLifecycle(configDir: string): SpendLedge owner.release(); }; return { - configure(spend): void { - configureSharedSpendLedger(spendPolicyFromConfig(spend)); + configure(spend, poolAliases): void { + configureSharedSpendLedger(spendPolicyFromConfig(spend, poolAliases)); }, track }>(server: T): T { // Capture the raw stop before startServer replaces the public method with full teardown. diff --git a/src/server/responses/request-send-budget.ts b/src/server/responses/request-send-budget.ts index b8bc18e4fd9..bad16cf6426 100644 --- a/src/server/responses/request-send-budget.ts +++ b/src/server/responses/request-send-budget.ts @@ -5,7 +5,7 @@ import { workflowSendCeilingReached, workflowSpendCeilingReached, } from "../../lib/workflow-budget"; -import { workflowRefusalResponse } from "../workflow-refusal"; +import { poolContinuityRefusalReason, workflowRefusalResponse } from "../workflow-refusal"; import type { AttemptRecoveryKind, AttemptRecoveryWithheld } from "../../usage/log"; import { noteAttemptRecoveryWithheld, noteAttemptSend } from "../request-log"; import { TRANSIENT_RETRY_MAX_ATTEMPTS } from "../../lib/upstream-retry"; @@ -78,7 +78,13 @@ export function createResponsesSendBudget( // returns -- a refusal an operator cannot tell from an ordinary budget exhaustion, on a // ceiling they configured themselves. Asked before dispatch, it names the scope and the // number instead. Returns undefined and touches no ledger when no ceiling is configured. - const spentCeiling = workflowSpendCeilingReached(workflowRootId); + // Passthrough reports sends after they leave. Historical identity uncertainty must + // refuse here as well as in reserve(), including requests with no workflow root. + const continuityRefusal = poolContinuityRefusalReason(); + if (continuityRefusal) { + return workflowRefusalResponse(continuityRefusal, logCtx, undefined, workflowRootId); + } + const spentCeiling = workflowSpendCeilingReached(workflowRootId, undefined, logCtx.spendPoolId ?? logCtx.provider); if (spentCeiling) { return workflowRefusalResponse( "workflow-spend-exhausted", diff --git a/src/server/responses/request-spend.ts b/src/server/responses/request-spend.ts index f18d031c36b..8e86aac475d 100644 --- a/src/server/responses/request-spend.ts +++ b/src/server/responses/request-spend.ts @@ -106,7 +106,14 @@ export function createRequestSpendTracker( // configured -- so an install that configured nothing is still never refused here. // Capacity and a duplicate send id stay permissive: they say the ledger cannot account // for this send, which is a degradation to report, not an outage to cause. - if (denial.reason === "reserve-not-durable" || denial.reason === "journal-corrupt") { + if (denial.reason === "reserve-not-durable" || denial.reason === "journal-corrupt" + || denial.reason === "pool-history-unresolved") { + if (denial.reason === "pool-history-unresolved" && !alreadySent) { + const summary = workflowDenialSummary("workflow-pool-history-unresolved"); + markLocalRequestLogRefusal(logCtx, summary.code); + logCtx.errorCode = summary.code; + recordWorkflowRefusalEvent(rootId, "workflow-pool-history-unresolved", Date.now()); + } return alreadySent; } if (denial.reason !== "spend-limit-exceeded") return true; diff --git a/src/server/workflow-refusal.ts b/src/server/workflow-refusal.ts index 38bfd559584..e20c8e178d1 100644 --- a/src/server/workflow-refusal.ts +++ b/src/server/workflow-refusal.ts @@ -1,3 +1,4 @@ +import { sharedPoolContinuityDenial } from "../lib/spend-reservation-ledger"; /** * The one place that knows how this proxy refuses a turn on its own workflow budget. * @@ -104,8 +105,19 @@ export function workflowRefusalResponse( * children. A request that names a parent is treated as that fan-out; a top-level request is * the conversation and may use the reserved slots. */ +/** Keep storage/integrity failures distinct from unresolved identity evidence. */ +export function poolContinuityRefusalReason(): WorkflowDenial | undefined { + const denial = sharedPoolContinuityDenial(); + if (!denial) return undefined; + return denial.reason === "pool-history-unresolved" ? "workflow-pool-history-unresolved" : "workflow-spend-undurable"; +} + export function admitHttpWorkflowTurn(headers: Headers): WorkflowDecision | undefined { const rootId = headers.get("x-codex-parent-thread-id")?.trim() || undefined; + const continuityRefusal = poolContinuityRefusalReason(); + if (continuityRefusal) { + return { admitted: false, reason: continuityRefusal, rootId: rootId ?? "" }; + } const threadId = headers.get("thread-id")?.trim() || undefined; const lane: WorkflowLane = rootId !== undefined && threadId !== undefined && threadId !== rootId ? "worker" diff --git a/src/types/config.ts b/src/types/config.ts index 1c0f7c5c83c..b60d8e9f606 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -1033,6 +1033,8 @@ export interface OcxConfig { * upgrade against a number nobody chose. */ spend?: OcxSpendConfig; + /** Exact historical salted pool aliases mapped to canonical providers; retained verbatim for fail-closed validation. */ + spendPoolAliases?: Record; /** Opt-in per-account activation of newly reset Codex quota windows. */ codexQuotaAutoRefresh?: Record.proxy`, `providers..noProxy` | An absent `proxy` inherits global egress; `"direct"` or `null` forces direct egress; HTTP(S) and SOCKS5(H) URLs select a provider-owned proxy. `noProxy` uses NO_PROXY syntax and sends a matching destination direct across either a provider-owned or inherited global proxy. `src/lib/provider-egress.ts` owns parsing and request-local resolution. | diff --git a/structure/runtime.md b/structure/runtime.md index 85e0a73cb16..cb56f5ce573 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -216,7 +216,7 @@ and build a ledger by replaying that directory's own journal. A second process o directory is refused even for observe-only spend configuration, while a separate directory is independent. Ordinary stop releases the final reference after listener teardown, and every thrown startup path releases its reference. SQLite and the OS release a crashed owner; no PID, timestamp, -TTL or lock-file deletion participates in recovery. +TTL or lock-file deletion participates in recovery. Startup passes top-level pool-alias evidence with the spend policy. HTTP admission checks [historical continuity](transports/responses-spend.md#historical-pool-continuity-and-rollback) even without a root; failed accounting reads release the active-turn lease. An explicit Codex integration OFF skips startup cache invalidation before the user-scoped catalog serialization lock is resolved. Explicit `sync` and `sync-cache` retain their catalog-only override. diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index d401f8a35bc..74e82f1b33c 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -111,6 +111,47 @@ includes an account ordinal; root and account identity scopes remain independent Before reserving each combo hop, `core-combo.ts` updates the parent tracker's pool to the resolved target provider; child account labels and the logical `combo` label do not create separate pools. +### Historical pool continuity and rollback + +`src/lib/spend-pool-continuity.ts` accepts only explicit operator mappings from exact salted +historical pool aliases to canonical provider IDs, configured in top-level `spendPoolAliases`. +Current account rosters, label prefixes, short IDs, and renamed/deleted providers are not evidence +for automatically assigning old balances. Old canonical-looking aliases also need explicit +mapping when their positive history predates identity metadata. Zero-balance history needs none. + +The ledger retains original scope balances and reservation targets. The canonical view adds each +member once, keeping settled, reserved and unresolved buckets separate. Explicit links can join +previously canonical groups for a verified rename; an already redirected alias cannot be assigned +to a different group. A removed config entry never removes a journaled link. Group activity, +last-seen time and exhaustion govern retention; unidentified positive balances cannot be evicted. +Identity evidence is bounded and retained even when dormant under-limit scopes are evicted. + +Before a mapping authorizes admission, a v1 checkpoint durably carries both unchanged accounting +and optional salted `poolContinuity` metadata. No raw provider/account names are added to the +journal. New reservations still use the routed canonical pool ID. Replays and compaction preserve +the links; complete invalid metadata fails closed, including at the final line, and corruption is +never compacted away. Unparseable torn final JSON keeps the existing conservative replay rule. + +With a configured pool ceiling, any remaining unidentified positive pool history or invalid/ +conflicting mapping refuses admission. HTTP workflow admission and the Responses pre-dispatch +seam check this even without a root ID, before passthrough transports that report sends afterwards. +After routing, that seam also refuses an already-exhausted canonical pool, including mapped historical totals. +It is a snapshot check, not a new atomic reservation for report-only transports: crossing sends, +concurrent preflight admissions and retries reported afterwards retain their existing limitations. +`workflow_pool_history_unresolved` identifies the local 429 without disclosing aliases; storage +or replay failures keep `workflow_spend_undurable`. Already-sent reports still book actual spend. +Observe-only mode has no new token refusal, and root/identity accounting remains independent. + +Supported rollback retains/backports **both** canonical route attribution and the continuity-aware +reader/writer, using the same journal, salt and verified bindings. Restoring an older journal or +salt loses newer spend and is not a supported rollback. Unmodified older binaries are unsupported: +they can read v1 counters but do not enforce the canonical aggregate, can introduce fresh account +label pools, and can discard optional identity metadata during compaction. There is no automatic +downgrade barrier and no unknown-record fence. If that unsupported write has happened, the current +reader requires explicit mappings for the remaining unidentified balances rather than assuming +zero. `tests/lib/spend-pool-continuity.test.ts` exercises this using the frozen pre-change reader, +as well as exact aggregation, active/unresolved sends, retention, failures and rootless preflight. + A booking is confirmed dispatched only once a LATER send exists, because that later send proves the earlier one left. The newest booking stays open, so a reservation the budget hands back during this process's lifetime can still be released for free. diff --git a/tests/config/config-spend-ceilings.test.ts b/tests/config/config-spend-ceilings.test.ts index e80cc9f4154..87eab546feb 100644 --- a/tests/config/config-spend-ceilings.test.ts +++ b/tests/config/config-spend-ceilings.test.ts @@ -16,7 +16,7 @@ import { validateConfigCandidate, } from "../../src/config"; import { configDiagnosticsFromRaw } from "../../src/config/diagnostics"; -import { spendCeilingsConfigured, spendPolicyFromConfig } from "../../src/lib/spend-reservation-ledger"; +import { createSpendReservationLedger, spendCeilingsConfigured, spendPolicyFromConfig } from "../../src/lib/spend-reservation-ledger"; import { removeTreeWithRetry } from "../helpers/remove-tree"; let home = ""; @@ -125,3 +125,24 @@ test("a ceiling on any one scope is enough to turn enforcement on", () => { expect(spendCeilingsConfigured(spendPolicyFromConfig({ pool: { maxTokens: 1 } }))).toBe(true); expect(spendCeilingsConfigured(spendPolicyFromConfig({ retentionDays: 30 }))).toBe(false); }); + + +test("top-level pool aliases preserve all ceilings and malformed hand edits fail closed", () => { + const alias = "a".repeat(32); + const spend = { root: { maxTokens: 200 }, identity: { maxTokens: 150 }, pool: { maxTokens: 100 } }; + const valid = { ...candidate(spend), spendPoolAliases: { [alias]: "fixture-provider" } }; + expect(validateConfigCandidate(valid).ok).toBe(true); + for (const aliases of [null, [], { wrong: "fixture-provider" }, { [alias]: 1 }]) { + const raw = { ...valid, spendPoolAliases: aliases }; + expect(validateConfigCandidate(raw).ok).toBe(false); + writeFileSync(getConfigPath(), JSON.stringify(raw), "utf8"); + const loaded = loadConfig(); + expect(loaded.spend).toEqual(spend); + const resolved = spendPolicyFromConfig(loaded.spend, loaded.spendPoolAliases); + expect(resolved.pool.maxTokens).toBe(100); + expect(createSpendReservationLedger({ policy: resolved }).checkPoolContinuity()?.reason).toBe("pool-history-unresolved"); + } + // The old schema's passthrough top level retains this key without routing it through + // the strict spend object. A nested spelling remains rejected rather than blessed. + expect(validateConfigCandidate(candidate({ ...spend, poolAliases: { [alias]: "fixture-provider" } })).ok).toBe(false); +}); diff --git a/tests/fixtures/spend-ledger-f7a50dc3.ts.txt b/tests/fixtures/spend-ledger-f7a50dc3.ts.txt new file mode 100644 index 00000000000..48c5dae3d18 --- /dev/null +++ b/tests/fixtures/spend-ledger-f7a50dc3.ts.txt @@ -0,0 +1,586 @@ +// Unchanged parser and ledger function excerpts from lidge-jun/opencodex +// f7a50dc35b493d29f7a5c6bf2a459e5441549580:src/lib/spend-reservation-ledger.ts +// Frozen deliberately: tests execute the old implementation, never emulate it. +const isCountable = (value: unknown): value is number => + typeof value === "number" && Number.isFinite(value) && value >= 0; + +const isAlias = (value: unknown): value is string => + typeof value === "string" && value.length > 0 && value.length <= 256; + +const isScopeName = (value: unknown): value is SpendScope => + value === "root" || value === "identity" || value === "pool"; + +const isStatus = (value: unknown): value is ReservationStatus => + value === "open" || value === "dispatched" || value === "settled" + || value === "lost" || value === "abandoned"; + +const parseTargets = (value: unknown): ScopeRef[] | undefined => { + if (!Array.isArray(value) || value.length > 3) return undefined; + const targets: ScopeRef[] = []; + for (const entry of value) { + if (typeof entry !== "object" || entry === null) return undefined; + const { scope, alias } = entry as { scope?: unknown; alias?: unknown }; + if (!isScopeName(scope) || !isAlias(alias)) return undefined; + targets.push({ scope, alias }); + } + return targets; +}; + +/** + * Validate one journal line into a record, or reject it. + * + * Exported because this is the boundary where a hostile or damaged file meets the accounting: + * `JSON.parse(line) as JournalRecord` type-asserts a lie, and a bare `null` line or a + * `{"v":1,"kind":"reserve"}` with no fields crashed the rebuild rather than being rejected. + * Every field is checked, including that numbers are finite and non-negative. + */ +export function parseSpendJournalRecord(line: string): JournalRecord | undefined { + let raw: unknown; + try { + raw = JSON.parse(line); + } catch { + return undefined; + } + if (typeof raw !== "object" || raw === null || Array.isArray(raw)) return undefined; + const record = raw as Record; + if (record.v !== 1) return undefined; + if (!isCountable(record.at)) return undefined; + const at = record.at; + switch (record.kind) { + case "reserve": { + const targets = parseTargets(record.targets); + if (!isAlias(record.send) || targets === undefined || !isCountable(record.tokens)) return undefined; + return { v: 1, kind: "reserve", send: record.send, targets, tokens: record.tokens, at }; + } + case "settle": + if (!isAlias(record.send) || !isCountable(record.tokens)) return undefined; + return { v: 1, kind: "settle", send: record.send, tokens: record.tokens, at }; + case "dispatch": + if (!isAlias(record.send)) return undefined; + return { v: 1, kind: "dispatch", send: record.send, at }; + case "lost": + if (!isAlias(record.send)) return undefined; + return { v: 1, kind: "lost", send: record.send, at }; + case "abandon": + if (!isAlias(record.send)) return undefined; + return { v: 1, kind: "abandon", send: record.send, at }; + case "forget": + if (!isAlias(record.send)) return undefined; + return { v: 1, kind: "forget", send: record.send, at }; + case "drop": + if (!isScopeName(record.scope) || !isAlias(record.alias)) return undefined; + return { v: 1, kind: "drop", scope: record.scope, alias: record.alias, at }; + case "checkpoint": { + if (!Array.isArray(record.scopes) || !Array.isArray(record.sends)) return undefined; + const scopes: { scope: SpendScope; alias: string; settled: number; unresolved: number; seenAt: number }[] = []; + for (const entry of record.scopes) { + if (typeof entry !== "object" || entry === null) return undefined; + const e = entry as Record; + if (!isScopeName(e.scope) || !isAlias(e.alias)) return undefined; + if (!isCountable(e.settled) || !isCountable(e.unresolved) || !isCountable(e.seenAt)) return undefined; + scopes.push({ scope: e.scope, alias: e.alias, settled: e.settled, unresolved: e.unresolved, seenAt: e.seenAt }); + } + const sends: { send: string; status: ReservationStatus; targets: ScopeRef[]; tokens: number; at: number; resolvedAt: number }[] = []; + for (const entry of record.sends) { + if (typeof entry !== "object" || entry === null) return undefined; + const e = entry as Record; + const targets = parseTargets(e.targets); + if (!isAlias(e.send) || !isStatus(e.status) || targets === undefined) return undefined; + if (!isCountable(e.tokens) || !isCountable(e.at) || !isCountable(e.resolvedAt)) return undefined; + sends.push({ send: e.send, status: e.status, targets, tokens: e.tokens, at: e.at, resolvedAt: e.resolvedAt }); + } + return { v: 1, kind: "checkpoint", at, scopes, sends }; + } + default: + return undefined; + } +} + +const scopeKey = (scope: SpendScope, alias: string): string => scope + "\0" + alias; + +const sanitizeTokens = (value: number): number => + Number.isFinite(value) ? Math.max(0, Math.trunc(value)) : 0; + +export function createSpendReservationLedger(options: { + readonly journal?: SpendJournal; + readonly policy?: SpendReservationPolicy; + readonly now?: () => number; + /** + * Per-install alias salt. Production passes the file-backed value from + * `loadOrCreateSpendLedgerSalt`; an empty default is for in-memory journals, which have + * no file anyone could correlate. + */ + readonly salt?: string; + /** + * Shared production ledgers prove their exact ownership before reading or changing + * accounting. Identity, not just the directory name: a handle kept across a release and a + * reacquire describes a journal another writer may have changed in between. + */ + readonly assertOwnedAccounting?: () => void; +} = {}): SpendReservationLedger { + // Mutable because the ceilings are operator configuration, and configuration is reloadable. + // The three bounds below are read through functions for the same reason: a value captured + // at construction would answer for the policy this ledger was BUILT with, and an operator + // who raised a bound would keep the old one until the process restarted. + let policy = options.policy ?? DEFAULT_SPEND_RESERVATION_POLICY; + const journal = options.journal; + const now = options.now ?? (() => Date.now()); + const salt = options.salt ?? ""; + const assertOwnedAccounting = options.assertOwnedAccounting; + const maxTrackedScopes = (): number => policy.maxTrackedScopes ?? DEFAULT_MAX_TRACKED_SCOPES; + const maxTrackedSends = (): number => policy.maxTrackedSends ?? DEFAULT_MAX_TRACKED_SENDS; + const compactAfterRecords = (): number => policy.compactAfterRecords ?? DEFAULT_COMPACT_AFTER_RECORDS; + const scopes = new Map(); + const reservations = new Map(); + let persistFailures = 0; + let corruptRecords = 0; + let recordsOnDisk = 0; + + /** + * Salted alias for one identifier. The raw value -- a client-supplied root header, a + * credential id, a pool name -- never leaves this function, so nothing identifying is + * written to disk or held in a map key. + */ + const aliasFor = (kind: SpendScope | "send", id: string): string => + createHash("sha256").update(salt).update("\u0000").update(kind).update("\u0000").update(id) + .digest("hex").slice(0, 32); + + const scopeState = (scope: SpendScope, alias: string): ScopeState => { + const key = scopeKey(scope, alias); + let state = scopes.get(key); + if (!state) { + state = { settled: 0, reserved: 0, unresolved: 0, lastSeenAt: 0 }; + scopes.set(key, state); + } + return state; + }; + + const limitFor = (scope: SpendScope): number | undefined => policy[scope].maxTokens; + + const isExhausted = (scope: SpendScope, state: ScopeState): boolean => { + const limit = limitFor(scope); + return limit !== undefined && state.settled + state.reserved + state.unresolved >= limit; + }; + + /** The scopes a request touches, as aliases. Creates no state: a refusal must leave none. */ + const refsFor = (targets: SpendScopes): ScopeRef[] => { + const refs: ScopeRef[] = []; + if (targets.rootId !== undefined) refs.push({ scope: "root", alias: aliasFor("root", targets.rootId) }); + if (targets.identityId !== undefined) refs.push({ scope: "identity", alias: aliasFor("identity", targets.identityId) }); + if (targets.poolId !== undefined) refs.push({ scope: "pool", alias: aliasFor("pool", targets.poolId) }); + return refs; + }; + + /** + * Returns whether the record reached storage. With no journal there is nothing to fail, + * and the caller's durability question is vacuously satisfied. + */ + const append = (record: JournalRecord): boolean => { + if (!journal) return true; + try { + journal.append(JSON.stringify(record)); + recordsOnDisk += 1; + return true; + } catch (error) { + if (error instanceof SpendLedgerOwnerError) throw error; + // In-memory state still bounds this process; the counter is how a caller learns the + // restart guarantee degraded instead of discovering it after the fact. + persistFailures += 1; + return false; + } + }; + + const applyReserve = (send: string, targets: readonly ScopeRef[], tokens: number, at: number): void => { + if (reservations.has(send)) return; + reservations.set(send, { targets, tokens, status: "open", at, resolvedAt: at }); + for (const ref of targets) { + const state = scopeState(ref.scope, ref.alias); + state.reserved += tokens; + state.lastSeenAt = Math.max(state.lastSeenAt, at); + } + }; + + const isLive = (status: ReservationStatus): boolean => status === "open" || status === "dispatched"; + + /** + * Resolve a live reservation. `settled` books the real figure, `lost` keeps the whole + * reservation as unresolved spend because it may have been billed, and `abandoned` + * releases it because no byte ever left this process. + */ + const applyResolve = (send: string, outcome: "settled" | "lost" | "abandoned", tokens: number, at: number): void => { + const reservation = reservations.get(send); + if (!reservation || !isLive(reservation.status)) return; + reservation.status = outcome; + reservation.resolvedAt = at; + for (const ref of reservation.targets) { + const state = scopeState(ref.scope, ref.alias); + state.reserved = Math.max(0, state.reserved - reservation.tokens); + if (outcome === "lost") state.unresolved += reservation.tokens; + else if (outcome === "settled") state.settled += tokens; + state.lastSeenAt = Math.max(state.lastSeenAt, at); + } + }; + + const applyDispatch = (send: string, at: number): void => { + const reservation = reservations.get(send); + if (!reservation || reservation.status !== "open") return; + reservation.status = "dispatched"; + reservation.resolvedAt = at; + }; + + /** Tombstone replay: the entry is gone, so a later reuse of the id books a fresh charge. */ + const applyForget = (send: string): void => { + const reservation = reservations.get(send); + if (!reservation || isLive(reservation.status)) return; + reservations.delete(send); + }; + + const applyDrop = (scope: SpendScope, alias: string): void => { + const state = scopes.get(scopeKey(scope, alias)); + if (!state || state.reserved > 0) return; + scopes.delete(scopeKey(scope, alias)); + }; + + const applyCheckpoint = (record: Extract): void => { + scopes.clear(); + reservations.clear(); + for (const entry of record.scopes) { + scopes.set(scopeKey(entry.scope, entry.alias), { + settled: entry.settled, + reserved: 0, + unresolved: entry.unresolved, + lastSeenAt: entry.seenAt, + }); + } + for (const entry of record.sends) { + // `reserved` is rebuilt from the live entries rather than trusted from the snapshot, + // so the two can never disagree about the same tokens. + if (isLive(entry.status)) { + applyReserve(entry.send, entry.targets, entry.tokens, entry.at); + if (entry.status === "dispatched") applyDispatch(entry.send, entry.resolvedAt); + continue; + } + reservations.set(entry.send, { + targets: entry.targets, + tokens: entry.tokens, + status: entry.status, + at: entry.at, + resolvedAt: entry.resolvedAt, + }); + } + }; + + // Rebuild from the journal before serving: an exhausted scope must still be exhausted + // after a restart, which is the whole reason this store exists. + if (journal) { + const lines = journal.read(); + recordsOnDisk = lines.length; + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index] as string; + const record = parseSpendJournalRecord(line); + if (!record) { + // A rejected FINAL line is a torn tail write -- the process died between the write + // and its newline -- and is dropped quietly, because that record never completed and + // therefore never authorised anything. A rejected line ANYWHERE ELSE is different: + // the records after it did complete, so skipping it silently undercounts a scope and + // hands back budget. It is counted, and a configured limit refuses on it below. + if (index < lines.length - 1) corruptRecords += 1; + continue; + } + switch (record.kind) { + case "reserve": applyReserve(record.send, record.targets, sanitizeTokens(record.tokens), record.at); break; + case "dispatch": applyDispatch(record.send, record.at); break; + case "settle": applyResolve(record.send, "settled", sanitizeTokens(record.tokens), record.at); break; + case "lost": applyResolve(record.send, "lost", 0, record.at); break; + case "abandon": applyResolve(record.send, "abandoned", 0, record.at); break; + case "forget": applyForget(record.send); break; + case "drop": applyDrop(record.scope, record.alias); break; + case "checkpoint": applyCheckpoint(record); break; + } + } + // A reservation that survived replay has no owner left. The process that made it is gone, + // so nothing in this one can ever settle it, and leaving it live means the send stays + // pending forever against a scope that can never resolve it. Deleting the entry is not the + // alternative either: that would hand the same send id a second reservation. + // + // Both live states resolve to UNRESOLVED, including an undispatched one. The tempting + // distinction -- open never reached the wire, so give its tokens back -- assumes the + // journal is complete up to the crash, and the torn-tail handling above says it is not: a + // send can dispatch and die before its dispatch record lands. Abandoning that reservation + // returns tokens for a send that may have been billed, and worse, it RESETS a ceiling that + // had already fired. An exhausted scope staying exhausted across a restart is the whole + // reason this store is on disk. + const reconciledAt = now(); + for (const [send, reservation] of reservations) { + if (!isLive(reservation.status)) continue; + applyResolve(send, "lost", 0, reconciledAt); + append({ v: 1, kind: "lost", send, at: reconciledAt }); + } + } + + /** + * Bounded cleanup. It runs before every admission, so nothing depends on a caller + * remembering `prune()` -- the first draft exported one and no production path called it. + * Every removal writes a tombstone: without one, replay rebuilds precisely what cleanup + * removed and the file keeps growing while the maps look bounded. + * + * `force` is the at-capacity pass. It ignores the retention window but never the safety + * rule: an ACTIVE or EXHAUSTED scope is not a candidate at any pressure, because dropping + * one hands it a fresh allowance under the same id. When that leaves nothing to remove, + * the caller refuses admission rather than making room by forgetting a spent scope. + */ + const evictScopes = (at: number, force: boolean): number => { + const cutoff = at - policy.retentionMs; + const candidates: { key: string; scope: SpendScope; alias: string; seenAt: number }[] = []; + for (const [key, state] of scopes) { + const separator = key.indexOf("\0"); + const scope = key.slice(0, separator) as SpendScope; + if (state.reserved > 0) continue; + if (isExhausted(scope, state)) continue; + if (!force && state.lastSeenAt >= cutoff) continue; + candidates.push({ key, scope, alias: key.slice(separator + 1), seenAt: state.lastSeenAt }); + } + if (force) { + candidates.sort((a, b) => a.seenAt - b.seenAt); + candidates.length = Math.min(candidates.length, 1); + } + for (const candidate of candidates) { + scopes.delete(candidate.key); + append({ v: 1, kind: "drop", scope: candidate.scope, alias: candidate.alias, at }); + } + return candidates.length; + }; + + /** + * Forget resolved send ids. A forgotten id is forgotten COMPLETELY: reusing it later books + * a fresh reservation against every scope, which is conservative. The state this must never + * produce is the middle one -- an id the ledger recognises but charges nothing for. + */ + const evictSends = (at: number, force: boolean): number => { + const cutoff = at - policy.retentionMs; + const candidates: { send: string; resolvedAt: number }[] = []; + for (const [send, reservation] of reservations) { + if (isLive(reservation.status)) continue; + if (!force && reservation.resolvedAt >= cutoff) continue; + candidates.push({ send, resolvedAt: reservation.resolvedAt }); + } + if (force) { + candidates.sort((a, b) => a.resolvedAt - b.resolvedAt); + candidates.length = Math.min(candidates.length, 1); + } + for (const candidate of candidates) { + reservations.delete(candidate.send); + append({ v: 1, kind: "forget", send: candidate.send, at }); + } + return candidates.length; + }; + + /** + * Replace the journal with a single checkpoint once it has grown past its record budget. + * Bounded maps are not enough on their own: the file behind them is what replay reads, and + * an uncompacted file grows forever on unique root and send ids. + */ + const compact = (at: number): void => { + const rewrite = journal?.rewrite; + if (!journal || !rewrite || recordsOnDisk < compactAfterRecords()) return; + const checkpoint: JournalRecord = { + v: 1, + kind: "checkpoint", + at, + scopes: [...scopes].map(([key, state]) => { + const separator = key.indexOf("\0"); + return { + scope: key.slice(0, separator) as SpendScope, + alias: key.slice(separator + 1), + settled: state.settled, + unresolved: state.unresolved, + seenAt: state.lastSeenAt, + }; + }), + sends: [...reservations].map(([send, reservation]) => ({ + send, + status: reservation.status, + targets: [...reservation.targets], + tokens: reservation.tokens, + at: reservation.at, + resolvedAt: reservation.resolvedAt, + })), + }; + try { + rewrite.call(journal, [JSON.stringify(checkpoint)]); + recordsOnDisk = 1; + } catch (error) { + if (error instanceof SpendLedgerOwnerError) throw error; + // Compaction is maintenance, not accounting: a failed rewrite leaves the previous + // journal intact and every figure in it still replayable. + persistFailures += 1; + } + }; + + /** The denial when tracking cannot fit this request, or undefined when it can. */ + const makeRoom = (refs: readonly ScopeRef[], at: number): SpendDenial | undefined => { + evictSends(at, false); + evictScopes(at, false); + while (reservations.size >= maxTrackedSends()) { + if (evictSends(at, true) === 0) return { reason: "tracking-capacity-exhausted" }; + } + let fresh = 0; + for (const ref of refs) if (!scopes.has(scopeKey(ref.scope, ref.alias))) fresh += 1; + while (scopes.size + fresh > maxTrackedScopes()) { + if (evictScopes(at, true) === 0) { + return { reason: "tracking-capacity-exhausted", scope: refs[0]?.scope }; + } + } + return undefined; + }; + + return { + // Every figure this ledger reports describes a journal it must still own. Reporting one + // after ownership ended is the same error as writing then, with a quieter symptom. + get persistFailures() { assertOwnedAccounting?.(); return persistFailures; }, + get corruptRecords() { assertOwnedAccounting?.(); return corruptRecords; }, + get degraded() { assertOwnedAccounting?.(); return persistFailures > 0 || corruptRecords > 0; }, + get policy() { assertOwnedAccounting?.(); return policy; }, + + reserve(request: SpendReservationRequest): SpendReservationDecision { + assertOwnedAccounting?.(); + const tokens = sanitizeTokens(request.inputTokens) + sanitizeTokens(request.outputCeilingTokens); + const at = request.at ?? now(); + const send = aliasFor("send", request.sendId); + const refs = refsFor(request.scopes); + const enforced = refs.some((ref) => limitFor(ref.scope) !== undefined); + + // A send id this ledger already knows is REFUSED. Returning success while booking + // nothing -- the old behaviour -- let one id authorise an unlimited number of physical + // sends with the scope totals never moving. + if (reservations.has(send)) { + return { reserved: false, denial: { reason: "duplicate-send-id", sendId: request.sendId } }; + } + // Replay could not prove these totals are complete, so a configured ceiling cannot be + // enforced on them. Observe-only accounting continues and reports the degradation. + if (enforced && corruptRecords > 0) { + return { reserved: false, denial: { reason: "journal-corrupt", corruptRecords } }; + } + const capacity = makeRoom(refs, at); + if (capacity) return { reserved: false, denial: capacity }; + + // Check every scope before mutating any: a refusal must not leave a partial + // reservation booked on the scopes that would have passed. Reading state without + // creating it matters here -- a denied request must not leave a tracked scope behind. + for (const ref of refs) { + // A recorded send has no limit to fail: it already happened, and the point of booking + // it is to let the total go OVER the ceiling so the next request can be refused. + const limit = request.alreadySent === true ? undefined : limitFor(ref.scope); + if (limit === undefined) continue; + const state = scopes.get(scopeKey(ref.scope, ref.alias)); + const projected = (state ? state.settled + state.reserved + state.unresolved : 0) + tokens; + if (projected > limit) { + const scopeId = ref.scope === "root" + ? request.scopes.rootId + : ref.scope === "identity" ? request.scopes.identityId : request.scopes.poolId; + return { + reserved: false, + denial: { reason: "spend-limit-exceeded", scope: ref.scope, scopeId: scopeId ?? "", limit, projected }, + }; + } + } + + // Durability BEFORE admission. The record goes to disk first, and under a configured + // limit a failed write refuses the request rather than admitting one that a restart + // would forget -- which is exactly the disk-full and permission case durability is for. + const durable = append({ v: 1, kind: "reserve", send, targets: refs, tokens, at }); + if (!durable && enforced && request.alreadySent !== true) { + return { reserved: false, denial: { reason: "reserve-not-durable", sendId: request.sendId } }; + } + applyReserve(send, refs, tokens, at); + compact(at); + return { reserved: true, sendId: request.sendId, tokens, durable }; + }, + + markDispatched(sendId: string): boolean { + assertOwnedAccounting?.(); + const send = aliasFor("send", sendId); + const reservation = reservations.get(send); + if (!reservation || reservation.status !== "open") return false; + const at = now(); + applyDispatch(send, at); + append({ v: 1, kind: "dispatch", send, at }); + return true; + }, + + abandon(sendId: string): boolean { + assertOwnedAccounting?.(); + const send = aliasFor("send", sendId); + const reservation = reservations.get(send); + // Only an UNDISPATCHED reservation may be released for free. Once bytes have left for + // upstream the tokens may already be billed, so the caller owes settle or markLost. + if (!reservation || reservation.status !== "open") return false; + const at = now(); + applyResolve(send, "abandoned", 0, at); + append({ v: 1, kind: "abandon", send, at }); + return true; + }, + + settle(sendId: string, usage: SpendUsage): boolean { + assertOwnedAccounting?.(); + const send = aliasFor("send", sendId); + const reservation = reservations.get(send); + if (!reservation || !isLive(reservation.status)) return false; + const tokens = sanitizeTokens(usage.inputTokens) + sanitizeTokens(usage.outputTokens); + const at = now(); + applyResolve(send, "settled", tokens, at); + append({ v: 1, kind: "settle", send, tokens, at }); + return true; + }, + + markLost(sendId: string): boolean { + assertOwnedAccounting?.(); + const send = aliasFor("send", sendId); + const reservation = reservations.get(send); + if (!reservation || !isLive(reservation.status)) return false; + const at = now(); + applyResolve(send, "lost", 0, at); + append({ v: 1, kind: "lost", send, at }); + return true; + }, + + knows(sendId: string): boolean { + assertOwnedAccounting?.(); + return reservations.has(aliasFor("send", sendId)); + }, + + snapshot(scope: SpendScope, scopeId: string): ScopeSpendSnapshot | undefined { + // Reading accounting from a handle whose ownership has ended is as wrong as writing it: + // the figures describe a journal this process no longer owns. + assertOwnedAccounting?.(); + const state = scopes.get(scopeKey(scope, aliasFor(scope, scopeId))); + if (!state) return undefined; + return { + settled: state.settled, + reserved: state.reserved, + unresolved: state.unresolved, + exhausted: isExhausted(scope, state), + }; + }, + + exhausted(scope: SpendScope, scopeId: string): boolean { + assertOwnedAccounting?.(); + const state = scopes.get(scopeKey(scope, aliasFor(scope, scopeId))); + return state !== undefined && isExhausted(scope, state); + }, + + prune(at: number = now()): void { + assertOwnedAccounting?.(); + // Removal requires BOTH inactive and not exhausted inside the window. An + // exhausted-but-idle scope that was dropped would be recreated fresh under the + // same id -- the exact laundering the ceiling exists to stop. + evictSends(at, false); + evictScopes(at, false); + }, + + reconfigure(next: SpendReservationPolicy): void { + assertOwnedAccounting?.(); + policy = next; + }, + }; +} diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 4303c879bac..22cd290dade 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1646,6 +1646,7 @@ "spend-ledger-lifecycle.test.ts": "server", "spend-ledger-owner-startup.test.ts": "server", "spend-ledger-owner.test.ts": "lib", + "spend-pool-continuity.test.ts": "lib", "spend-reservation-ledger.test.ts": "lib", "sponsor-presets.test.ts": "providers", "sse-client-frame-bounds.test.ts": "responses", diff --git a/tests/helpers/legacy-spend-ledger.ts b/tests/helpers/legacy-spend-ledger.ts new file mode 100644 index 00000000000..2de285d36ed --- /dev/null +++ b/tests/helpers/legacy-spend-ledger.ts @@ -0,0 +1,16 @@ +import { readFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { fixturePath } from "./repo-root"; +import { DEFAULT_SPEND_RESERVATION_POLICY, type SpendJournal, type SpendReservationLedger, type SpendReservationPolicy } from "../../src/lib/spend-reservation-ledger"; + +/** Execute immutable pre-continuity production code, with only its ordinary imports supplied. */ +const source = readFileSync(fixturePath("spend-ledger-f7a50dc3.ts.txt"), "utf8"); +const code = new Bun.Transpiler({ loader: "ts", target: "bun" }) + .transformSync(source.replaceAll("export function", "function")); +export const createLegacySpendLedger = new Function( + "createHash", "DEFAULT_SPEND_RESERVATION_POLICY", "DEFAULT_MAX_TRACKED_SCOPES", + "DEFAULT_MAX_TRACKED_SENDS", "DEFAULT_COMPACT_AFTER_RECORDS", "SpendLedgerOwnerError", + `${code}\nreturn createSpendReservationLedger;`, +)(createHash, DEFAULT_SPEND_RESERVATION_POLICY, 4_096, 16_384, 8_192, class SpendLedgerOwnerError extends Error {}) as + (options: { journal: SpendJournal; salt?: string; policy: SpendReservationPolicy; now: () => number }) => + Pick; diff --git a/tests/lib/spend-pool-continuity.test.ts b/tests/lib/spend-pool-continuity.test.ts new file mode 100644 index 00000000000..5f47660df84 --- /dev/null +++ b/tests/lib/spend-pool-continuity.test.ts @@ -0,0 +1,249 @@ +import { createLegacySpendLedger } from "../helpers/legacy-spend-ledger"; +import { describe, expect, test } from "bun:test"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createHash } from "node:crypto"; +import { + createSpendReservationLedger, configureSharedSpendLedger, DEFAULT_SPEND_RESERVATION_POLICY, parseSpendJournalRecord, + type SpendJournal, type SpendReservationPolicy, +} from "../../src/lib/spend-reservation-ledger"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { admitHttpWorkflowTurn, workflowDecisionRefusalResponse } from "../../src/server/workflow-refusal"; +import { createResponsesSendBudget } from "../../src/server/responses/request-send-budget"; +import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; + +const salt = "5".repeat(64); +const alias = (kind: string, id: string) => createHash("sha256").update(salt).update("\0").update(kind).update("\0").update(id).digest("hex").slice(0, 32); +const pool = (id: string) => alias("pool", id); +const policy = (poolAliases?: unknown, overrides: Partial = {}): SpendReservationPolicy => ({ + ...DEFAULT_SPEND_RESERVATION_POLICY, pool: { maxTokens: 100 }, poolAliases, ...overrides, +}); +const journal = (records: unknown[] = []): SpendJournal & { lines: string[] } => { + const lines = records.map(record => JSON.stringify(record)); + return { lines, read: () => [...lines], append: line => { lines.push(line); }, + rewrite: next => { lines.splice(0, lines.length, ...next); } }; +}; +const checkpoint = (entries: Array<[string, number, number]>) => ({ + v: 1, kind: "checkpoint", at: 1, + scopes: entries.map(([id, settled, unresolved]) => ({ scope: "pool", alias: pool(id), settled, unresolved, seenAt: 1 })), sends: [], +}); +const reserve = (ledger: ReturnType, sendId: string, poolId = "provider", tokens = 1, alreadySent = false) => + ledger.reserve({ sendId, scopes: { poolId }, inputTokens: tokens, outputCeilingTokens: 0, alreadySent }); + +describe("historical pool identity continuity", () => { + test("unmapped historical debt fails closed across labels, providers, pruning and restart", () => { + const disk = journal([checkpoint([["provider-old-label", 100, 0]])]); + for (let restart = 0; restart < 2; restart += 1) { + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 100_000 }); + ledger.prune(); + for (const id of ["provider", "provider-old-label", "unrelated-provider"]) { + expect(reserve(ledger, `${restart}-${id}`, id)).toMatchObject({ reserved: false, denial: { reason: "pool-history-unresolved" } }); + } + expect(ledger.snapshot("pool", "provider-old-label")?.settled).toBe(100); + } + }); + + test("explicit aliases aggregate each original balance once, including canonical history", () => { + const disk = journal([checkpoint([["label-a", 40, 0], ["label-b", 0, 30], ["provider", 20, 0]])]); + const aliases = { [pool("label-a")]: "provider", [pool("label-b")]: "provider", [pool("provider")]: "provider" }; + let ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(aliases, { compactAfterRecords: 1 }), now: () => 2 }); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + expect(ledger.snapshot("pool", "provider")).toEqual({ settled: 60, unresolved: 30, reserved: 0, exhausted: false }); + expect(reserve(ledger, "new", "provider", 10).reserved).toBe(true); + expect(ledger.settle("new", { inputTokens: 10, outputTokens: 0 })).toBe(true); + expect(ledger.settle("new", { inputTokens: 10, outputTokens: 0 })).toBe(false); + // Clearing config never removes durable evidence. Repeated replay/compaction never adds + // a migrated copy of a balance or charges a canonical self-alias twice. + for (let restart = 0; restart < 3; restart += 1) { + ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(undefined, { compactAfterRecords: 1 }), now: () => 3 }); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + expect(ledger.snapshot("pool", "provider")).toEqual({ settled: 70, unresolved: 30, reserved: 0, exhausted: true }); + expect(reserve(ledger, `denied-${restart}`)).toMatchObject({ reserved: false, denial: { reason: "spend-limit-exceeded", projected: 101 } }); + } + expect(disk.lines.join("\n")).not.toContain("label-a"); + expect(disk.lines.join("\n")).not.toContain("provider"); + }); + + test("old open/dispatched sends become unresolved exactly once; duplicate send IDs remain refused", () => { + const old = ["a", "b"].flatMap(id => [ + { v: 1, kind: "reserve", send: alias("send", id), targets: [{ scope: "pool", alias: pool(`label-${id}`) }], tokens: 30, at: 1 }, + ...(id === "b" ? [{ v: 1, kind: "dispatch", send: alias("send", id), at: 1 }] : []), + ]); + const disk = journal(old); + const aliases = { [pool("label-a")]: "provider", [pool("label-b")]: "provider" }; + for (let count = 0; count < 2; count += 1) { + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(aliases), now: () => 2 }); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + expect(ledger.snapshot("pool", "provider")?.unresolved).toBe(60); + expect(reserve(ledger, "a")).toMatchObject({ reserved: false, denial: { reason: "duplicate-send-id" } }); + } + }); + + test("a live original reservation settles/refunds once after explicit linking", () => { + const ledger = createSpendReservationLedger({ salt, policy: policy(), now: () => 2 }); + expect(reserve(ledger, "pending", "label", 40).reserved).toBe(true); + expect(reserve(ledger, "refund", "label", 10).reserved).toBe(true); + ledger.reconfigure(policy({ [pool("label")]: "provider" })); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + expect(ledger.abandon("refund")).toBe(true); + expect(ledger.settle("pending", { inputTokens: 30, outputTokens: 0 })).toBe(true); + expect(ledger.snapshot("pool", "provider")).toEqual({ settled: 30, reserved: 0, unresolved: 0, exhausted: false }); + }); + + test("zero/abandoned-only historical scopes do not create debt", () => { + const ledger = createSpendReservationLedger({ journal: journal([checkpoint([["empty-old", 0, 0]])]), salt, policy: policy(), now: () => 2 }); + expect(reserve(ledger, "new").reserved).toBe(true); + }); + + test("unknown history and aggregate exhaustion survive retention and capacity pressure", () => { + const disk = journal([checkpoint([["label-a", 60, 0], ["label-b", 40, 0]])]); + const ledger = createSpendReservationLedger({ journal: disk, salt, + policy: policy({ [pool("label-a")]: "provider", [pool("label-b")]: "provider" }, { retentionMs: 1, maxTrackedScopes: 2 }), now: () => 100 }); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + ledger.prune(); + expect(ledger.snapshot("pool", "provider")?.settled).toBe(100); + expect(reserve(ledger, "new").reserved).toBe(false); + expect(ledger.snapshot("pool", "provider")?.settled).toBe(100); + }); + + test("under-limit historical components remain while their canonical group is active", () => { + const disk = journal([checkpoint([["label", 40, 0]])]); + let now = 2; + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy({ [pool("label")]: "provider" }, { retentionMs: 5 }), now: () => now }); + expect(reserve(ledger, "pending", "provider", 10).reserved).toBe(true); + now = 100; + ledger.prune(); + expect(ledger.snapshot("pool", "provider")).toMatchObject({ settled: 40, reserved: 10 }); + }); + + test("observe-only records already-sent requests while ambiguity still blocks new dispatches", () => { + const disk = journal([checkpoint([["unknown-label", 40, 0]])]); + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 2 }); + const tracker = createRequestSpendTracker({ provider: "provider-display", spendPoolId: "provider", usageLogInputTokens: 10 }, undefined, ledger); + expect(tracker.charge()).toBe(false); + expect(tracker.charge({ alreadySent: true })).toBe(true); + tracker.settle({ inputTokens: 8, outputTokens: 0 }); + expect(ledger.snapshot("pool", "provider")?.settled).toBe(8); + expect(ledger.snapshot("pool", "unknown-label")?.settled).toBe(40); + expect(reserve(ledger, "new").reserved).toBe(false); + }); + + test("malformed or conflicting maps fail closed without clearing ceilings or saved links", () => { + const disk = journal([checkpoint([["label", 40, 0]])]); + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy({ [pool("label")]: "provider" }), now: () => 2 }); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + for (const aliases of [null, [], { label: "provider" }, { [pool("label")]: "different-provider" }]) { + ledger.reconfigure(policy(aliases)); + expect(ledger.policy.pool.maxTokens).toBe(100); + expect(ledger.checkPoolContinuity()?.reason).toBe("pool-history-unresolved"); + expect(reserve(ledger, "new").reserved).toBe(false); + expect(ledger.snapshot("pool", "provider")?.settled).toBe(40); + } + ledger.reconfigure(policy()); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + }); + + test("evidence write failure cannot publish a mapping or erase historical balances", () => { + const disk = journal([checkpoint([["label", 40, 0]])]); + disk.append = () => { throw new Error("synthetic write failure"); }; + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy({ [pool("label")]: "provider" }), now: () => 2 }); + expect(reserve(ledger, "new")).toMatchObject({ reserved: false, denial: { reason: "reserve-not-durable" } }); + expect(ledger.snapshot("pool", "provider")).toBeUndefined(); + expect(ledger.snapshot("pool", "label")?.settled).toBe(40); + expect(disk.lines).toHaveLength(1); + }); + + test("complete invalid metadata at the final line fails closed and survives attempted compaction", () => { + const invalid = { ...checkpoint([["label", 40, 0]]), poolContinuity: { v: 1, kind: "pool-continuity", at: 1, bindings: [{ alias: "bad", canonical: pool("provider") }] } }; + const disk = journal([invalid]); + let ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 2 }); + expect(ledger.corruptRecords).toBe(1); + expect(ledger.checkPoolContinuity()?.reason).toBe("journal-corrupt"); + ledger.reconfigure(policy(undefined, { pool: {}, compactAfterRecords: 1 })); + reserve(ledger, "observed", "provider", 1, true); + ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 3 }); + expect(ledger.corruptRecords).toBe(1); + }); + + test("a v1 checkpoint remains parseable when compatibility metadata is omitted by an old reader", () => { + const disk = journal(); + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(undefined, { compactAfterRecords: 1 }), now: () => 2 }); + reserve(ledger, "new", "provider", 30); + ledger.settle("new", { inputTokens: 25, outputTokens: 0 }); + expect(disk.lines.every(line => parseSpendJournalRecord(line) !== undefined)).toBe(true); + const oldView = JSON.parse(disk.lines[0]!); + delete oldView.poolContinuity; + // Unmodified old readers retain raw v1 counters, but cannot prove canonical continuity. + expect(parseSpendJournalRecord(JSON.stringify(oldView))).toBeDefined(); + const restart = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 3 }); + expect(restart.checkPoolContinuity()).toBeUndefined(); + expect(restart.snapshot("pool", "provider")?.settled).toBe(25); + }); +}); + + +test("rootless HTTP and passthrough preflight refuse before any synthetic fetch", () => { + const previous = process.env.OPENCODEX_HOME; + const home = mkdtempSync(join(tmpdir(), "ocx-pool-history-")); + process.env.OPENCODEX_HOME = home; + const release = acquireOwnedSpendHome(); + try { + writeFileSync(join(home, "spend-ledger.salt"), salt + "\n", { mode: 0o600 }); + writeFileSync(join(home, "spend-ledger.jsonl"), JSON.stringify(checkpoint([["old-label", 100, 0]])) + "\n", { mode: 0o600 }); + configureSharedSpendLedger(policy()); + let syntheticFetches = 0; + const decision = admitHttpWorkflowTurn(new Headers()); + expect(decision).toMatchObject({ admitted: false, reason: "workflow-pool-history-unresolved" }); + if (!decision || decision.admitted) syntheticFetches += 1; + else { + const response = workflowDecisionRefusalResponse(decision); + expect(response.status).toBe(429); + expect(response.headers.get("x-opencodex-local-refusal")).toBe("workflow_pool_history_unresolved"); + } + const budget = createResponsesSendBudget({ req: new Request("https://fixture.example.test/v1/responses"), options: {}, logCtx: { model: "fixture", provider: "provider" } }); + expect(budget).toBeInstanceOf(Response); + if (!(budget instanceof Response)) syntheticFetches += 1; + expect(syntheticFetches).toBe(0); + configureSharedSpendLedger(policy({ [pool("old-label")]: "provider" })); + expect(admitHttpWorkflowTurn(new Headers())).toBeUndefined(); + const mappedBudget = createResponsesSendBudget({ req: new Request("https://fixture.example.test/v1/responses"), options: {}, logCtx: { model: "fixture", provider: "provider-display", spendPoolId: "provider" } }); + expect(mappedBudget).toBeInstanceOf(Response); + if (mappedBudget instanceof Response) { + expect(mappedBudget.status).toBe(429); + expect(mappedBudget.headers.get("x-opencodex-local-refusal")).toBe("workflow_spend_exhausted"); + } else syntheticFetches += 1; + expect(syntheticFetches).toBe(0); + const otherPool = createResponsesSendBudget({ req: new Request("https://fixture.example.test/v1/responses"), options: {}, logCtx: { model: "fixture", provider: "provider", spendPoolId: "unspent-provider" } }); + expect(otherPool).not.toBeInstanceOf(Response); + } finally { + release(); + if (previous === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previous; + removeTreeWithRetry(home); + } +}); + +test("actual old-reader compaction preserves raw spend; compatible rollback resolves every alias exactly once", () => { + const disk = journal([checkpoint([["old-label", 40, 0]])]); + const modern = createSpendReservationLedger({ journal: disk, salt, + policy: policy({ [pool("old-label")]: "provider" }, { compactAfterRecords: 1 }), now: () => 2 }); + expect(reserve(modern, "modern", "provider", 8).reserved).toBe(true); + modern.settle("modern", { inputTokens: 8, outputTokens: 0 }); + expect(modern.snapshot("pool", "provider")?.settled).toBe(48); + const old = createLegacySpendLedger({ journal: disk, salt, + policy: { ...DEFAULT_SPEND_RESERVATION_POLICY, compactAfterRecords: 1 }, now: () => 3 }); + expect(old.corruptRecords).toBe(0); + // Unsupported old binaries can still write a newly account-qualified label and strip + // compatibility metadata. Do not claim an automatic downgrade barrier or its enforcement. + expect(old.reserve({ sendId: "old-again", scopes: { poolId: "new-old-label" }, inputTokens: 12, outputCeilingTokens: 0 }).reserved).toBe(true); + old.settle("old-again", { inputTokens: 12, outputTokens: 0 }); + const returned = createSpendReservationLedger({ journal: disk, salt, + policy: policy({ [pool("old-label")]: "provider" }), now: () => 4 }); + expect(returned.checkPoolContinuity()?.reason).toBe("pool-history-unresolved"); + expect(returned.snapshot("pool", "new-old-label")?.settled).toBe(12); + returned.reconfigure(policy({ [pool("old-label")]: "provider", [pool("provider")]: "provider", [pool("new-old-label")]: "provider" })); + expect(returned.checkPoolContinuity()).toBeUndefined(); + expect(returned.snapshot("pool", "provider")?.settled).toBe(60); +}); From 22027eb28bf0f6c545bfa06f69693080ea2e9f07 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Thu, 1 Oct 2026 05:50:31 -0700 Subject: [PATCH 3/7] fix(spend): honor verified pool merges and prepaid dispatches Validate historical identity proposals atomically, retain exact reservation ownership through child preflight, and record rooted continuity refusals once. Add synthetic regressions for graph ordering, permit lifecycle and scope, exact-limit recovery/combo dispatch, and refusal event accounting. --- .../docs/reference/configuration/server.md | 7 +- src/lib/request-execution-budget.ts | 110 +++++---- src/lib/spend-pool-continuity.ts | 36 ++- src/lib/spend-reservation-ledger.ts | 20 +- src/lib/workflow-budget.ts | 10 +- src/server/responses/core-combo.ts | 1 + src/server/responses/core-options.ts | 2 + src/server/responses/request-send-budget.ts | 6 +- src/server/responses/request-spend.ts | 6 +- src/server/workflow-refusal.ts | 1 + structure/runtime.md | 2 +- structure/transports/responses-spend.md | 7 +- tests/lib/spend-pool-continuity.test.ts | 231 +++++++++++++++++- 13 files changed, 358 insertions(+), 81 deletions(-) diff --git a/docs-site/src/content/docs/reference/configuration/server.md b/docs-site/src/content/docs/reference/configuration/server.md index ca72348a5fb..429123ac244 100644 --- a/docs-site/src/content/docs/reference/configuration/server.md +++ b/docs-site/src/content/docs/reference/configuration/server.md @@ -907,7 +907,9 @@ inference admission returns local HTTP 429 with `x-opencodex-local-refusal: workflow_pool_history_unresolved` before contacting a provider. This can temporarily block otherwise valid requests, including requests without a workflow root. After routing, preflight also refuses a canonical pool that is already exhausted by its combined -balances. This check does not turn post-reported passthrough sends into atomic reservations: +balances. A recovery or combo send already admitted by a reservation does not count that same +reservation against itself a second time; all other reservations remain counted. +This check does not turn post-reported passthrough sends into atomic reservations: a crossing send, concurrent admissions or retries reported afterwards retain their existing limits. Observe-only installs remain observe-only. Root and identity ceilings remain in force independently. @@ -924,7 +926,8 @@ disable the entire section. Invalid top-level mappings are rejected on configura malformed `spendPoolAliases` hand edits retain existing ceilings and fail pool admission closed. Correct the mapping and restart through the ordinary configuration workflow. Empty/removing mappings does not erase links already recorded durably. A previously redirected alias cannot be reassigned to a different -group; verified canonical renames can join groups without splitting existing spend. +group; verified canonical renames can join groups without splitting existing spend. The complete +mapping is checked together, so a valid group merge does not depend on alias-key order. Each original balance is counted once. Settled usage, in-flight reservations and unresolved usage all count; unknown usage is never treated as a refund. Original reservation targets are retained. diff --git a/src/lib/request-execution-budget.ts b/src/lib/request-execution-budget.ts index cd74c808598..2bb250d30d6 100644 --- a/src/lib/request-execution-budget.ts +++ b/src/lib/request-execution-budget.ts @@ -14,6 +14,7 @@ * sends plus one alternate -- by funding the alternate from a reserve that a validated * sanitized repair can spend instead, but never both. */ +import type { SpendReservationProof } from "./spend-reservation-ledger"; import type { TransientSendBudget } from "./upstream-retry"; export type SendClass = @@ -137,7 +138,7 @@ export interface RequestSendObserver { * would cross a ceiling never joins the total, the total stays just under, and the ceiling * never fires for any later request either. */ - charge(options?: { alreadySent?: boolean }): boolean; + charge(options?: { alreadySent?: boolean; onReserved?: (proof: SpendReservationProof) => void }): boolean; /** Give back a booking whose send never happened. */ refund(): void; } @@ -215,7 +216,7 @@ let logicalRequestSeq = 0; */ interface SharedSendLedger { spent: number; - pendingExternalSends: number; + pendingExternalSends: Set; /** * Spend one of this logical request's replacements for an ambiguous failure. Beside `spent` * for the same reason `pendingExternalSends` is: a derived scope that shared one without the @@ -231,6 +232,16 @@ interface SharedSendLedger { } const sharedSendLedgers = new WeakMap(); +const dispatchSpendProofs = new WeakMap(); + +/** One preflight for the exact prepaid dispatch, never another budget or a replayed permit. */ +export function claimDispatchSpendProof(budget: RequestExecutionBudget, permit?: SingleUseDispatchPermit): SpendReservationProof | undefined { + const proof = permit && dispatchSpendProofs.get(permit); + return proof && proof.owner === sharedSendLedgers.get(budget) ? proof.claim() : undefined; +} /** * One logical request's replacement grant: how many it has spent, and the ceiling it is held @@ -285,8 +296,10 @@ function createRequestExecutionBudgetWithLedger( counter.spent = Math.max(0, next); return; } - const settled = Math.min(delta, counter.pendingExternalSends); - counter.pendingExternalSends -= settled; + const settled = Math.min(delta, counter.pendingExternalSends.size); + for (let index = 0; index < settled; index += 1) { + counter.pendingExternalSends.delete(counter.pendingExternalSends.values().next().value!); + } const charged = delta - settled; counter.spent += charged; // These sends have already left. The ledger records them even past a ceiling it would @@ -339,7 +352,10 @@ function createRequestExecutionBudgetWithLedger( // Consulted last, because it is the only bound here that WRITES. A ledger entry booked // for a dispatch a cheaper check above would have refused is spend this request never // makes, and it would hold those tokens against the scope until retention expired. - if (observer && !observer.charge()) return { allowed: false, reason: "spend-exhausted" }; + let spendProof: SpendReservationProof | undefined; + if (observer && !observer.charge({ onReserved: proof => { spendProof = proof; } })) { + return { allowed: false, reason: "spend-exhausted" }; + } // THE RESERVATION IS THE CHARGE. Deciding here and charging in `use()` left a window in // which two legs read the same remainder, both received a permit, and both dispatched: @@ -347,51 +363,54 @@ function createRequestExecutionBudgetWithLedger( // this budget exists to stop. Everything is booked now; `release()` is the way back. const previousTargetKey = lastTargetKey; counter.spent += 1; - if (intent.countedExternally === true) counter.pendingExternalSends += 1; + const receipt = {}; + if (intent.countedExternally === true) counter.pendingExternalSends.add(receipt); if (drawsReserve) reserveSpent = true; if (isAlternateTarget) alternateTargetSends += 1; if (changesTarget) targetTransitions += 1; lastTargetKey = intent.targetKey; let settled: "open" | "used" | "released" = "open"; - return { - allowed: true, - permit: { - sendClass: intent.sendClass, - use(): boolean { - if (settled !== "open") return false; - settled = "used"; - return true; - }, - assumeCharge(): boolean { - if (settled !== "open") return false; - settled = "used"; - // The booking this reservation made for an external reporter is now owned by the - // caller. Leaving it pending is not harmless: the next `used` report of this request - // would settle against it and one real send would go uncharged. - if (intent.countedExternally === true && counter.pendingExternalSends > 0) { - counter.pendingExternalSends -= 1; - } - return true; - }, - release(): void { - if (settled !== "open") return; - settled = "released"; - // An externally counted reservation the reporter already settled paid for a send - // that physically happened. Refunding it would hand the request a free send back. - if (intent.countedExternally === true) { - if (counter.pendingExternalSends === 0) return; - counter.pendingExternalSends -= 1; - } - counter.spent -= 1; - observer?.refund(); - if (drawsReserve) reserveSpent = false; - if (isAlternateTarget) alternateTargetSends -= 1; - if (changesTarget) targetTransitions -= 1; - lastTargetKey = previousTargetKey; - }, + const permit: SingleUseDispatchPermit = { + sendClass: intent.sendClass, + use(): boolean { + if (settled !== "open") return false; + settled = "used"; + return true; + }, + assumeCharge(): boolean { + if (settled !== "open") return false; + settled = "used"; + // The booking this reservation made for an external reporter is now owned by the + // caller. Leaving it pending is not harmless: the next `used` report of this request + // would settle against it and one real send would go uncharged. + if (intent.countedExternally === true) counter.pendingExternalSends.delete(receipt); + return true; + }, + release(): void { + if (settled !== "open") return; + settled = "released"; + // An externally counted reservation the reporter already settled paid for a send + // that physically happened. Refunding it would hand the request a free send back. + if (intent.countedExternally === true) { + if (!counter.pendingExternalSends.delete(receipt)) return; + } + counter.spent -= 1; + observer?.refund(); + if (drawsReserve) reserveSpent = false; + if (isAlternateTarget) alternateTargetSends -= 1; + if (changesTarget) targetTransitions -= 1; + lastTargetKey = previousTargetKey; }, }; + let preflightClaimed = false; + dispatchSpendProofs.set(permit, { owner: counter, claim: () => { + if (preflightClaimed || settled === "released" + || (intent.countedExternally === true ? !counter.pendingExternalSends.has(receipt) : settled !== "open")) return undefined; + preflightClaimed = true; + return spendProof; + } }); + return { allowed: true, permit }; }, }; sharedSendLedgers.set(budget, counter); @@ -406,7 +425,7 @@ export function createRequestExecutionBudget( const grant = createAmbiguousResendGrant(); return createRequestExecutionBudgetWithLedger(policy, logicalRequestId, { spent: 0, - pendingExternalSends: 0, + pendingExternalSends: new Set(), claimAmbiguousResend: grant.claimAmbiguousResend, get ambiguousResendSpent(): boolean { return grant.ambiguousResendSpent; }, ...(observer ? { observer } : {}), @@ -450,7 +469,7 @@ const bridgedGrantClaims = new WeakMap(); let bridged = bridgedGrantClaims.get(parent); if (!bridged) { bridged = { claimed: false }; @@ -460,8 +479,7 @@ function ledgerFor(parent: RequestExecutionBudget): SharedSendLedger { return { get spent(): number { return parent.used; }, set spent(next: number) { parent.used = next; }, - get pendingExternalSends(): number { return pendingExternalSends; }, - set pendingExternalSends(next: number) { pendingExternalSends = next; }, + pendingExternalSends, // Asked of the parent rather than counted here. A local counter is a SECOND grant: two // scopes derived from one bridged parent, or one scope beside the parent it was derived // from, each replaced an unknown-state send once. Pending bookings and the durable-spend diff --git a/src/lib/spend-pool-continuity.ts b/src/lib/spend-pool-continuity.ts index 991f8fe4541..89b8243afe1 100644 --- a/src/lib/spend-pool-continuity.ts +++ b/src/lib/spend-pool-continuity.ts @@ -53,14 +53,19 @@ export function createPoolContinuity() { } return alias; }; - const link = (map: Map, alias: string, canonical: string): boolean => { - const existing = map.get(alias); - const target = resolve(canonical, map); - if (existing !== undefined && existing !== alias && resolve(alias, map) !== target) return false; - // An explicit rename can join a formerly canonical alias to its successor. The old - // name still resolves to this same group, so neither a reload nor an old caller splits it. - if (target !== alias || existing === undefined) map.set(alias, target); - return true; + const merge = (proposed: Iterable): Map | undefined => { + const next = new Map(bindings); + for (const [alias, canonical] of proposed) next.set(alias, canonical); + try { + // Validate the whole proposed graph, not an intermediate sorted prefix. A verified + // A -> B -> C rename may repeat A -> C without splitting the existing A/B group. + for (const alias of next.keys()) resolve(alias, next); + for (const [alias, canonical] of bindings) { + if (resolve(alias, next) !== resolve(canonical, next)) return undefined; + } + for (const alias of next.keys()) next.set(alias, resolve(alias, next)); + } catch { return undefined; } + return next; }; return { resolve: (alias: string): string => resolve(alias), @@ -71,23 +76,16 @@ export function createPoolContinuity() { bindings: [...bindings].map(([alias, canonical]) => ({ alias, canonical })), }), restore(record: PoolContinuityRecord): boolean { - const next = new Map(bindings); - try { - for (const { alias, canonical } of record.bindings) if (!link(next, alias, canonical)) return false; - for (const alias of next.keys()) resolve(alias, next); - } catch { return false; } + const next = merge(record.bindings.map(({ alias, canonical }) => [alias, canonical] as const)); + if (!next) return false; bindings = next; return true; }, prepare(config: unknown, requested: string | undefined, historical: ReadonlySet, hash: (provider: string) => string, at: number, capacity: number): PoolContinuityRecord | false | undefined { if (spendPoolAliasesError(config)) return false; - const next = new Map(bindings); - // Sort for deterministic multi-hop mappings regardless of JSON property order. Link - // conflicts fail closed; repeat entries and already-joined destinations are idempotent. - for (const [alias, provider] of Object.entries(config ?? {}).sort(([a], [b]) => a.localeCompare(b))) { - if (!link(next, alias, hash(provider as string))) return false; - } + const next = merge(Object.entries(config ?? {}).map(([alias, provider]) => [alias, hash(provider as string)] as const)); + if (!next) return false; if (requested !== undefined && !next.has(requested) && !historical.has(requested)) next.set(requested, requested); if (next.size > Math.min(16_384, capacity)) return false; if (next.size === bindings.size && [...next].every(([key, value]) => bindings.get(key) === value)) return undefined; diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index 04a5a313339..1e774d50f72 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -579,6 +579,12 @@ export interface ScopeSpendSnapshot { readonly exhausted: boolean; } +/** In-memory proof of one booked send; never journaled or exposed in request logs. */ +export interface SpendReservationProof { + readonly ledger: SpendReservationLedger; + readonly sendId: string; +} + export interface SpendReservationLedger { reserve(request: SpendReservationRequest): SpendReservationDecision; /** Pre-dispatch guard, including transports which report their sends after dispatch. */ @@ -607,7 +613,8 @@ export interface SpendReservationLedger { */ markLost(sendId: string): boolean; snapshot(scope: SpendScope, scopeId: string): ScopeSpendSnapshot | undefined; - exhausted(scope: SpendScope, scopeId: string): boolean; + /** Exclude only a proven current dispatch's still-open reservation in this scope. */ + exhausted(scope: SpendScope, scopeId: string, excludingSendId?: string): boolean; /** * Drop dormant scopes per the retention rule in SpendReservationPolicy. Cleanup also runs * automatically on every reservation, so nothing depends on a caller remembering this. @@ -1161,10 +1168,15 @@ export function createSpendReservationLedger(options: { }; }, - exhausted(scope: SpendScope, scopeId: string): boolean { + exhausted(scope: SpendScope, scopeId: string, excludingSendId?: string): boolean { assertOwnedAccounting?.(); - const state = stateFor(scope, aliasFor(scope, scopeId)); - return state !== undefined && isExhausted(scope, state); + const alias = aliasFor(scope, scopeId); + const state = stateFor(scope, alias); + if (!state) return false; + const own = excludingSendId === undefined ? undefined : reservations.get(aliasFor("send", excludingSendId)); + const matches = own?.status === "open" && own.targets.some(target => target.scope === scope + && (scope === "pool" ? poolContinuity.resolve(target.alias) === poolContinuity.resolve(alias) : target.alias === alias)); + return isExhausted(scope, matches ? { ...state, reserved: Math.max(0, state.reserved - own.tokens) } : state); }, prune(at: number = now()): void { diff --git a/src/lib/workflow-budget.ts b/src/lib/workflow-budget.ts index 41b72265b36..0c2e518045d 100644 --- a/src/lib/workflow-budget.ts +++ b/src/lib/workflow-budget.ts @@ -23,6 +23,7 @@ import { sharedSpendLedger, spendCeilingsConfigured, type SpendReservationLedger, + type SpendReservationProof, type SpendScope, type SpendUsage, } from "./spend-reservation-ledger"; @@ -715,10 +716,11 @@ export function workflowSendCeilingReached( function spentRootCeiling( rootId: string, ledger: SpendReservationLedger, + excludingSendId?: string, ): WorkflowSpendDenialDetail | undefined { const limit = ledger.policy.root.maxTokens; if (limit === undefined) return undefined; - return ledger.exhausted("root", rootId) ? { scope: "root", limit } : undefined; + return ledger.exhausted("root", rootId, excludingSendId) ? { scope: "root", limit } : undefined; } /** @@ -736,14 +738,16 @@ export function workflowSpendCeilingReached( rootId: string | undefined, spendLedger?: SpendReservationLedger, poolId?: string, + reservation?: SpendReservationProof, ): WorkflowSpendDenialDetail | undefined { if (!rootId && !poolId) return undefined; const ledger = spendLedger ?? (spendCeilingsConfigured() ? sharedSpendLedger() : undefined); if (!ledger) return undefined; - const root = rootId ? spentRootCeiling(rootId, ledger) : undefined; + const excludingSendId = reservation?.ledger === ledger ? reservation.sendId : undefined; + const root = rootId ? spentRootCeiling(rootId, ledger, excludingSendId) : undefined; if (root) return root; const limit = ledger.policy.pool.maxTokens; - return poolId && limit !== undefined && ledger.exhausted("pool", poolId) ? { scope: "pool", limit } : undefined; + return poolId && limit !== undefined && ledger.exhausted("pool", poolId, excludingSendId) ? { scope: "pool", limit } : undefined; } export interface WorkflowBudgetSnapshot { diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index a9669bbe327..d1f9466853f 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -791,6 +791,7 @@ export async function executeComboResponses( // parent arrived with. sendBudget: targetSendBudget, comboAttempt: true, + comboDispatchPermit: hopDecision?.allowed ? hopDecision.permit : undefined, comboReplaySnapshot, deferCodexResetDerivedCooldown, // Attempt-relative TTFT is recorded HERE (not via childLog.firstOutputMs — a later diff --git a/src/server/responses/core-options.ts b/src/server/responses/core-options.ts index aea7f850800..0fa5ba9e0fb 100644 --- a/src/server/responses/core-options.ts +++ b/src/server/responses/core-options.ts @@ -143,6 +143,8 @@ export interface HandleResponsesOptions { callerDirectAuth?: CallerDirectAuth | null; /** Internal recursion guard; callers outside this module must not set it. */ comboAttempt?: boolean; + /** Exact externally booked combo hop, used only for its child's spend preflight. */ + comboDispatchPermit?: SingleUseDispatchPermit; /** Internal handoff: this combo was selected by shadow-call interception. */ shadowCallIntercepted?: boolean; /** Internal handoff: the memory phase this turn belongs to, so combo children keep its routing. */ diff --git a/src/server/responses/request-send-budget.ts b/src/server/responses/request-send-budget.ts index bad16cf6426..33f1cf954b2 100644 --- a/src/server/responses/request-send-budget.ts +++ b/src/server/responses/request-send-budget.ts @@ -1,5 +1,5 @@ import type { ResponsesRequestContext } from "./core-options"; -import { createRequestExecutionBudget, isRequestExecutionBudget } from "../../lib/request-execution-budget"; +import { claimDispatchSpendProof, createRequestExecutionBudget, isRequestExecutionBudget } from "../../lib/request-execution-budget"; import { chargeWorkflowSends, workflowSendCeilingReached, @@ -84,7 +84,9 @@ export function createResponsesSendBudget( if (continuityRefusal) { return workflowRefusalResponse(continuityRefusal, logCtx, undefined, workflowRootId); } - const spentCeiling = workflowSpendCeilingReached(workflowRootId, undefined, logCtx.spendPoolId ?? logCtx.provider); + const prepaid = isRequestExecutionBudget(sendBudget) + ? claimDispatchSpendProof(sendBudget, options.compactionRecoveryPermit ?? options.comboDispatchPermit) : undefined; + const spentCeiling = workflowSpendCeilingReached(workflowRootId, undefined, logCtx.spendPoolId ?? logCtx.provider, prepaid); if (spentCeiling) { return workflowRefusalResponse( "workflow-spend-exhausted", diff --git a/src/server/responses/request-spend.ts b/src/server/responses/request-spend.ts index 8e86aac475d..9ddab48c3e5 100644 --- a/src/server/responses/request-spend.ts +++ b/src/server/responses/request-spend.ts @@ -73,7 +73,7 @@ export function createRequestSpendTracker( for (let index = 0; index < live.length - 1; index += 1) ledger().markDispatched(live[index] as string); }; return { - charge(options?: { alreadySent?: boolean }): boolean { + charge(options?: Parameters[0]): boolean { // A send that has already left is RECORDED, never refused: the tokens are spent, and a // booking the ledger drops is a booking the ceiling can never see. This is the reporting // transports' path -- the passthrough ladder reports through `onSendsConsumed` after the @@ -81,7 +81,8 @@ export function createRequestSpendTracker( // short of its limit forever and refuse nothing. const alreadySent = options?.alreadySent === true; const sendId = randomUUID(); - const decision = ledger().reserve({ + const bookedLedger = ledger(); + const decision = bookedLedger.reserve({ sendId, scopes: { ...(rootId !== undefined ? { rootId } : {}), @@ -130,6 +131,7 @@ export function createRequestSpendTracker( recordWorkflowRefusalEvent(rootId, "workflow-spend-exhausted", Date.now(), detail); return false; } + if (!alreadySent) options?.onReserved?.({ ledger: bookedLedger, sendId }); live.push(sendId); confirmOlderSends(); // It has already left, so the reservation cannot be handed back for free: from here only diff --git a/src/server/workflow-refusal.ts b/src/server/workflow-refusal.ts index e20c8e178d1..a1457fed130 100644 --- a/src/server/workflow-refusal.ts +++ b/src/server/workflow-refusal.ts @@ -116,6 +116,7 @@ export function admitHttpWorkflowTurn(headers: Headers): WorkflowDecision | unde const rootId = headers.get("x-codex-parent-thread-id")?.trim() || undefined; const continuityRefusal = poolContinuityRefusalReason(); if (continuityRefusal) { + recordWorkflowRefusalEvent(rootId, continuityRefusal); return { admitted: false, reason: continuityRefusal, rootId: rootId ?? "" }; } const threadId = headers.get("thread-id")?.trim() || undefined; diff --git a/structure/runtime.md b/structure/runtime.md index cb56f5ce573..82983125301 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -216,7 +216,7 @@ and build a ledger by replaying that directory's own journal. A second process o directory is refused even for observe-only spend configuration, while a separate directory is independent. Ordinary stop releases the final reference after listener teardown, and every thrown startup path releases its reference. SQLite and the OS release a crashed owner; no PID, timestamp, -TTL or lock-file deletion participates in recovery. Startup passes top-level pool-alias evidence with the spend policy. HTTP admission checks [historical continuity](transports/responses-spend.md#historical-pool-continuity-and-rollback) even without a root; failed accounting reads release the active-turn lease. +TTL or lock-file deletion participates in recovery. Startup passes top-level pool-alias evidence with the spend policy. HTTP admission checks [historical continuity](transports/responses-spend.md#historical-pool-continuity-and-rollback) even without a root; failed accounting reads release the active-turn lease. Rooted continuity denials record exactly one workflow refusal event before response formatting. An explicit Codex integration OFF skips startup cache invalidation before the user-scoped catalog serialization lock is resolved. Explicit `sync` and `sync-cache` retain their catalog-only override. diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index 74e82f1b33c..a7e43c36ccb 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -122,7 +122,9 @@ mapping when their positive history predates identity metadata. Zero-balance his The ledger retains original scope balances and reservation targets. The canonical view adds each member once, keeping settled, reserved and unresolved buckets separate. Explicit links can join previously canonical groups for a verified rename; an already redirected alias cannot be assigned -to a different group. A removed config entry never removes a journaled link. Group activity, +to a different group. The complete proposed graph is validated atomically, so a verified merge +that repeats existing member aliases is independent of salted-key order. A removed config entry +never removes a journaled link. Group activity, last-seen time and exhaustion govern retention; unidentified positive balances cannot be evicted. Identity evidence is bounded and retained even when dormant under-limit scopes are evicted. @@ -136,6 +138,9 @@ With a configured pool ceiling, any remaining unidentified positive pool history conflicting mapping refuses admission. HTTP workflow admission and the Responses pre-dispatch seam check this even without a root ID, before passthrough transports that report sends afterwards. After routing, that seam also refuses an already-exhausted canonical pool, including mapped historical totals. +A prepaid child excludes only its own open reservation, proven by its exact permit, shared send +ledger and still-pending receipt. Proof is single-use; unrelated reservations and other pool groups +remain counted. HTTP continuity refusals record one rooted workflow event; rootless ones record none. It is a snapshot check, not a new atomic reservation for report-only transports: crossing sends, concurrent preflight admissions and retries reported afterwards retain their existing limitations. `workflow_pool_history_unresolved` identifies the local 429 without disclosing aliases; storage diff --git a/tests/lib/spend-pool-continuity.test.ts b/tests/lib/spend-pool-continuity.test.ts index 5f47660df84..dffaf3ea575 100644 --- a/tests/lib/spend-pool-continuity.test.ts +++ b/tests/lib/spend-pool-continuity.test.ts @@ -1,3 +1,7 @@ +import { executeComboResponses } from "../../src/server/responses/core-combo"; +import { clearComboSelectionState, clearComboTargetCooldowns } from "../../src/combos"; +import { createTranslatorBudget } from "../../src/lib/translator-budget"; +import type { OcxConfig } from "../../src/types"; import { createLegacySpendLedger } from "../helpers/legacy-spend-ledger"; import { describe, expect, test } from "bun:test"; import { mkdtempSync, writeFileSync } from "node:fs"; @@ -6,12 +10,15 @@ import { join } from "node:path"; import { createHash } from "node:crypto"; import { createSpendReservationLedger, configureSharedSpendLedger, DEFAULT_SPEND_RESERVATION_POLICY, parseSpendJournalRecord, - type SpendJournal, type SpendReservationPolicy, + sharedSpendLedger, type SpendJournal, type SpendReservationPolicy, } from "../../src/lib/spend-reservation-ledger"; import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; import { admitHttpWorkflowTurn, workflowDecisionRefusalResponse } from "../../src/server/workflow-refusal"; import { createResponsesSendBudget } from "../../src/server/responses/request-send-budget"; +import { claimDispatchSpendProof, createRequestExecutionBudget, deriveRequestExecutionBudget } from "../../src/lib/request-execution-budget"; +import { createPoolContinuity } from "../../src/lib/spend-pool-continuity"; +import { listWorkflowBudgetEvents, resetWorkflowBudgetsForTest, workflowSpendCeilingReached } from "../../src/lib/workflow-budget"; import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; const salt = "5".repeat(64); @@ -33,6 +40,30 @@ const reserve = (ledger: ReturnType, sendId ledger.reserve({ sendId, scopes: { poolId }, inputTokens: tokens, outputCeilingTokens: 0, alreadySent }); describe("historical pool identity continuity", () => { + test("verified group merges are atomic and independent of salted key order", () => { + const low = "1".repeat(32), high = "2".repeat(32), target = "3".repeat(32); + for (const [a, b] of [[low, high], [high, low]]) { + const continuity = createPoolContinuity(); + expect(continuity.restore({ v: 1, kind: "pool-continuity", at: 1, + bindings: [{ alias: a!, canonical: b! }] })).toBe(true); + const record = continuity.prepare({ [a!]: target, [b!]: target }, undefined, new Set(), id => id, 2, 100); + expect(record).not.toBe(false); + if (!record) throw new Error("merge was refused"); + expect(continuity.resolve(a!)).toBe(b!); // prepare does not publish evidence + expect(continuity.restore(record)).toBe(true); + expect(continuity.resolve(a!)).toBe(target); + expect(continuity.resolve(b!)).toBe(target); + const before = continuity.record(3); + expect(continuity.prepare({ [a!]: "4".repeat(32) }, undefined, new Set(), id => id, 3, 100)).toBe(false); + expect(continuity.record(3)).toEqual(before); + expect(continuity.restore({ v: 1, kind: "pool-continuity", at: 3, + bindings: [{ alias: a!, canonical: "4".repeat(32) }] })).toBe(false); + expect(continuity.record(3)).toEqual(before); + expect(continuity.prepare({ [target]: a!, [a!]: target }, undefined, new Set(), id => id, 3, 100)).toBe(false); + expect(continuity.record(3)).toEqual(before); + } + }); + test("unmapped historical debt fails closed across labels, providers, pruning and restart", () => { const disk = journal([checkpoint([["provider-old-label", 100, 0]])]); for (let restart = 0; restart < 2; restart += 1) { @@ -193,6 +224,12 @@ test("rootless HTTP and passthrough preflight refuse before any synthetic fetch" writeFileSync(join(home, "spend-ledger.salt"), salt + "\n", { mode: 0o600 }); writeFileSync(join(home, "spend-ledger.jsonl"), JSON.stringify(checkpoint([["old-label", 100, 0]])) + "\n", { mode: 0o600 }); configureSharedSpendLedger(policy()); + resetWorkflowBudgetsForTest(); + const rooted = admitHttpWorkflowTurn(new Headers({ "x-codex-parent-thread-id": "synthetic-root" })); + expect(rooted).toMatchObject({ admitted: false, reason: "workflow-pool-history-unresolved" }); + expect(listWorkflowBudgetEvents()).toMatchObject([{ rootId: "synthetic-root", reason: "workflow-pool-history-unresolved" }]); + if (rooted && !rooted.admitted) workflowDecisionRefusalResponse(rooted); + expect(listWorkflowBudgetEvents()).toHaveLength(1); let syntheticFetches = 0; const decision = admitHttpWorkflowTurn(new Headers()); expect(decision).toMatchObject({ admitted: false, reason: "workflow-pool-history-unresolved" }); @@ -206,6 +243,7 @@ test("rootless HTTP and passthrough preflight refuse before any synthetic fetch" expect(budget).toBeInstanceOf(Response); if (!(budget instanceof Response)) syntheticFetches += 1; expect(syntheticFetches).toBe(0); + expect(listWorkflowBudgetEvents()).toHaveLength(1); // rootless refusals add no event configureSharedSpendLedger(policy({ [pool("old-label")]: "provider" })); expect(admitHttpWorkflowTurn(new Headers())).toBeUndefined(); const mappedBudget = createResponsesSendBudget({ req: new Request("https://fixture.example.test/v1/responses"), options: {}, logCtx: { model: "fixture", provider: "provider-display", spendPoolId: "provider" } }); @@ -218,6 +256,7 @@ test("rootless HTTP and passthrough preflight refuse before any synthetic fetch" const otherPool = createResponsesSendBudget({ req: new Request("https://fixture.example.test/v1/responses"), options: {}, logCtx: { model: "fixture", provider: "provider", spendPoolId: "unspent-provider" } }); expect(otherPool).not.toBeInstanceOf(Response); } finally { + resetWorkflowBudgetsForTest(); release(); if (previous === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previous; @@ -247,3 +286,193 @@ test("actual old-reader compaction preserves raw spend; compatible rollback reso expect(returned.checkPoolContinuity()).toBeUndefined(); expect(returned.snapshot("pool", "provider")?.settled).toBe(60); }); + +for (const kind of ["compaction", "combo"] as const) { + for (const rootId of [undefined, "reservation-root"]) { + test(`${kind} child spends its own exact-limit reservation (${rootId ?? "rootless"})`, () => { + const previous = process.env.OPENCODEX_HOME; + const home = mkdtempSync(join(tmpdir(), "ocx-prepaid-spend-")); + process.env.OPENCODEX_HOME = home; + const release = acquireOwnedSpendHome(); + try { + configureSharedSpendLedger(policy(undefined, { root: { maxTokens: 100 } })); + const logCtx = { model: "fixture", provider: "provider-display", spendPoolId: "provider", usageLogInputTokens: 100 }; + const tracker = createRequestSpendTracker(logCtx, rootId); + const sendBudget = createRequestExecutionBudget(undefined, undefined, tracker); + const reservation = sendBudget.reserveDispatch({ sendClass: "initial", targetKey: "provider/fixture", countedExternally: true }); + expect(reservation.allowed).toBe(true); + if (!reservation.allowed) throw new Error("synthetic reservation refused"); + if (kind === "combo") expect(reservation.permit.use()).toBe(true); + expect(sharedSpendLedger().snapshot("pool", "provider")?.reserved).toBe(100); + const req = new Request("https://fixture.example.test/v1/responses", { + headers: rootId ? { "x-codex-parent-thread-id": rootId } : {}, + }); + const options = { sendBudget, ...(kind === "compaction" + ? { compactionRecoveryPermit: reservation.permit } : { comboDispatchPermit: reservation.permit }) }; + const child = createResponsesSendBudget({ req, options, logCtx }); + expect(child).not.toBeInstanceOf(Response); + if (child instanceof Response) throw new Error("own reservation refused"); + expect(createResponsesSendBudget({ req, options, logCtx })).toBeInstanceOf(Response); // proof is single-use + if (kind === "compaction") { + const dispatch = child.adapterDispatchBudget!.reserveDispatch({ sendClass: "initial", targetKey: "provider/fixture" }); + expect(dispatch.allowed).toBe(true); + if (dispatch.allowed) expect(dispatch.permit.use()).toBe(true); + } else child.noteTransientSends(1); // synthetic report-only combo dispatch, no fetch + expect(sendBudget.used).toBe(1); + tracker.settle({ inputTokens: 100, outputTokens: 0 }); + expect(sharedSpendLedger().snapshot("pool", "provider")).toMatchObject({ settled: 100, reserved: 0 }); + expect(createResponsesSendBudget({ req, options: {}, logCtx })).toBeInstanceOf(Response); + } finally { + release(); + if (previous === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previous; + removeTreeWithRetry(home); + } + }); + } +} + +function prepaidFixture(tokens = 100, ceiling = 100) { + const ledger = createSpendReservationLedger({ salt, policy: policy(undefined, { root: { maxTokens: ceiling }, pool: { maxTokens: ceiling } }) }); + const tracker = createRequestSpendTracker({ provider: "provider", usageLogInputTokens: tokens }, "root", ledger); + const budget = createRequestExecutionBudget(undefined, undefined, tracker); + const reservePermit = () => { + const decision = budget.reserveDispatch({ sendClass: "initial", targetKey: "provider/fixture", countedExternally: true }); + if (!decision.allowed) throw new Error("synthetic permit refused"); + return decision.permit; + }; + return { ledger, tracker, budget, reservePermit }; +} + +test("prepaid proof belongs to one shared budget and one still-pending dispatch", () => { + for (const end of ["release", "report", "assume"] as const) { + const { budget, reservePermit } = prepaidFixture(); + const permit = reservePermit(); + if (end === "release") permit.release(); + if (end === "report") budget.used += 1; + if (end === "assume") expect(permit.assumeCharge()).toBe(true); + expect(claimDispatchSpendProof(budget, permit)).toBeUndefined(); + } + const { ledger, budget, reservePermit } = prepaidFixture(); + const permit = reservePermit(); + expect(permit.use()).toBe(true); // combo use leaves its external receipt pending + expect(claimDispatchSpendProof(createRequestExecutionBudget(), permit)).toBeUndefined(); + const childBudget = deriveRequestExecutionBudget(budget, budget.policy); + const proof = claimDispatchSpendProof(childBudget, permit); + expect(proof).toBeDefined(); + expect(workflowSpendCeilingReached("root", ledger, "provider", proof)).toBeUndefined(); + expect(claimDispatchSpendProof(budget, permit)).toBeUndefined(); + expect(workflowSpendCeilingReached("root", ledger, "provider")).toMatchObject({ scope: "root" }); +}); + +test("receipt identity survives out-of-order handoff and reports cannot authorize another permit", () => { + const first = prepaidFixture(10, 100); + const a = first.reservePermit(), b = first.reservePermit(); + expect(b.assumeCharge()).toBe(true); + expect(claimDispatchSpendProof(first.budget, b)).toBeUndefined(); + expect(claimDispatchSpendProof(first.budget, a)).toBeDefined(); + first.budget.used += 1; + expect(first.budget.used).toBe(2); // report consumed A; B was already assumed + + const second = prepaidFixture(10, 100); + const reported = second.reservePermit(), pending = second.reservePermit(); + second.budget.used += 1; + expect(claimDispatchSpendProof(second.budget, reported)).toBeUndefined(); + reported.release(); + expect(second.budget.used).toBe(2); // cannot refund a different pending receipt + expect(claimDispatchSpendProof(second.budget, pending)).toBeDefined(); + pending.release(); + expect(second.budget.used).toBe(1); + expect(second.ledger.snapshot("pool", "provider")).toMatchObject({ reserved: 10 }); +}); + +test("preflight retains unrelated reservations, debt and mismatched scope or ledger", () => { + const { ledger, budget, reservePermit } = prepaidFixture(100, 200); + const permit = reservePermit(); + expect(reserve(ledger, "unrelated", "provider", 100).reserved).toBe(true); + ledger.reconfigure(policy(undefined, { root: { maxTokens: 200 } })); + const proof = claimDispatchSpendProof(budget, permit)!; + expect(workflowSpendCeilingReached(undefined, ledger, "provider", proof)).toMatchObject({ scope: "pool" }); + const foreign = prepaidFixture(); + foreign.reservePermit(); + expect(workflowSpendCeilingReached(undefined, foreign.ledger, "provider", proof)).toMatchObject({ scope: "pool" }); + expect(ledger.markDispatched(proof.sendId)).toBe(true); + expect(ledger.exhausted("pool", "provider", proof.sendId)).toBe(true); + ledger.settle(proof.sendId, { inputTokens: 100, outputTokens: 0 }); + expect(ledger.exhausted("pool", "provider", proof.sendId)).toBe(true); + + const zero = prepaidFixture(0, 100); + expect(reserve(zero.ledger, "full", "provider", 100).reserved).toBe(true); + const zeroProof = claimDispatchSpendProof(zero.budget, zero.reservePermit()); + expect(workflowSpendCeilingReached(undefined, zero.ledger, "provider", zeroProof)).toMatchObject({ scope: "pool" }); +}); + +test("prepaid exclusion follows only its canonical pool and retains settled/unresolved history", () => { + const disk = journal([checkpoint([["historical", 30, 10]])]); + const ledger = createSpendReservationLedger({ salt, journal: disk, policy: policy({ [pool("historical")]: "provider" }), now: () => 2 }); + const tracker = createRequestSpendTracker({ provider: "provider", usageLogInputTokens: 60 }, undefined, ledger); + const budget = createRequestExecutionBudget(undefined, undefined, tracker); + const decision = budget.reserveDispatch({ sendClass: "initial", targetKey: "provider", countedExternally: true }); + if (!decision.allowed) throw new Error("synthetic permit refused"); + const proof = claimDispatchSpendProof(budget, decision.permit)!; + expect(ledger.snapshot("pool", "provider")).toMatchObject({ settled: 30, unresolved: 10, reserved: 60 }); + expect(workflowSpendCeilingReached(undefined, ledger, "provider", proof)).toBeUndefined(); + ledger.reconfigure(policy({ [pool("historical")]: "renamed", [pool("provider")]: "renamed" })); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + expect(workflowSpendCeilingReached(undefined, ledger, "renamed", proof)).toBeUndefined(); + expect(reserve(ledger, "other", "unrelated", 100).reserved).toBe(true); + expect(workflowSpendCeilingReached(undefined, ledger, "unrelated", proof)).toMatchObject({ scope: "pool" }); + ledger.reconfigure(policy(undefined, { pool: { maxTokens: 40 } })); + expect(workflowSpendCeilingReached(undefined, ledger, "renamed", proof)).toMatchObject({ scope: "pool" }); +}); + + +test("actual combo dispatch forwards only its own prepaid permit into the child preflight", async () => { + const previous = process.env.OPENCODEX_HOME; + const home = mkdtempSync(join(tmpdir(), "ocx-combo-prepaid-")); + process.env.OPENCODEX_HOME = home; + const release = acquireOwnedSpendHome(); + const originalFetch = globalThis.fetch; + globalThis.fetch = (() => { throw new Error("unexpected network in synthetic combo"); }) as typeof fetch; + const translatorBudget = createTranslatorBudget(); + clearComboSelectionState(); + clearComboTargetCooldowns(); + try { + configureSharedSpendLedger(policy()); + const config: OcxConfig = { port: 0, defaultProvider: "provider", providers: { + provider: { adapter: "openai-chat", apiKey: "fixture-only", baseUrl: "https://provider.example.test/v1" }, + }, combos: { prepaid: { strategy: "failover", targets: [{ provider: "provider", model: "fixture" }] } } }; + const logCtx = { model: "", provider: "", usageLogInputTokens: 100 }; + const tracker = createRequestSpendTracker(logCtx, undefined); + const sendBudget = createRequestExecutionBudget(undefined, undefined, tracker); + const body = { model: "combo/prepaid", input: [] }; + let sends = 0; + const response = await executeComboResponses(new Request("https://fixture.example.test/v1/responses", { + method: "POST", body: JSON.stringify(body), headers: { "content-type": "application/json" }, + }), body, "prepaid", config, logCtx, { sendBudget, translatorBudget }, { + handleResponses: async (req, _config, childLog, options) => { + const child = createResponsesSendBudget({ req, logCtx: childLog, options: options! }); + expect(child).not.toBeInstanceOf(Response); + if (child instanceof Response) return child; + sends += 1; + child.noteTransientSends(1); + return Response.json({ id: "synthetic-response", output: [] }); + }, + handleComboResponses: async () => { throw new Error("unexpected nested combo"); }, + }); + expect(response.status).toBe(200); + expect(sends).toBe(1); + expect(sendBudget.used).toBe(1); + tracker.settle({ inputTokens: 100, outputTokens: 0 }); + expect(sharedSpendLedger().snapshot("pool", "provider")).toMatchObject({ settled: 100, reserved: 0 }); + } finally { + globalThis.fetch = originalFetch; + translatorBudget.dispose(); + clearComboSelectionState(); + clearComboTargetCooldowns(); + release(); + if (previous === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previous; + removeTreeWithRetry(home); + } +}); From ade5bcf0a064709de05e03186c06e5952283b969 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Thu, 1 Oct 2026 22:13:52 -0700 Subject: [PATCH 4/7] fix(spend): attribute external reports to exact dispatch receipts --- src/lib/request-execution-budget.ts | 68 +++++++----- src/server/chat-native.ts | 6 +- src/server/responses/adapter-continuation.ts | 18 +-- src/server/responses/adapter-dispatch.ts | 8 +- src/server/responses/core-combo-native.ts | 5 +- src/server/responses/core-combo.ts | 1 + src/server/responses/passthrough-dispatch.ts | 12 +- src/server/responses/request-send-budget.ts | 14 ++- src/server/responses/request-spend.ts | 34 ++++-- structure/transports/responses-spend.md | 17 ++- tests/lib/execution-budget-permits.test.ts | 29 ++++- tests/lib/spend-pool-continuity.test.ts | 105 +++++++++++++++++- .../lib/transient-budget-scope-source.test.ts | 20 +++- tests/responses/chat-native-combo.test.ts | 2 + .../responses/responses-core-modules.test.ts | 2 +- .../responses-spend-ledger-wiring.test.ts | 4 +- tests/server/inference-send-budget.test.ts | 2 +- 17 files changed, 268 insertions(+), 79 deletions(-) diff --git a/src/lib/request-execution-budget.ts b/src/lib/request-execution-budget.ts index fcc85734f6f..b2da4cf8b80 100644 --- a/src/lib/request-execution-budget.ts +++ b/src/lib/request-execution-budget.ts @@ -146,9 +146,11 @@ export interface RequestSendObserver { * would cross a ceiling never joins the total, the total stays just under, and the ceiling * never fires for any later request either. */ - charge(options?: { alreadySent?: boolean; onReserved?: (proof: SpendReservationProof) => void }): boolean; + charge(options?: { alreadySent?: boolean; deferDispatch?: boolean; onReserved?: (proof: SpendReservationProof) => void }): boolean; + /** Confirm the exact reservation once its dispatch is known. */ + dispatch?(proof: SpendReservationProof): void; /** Give back a booking whose send never happened. */ - refund(): void; + refund(proof?: SpendReservationProof | null): void; } /** @@ -243,6 +245,7 @@ const sharedSendLedgers = new WeakMap( const dispatchSpendProofs = new WeakMap(); /** One preflight for the exact prepaid dispatch, never another budget or a replayed permit. */ @@ -251,6 +254,21 @@ export function claimDispatchSpendProof(budget: RequestExecutionBudget, permit?: return proof && proof.owner === sharedSendLedgers.get(budget) ? proof.claim() : undefined; } +/** Reports actual sends against only the named permit on this shared ledger. */ +export function reportDispatchSends( + budget: TransientSendBudget, + sends: number, + permit?: SingleUseDispatchPermit, +): void { + const count = Number.isFinite(sends) ? Math.max(0, Math.trunc(sends)) : 0; + if (count === 0) return; + const receipt = permit && dispatchSpendProofs.get(permit); + const prepaid = receipt && receipt.owner === sharedSendLedgers.get(budget as RequestExecutionBudget) + && receipt.report() ? 1 : 0; + // Unnamed, foreign, released or already-reported permits cannot consume another receipt. + budget.used += count - prepaid; +} + /** * One logical request's replacement grant: how many it has spent, and the ceiling it is held * to. @@ -292,24 +310,15 @@ function createRequestExecutionBudgetWithLedger( let alternateTargetSends = 0; let targetTransitions = 0; let lastTargetKey: string | undefined; + const targetReservations: Array<{ targetKey: string }> = []; const budget: RequestExecutionBudget = { get used(): number { return counter.spent; }, set used(next: number) { - // The retry helpers report their real send count by assigning through this field. A - // reservation taken with `countedExternally` has already booked one of those sends, so - // the report settles the pending booking first and only the surplus is charged. - const delta = next - counter.spent; - if (delta <= 0) { - counter.spent = Math.max(0, next); - return; - } - const settled = Math.min(delta, counter.pendingExternalSends.size); - for (let index = 0; index < settled; index += 1) { - counter.pendingExternalSends.delete(counter.pendingExternalSends.values().next().value!); - } - const charged = delta - settled; - counter.spent += charged; + // A numeric report has no receipt identity. Only reportDispatchSends may settle a + // prepaid permit; guessing by reservation order can refund another leg's actual send. + const charged = next - counter.spent; + counter.spent = Math.max(0, next); // These sends have already left. The ledger records them even past a ceiling it would // have refused, because refusing after the fact only hides spend that was really // incurred -- the refusal has to happen at the reservation below, or not at all. @@ -363,7 +372,7 @@ function createRequestExecutionBudgetWithLedger( // for a dispatch a cheaper check above would have refused is spend this request never // makes, and it would hold those tokens against the scope until retention expired. let spendProof: SpendReservationProof | undefined; - if (observer && !observer.charge({ onReserved: proof => { spendProof = proof; } })) { + if (observer && !observer.charge({ deferDispatch: true, onReserved: proof => { spendProof = proof; } })) { return { allowed: false, reason: "spend-exhausted" }; } @@ -371,9 +380,9 @@ function createRequestExecutionBudgetWithLedger( // which two legs read the same remainder, both received a permit, and both dispatched: // one remaining send admitted two physical sends, which is the per-request multiplication // this budget exists to stop. Everything is booked now; `release()` is the way back. - const previousTargetKey = lastTargetKey; counter.spent += 1; - const receipt = {}; + const receipt = { targetKey: intent.targetKey }; + targetReservations.push(receipt); if (intent.countedExternally === true) counter.pendingExternalSends.add(receipt); if (drawsReserve) reserveSpent = true; if (chargesAlternateTarget) alternateTargetSends += 1; @@ -386,15 +395,16 @@ function createRequestExecutionBudgetWithLedger( use(): boolean { if (settled !== "open") return false; settled = "used"; + if (intent.countedExternally !== true && spendProof) observer?.dispatch?.(spendProof); return true; }, assumeCharge(): boolean { - if (settled !== "open") return false; + if (settled !== "open" || (intent.countedExternally === true && !counter.pendingExternalSends.has(receipt))) return false; settled = "used"; // The booking this reservation made for an external reporter is now owned by the - // caller. Leaving it pending is not harmless: the next `used` report of this request - // would settle against it and one real send would go uncharged. + // caller. Close only its own receipt so a later reporter cannot spend it again. if (intent.countedExternally === true) counter.pendingExternalSends.delete(receipt); + if (spendProof) observer?.dispatch?.(spendProof); return true; }, release(): void { @@ -406,15 +416,23 @@ function createRequestExecutionBudgetWithLedger( if (!counter.pendingExternalSends.delete(receipt)) return; } counter.spent -= 1; - observer?.refund(); + observer?.refund(spendProof ?? null); if (drawsReserve) reserveSpent = false; if (chargesAlternateTarget) alternateTargetSends -= 1; if (chargesTransition) targetTransitions -= 1; - lastTargetKey = previousTargetKey; + targetReservations.splice(targetReservations.indexOf(receipt), 1); + lastTargetKey = targetReservations.at(-1)?.targetKey; }, }; let preflightClaimed = false; - dispatchSpendProofs.set(permit, { owner: counter, claim: () => { + dispatchSpendProofs.set(permit, { owner: counter, report: () => { + if (!counter.pendingExternalSends.has(receipt)) return false; + if (spendProof) observer?.dispatch?.(spendProof); + counter.pendingExternalSends.delete(receipt); + // Reset-only helpers report just BEFORE calling the dispatch thunk. Its one use() + // remains available, but release/proof cannot refund or reuse this reported receipt. + return true; + }, claim: () => { if (preflightClaimed || settled === "released" || (intent.countedExternally === true ? !counter.pendingExternalSends.has(receipt) : settled !== "open")) return undefined; preflightClaimed = true; diff --git a/src/server/chat-native.ts b/src/server/chat-native.ts index 3e50b5e02c0..2e394d23350 100644 --- a/src/server/chat-native.ts +++ b/src/server/chat-native.ts @@ -17,7 +17,7 @@ import { isCyberPolicyMessage, SEND_BUDGET_EXHAUSTED_CODE, } from "../lib/errors"; -import type { RequestExecutionBudget } from "../lib/request-execution-budget"; +import { reportDispatchSends, type RequestExecutionBudget, type SingleUseDispatchPermit } from "../lib/request-execution-budget"; import type { AdmissionLease } from "../lib/admission"; import { readBoundedResponseBody } from "../lib/bounded-body"; import { redactSecretString } from "../lib/redact"; @@ -198,6 +198,7 @@ export interface NativeChatExecution extends HandleNativeChatOptions { * the parent row, replace the one the final log settles. */ sendBudget?: RequestExecutionBudget; + comboDispatchPermit?: SingleUseDispatchPermit; /** Replaces the request-relative first-output mark; a combo child records its own. */ onFirstOutput?: () => void; /** The lease a streamed body holds; defaults to `logIds.turnAdmissionLease`. */ @@ -254,6 +255,7 @@ export function createNativeChatComboSource(input: { translatorBudget: input.translatorBudget, finishLog: child.finishLog, sendBudget: child.sendBudget, + comboDispatchPermit: child.comboDispatchPermit, onFirstOutput: child.onFirstOutput, ...(child.turnAdmissionLease ? { turnAdmissionLease: child.turnAdmissionLease } : {}), }, child.attemptHandle), @@ -457,7 +459,7 @@ export async function runNativeChatAttempt( throw new SendBudgetExhaustedError(safeHostLabel(request.url)); } physicalSends += 1; - sendBudget.used += 1; + reportDispatchSends(sendBudget, 1, execution.comboDispatchPermit); } else if (!spendTracker?.charge()) throw new NativeChatSpendRefusal(); noteProviderAttemptSend(logCtx, route.providerName, activeProvider, logCtx.usageLogInputTokens, transportRecovery ?? recovery); // A reselected provider transport is still a physical send: the connection policy diff --git a/src/server/responses/adapter-continuation.ts b/src/server/responses/adapter-continuation.ts index 2d3aa39e5e7..a9fcab36385 100644 --- a/src/server/responses/adapter-continuation.ts +++ b/src/server/responses/adapter-continuation.ts @@ -102,7 +102,7 @@ export function createAdapterContinuations( | "noteAdapterPhysicalSend" | "noteAdapterRecoveryWithheld" | "remainingTransientSendBudget" - | "noteTransientSends" + | "transientSendReporter" | "reserveCredentialHop" | "pendingHopPermit" | "sendBudgetExhausted" @@ -135,7 +135,7 @@ export function createAdapterContinuations( noteAdapterPhysicalSend, noteAdapterRecoveryWithheld, remainingTransientSendBudget, - noteTransientSends, + transientSendReporter, reserveCredentialHop, sendBudgetExhausted, } = sendBudgetState; @@ -264,7 +264,7 @@ export function createAdapterContinuations( ...(continuationTransientPolicy ? { attempts: remainingTransientSendBudget(continuationTransientPolicy.attempts), - onSendsConsumed: noteTransientSends, + onSendsConsumed: transientSendReporter(), } : {}), }, @@ -513,9 +513,9 @@ export function createAdapterContinuations( recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, transportState.activeAdapter.name); // The replay goes out on the next iteration. An adapter that owns its ladder // reserves for that send itself, so hand this reservation down rather than let it - // take a second one for the same replay. A helper-routed replay needs no handoff: - // its reporter settles the booking made above. - if (adapterOwnsDispatch) sendBudgetState.pendingHopPermit = hop.permit; + // take a second one for the same replay. A helper-routed replay carries the + // same permit so its reporter settles only this booking. + sendBudgetState.pendingHopPermit = hop.permit; nextContinuationRecoveryKind = "oauth-account-429"; kiroRefusalPendingReplay = response; continue; @@ -604,9 +604,9 @@ export function createAdapterContinuations( recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, transportState.activeAdapter.name); // The replay goes out on the next iteration. An adapter that owns its ladder // reserves for that send itself, so hand this reservation down rather than let it - // take a second one for the same replay. A helper-routed replay needs no handoff: - // its reporter settles the booking made above. - if (adapterOwnsDispatch) sendBudgetState.pendingHopPermit = hop.permit; + // take a second one for the same replay. A helper-routed replay carries the + // same permit so its reporter settles only this booking. + sendBudgetState.pendingHopPermit = hop.permit; nextContinuationRecoveryKind = "oauth-account-429"; continue; } diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index 3cdede083cd..f228eff3886 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -147,7 +147,7 @@ export async function prepareAdapterExchange( | "noteAdapterPhysicalSend" | "noteAdapterRecoveryWithheld" | "remainingTransientSendBudget" - | "noteTransientSends" + | "transientSendReporter" | "recoverySendAllowance" | "recoveryClassFor" | "sendBudgetExhausted" @@ -182,7 +182,7 @@ export async function prepareAdapterExchange( noteAdapterPhysicalSend, noteAdapterRecoveryWithheld, remainingTransientSendBudget, - noteTransientSends, + transientSendReporter, recoverySendAllowance, recoveryClassFor, sendBudgetExhausted, @@ -374,7 +374,7 @@ export async function prepareAdapterExchange( // remaining allowance; treating the booking as unavailable blocks a cap of two. attempts: Math.min(transientPolicy?.attempts ?? 1, remainingTransientSendBudget(transientPolicy?.attempts ?? 1) + (compactPrepaid ? 1 : 0)), - onSendsConsumed: noteTransientSends, + onSendsConsumed: transientSendReporter(compactPrepaid), } : {}), }, @@ -581,7 +581,7 @@ export async function prepareAdapterExchange( ...(refetchAllowance ? { attempts: refetchAllowance.attempts, - onSendsConsumed: noteTransientSends, + onSendsConsumed: transientSendReporter(refetchAllowance.permit), } : {}), }, diff --git a/src/server/responses/core-combo-native.ts b/src/server/responses/core-combo-native.ts index c70af618f51..3a039f741db 100644 --- a/src/server/responses/core-combo-native.ts +++ b/src/server/responses/core-combo-native.ts @@ -18,7 +18,7 @@ import type { OcxConfig } from "../../types"; import type { RouteResult } from "../../router"; import type { DataPlaneAdmission } from "../auth-cors"; import type { AdmissionLease } from "../../lib/admission"; -import type { RequestExecutionBudget } from "../../lib/request-execution-budget"; +import type { RequestExecutionBudget, SingleUseDispatchPermit } from "../../lib/request-execution-budget"; import type { TransientSendBudget } from "../../lib/upstream-retry"; import type { ResponsesTerminalStatus } from "../../bridge"; import type { NativeChatFinishLog } from "../chat-native"; @@ -61,6 +61,7 @@ export interface NativeComboChildRun { childLog: RequestLogContext; attemptHandle: InferenceAttempt; sendBudget: RequestExecutionBudget; + comboDispatchPermit?: SingleUseDispatchPermit; turnAdmissionLease?: AdmissionLease; onFirstOutput: () => void; /** Reports to the combo's child callbacks; it never writes the parent's final row itself. */ @@ -266,6 +267,7 @@ export interface ComboChildCallbacks { export async function dispatchNativeComboChild(input: { source: ComboProtocolSource; plan: NativeComboChildPlan; + comboDispatchPermit?: SingleUseDispatchPermit; logCtx: RequestLogContext; childLog: RequestLogContext; attempt: PersistedUsageAttempt; @@ -311,6 +313,7 @@ export async function dispatchNativeComboChild(input: { childLog, attemptHandle, sendBudget: plan.sendBudget, + comboDispatchPermit: input.comboDispatchPermit, ...(input.turnAdmissionLease ? { turnAdmissionLease: input.turnAdmissionLease } : {}), onFirstOutput: () => { outputSeen = true; diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index d730349e086..7a9accdd518 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -816,6 +816,7 @@ export async function executeComboResponses( response = nativeChild ? await dispatchNativeComboChild({ source: options.protocolSource!, plan: nativeChild, + comboDispatchPermit: hopDecision?.allowed ? hopDecision.permit : undefined, logCtx, childLog, attempt, diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index ec47d4e6e77..bdda62fdfb1 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -215,6 +215,7 @@ export async function preparePassthroughExchange( ResponsesSendBudget, | "remainingTransientSendBudget" | "noteTransientSends" + | "transientSendReporter" | "recoverySendAllowance" | "recoveryClassFor" | "sendBudgetExhausted" @@ -250,6 +251,7 @@ export async function preparePassthroughExchange( const { remainingTransientSendBudget, noteTransientSends, + transientSendReporter, recoverySendAllowance, recoveryClassFor, sendBudgetExhausted, @@ -992,7 +994,7 @@ export async function preparePassthroughExchange( .then(adoptObservedResponse); }, { abortSignal: upstream.signal, label: safeHostLabel(request.url), - attempts: remainingTransientSendBudget(transientSendAttempts()), onSendsConsumed: noteTransientSends, + attempts: remainingTransientSendBudget(transientSendAttempts()), onSendsConsumed: transientSendReporter(), claimAmbiguousResend: claimPreHeaderResend, }, ); @@ -1095,7 +1097,7 @@ export async function preparePassthroughExchange( .then(adoptObservedResponse); }, { abortSignal: upstream.signal, label: safeHostLabel(request.url), attempts: allowance.attempts, - onSendsConsumed: noteTransientSends, claimAmbiguousResend: claimPreHeaderResend }, + onSendsConsumed: transientSendReporter(allowance.permit), claimAmbiguousResend: claimPreHeaderResend }, ); } catch (err) { return { failed: transportFailureResponse(err) }; @@ -1224,7 +1226,7 @@ export async function preparePassthroughExchange( }, { abortSignal: upstream.signal, label: safeHostLabel(request.url), attempts: remainingTransientSendBudget(transientSendAttempts()), - onSendsConsumed: noteTransientSends, claimAmbiguousResend: claimPreHeaderResend }, + onSendsConsumed: transientSendReporter(), claimAmbiguousResend: claimPreHeaderResend }, ); } catch (err) { return transportFailureResponse(err); @@ -1352,7 +1354,7 @@ export async function preparePassthroughExchange( .then(adoptObservedResponse); }, { abortSignal: upstream.signal, label: safeHostLabel(request.url), attempts: remainingTransientSendBudget(transientSendAttempts()), - onSendsConsumed: noteTransientSends, claimAmbiguousResend: claimPreHeaderResend }, + onSendsConsumed: transientSendReporter(), claimAmbiguousResend: claimPreHeaderResend }, ); } catch (err) { return transportFailureResponse(err); @@ -1487,7 +1489,7 @@ export async function preparePassthroughExchange( .then(adoptObservedResponse); }, { abortSignal: upstream.signal, label: safeHostLabel(request.url), attempts: remainingTransientSendBudget(transientSendAttempts()), - onSendsConsumed: noteTransientSends, claimAmbiguousResend: claimPreHeaderResend }, + onSendsConsumed: transientSendReporter(), claimAmbiguousResend: claimPreHeaderResend }, ); } catch (err) { return transportFailureResponse(err); diff --git a/src/server/responses/request-send-budget.ts b/src/server/responses/request-send-budget.ts index 33f1cf954b2..5b4d3e26eaa 100644 --- a/src/server/responses/request-send-budget.ts +++ b/src/server/responses/request-send-budget.ts @@ -1,5 +1,5 @@ import type { ResponsesRequestContext } from "./core-options"; -import { claimDispatchSpendProof, createRequestExecutionBudget, isRequestExecutionBudget } from "../../lib/request-execution-budget"; +import { claimDispatchSpendProof, createRequestExecutionBudget, isRequestExecutionBudget, reportDispatchSends } from "../../lib/request-execution-budget"; import { chargeWorkflowSends, workflowSendCeilingReached, @@ -57,12 +57,18 @@ export function createResponsesSendBudget( // sends once per child seven hundred times, so every send charged to the request is charged // to the root as well (#4546). const workflowRootId = req.headers.get("x-codex-parent-thread-id")?.trim() || undefined; - const noteTransientSends = (used: number): void => { + const inheritedPermit = options.compactionRecoveryPermit ?? options.comboDispatchPermit; + let pendingHopPermit: SingleUseDispatchPermit | undefined = options.compactionRecoveryPermit; + const recordTransientSends = (used: number, permit?: SingleUseDispatchPermit): void => { const charged = Math.max(0, used); - sendBudget.used += charged; + reportDispatchSends(sendBudget, charged, permit); options.onCompactionRecoverySendsReported?.(charged); chargeWorkflowSends(workflowRootId, charged); }; + const noteTransientSends = (used: number): void => recordTransientSends(used, pendingHopPermit ?? inheritedPermit); + // Capture at helper creation, before another leg can replace the pending handoff. + const transientSendReporter = (permit = pendingHopPermit ?? inheritedPermit) => + (used: number): void => recordTransientSends(used, permit); // Refused before any dispatch, and deliberately not by evicting the root's ledger entry: // dropping the record to make room would hand the fan-out a fresh allowance, which is the // laundering this ceiling exists to stop. The client is told the task needs a new grant @@ -165,7 +171,6 @@ export function createResponsesSendBudget( * refused and the request would answer with a synthetic 502 in place of the real 429 the hop * was recovering from. */ - let pendingHopPermit: SingleUseDispatchPermit | undefined = options.compactionRecoveryPermit; /** * The budget an adapter's OWN dispatch ladder reserves against. * @@ -276,6 +281,7 @@ export function createResponsesSendBudget( return { workflowRootId, noteTransientSends, + transientSendReporter, remainingTransientSendBudget, /** * Physical sends this logical request has already made. diff --git a/src/server/responses/request-spend.ts b/src/server/responses/request-spend.ts index 9ddab48c3e5..ae6d896a6fb 100644 --- a/src/server/responses/request-spend.ts +++ b/src/server/responses/request-spend.ts @@ -54,23 +54,25 @@ export function createRequestSpendTracker( // one. It also means the home in effect at dispatch is the one that gets written. let ledgerRef: SpendReservationLedger | undefined = injected; const ledger = (): SpendReservationLedger => (ledgerRef ??= sharedSpendLedger()); - // Every send this request still owes the ledger an answer for, oldest first. + // Outstanding entries; exact dispatch reports move their reservation to the end. const live: string[] = []; + const pendingDispatch = new Set(); let refusals = 0; let resolved = false; let terminalProcessed = false; /** * Confirm the sends this request has already moved past. * - * A booking is only marked dispatched once a LATER send exists, because that later send - * proves the earlier one left. The newest booking stays open until it is settled, so a - * reservation the budget hands back -- a rotation that found no alternate, a rebuild - * abandoned before the wire -- can still be released for free while this process is alive. + * Legacy direct charges infer dispatch from a later send. Exact budget reservations wait + * for their own dispatch/report instead: reserving B does not prove that A left, and A + * must remain refundable if B reports first. The newest direct charge stays open. * A crash resolves every surviving reservation as unresolved spend regardless of this mark, * because a journal that lost its tail cannot prove a send never left. */ const confirmOlderSends = (): void => { - for (let index = 0; index < live.length - 1; index += 1) ledger().markDispatched(live[index] as string); + for (const sendId of live.slice(0, -1)) { + if (!pendingDispatch.has(sendId)) ledger().markDispatched(sendId); + } }; return { charge(options?: Parameters[0]): boolean { @@ -133,15 +135,29 @@ export function createRequestSpendTracker( } if (!alreadySent) options?.onReserved?.({ ledger: bookedLedger, sendId }); live.push(sendId); - confirmOlderSends(); + if (options?.deferDispatch) pendingDispatch.add(sendId); + else confirmOlderSends(); // It has already left, so the reservation cannot be handed back for free: from here only // a settlement or unresolved spend is honest about it. if (alreadySent) ledger().markDispatched(sendId); return true; }, - refund(): void { - const sendId = live.pop(); + dispatch(proof): void { + if (proof.ledger !== ledgerRef || !pendingDispatch.has(proof.sendId)) return; + ledger().markDispatched(proof.sendId); + pendingDispatch.delete(proof.sendId); + // Terminal usage follows dispatch/report order, not reservation order. + const index = live.indexOf(proof.sendId); + if (index >= 0) live.push(...live.splice(index, 1)); + }, + refund(proof): void { + // null is an exact budget reservation that obtained no durable booking. + if (proof === null || (proof && proof.ledger !== ledgerRef)) return; + const index = proof ? live.indexOf(proof.sendId) : live.length - 1; + if (index < 0) return; + const [sendId] = live.splice(index, 1); if (sendId === undefined) return; + pendingDispatch.delete(sendId); // Undispatched, so this returns the tokens. If the send was already confirmed by a later // one, `abandon` refuses and unresolved is the only honest outcome left. if (!ledger().abandon(sendId)) ledger().markLost(sendId); diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index 1e4e06bfa90..53afd9bb089 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -38,7 +38,13 @@ refunds only charges it made; reporting or assuming that receipt prevents a late The hop pays for a replay that some *other* layer dispatches, so which layer settles the reservation follows the dispatcher, not the ladder. A helper-routed replay reports the same physical send back through `onSendsConsumed`; that is what `countedExternally: true` names, and the -reporter's first send settles the pending booking instead of adding a second charge. An adapter +reporter's first send settles only its named pending permit instead of adding a second charge. +`transientSendReporter` captures that permit before entering a helper; a later handoff cannot +replace it. `reportDispatchSends` verifies shared-ledger ownership and consumes one receipt only. +Native Chat combo children carry the same exact permit to their physical-send boundary. +Numeric `used` updates and unnamed, foreign, released or already-reported permits charge actual +sends without consuming another reservation. Extra retries remain full charges. Reports may arrive +out of reservation order; no FIFO ordering is required. An adapter that owns its transport — Kiro's reset ladder, Cursor's transport ladder, or Devin's bounded pre-output stated-reset replay — reserves once per physical send instead, so no reporter ever arrives. Those ladders are handed @@ -166,9 +172,12 @@ reader requires explicit mappings for the remaining unidentified balances rather zero. `tests/lib/spend-pool-continuity.test.ts` exercises this using the frozen pre-change reader, as well as exact aggregation, active/unresolved sends, retention, failures and rootless preflight. -A booking is confirmed dispatched only once a LATER send exists, because that later send proves -the earlier one left. The newest booking stays open, so a reservation the budget hands back -during this process's lifetime can still be released for free. +Budget reservations retain their exact durable proof until their own dispatch/report confirms +them. A later reservation does not confirm an earlier pending send. Refunds remove the exact +send and original pool, and releasing an older permit preserves the latest surviving target. +Legacy direct charges still infer dispatch from a later charge and leave their newest booking +open. Report order updates terminal attribution; unrelated pending reservations remain independent. +`tests/lib/spend-pool-continuity.test.ts` covers reversed child reports and exact-pool cancellation. Settlement follows what the request learned. The terminal usage belongs to the last send that left, so that one settles with the real figure; every earlier send failed without reporting usage diff --git a/tests/lib/execution-budget-permits.test.ts b/tests/lib/execution-budget-permits.test.ts index 0084f8798f0..a5973d1c231 100644 --- a/tests/lib/execution-budget-permits.test.ts +++ b/tests/lib/execution-budget-permits.test.ts @@ -3,6 +3,7 @@ import { CODEX_TEXT_GUARDED_BUDGET_POLICY, createRequestExecutionBudget, deriveRequestExecutionBudget, + reportDispatchSends, type RequestExecutionBudgetPolicy, } from "../../src/lib/request-execution-budget"; @@ -102,7 +103,7 @@ describe("atomic dispatch permits", () => { expect(leg.permit.use()).toBe(true); // `onSendsConsumed` reporting one physical send settles the pending booking instead of // charging a second time. Charging both is how a four-send cap became a two-send cap. - budget.used += 1; + reportDispatchSends(budget, 1, leg.permit); expect(budget.used).toBe(1); // Sends the helper made beyond the reserved one are still charged in full. @@ -118,7 +119,7 @@ describe("atomic dispatch permits", () => { countedExternally: true, }); if (!leg.allowed) throw new Error("unreachable"); - budget.used += 1; + reportDispatchSends(budget, 1, leg.permit); expect(budget.used).toBe(1); // The send physically happened. A refund here would hand the request a free one back. leg.permit.release(); @@ -293,7 +294,8 @@ describe("a credential hop is settled by whichever layer dispatches its replay", expect(budget.used).toBe(1); // The helper names the same physical send the hop already booked. - budget.used += 1; + if (!hop.allowed) throw new Error("unreachable"); + reportDispatchSends(budget, 1, hop.permit); expect(budget.used).toBe(1); // A genuinely second send is charged in full. budget.used += 1; @@ -373,7 +375,8 @@ describe("derived policy scopes", () => { const target = deriveRequestExecutionBudget(scope, wide); // The reporter names the send that the booking above already paid for. - target.used += 1; + if (!hop.allowed) throw new Error("unreachable"); + reportDispatchSends(target, 1, hop.permit); expect(parent.used).toBe(1); // Anything beyond it is a genuinely new send. target.used += 2; @@ -455,8 +458,10 @@ describe("derived scopes and the durable spend observer", () => { const spy = recordingObserver(); const parent = createRequestExecutionBudget(wide, "lr-once", spy.observer); const scope = deriveRequestExecutionBudget(parent, wide); - expect(scope.reserveDispatch({ sendClass: "initial", targetKey: "a/m", countedExternally: true }).allowed).toBe(true); - deriveRequestExecutionBudget(scope, wide).used += 1; + const hop = scope.reserveDispatch({ sendClass: "initial", targetKey: "a/m", countedExternally: true }); + expect(hop.allowed).toBe(true); + if (!hop.allowed) throw new Error("unreachable"); + reportDispatchSends(deriveRequestExecutionBudget(scope, wide), 1, hop.permit); expect(spy.events).toEqual(["charge"]); expect(parent.used).toBe(1); }); @@ -559,3 +564,15 @@ test("a validated rebase remains admissible after alternate-target spend and ref expect(budget.alternateTargetSends).toBe(1); expect(budget.targetTransitions).toBe(1); }); + +test("a reported external receipt cannot be taken over by an adapter", () => { + const budget = createRequestExecutionBudget(); + const decision = budget.reserveDispatch({ sendClass: "initial", targetKey: "same", countedExternally: true }); + if (!decision.allowed) throw new Error("synthetic reservation refused"); + reportDispatchSends(budget, 1, decision.permit); + expect(decision.permit.assumeCharge()).toBe(false); + expect(decision.permit.use()).toBe(true); // reset helper reports before its dispatch thunk + expect(decision.permit.use()).toBe(false); + decision.permit.release(); + expect(budget.used).toBe(1); +}); diff --git a/tests/lib/spend-pool-continuity.test.ts b/tests/lib/spend-pool-continuity.test.ts index 3ee0ba59051..34923a6445c 100644 --- a/tests/lib/spend-pool-continuity.test.ts +++ b/tests/lib/spend-pool-continuity.test.ts @@ -16,7 +16,7 @@ import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; import { admitHttpWorkflowTurn, workflowDecisionRefusalResponse } from "../../src/server/workflow-refusal"; import { createResponsesSendBudget } from "../../src/server/responses/request-send-budget"; -import { claimDispatchSpendProof, createRequestExecutionBudget, deriveRequestExecutionBudget } from "../../src/lib/request-execution-budget"; +import { claimDispatchSpendProof, createRequestExecutionBudget, deriveRequestExecutionBudget, reportDispatchSends } from "../../src/lib/request-execution-budget"; import { createPoolContinuity } from "../../src/lib/spend-pool-continuity"; import { listWorkflowBudgetEvents, resetWorkflowBudgetsForTest, workflowSpendCeilingReached } from "../../src/lib/workflow-budget"; import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; @@ -377,7 +377,7 @@ for (const end of ["release", "report", "assume"] as const) { expect(workflowSpendCeilingReached(undefined, ledger, "provider", proof)).toBeUndefined(); expect(claimDispatchSpendProof(budget, permit)).toBeUndefined(); } - if (end === "report") budget.used += 1; + if (end === "report") reportDispatchSends(budget, 1, permit); if (end === "assume") expect(permit.assumeCharge()).toBe(true); permit.release(); permit.release(); @@ -398,7 +398,7 @@ test("prepaid proof belongs to one shared budget and one still-pending dispatch" const { budget, reservePermit } = prepaidFixture(); const permit = reservePermit(); if (end === "release") permit.release(); - if (end === "report") budget.used += 1; + if (end === "report") reportDispatchSends(budget, 1, permit); if (end === "assume") expect(permit.assumeCharge()).toBe(true); expect(claimDispatchSpendProof(budget, permit)).toBeUndefined(); } @@ -420,12 +420,12 @@ test("receipt identity survives out-of-order handoff and reports cannot authoriz expect(b.assumeCharge()).toBe(true); expect(claimDispatchSpendProof(first.budget, b)).toBeUndefined(); expect(claimDispatchSpendProof(first.budget, a)).toBeDefined(); - first.budget.used += 1; + reportDispatchSends(first.budget, 1, a); expect(first.budget.used).toBe(2); // report consumed A; B was already assumed const second = prepaidFixture(10, 100); const reported = second.reservePermit(), pending = second.reservePermit(); - second.budget.used += 1; + reportDispatchSends(second.budget, 1, reported); expect(claimDispatchSpendProof(second.budget, reported)).toBeUndefined(); reported.release(); expect(second.budget.used).toBe(2); // cannot refund a different pending receipt @@ -525,3 +525,98 @@ test("actual combo dispatch forwards only its own prepaid permit into the child removeTreeWithRetry(home); } }); + +test("a later child report preserves the earlier receipt and refunds its exact pool", () => { + const ledger = createSpendReservationLedger({ salt }); + const logCtx = { provider: "earlier", usageLogInputTokens: 10 }; + const tracker = createRequestSpendTracker(logCtx, "root", ledger); + const budget = createRequestExecutionBudget(undefined, undefined, tracker); + const first = budget.reserveDispatch({ sendClass: "initial", targetKey: "same", countedExternally: true }); + logCtx.provider = "later"; + logCtx.usageLogInputTokens = 30; + const second = budget.reserveDispatch({ sendClass: "initial", targetKey: "same", countedExternally: true }); + if (!first.allowed || !second.allowed) throw new Error("synthetic reservation refused"); + const child = createResponsesSendBudget({ + req: new Request("http://localhost/v1/responses"), logCtx: {}, + options: { sendBudget: deriveRequestExecutionBudget(budget, budget.policy), comboDispatchPermit: second.permit }, + }); + if (child instanceof Response) throw new Error("synthetic child refused"); + child.noteTransientSends(1); + expect(claimDispatchSpendProof(budget, first.permit)).toBeDefined(); + expect(claimDispatchSpendProof(budget, second.permit)).toBeUndefined(); + second.permit.release(); + expect(budget.used).toBe(2); + first.permit.release(); + expect(budget.used).toBe(1); + expect(ledger.snapshot("pool", "earlier")).toMatchObject({ reserved: 0, unresolved: 0 }); + expect(ledger.snapshot("pool", "later")).toMatchObject({ reserved: 30, unresolved: 0 }); + tracker.settle({ inputTokens: 25, outputTokens: 0 }); + expect(ledger.snapshot("pool", "later")).toMatchObject({ settled: 25, reserved: 0, unresolved: 0 }); +}); + +test("captured reporters retain receipt ownership across handoffs, cancellation and retries", () => { + for (const finish of ["release", "report", "assume"] as const) { + const { ledger, tracker, budget, reservePermit } = prepaidFixture(10, 100); + const a = reservePermit(); + const owner = createResponsesSendBudget({ + req: new Request("http://localhost/v1/responses"), logCtx: {}, options: { sendBudget: budget }, + }); + if (owner instanceof Response) throw new Error("synthetic owner refused"); + owner.pendingHopPermit = a; + const reportA = owner.transientSendReporter(); + const b = reservePermit(); + owner.pendingHopPermit = b; + const reportB = owner.transientSendReporter(); + reportB(0); // A cancelled/no-send helper cannot settle a receipt. + reportB(1); + b.release(); + expect(budget.used).toBe(2); + expect(claimDispatchSpendProof(budget, b)).toBeUndefined(); + expect(claimDispatchSpendProof(budget, a)).toBeDefined(); + if (finish === "report") reportA(1); + if (finish === "assume") expect(a.assumeCharge()).toBe(true); + a.release(); + a.release(); + expect(budget.used).toBe(finish === "release" ? 1 : 2); + // The same reporter's next count is a real retry, never another prepaid receipt. + reportB(1); + expect(budget.used).toBe(finish === "release" ? 2 : 3); + tracker.settle(undefined); + expect(ledger.snapshot("pool", "provider")).toMatchObject({ + reserved: 0, settled: 0, unresolved: finish === "release" ? 20 : 30, + }); + } +}); + +test("unnamed and foreign reports cannot consume a pending receipt", () => { + const { ledger, budget, reservePermit } = prepaidFixture(10, 100); + const a = reservePermit(), b = reservePermit(); + const foreign = prepaidFixture(10, 100).reservePermit(); + budget.used += 1; + reportDispatchSends(deriveRequestExecutionBudget(budget, budget.policy), 1, foreign); + expect(budget.used).toBe(4); + expect(claimDispatchSpendProof(budget, a)).toBeDefined(); + expect(claimDispatchSpendProof(budget, b)).toBeDefined(); + a.release(); + b.release(); + expect(budget.used).toBe(2); + expect(ledger.snapshot("pool", "provider")).toMatchObject({ reserved: 20, unresolved: 0 }); + reportDispatchSends(budget, 1, a); // A late physical report is counted; no proof is revived. + expect(budget.used).toBe(3); + expect(claimDispatchSpendProof(budget, a)).toBeUndefined(); +}); + +test("releasing an older receipt preserves the later target and its recovery charges", () => { + const budget = createRequestExecutionBudget(); + const a = budget.reserveDispatch({ sendClass: "initial", targetKey: "a", countedExternally: true }); + const b = budget.reserveDispatch({ sendClass: "account-failover", targetKey: "b", countedExternally: true }); + if (!a.allowed || !b.allowed) throw new Error("synthetic reservation refused"); + reportDispatchSends(budget, 1, b.permit); + a.permit.release(); + expect(budget.used).toBe(1); + expect(budget.lastTargetKey).toBe("b"); + expect(budget.alternateTargetSends).toBe(1); + expect(budget.targetTransitions).toBe(1); + b.permit.release(); + expect(budget.used).toBe(1); +}); diff --git a/tests/lib/transient-budget-scope-source.test.ts b/tests/lib/transient-budget-scope-source.test.ts index ecf41ca3613..9f314bbc5d8 100644 --- a/tests/lib/transient-budget-scope-source.test.ts +++ b/tests/lib/transient-budget-scope-source.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test"; import { createRequestExecutionBudget, deriveRequestExecutionBudget, + reportDispatchSends, type RequestExecutionBudget, type RequestExecutionBudgetPolicy, type RequestSendObserver, @@ -91,7 +92,7 @@ describe("transient send accounting stays request-scoped", () => { return new Response("ok"); }, { attempts: 1, - onSendsConsumed: (count) => { budget.used += count; }, + onSendsConsumed: (count) => { reportDispatchSends(budget, count, reserved.permit); }, }); expect(response.status).toBe(200); @@ -102,6 +103,23 @@ describe("transient send accounting stays request-scoped", () => { expect(reserved.permit.use()).toBe(false); }); + test("a reset helper can report its exact receipt before the single-use dispatch thunk", async () => { + const budget = createRequestExecutionBudget(THREE_SEND_POLICY); + const earlier = budget.reserveDispatch({ sendClass: "initial", targetKey: "same", countedExternally: true }); + const later = budget.reserveDispatch({ sendClass: "initial", targetKey: "same", countedExternally: true }); + if (!earlier.allowed || !later.allowed) throw new Error("synthetic reservation refused"); + let physicalSends = 0; + await fetchWithResetRetry(async () => { + expect(later.permit.use()).toBe(true); + physicalSends += 1; + return new Response("ok"); + }, { attempts: 1, onSendsConsumed: count => reportDispatchSends(budget, count, later.permit) }); + later.permit.release(); + earlier.permit.release(); + expect(budget.used).toBe(physicalSends); + expect(physicalSends).toBe(1); + }); + test("the transient wrapper reports a rejected physical send exactly once", async () => { const observer = recordingObserver(); const budget = createRequestExecutionBudget(THREE_SEND_POLICY, "lr-rejected", observer); diff --git a/tests/responses/chat-native-combo.test.ts b/tests/responses/chat-native-combo.test.ts index 55b9d15a679..c527364b37e 100644 --- a/tests/responses/chat-native-combo.test.ts +++ b/tests/responses/chat-native-combo.test.ts @@ -168,6 +168,7 @@ describe("native Chat candidates in a combo", () => { expect(b.bodies).toHaveLength(0); expect(rows).toHaveLength(1); expect(rows[0]!.status).toBe(200); + expect(rows[0]!.spend).toMatchObject({ sends: 1, unresolved: 0 }); expect(rows[0]!.provider).toBe("combo"); expect(rows[0]!.protocolTrace).toMatchObject({ inbound: "chat", mode: "native", requestPath: ["chat", "chat"], @@ -201,6 +202,7 @@ describe("native Chat candidates in a combo", () => { expect(b.bodies[0]).not.toHaveProperty("messages"); expect(rows).toHaveLength(1); expect(rows[0]!.attempts?.map(attempt => attempt.status)).toEqual([503, 200]); + expect(rows[0]!.spend).toMatchObject({ sends: 4, unresolved: 0 }); expect(rows[0]!.protocolTrace).toMatchObject({ inbound: "chat", requestPath: ["chat", "responses"], diff --git a/tests/responses/responses-core-modules.test.ts b/tests/responses/responses-core-modules.test.ts index 6d92a3ab551..426889d924e 100644 --- a/tests/responses/responses-core-modules.test.ts +++ b/tests/responses/responses-core-modules.test.ts @@ -161,7 +161,7 @@ describe("Responses request-owned send budget after extraction", () => { expect(owner.pendingHopPermit).toBeUndefined(); expect(hop.permit.use()).toBe(true); expect(hop.permit.use()).toBe(false); - owner.noteTransientSends(1); + owner.transientSendReporter(allowance.permit)(1); expect(holder.used).toBe(4); expect(owner.remainingTransientSendBudget(3)).toBe(0); } finally { dispose(); } diff --git a/tests/responses/responses-spend-ledger-wiring.test.ts b/tests/responses/responses-spend-ledger-wiring.test.ts index 4fa5fc1d6ee..152e69f0f6e 100644 --- a/tests/responses/responses-spend-ledger-wiring.test.ts +++ b/tests/responses/responses-spend-ledger-wiring.test.ts @@ -7,7 +7,7 @@ import { DEFAULT_SPEND_RESERVATION_POLICY, type SpendJournal, } from "../../src/lib/spend-reservation-ledger"; -import { createRequestExecutionBudget } from "../../src/lib/request-execution-budget"; +import { createRequestExecutionBudget, reportDispatchSends } from "../../src/lib/request-execution-budget"; import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; import { SpendLedgerOwnerError, type SpendLedgerOwnerErrorCode } from "../../src/lib/spend-ledger-owner"; import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; @@ -189,7 +189,7 @@ describe("the request path books every physical send on the durable ledger", () }), body, "spend", config, logCtx, { sendBudget, translatorBudget }, { handleResponses: async (_req, _config, childLog, options) => { children += 1; - options!.sendBudget!.used += 1; // Report the send already reserved by the combo. + reportDispatchSends(options!.sendBudget!, 1, options!.comboDispatchPermit); childLog.provider += `-account-${children}`; return children === 1 ? Response.json({ error: { message: "fixture outage" } }, { status: 503 }) diff --git a/tests/server/inference-send-budget.test.ts b/tests/server/inference-send-budget.test.ts index 0e3ca1597a9..4870996cf16 100644 --- a/tests/server/inference-send-budget.test.ts +++ b/tests/server/inference-send-budget.test.ts @@ -64,7 +64,7 @@ describe("createInferenceSendBudget", () => { if (sends === 0) hop.permit?.use(); sends++; return new Response("", { status: sends === 3 ? 200 : 503 }); - }, { attempts: allowance.attempts, onSendsConsumed: owner.noteTransientSends }); + }, { attempts: allowance.attempts, onSendsConsumed: owner.transientSendReporter(allowance.permit) }); expect(response.status).toBe(200); expect(sends).toBe(3); expect(budget.used).toBe(12); From 056941866fdc815708313a6d2744bf1e4e221c49 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Thu, 1 Oct 2026 22:32:02 -0700 Subject: [PATCH 5/7] fix(spend): report prepaid reset-only combo sends --- src/server/responses/adapter-dispatch.ts | 3 +++ src/server/responses/core-options.ts | 4 ++-- structure/transports/responses-spend.md | 2 ++ .../lib/ambiguous-resend-composition.test.ts | 22 ++++++++++--------- .../responses-compaction-recovery.test.ts | 8 ++++++- 5 files changed, 26 insertions(+), 13 deletions(-) diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index f228eff3886..500039924b0 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -365,6 +365,9 @@ export async function prepareAdapterExchange( { abortSignal: upstream.signal, label: safeHostLabel(builtInitialRequest.url), + // Even the reset-only combo leg must settle its named prepaid receipt. + ...(options.comboDispatchPermit + ? { onSendsConsumed: transientSendReporter(options.comboDispatchPermit) } : {}), ...(transientPolicy || compactPrepaid // Draws the remainder, not the raw policy. A combo child inherits the parent's // holder but used to take a fresh full allowance on its own first send, so the diff --git a/src/server/responses/core-options.ts b/src/server/responses/core-options.ts index 445b004ef40..68e7a9eb062 100644 --- a/src/server/responses/core-options.ts +++ b/src/server/responses/core-options.ts @@ -68,7 +68,7 @@ export interface HandleResponsesOptions { compactionRecoveryKind?: "compaction-v1" | "compaction-v2"; onCompactionRecoveryRoute?: (route: RouteResult) => void; onCompactionRecoveryAdapterEvent?: (event: AdapterEvent) => void; - /** Physical-send reports already delivered to the shared used setter, including booking settlement. */ + /** Physical-send reports already delivered to the shared ledger, including exact booking settlement. */ onCompactionRecoverySendsReported?: (count: number) => void; /** Private holder for the Kiro serving-account lease. */ accountLoad?: { lease: AccountLease | null; cancelled: boolean }; @@ -153,7 +153,7 @@ export interface HandleResponsesOptions { callerDirectAuth?: CallerDirectAuth | null; /** Internal recursion guard; callers outside this module must not set it. */ comboAttempt?: boolean; - /** Exact externally booked combo hop, used only for its child's spend preflight. */ + /** Exact externally booked combo hop, used for its child's spend preflight and send reports. */ comboDispatchPermit?: SingleUseDispatchPermit; /** Internal handoff: this combo was selected by shadow-call interception. */ shadowCallIntercepted?: boolean; diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index 53afd9bb089..339349a5844 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -42,6 +42,8 @@ reporter's first send settles only its named pending permit instead of adding a `transientSendReporter` captures that permit before entering a helper; a later handoff cannot replace it. `reportDispatchSends` verifies shared-ledger ownership and consumes one receipt only. Native Chat combo children carry the same exact permit to their physical-send boundary. +Reset-only generic combo helpers report their named prepaid receipt too, without changing +the selected retry cap; compaction reconciliation therefore does not count that source again. Numeric `used` updates and unnamed, foreign, released or already-reported permits charge actual sends without consuming another reservation. Extra retries remain full charges. Reports may arrive out of reservation order; no FIFO ordering is required. An adapter diff --git a/tests/lib/ambiguous-resend-composition.test.ts b/tests/lib/ambiguous-resend-composition.test.ts index f82bf06bcd2..093e4a61932 100644 --- a/tests/lib/ambiguous-resend-composition.test.ts +++ b/tests/lib/ambiguous-resend-composition.test.ts @@ -5,6 +5,8 @@ import { CODEX_TEXT_GUARDED_BUDGET_POLICY, createRequestExecutionBudget, deriveRequestExecutionBudget, + reportDispatchSends, + type SingleUseDispatchPermit, type RequestExecutionBudget, type SendClass, } from "../../src/lib/request-execution-budget"; @@ -96,23 +98,23 @@ function oneLogicalRequest() { * What src/server/responses/request-send-budget.ts hands a recovery leg: the base allowance * while it lasts, then the single final-recovery reserve, and nothing after that. */ - const recoveryAttempts = (sendClass: SendClass, targetKey: string): number => { + const recoveryAllowance = (sendClass: SendClass, targetKey: string): { attempts: number; permit?: SingleUseDispatchPermit } => { const base = budget.remainingBaseSends(TRANSIENT_RETRY_MAX_ATTEMPTS); - if (base > 0) return base; + if (base > 0) return { attempts: base }; const decision = budget.reserveDispatch({ sendClass, targetKey, countedExternally: true }); - return decision.allowed ? 1 : 0; + return decision.allowed ? { attempts: 1, permit: decision.permit } : { attempts: 0 }; }; return { budget, get physicalSends(): number { return counts.physical; }, get ambiguousSends(): number { return counts.ambiguous; }, - recoveryAttempts, + recoveryAllowance, /** A leg that fails before any response head, through the real reset ladder. */ - preHeader: (outcomes: Array, row: ProviderRow, attempts?: number): Promise => + preHeader: (outcomes: Array, row: ProviderRow, allowance?: { attempts: number; permit?: SingleUseDispatchPermit }): Promise => fetchWithResetRetry(dispatcher(outcomes), { - attempts: attempts ?? budget.remainingBaseSends(TRANSIENT_RETRY_MAX_ATTEMPTS), - onSendsConsumed: sends => { budget.used += sends; }, + attempts: allowance?.attempts ?? budget.remainingBaseSends(TRANSIENT_RETRY_MAX_ATTEMPTS), + onSendsConsumed: sends => reportDispatchSends(budget, sends, allowance?.permit), claimAmbiguousResend: () => authorize("pre-header", row).allowed, }), /** A stream that died after the head while carrying only control events. */ @@ -191,9 +193,9 @@ describe("one resend budget across composed recovery legs", () => { // the account it moves to is a row the operator tuned HIGHER. A ceiling read from the // asking leg let that row buy a second duplicate inference on the way out; the ceiling // is the smallest any leg presented, so it buys nothing. - const attempts = request.recoveryAttempts("account-failover", "other-account"); - expect(attempts).toBe(1); - const lastSend = await request.preHeader([reset(), new Response("duplicate")], MORE_PERMISSIVE, attempts); + const allowance = request.recoveryAllowance("account-failover", "other-account"); + expect(allowance.attempts).toBe(1); + const lastSend = await request.preHeader([reset(), new Response("duplicate")], MORE_PERMISSIVE, allowance); expect(isNonReplayableResponse(lastSend)).toBe(true); expect(request.ambiguousSends).toBe(GRANT); diff --git a/tests/responses/responses-compaction-recovery.test.ts b/tests/responses/responses-compaction-recovery.test.ts index 076909a06d3..0fc68284d06 100644 --- a/tests/responses/responses-compaction-recovery.test.ts +++ b/tests/responses/responses-compaction-recovery.test.ts @@ -322,11 +322,17 @@ describe("routed compaction emergency integration", () => { const budget = createRequestExecutionBudget({ maxTotalModelSends: 2, baseSendAllowance: 2, finalRecoveryAllowance: 0, maxAlternateTargetSends: 1, maxTargetTransitions: 1 }); const reservation = budget.reserveDispatch({ sendClass: "initial", targetKey: "source/swe-2", countedExternally: true }); expect(reservation.allowed).toBe(true); - const response = await handleResponses(request(), config, { model: "", provider: "" }, { sendBudget: budget }); + if (!reservation.allowed) throw new Error("synthetic source reservation refused"); + const response = await handleResponses(request(), config, { model: "", provider: "" }, { + sendBudget: budget, comboDispatchPermit: reservation.permit, + }); expect((await response.json()).status).toBe("completed"); expect(sourceRequests).toBe(1); expect(calls.map(call => call.model)).toEqual(["rescue"]); expect(budget.used).toBe(2); + expect(reservation.permit.assumeCharge()).toBe(false); + reservation.permit.release(); + expect(budget.used).toBe(2); }); test("runTurn emergency consumes its prepaid permit once rather than taking another send", async () => { From 023c3d8c4c1fc7a9fabda8df8a1b2fe05e80384b Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Fri, 2 Oct 2026 00:00:59 -0700 Subject: [PATCH 6/7] fix(spend): aggregate pool eviction once and sync locale guidance --- .../docs/ja/reference/configuration/server.md | 57 +++++++++++++++++ .../docs/ko/reference/configuration/server.md | 53 ++++++++++++++++ .../docs/ru/reference/configuration/server.md | 63 +++++++++++++++++++ .../zh-cn/reference/configuration/server.md | 47 ++++++++++++++ src/lib/spend-reservation-ledger.ts | 15 ++++- structure/transports/responses-spend.md | 3 + tests/lib/spend-pool-continuity.test.ts | 44 +++++++++++++ 7 files changed, 281 insertions(+), 1 deletion(-) diff --git a/docs-site/src/content/docs/ja/reference/configuration/server.md b/docs-site/src/content/docs/ja/reference/configuration/server.md index c11d7af21c0..0c062f1befb 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/server.md +++ b/docs-site/src/content/docs/ja/reference/configuration/server.md @@ -187,3 +187,60 @@ Anthropic OAuth サイドカーは、opencodex の既存のクロード コー メイン Codex アカウント行の `quotaRefresh` はクォータ取得の診断情報であり、残量やモデルへのアクセス権を示すものではありません。キャッシュ利用時や取得を行わない場合は省略されることがあります。取得には操作中のシェルではなく、実行中のプロキシサービスの環境が使われます。`proxy` 未設定では既存の環境を維持し、`"auto"` は起動時の Windows または macOS の静的 HTTP/HTTPS 設定を読みます。macOS では継承したプロキシがある場合、読み取りを行いません。macOS では有効な `*.` を `.` に変換します。`*.local` は `foo.local` と基底名 `local` を直接接続にしますが、`xlocal` は対象外です。`169.254/16`、`169.254.0.0/16`、`fe80::/10` は診断を出して省略し、リンクローカル IP アドレスはプロキシを使います。IP アドレスと `*` は受け入れますが、その他の CIDR、glob、単純ホスト名の例外では環境を変更せず検出を中止します。PAC/WPAD、SOCKS のみの設定、実行中の変更は自動反映されません。TUN での成功だけでは HTTP プロキシ経路の正常性は確認できません。[コマンドと状態の説明（英語）](/reference/configuration/server/#codex-quota-network-diagnostics)を参照してください。 `dropCodexSafetyBuffering`: プロバイダーの安全性の適用と拒否応答は変更しません。native `codex.response.metadata.headers` WebSocket メタデータと `/responses/compact` は対象外です。 + +## 過去のプール使用量の継承 + +プロバイダープールの上限にはルーティング先の正規プロバイダーを使い、リクエストログにはアカウント別の +表示ラベルを残します。古いジャーナルでは、その表示ラベルに使用量が計上されている場合があります。 +ジャーナルに保存されるのはソルト付きのエイリアスであり、元のプロバイダー名やアカウント名ではありません。 +そのため、OpenCodex は現在のアカウント一覧、ラベルの接頭辞、短縮されたアカウント ID から対応関係を推測しません。 + +`spend.pool.maxTokens` が設定されていて、正の使用量が残る過去のプールを識別できない場合、推論リクエストは +プロバイダーへ接続する前にローカルの HTTP 429 と +`x-opencodex-local-refusal: workflow_pool_history_unresolved` で拒否されます。 +これにより、ワークフロールートを持たないリクエストを含め、本来は有効なリクエストが一時的にブロックされることがあります。 +ルーティング後の事前確認でも、合算した使用量がすでに上限に達している正規プールを拒否します。 +予約によってすでに許可されたリカバリー送信やコンボ送信では、その予約を自分自身に対して再度計上しません。 +それ以外の予約はすべて計上されます。この確認により、送信後に使用量を報告するパススルー送信が +アトミックな予約に変わるわけではありません。上限をまたぐ送信、同時に許可されるリクエスト、 +事後報告される再試行には、従来の制約が残ります。監視のみの構成は引き続き監視のみです。 +ルートと ID の上限もそれぞれ独立して適用されます。 + +解決するには、運用者自身が保持している証拠を使い、過去の各ソルト付きプールエイリアスが +どの正規プロバイダーに属するかを確認する必要があります。`config.json` のトップレベルに +`spendPoolAliases` オブジェクトを追加してください。各キーには、その同じインストールのジャーナルにある +32 文字の小文字 16 進数のプールエイリアスを正確に指定し、各値には確認済みの正規プロバイダー ID を指定します。 +古いエイリアスがすでに正規プロバイダーを表していても、ID メタデータがなければ明示的なエントリが必要です。 +正の使用量を持ち、識別できないエイリアスはブロックされたままです。似た名前、アカウントの削除、 +短縮ラベルの衝突から対応関係を推測しないでください。ジャーナルやソルトを公開しないでください。 + +`spendPoolAliases` は `spend` の外に置いてください。古いバージョンは `spend` 内の未知のキーを拒否し、 +セクション全体を無効にする場合があります。無効なトップレベルのマッピングは設定の書き込み時に拒否されます。 +手動編集で `spendPoolAliases` が不正になった場合、既存の上限を維持したままプールへのリクエスト受け入れを拒否します。 +通常の設定手順でマッピングを修正し、再起動してください。マッピングを空にしたり削除したりしても、 +すでに永続的に記録された対応関係は消えません。一度リダイレクトされたエイリアスを別のグループへ再割り当てすることはできません。 +確認済みの正規プロバイダー名の変更では、既存の使用量を分割せずにグループを統合できます。 +マッピング全体をまとめて検証するため、有効なグループ統合はエイリアスキーの順序に依存しません。 + +元の使用量はそれぞれ一度だけ計上されます。確定済みの使用量、処理中の予約、未解決の使用量はすべて計上し、 +不明な使用量を払い戻しとして扱うことはありません。元の予約先は保持されます。 +新しいリクエストでは引き続き正規プロバイダーのプールを記録し、チェックポイントには生のアカウント名や +プロバイダー名を追加せず、ソルト付きの ID 識別情報を保持します。識別できない正の使用履歴と、稼働中または +上限に達したグループはクリーンアップから保護されます。休眠中で上限未満の項目には従来の保持ルールが引き続き適用されます。 +ID 識別情報の保存容量には上限があり、容量を使い切った場合は情報を忘れるのではなくプールへの受け入れを停止します。 +検証できない履歴を復元する自動ツールはありません。 + +### 安全なロールバックの要件 + +サポートされるロールバックでは、リクエストを正規プロバイダーに帰属させる処理と、プール間の継承に対応した +台帳の読み書き処理の両方を維持するかバックポートし、同じジャーナル、ソルト、確認済みマッピングを使う必要があります。 +最新のジャーナルを保持してください。アップグレード前のコピーを復元すると、それ以降の使用量が欠落します。 +ダウングレードを互換性があるように見せるために、記録の削除、ソルトの変更、上限の引き上げ、制限の無効化をしないでください。 + +変更を加えていない古いバイナリへの切り戻しは、**サポート対象のロールバックではありません**。 +古いバイナリは v1 の生の計上値を読めても、エイリアスをまたぐプロバイダー合計に上限を適用せず、 +新たなアカウントラベルのプールを作成したり、圧縮時に任意の ID メタデータを破棄したりする場合があります。 +ダウングレードを自動的に防ぐ仕組みはありません。そのような古い書き込み処理が実行された場合、新しい読み取り処理は、 +運用者が残っているすべてのマッピングを明示的に再確認するまで、識別できない正の使用履歴を拒否します。 +ストレージやジャーナルの整合性による拒否はエイリアスの問題とは別で、`workflow_spend_undurable` のままです。 +安全でないファイルや所有権の問題は、代わりにストレージエラーとして伝播する場合がありますが、リクエストは許可されません。 diff --git a/docs-site/src/content/docs/ko/reference/configuration/server.md b/docs-site/src/content/docs/ko/reference/configuration/server.md index ee24fd4f16e..d0980a0d407 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/server.md +++ b/docs-site/src/content/docs/ko/reference/configuration/server.md @@ -244,3 +244,56 @@ Anthropic OAuth 사이드카는 opencodex의 기존 Claude Code OAuth fingerprin ## Codex 할당량 네트워크 진단 메인 Codex 계정 행의 `quotaRefresh`는 할당량 조회 결과를 분류하는 진단값입니다. 남은 할당량이나 모델 접근 권한을 뜻하지 않으며, 캐시를 쓰거나 조회하지 않았다면 생략될 수 있습니다. 요청은 명령을 입력한 터미널이 아니라 실행 중인 프록시 서비스의 환경을 따릅니다. `proxy`를 지정하지 않으면 기존 환경을 유지하고, `"auto"`는 시작 시 Windows 또는 macOS의 정적 HTTP/HTTPS 설정을 읽습니다. macOS에서는 상속된 프록시가 있으면 읽지 않습니다. macOS에서는 유효한 `*.`을 `.`으로 바꿉니다. `*.local`은 `foo.local`과 최상위 이름 `local`을 직접 연결하지만 `xlocal`은 제외합니다. `169.254/16`, `169.254.0.0/16`, `fe80::/10`은 진단 메시지와 함께 생략하므로 링크 로컬 IP 주소는 프록시를 사용합니다. IP 주소와 `*`는 허용하지만 다른 CIDR, glob, 단순 호스트명 예외는 환경 변경 전에 탐색을 거부합니다. PAC/WPAD, SOCKS 전용 설정과 실행 중 변경은 자동으로 반영하지 않습니다. TUN에서 성공했다고 HTTP 프록시 경로도 정상이라는 뜻은 아닙니다. 명령과 상태값은 [네트워크 진단(영문)](/reference/configuration/server/#codex-quota-network-diagnostics)에서 확인하세요. + +## 과거 풀 사용량의 연속성 + +제공자 풀 상한은 라우팅된 정규 제공자를 기준으로 적용하고, 요청 로그에는 계정별 표시 레이블을 유지합니다. +이전 저널에는 이 표시 레이블에 사용량이 기록되어 있을 수 있습니다. 저널은 원래 제공자·계정 이름이 아닌 +솔트가 적용된 별칭을 저장하므로, OpenCodex는 현재 계정 목록, 레이블 접두사, 축약된 계정 ID로 매핑을 추측하지 않습니다. + +`spend.pool.maxTokens`가 설정되어 있고 양수의 사용량이 남은 과거 풀을 식별하지 못하면, 추론 요청은 +제공자에 접속하기 전에 로컬 HTTP 429와 +`x-opencodex-local-refusal: workflow_pool_history_unresolved`로 거부됩니다. +워크플로 루트가 없는 요청을 포함해 원래 유효한 요청도 일시적으로 차단될 수 있습니다. +라우팅 후 사전 검사에서도 합산 사용량이 이미 한도에 도달한 정규 풀을 거부합니다. +예약을 통해 이미 허용된 복구 또는 콤보 전송에서는 그 예약을 자신에게 다시 계산하지 않습니다. +다른 예약은 모두 계산합니다. 이 검사가 사용량을 사후 보고하는 패스스루 전송을 원자적 예약으로 바꾸지는 않습니다. +한도를 넘기는 전송, 동시 요청 허용, 사후 보고되는 재시도에는 기존 제약이 그대로 남습니다. +관찰 전용 구성은 계속 관찰 전용으로 동작합니다. 루트와 ID 상한도 각각 독립적으로 적용됩니다. + +해결하려면 운영자가 보관한 근거를 사용해 과거의 각 솔트 적용 풀 별칭이 어느 정규 제공자에 속하는지 확인해야 합니다. +`config.json`의 최상위에 `spendPoolAliases` 객체를 추가하세요. 각 키는 동일 설치의 저널에 있는 +32자리 소문자 16진수 풀 별칭과 정확히 일치해야 하고, 각 값은 확인된 정규 제공자 ID여야 합니다. +이전 별칭이 이미 정규 제공자를 나타내더라도 ID 메타데이터가 없으면 명시적 항목이 필요합니다. +식별할 수 없는 양수 사용량의 별칭은 계속 차단됩니다. 비슷한 이름, 계정 삭제, 축약 레이블 충돌로 +매핑을 추측하지 말고, 저널이나 솔트를 공개하지 마세요. + +`spendPoolAliases`는 `spend` 밖에 두세요. 이전 버전은 `spend` 안의 알 수 없는 키를 거부하며 +섹션 전체를 비활성화할 수 있습니다. 잘못된 최상위 매핑은 설정 쓰기 시 거부됩니다. +수동 편집으로 `spendPoolAliases`가 잘못된 형식이 되면 기존 상한을 유지한 채 풀 요청 허용을 차단합니다. +일반적인 설정 절차로 매핑을 수정하고 재시작하세요. 매핑을 비우거나 삭제해도 이미 영구 기록된 연결은 지워지지 않습니다. +이전에 다른 대상으로 연결한 별칭은 다른 그룹에 재할당할 수 없습니다. 확인된 정규 제공자 이름 변경은 +기존 사용량을 나누지 않고 그룹을 합칠 수 있습니다. 전체 매핑을 함께 검증하므로 유효한 그룹 병합은 별칭 키 순서에 의존하지 않습니다. + +각 원래 사용량은 한 번만 계산합니다. 확정된 사용량, 진행 중 예약, 미해결 사용량을 모두 계산하며, +알 수 없는 사용량을 환급으로 취급하지 않습니다. 원래 예약 대상은 유지됩니다. +새 요청은 계속 정규 제공자 풀을 기록하고, 체크포인트는 원래 계정·제공자 이름을 추가하지 않은 채 +솔트가 적용된 ID 식별 근거를 보존합니다. 알 수 없는 양수 사용 이력과 활성 상태 또는 한도에 도달한 그룹은 +정리 대상에서 보호됩니다. 휴면 상태이고 한도 미만인 항목에는 기존 보존 규칙이 계속 적용됩니다. +ID 식별 근거의 저장 용량에는 한도가 있으며, 용량을 소진하면 근거를 잊는 대신 풀 요청 허용을 중단합니다. +검증할 수 없는 이력을 복원하는 자동 도구는 없습니다. + +### 안전한 롤백 요건 + +지원되는 롤백은 요청을 정규 제공자에 귀속시키는 처리와 연속성을 인식하는 원장 읽기·쓰기 처리를 모두 유지하거나 +백포트해야 하며, 동일한 저널, 솔트, 검증된 매핑을 사용해야 합니다. +최신 저널을 유지하세요. 업그레이드 전 사본을 복원하면 이후 사용량이 누락됩니다. +다운그레이드가 호환되는 것처럼 보이게 하려고 기록을 삭제하거나, 솔트를 바꾸거나, 상한을 높이거나, 제한 적용을 끄지 마세요. + +수정하지 않은 이전 바이너리로 되돌리는 것은 **지원되는 롤백이 아닙니다**. +이전 바이너리는 v1 원시 사용량을 읽을 수 있지만 별칭 간 제공자 합계에 상한을 적용하지 않으며, +새 계정 레이블 풀을 만들거나 압축할 때 선택적 ID 메타데이터를 버릴 수 있습니다. +다운그레이드를 자동으로 막는 장치는 없습니다. 이런 이전 쓰기 처리가 실행되었다면, 새 읽기 처리는 +운영자가 남아 있는 모든 매핑을 다시 명시적으로 확인할 때까지 식별되지 않은 양수 사용 이력을 거부합니다. +스토리지 또는 저널 무결성에 따른 거부는 별칭 문제와 별개이며 `workflow_spend_undurable`을 유지합니다. +안전하지 않은 파일이나 소유권 문제는 대신 스토리지 오류로 전파될 수 있으며, 이 경우에도 요청을 허용하지 않습니다. diff --git a/docs-site/src/content/docs/ru/reference/configuration/server.md b/docs-site/src/content/docs/ru/reference/configuration/server.md index 73af6c0fbe8..7ec344d4f92 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/server.md +++ b/docs-site/src/content/docs/ru/reference/configuration/server.md @@ -235,3 +235,66 @@ opencodex. Перед использованием прогоните soak-test Поле `quotaRefresh` в строке основного аккаунта Codex описывает получение квоты, а не её остаток или право доступа к модели. Оно может отсутствовать при чтении кэша или если запрос не выполнялся. Используется окружение работающего прокси-сервиса, а не текущего терминала. Если `proxy` не задан, существующее окружение сохраняется; `"auto"` при запуске читает статические настройки HTTP/HTTPS Windows или macOS. На macOS унаследованный прокси отменяет это чтение. На macOS допустимый шаблон `*.` преобразуется в `.`: для `*.local` прямое соединение получают `foo.local` и само имя `local`, но не `xlocal`. Точные диапазоны `169.254/16`, `169.254.0.0/16` и `fe80::/10` пропускаются с диагностикой: link-local IP-адреса используют прокси. IP-адреса и `*` принимаются; прочие CIDR, glob-шаблоны и исключения простых имён отменяют обнаружение без изменения окружения. PAC/WPAD, настройки только SOCKS и изменения во время работы автоматически не учитываются. Успех через TUN сам по себе не подтверждает исправность пути HTTP-прокси. См. [команды и состояния на английском](/reference/configuration/server/#codex-quota-network-diagnostics). `dropCodexSafetyBuffering`: не меняет проверки безопасности провайдера или отказы. Native WebSocket `codex.response.metadata.headers` и `/responses/compact` не входят в область фильтра. + +## Непрерывный учёт истории пулов + +Лимиты пулов провайдеров применяются к каноническому провайдеру, выбранному маршрутизацией, а журналы +запросов сохраняют отображаемые метки отдельных аккаунтов. В старых журналах расход мог учитываться +под этими метками. Журнал хранит псевдонимы с солью, а не исходные имена провайдеров и аккаунтов, +поэтому OpenCodex не выводит соответствия из текущего списка аккаунтов, префикса метки или сокращённого ID аккаунта. + +Если задан `spend.pool.maxTokens` и остаётся положительный расход в неопознанных исторических пулах, +допуск запроса на инференс завершается локальным HTTP 429 с +`x-opencodex-local-refusal: workflow_pool_history_unresolved` до обращения к провайдеру. +Это может временно блокировать корректные запросы, в том числе запросы без корня workflow. +После маршрутизации предварительная проверка также отклоняет канонический пул, суммарный расход которого +уже исчерпал лимит. Отправка при восстановлении или через combo, уже допущенная по резервированию, +не учитывает то же резервирование повторно против самой себя; все остальные резервирования учитываются. +Эта проверка не превращает passthrough-отправки с последующим отчётом об использовании в атомарные резервирования: +отправка, пересекающая лимит, одновременные допуски и повторы с последующим отчётом сохраняют существующие ограничения. +Установки в режиме только наблюдения остаются в этом режиме. Лимиты корня и идентичности продолжают действовать независимо. + +Для устранения блокировки оператор должен по собственным сохранённым подтверждениям установить, +какому каноническому провайдеру принадлежит каждый исторический псевдоним пула с солью. +Добавьте объект `spendPoolAliases` на верхнем уровне `config.json`: каждый ключ должен точно совпадать +с 32-символьным шестнадцатеричным псевдонимом пула в нижнем регистре из журнала той же установки, +а каждое значение должно быть подтверждённым каноническим ID провайдера. Даже старый псевдоним, +уже обозначающий канонического провайдера, требует явной записи, если у него нет метаданных идентичности. +Неопознанный псевдоним с положительным расходом остаётся заблокированным. Не выводите соответствие +из похожих имён, удаления аккаунта или совпадения коротких меток и не публикуйте журнал или соль. + +Размещайте `spendPoolAliases` вне `spend`: старые версии отклоняют неизвестные ключи внутри `spend` +и могут отключить весь раздел. Некорректные соответствия верхнего уровня отклоняются при записи конфигурации; +если ручная правка нарушила формат `spendPoolAliases`, существующие лимиты сохраняются, а допуск к пулам блокируется. +Исправьте соответствия и перезапустите сервис обычным способом изменения конфигурации. +Очистка или удаление соответствий не стирает уже надёжно записанные связи. Ранее перенаправленный псевдоним +нельзя назначить другой группе; подтверждённые переименования канонического провайдера могут объединять +группы без разделения существующего расхода. Все соответствия проверяются вместе, поэтому корректное +объединение групп не зависит от порядка ключей псевдонимов. + +Каждая исходная величина расхода учитывается ровно один раз. Учитываются подтверждённое использование, +резервирования в процессе выполнения и неразрешённое использование; неизвестное использование никогда +не считается возвратом. Исходные цели резервирования сохраняются. Новые запросы продолжают записываться +в пул канонического провайдера, а контрольные точки сохраняют сведения об идентичности с солью, +не добавляя исходные имена аккаунтов и провайдеров. Неопознанная история с положительным расходом, +активные группы и группы с исчерпанным лимитом защищены от очистки; существующее правило хранения +неактивных записей ниже лимита продолжает действовать. Объём сведений об идентичности ограничен: +при его исчерпании допуск к пулам прекращается, а сведения не забываются. +Автоматического инструмента для восстановления неподтверждаемой истории нет. + +### Требования к безопасному откату + +Поддерживаемый откат должен сохранять или переносить в старую версию как отнесение запросов к каноническому +провайдеру, так и чтение и запись реестра с учётом непрерывности истории, используя тот же журнал, +соль и подтверждённые соответствия. Сохраните актуальный журнал: восстановление копии до обновления +исключит более поздний расход. Не удаляйте записи, не меняйте соль, не повышайте лимиты и не отключайте +их применение, чтобы создать видимость совместимости при понижении версии. + +Неизменённый старый бинарный файл **не является поддерживаемым вариантом отката**. Он может читать +исходные величины расхода v1, но не применяет лимит к сумме по всем псевдонимам провайдера, +может создать новый пул с меткой аккаунта и отбросить необязательные метаданные идентичности при уплотнении. +Автоматического запрета понижения версии нет. Если такая старая версия выполняла запись, новая версия +при чтении отклоняет неопознанную историю с положительным расходом, пока оператор явно не подтвердит +все оставшиеся соответствия заново. Отказы из-за хранилища или целостности журнала не связаны с проблемой +псевдонимов и сохраняют `workflow_spend_undurable`; проблемы небезопасных файлов или прав владения +могут вместо этого передаваться как ошибки хранилища, при этом запрос не допускается. diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/server.md b/docs-site/src/content/docs/zh-cn/reference/configuration/server.md index dc96d8ff4b9..fed411d91fe 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/server.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/server.md @@ -201,3 +201,50 @@ Anthropic OAuth 侧车会复用 opencodex 现有的 Claude Code OAuth 指纹。 主 Codex 账户行中的 `quotaRefresh` 描述额度查询结果，并不代表剩余额度或模型访问权限。读取缓存或未执行查询时，该字段可能省略。查询使用正在运行的代理服务的环境，而不是当前终端的环境。未设置 `proxy` 时保留现有环境；`"auto"` 在启动时读取 Windows 或 macOS 静态 HTTP/HTTPS 设置；macOS 上若有继承代理则跳过读取。macOS 将有效的 `*.` 转为 `.`：`*.local` 使 `foo.local` 和裸域名 `local` 直连，但不匹配 `xlocal`。精确的 `169.254/16`、`169.254.0.0/16`、`fe80::/10` 网段会跳过并给出诊断，因此链路本地 IP 地址使用代理。IP 地址和 `*` 仍可用；其他 CIDR、通配形式和简单主机名例外会在修改环境前拒绝自动发现。不自动处理 PAC/WPAD、仅 SOCKS 的设置或运行中的更改。TUN 测试成功并不能单独证明 HTTP 代理路径正常。命令和状态说明见[英文网络诊断章节](/reference/configuration/server/#codex-quota-network-diagnostics)。 `dropCodexSafetyBuffering`: 不会改变供应商安全策略或拒绝响应。原生 WebSocket `codex.response.metadata.headers` 和 `/responses/compact` 不在过滤范围内。 + +## 历史池用量的连续性 + +提供方池上限按路由选定的规范提供方计算，而请求日志保留各账户的显示标签。 +旧日志可能将用量记在这些显示标签下。日志保存的是加盐别名，而不是原始提供方或账户名称， +因此 OpenCodex 不会根据当前账户列表、标签前缀或缩短的账户 ID 推测映射。 + +如果配置了 `spend.pool.maxTokens`，并且仍有已记账用量大于零的历史池无法识别， +推理请求会在联系提供方之前被本地 HTTP 429 拒绝，并返回 +`x-opencodex-local-refusal: workflow_pool_history_unresolved`。 +这可能暂时阻止原本有效的请求，包括没有工作流根节点的请求。 +路由后的预检也会拒绝合计用量已耗尽额度的规范池。 +已通过预留获准的恢复或组合发送，不会再次将同一预留计入自身；其他所有预留仍会计入。 +这项检查不会将事后报告用量的透传发送变成原子预留：跨越上限的发送、并发准入或事后报告的重试， +仍受现有机制的限制。仅观察模式的安装仍保持仅观察模式。根节点和身份上限继续独立生效。 + +要解除阻止，操作员必须使用自己保留的证据，确认每个历史加盐池别名所属的规范提供方。 +在 `config.json` 顶层添加 `spendPoolAliases` 对象：每个键必须与同一安装实例日志中的 +32 位小写十六进制池别名完全一致，每个值则是已经核实的规范提供方 ID。 +旧别名即使已代表规范提供方，只要缺少身份元数据，仍然需要显式条目。 +用量大于零且无法识别的别名会继续被阻止。不要根据名称相似、账户删除或短标签冲突推测映射， +也不要公开日志或盐值。 + +请将 `spendPoolAliases` 放在 `spend` 之外：旧版本会拒绝 `spend` 内的未知键，并可能禁用整个部分。 +无效的顶层映射会在写入配置时被拒绝；手动编辑导致 `spendPoolAliases` 格式错误时， +现有上限会被保留，池准入会关闭并拒绝请求。请按正常配置流程修正映射并重启。 +清空或删除映射不会抹去已经持久记录的关联。先前已重定向的别名不能重新分配给其他组； +经核实的规范提供方重命名可以合并组，而不会拆分已有用量。 +系统会一起检查完整映射，因此有效的分组合并不依赖别名键的顺序。 + +每份原始用量只计算一次。已结算用量、进行中的预留和未解决的用量都会计入；未知用量绝不会被视为退款。 +原始预留目标会被保留。新请求继续记录规范提供方池，检查点保留加盐身份依据， +而不加入原始账户或提供方名称。未知且用量大于零的历史记录、活跃组以及额度耗尽的组都受保护，不会被清理； +现有的休眠且未达上限记录保留规则仍然适用。身份依据的存储容量仍有上限；容量耗尽时， +系统会停止池准入，而不会遗忘这些依据。没有自动工具可以重建无法核实的历史记录。 + +### 安全回滚要求 + +受支持的回滚必须保留或回移两项功能：按规范提供方归属请求，以及识别历史连续性的账本读写， +并使用相同的日志、盐值和已核实的映射。请保留最新日志：恢复升级前的副本会遗漏之后的用量。 +不要通过删除记录、更改盐值、提高上限或关闭限制，来让降级看起来兼容。 + +未经修改的旧二进制文件**不是受支持的回滚方式**。它可以读取 v1 原始用量，但不会对提供方的跨别名合计用量实施限制， +可能创建新的账户标签池，也可能在压缩时丢弃可选身份元数据。没有自动阻止降级的机制。 +如果这样的旧写入程序已经运行，新读取程序会拒绝无法识别且用量大于零的历史记录， +直到操作员再次显式核实所有剩余映射。存储或日志完整性导致的拒绝与别名问题分开，仍使用 +`workflow_spend_undurable`；不安全文件或所有权问题可能改为作为存储错误向上传递，同样不会允许请求通过。 diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index e5fe7189379..ac100b30491 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -939,11 +939,24 @@ export function createSpendReservationLedger(options: { */ const evictScopes = (at: number, force: boolean): number => { const cutoff = at - policy.retentionMs; + // Candidate selection uses one snapshot of each pool group for the whole pass. + // Re-scanning all scopes per pool member repeats work on every reservation. + const pools = new Map(); + for (const [key, state] of scopes) { + if (!key.startsWith("pool\0")) continue; + const group = poolContinuity.resolve(key.slice(5)); + let total = pools.get(group); + if (!total) pools.set(group, total = { settled: 0, reserved: 0, unresolved: 0, lastSeenAt: 0 }); + total.settled += state.settled; + total.reserved += state.reserved; + total.unresolved += state.unresolved; + total.lastSeenAt = Math.max(total.lastSeenAt, state.lastSeenAt); + } const candidates: { key: string; scope: SpendScope; alias: string; seenAt: number }[] = []; for (const [key, state] of scopes) { const separator = key.indexOf("\0"); const scope = key.slice(0, separator) as SpendScope; - const effective = scope === "pool" ? poolState(key.slice(separator + 1))! : state; + const effective = scope === "pool" ? pools.get(poolContinuity.resolve(key.slice(separator + 1)))! : state; if (effective.reserved > 0) continue; if (scope === "pool" && state.settled + state.unresolved > 0 && !poolContinuity.known(key.slice(separator + 1))) continue; diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index 339349a5844..2c6eb430de8 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -144,6 +144,9 @@ that repeats existing member aliases is independent of salted-key order. A remov never removes a journaled link. Group activity, last-seen time and exhaustion govern retention; unidentified positive balances cannot be evicted. Identity evidence is bounded and retained even when dormant under-limit scopes are evicted. +Each eviction pass aggregates pool groups once before choosing candidates. Retention uses the +group's newest activity; capacity pressure still removes only the oldest eligible individual +scope per pass, and every removal journals its original scope alias. Before a mapping authorizes admission, a v1 checkpoint durably carries both unchanged accounting and optional salted `poolContinuity` metadata. No raw provider/account names are added to the diff --git a/tests/lib/spend-pool-continuity.test.ts b/tests/lib/spend-pool-continuity.test.ts index 34923a6445c..298fcbeb7b2 100644 --- a/tests/lib/spend-pool-continuity.test.ts +++ b/tests/lib/spend-pool-continuity.test.ts @@ -149,6 +149,50 @@ describe("historical pool identity continuity", () => { expect(ledger.snapshot("pool", "provider")).toMatchObject({ settled: 40, reserved: 10 }); }); + test("retention uses the newest pool member and journals every dormant member removal", () => { + const record = checkpoint([["older", 10, 0], ["newer", 0, 20]]); + record.scopes[1]!.seenAt = 95; + const disk = journal([record]); + const aliases = { [pool("older")]: "provider", [pool("newer")]: "provider" }; + const config = policy(aliases, { retentionMs: 5 }); + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: config, now: () => 100 }); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + ledger.prune(100); // The newest member is exactly at the retention cutoff. + expect(ledger.snapshot("pool", "provider")).toMatchObject({ settled: 10, unresolved: 20 }); + expect(disk.lines.map(line => JSON.parse(line)).filter(record => record.kind === "drop")).toEqual([]); + ledger.prune(101); + expect(ledger.snapshot("pool", "provider")).toBeUndefined(); + expect(disk.lines.map(line => JSON.parse(line)).filter(record => record.kind === "drop")).toEqual([ + { v: 1, kind: "drop", scope: "pool", alias: pool("older"), at: 101 }, + { v: 1, kind: "drop", scope: "pool", alias: pool("newer"), at: 101 }, + ]); + const restarted = createSpendReservationLedger({ journal: disk, salt, policy: config, now: () => 101 }); + expect(restarted.snapshot("pool", "provider")).toBeUndefined(); + expect(restarted.checkPoolContinuity()).toBeUndefined(); + }); + + test("capacity eviction chooses the oldest individual member and drops only one scope", () => { + const record = checkpoint([["oldest", 10, 0], ["recent", 20, 0], ["other", 0, 0]]); + record.scopes[1]!.seenAt = 40; + record.scopes[2]!.seenAt = 2; + const disk = journal([record]); + const config = policy({ [pool("oldest")]: "provider", [pool("recent")]: "provider" }, + { retentionMs: 100, maxTrackedScopes: 3 }); + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: config, now: () => 50 }); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + expect(ledger.reserve({ sendId: "new-root", scopes: { rootId: "new-root" }, inputTokens: 1, outputCeilingTokens: 0 }).reserved).toBe(true); + expect(disk.lines.map(line => JSON.parse(line)).filter(record => record.kind === "drop")).toEqual([ + { v: 1, kind: "drop", scope: "pool", alias: pool("oldest"), at: 50 }, + ]); + expect(ledger.snapshot("pool", "provider")?.settled).toBe(20); + expect(ledger.snapshot("pool", "other")?.settled).toBe(0); + expect(ledger.snapshot("root", "new-root")?.reserved).toBe(1); + const restarted = createSpendReservationLedger({ journal: disk, salt, policy: config, now: () => 50 }); + expect(restarted.snapshot("pool", "provider")?.settled).toBe(20); + expect(restarted.snapshot("pool", "other")?.settled).toBe(0); + expect(restarted.snapshot("root", "new-root")?.unresolved).toBe(1); + }); + test("observe-only records already-sent requests while ambiguity still blocks new dispatches", () => { const disk = journal([checkpoint([["unknown-label", 40, 0]])]); const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 2 }); From 9892f8ad57c506ac16ebcf5d2edfd65cc5942c68 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Sun, 4 Oct 2026 01:36:34 -0700 Subject: [PATCH 7/7] fix(spend): isolate legacy pool history and explain refusals --- .../docs/ja/reference/configuration/server.md | 18 +- .../docs/ko/reference/configuration/server.md | 18 +- .../docs/reference/configuration/server.md | 31 ++- .../docs/ru/reference/configuration/server.md | 22 +- .../zh-cn/reference/configuration/server.md | 18 +- src/lib/spend-reservation-ledger.ts | 99 ++++++-- src/lib/workflow-budget.ts | 32 ++- src/server/request-log.ts | 3 + src/server/responses/adapter-dispatch.ts | 7 +- src/server/responses/passthrough-dispatch.ts | 4 +- src/server/responses/request-send-budget.ts | 5 +- src/server/responses/request-spend.ts | 11 +- src/server/responses/run-turn-execution.ts | 3 +- src/server/workflow-refusal.ts | 17 ++ structure/transports/responses-spend.md | 46 ++-- tests/lib/spend-ceiling-enforcement.test.ts | 3 +- tests/lib/spend-pool-continuity.test.ts | 233 +++++++++++++++--- .../responses-send-budget-errors.test.ts | 9 + 18 files changed, 447 insertions(+), 132 deletions(-) diff --git a/docs-site/src/content/docs/ja/reference/configuration/server.md b/docs-site/src/content/docs/ja/reference/configuration/server.md index a345d9c50bf..dab41bbda05 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/server.md +++ b/docs-site/src/content/docs/ja/reference/configuration/server.md @@ -207,11 +207,12 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and ジャーナルに保存されるのはソルト付きのエイリアスであり、元のプロバイダー名やアカウント名ではありません。 そのため、OpenCodex は現在のアカウント一覧、ラベルの接頭辞、短縮されたアカウント ID から対応関係を推測しません。 -`spend.pool.maxTokens` が設定されていて、正の使用量が残る過去のプールを識別できない場合、推論リクエストは -プロバイダーへ接続する前にローカルの HTTP 429 と -`x-opencodex-local-refusal: workflow_pool_history_unresolved` で拒否されます。 -これにより、ワークフロールートを持たないリクエストを含め、本来は有効なリクエストが一時的にブロックされることがあります。 -ルーティング後の事前確認でも、合算した使用量がすでに上限に達している正規プールを拒否します。 +`spend.pool.maxTokens` が設定され、正の過去残高の所有先が不明な場合、元のスコープに残したまま、 +各残高を候補となるすべてのプロバイダープールに一度ずつ加算し、そのプロバイダーの確認済みグループと合算します。 +この合計と予約分が上限に収まる場合だけリクエストを許可します。そのため、未使用のプールも一時的に制限されることがあります。 +ローカル拒否では、エイリアスやジャーナル内容を公開せず、この保守的な加算を説明します。 +ルートが判明する前は、不明な履歴だけを理由に HTTP admission を拒否しません。 +ルーティング後の事前確認では、保守的に合算した使用量がすでに上限に達している正規プールを拒否します。 予約によってすでに許可されたリカバリー送信やコンボ送信では、その予約を自分自身に対して再度計上しません。 それ以外の予約はすべて計上されます。この確認により、送信後に使用量を報告するパススルー送信が アトミックな予約に変わるわけではありません。上限をまたぐ送信、同時に許可されるリクエスト、 @@ -223,8 +224,9 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and `spendPoolAliases` オブジェクトを追加してください。各キーには、その同じインストールのジャーナルにある 32 文字の小文字 16 進数のプールエイリアスを正確に指定し、各値には確認済みの正規プロバイダー ID を指定します。 古いエイリアスがすでに正規プロバイダーを表していても、ID メタデータがなければ明示的なエントリが必要です。 -正の使用量を持ち、識別できないエイリアスはブロックされたままです。似た名前、アカウントの削除、 -短縮ラベルの衝突から対応関係を推測しないでください。ジャーナルやソルトを公開しないでください。 +正の使用量を持ち、識別できないエイリアスは、すべての候補プールに保守的に加算されます。 +似た名前、アカウントの削除、現在のプロバイダー名やハッシュ、短縮ラベルの衝突から対応関係を推測しないでください。 +ジャーナルやソルトを公開しないでください。 `spendPoolAliases` は `spend` の外に置いてください。古いバージョンは `spend` 内の未知のキーを拒否し、 セクション全体を無効にする場合があります。無効なトップレベルのマッピングは設定の書き込み時に拒否されます。 @@ -253,7 +255,7 @@ ID 識別情報の保存容量には上限があり、容量を使い切った 古いバイナリは v1 の生の計上値を読めても、エイリアスをまたぐプロバイダー合計に上限を適用せず、 新たなアカウントラベルのプールを作成したり、圧縮時に任意の ID メタデータを破棄したりする場合があります。 ダウングレードを自動的に防ぐ仕組みはありません。そのような古い書き込み処理が実行された場合、新しい読み取り処理は、 -運用者が残っているすべてのマッピングを明示的に再確認するまで、識別できない正の使用履歴を拒否します。 +運用者が残っているマッピングを確認するまで、識別できない正の使用履歴を各候補プールに保守的に加算します。 ストレージやジャーナルの整合性による拒否はエイリアスの問題とは別で、`workflow_spend_undurable` のままです。 安全でないファイルや所有権の問題は、代わりにストレージエラーとして伝播する場合がありますが、リクエストは許可されません。 diff --git a/docs-site/src/content/docs/ko/reference/configuration/server.md b/docs-site/src/content/docs/ko/reference/configuration/server.md index 63d2f3be220..32dc569716b 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/server.md +++ b/docs-site/src/content/docs/ko/reference/configuration/server.md @@ -263,11 +263,12 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and 이전 저널에는 이 표시 레이블에 사용량이 기록되어 있을 수 있습니다. 저널은 원래 제공자·계정 이름이 아닌 솔트가 적용된 별칭을 저장하므로, OpenCodex는 현재 계정 목록, 레이블 접두사, 축약된 계정 ID로 매핑을 추측하지 않습니다. -`spend.pool.maxTokens`가 설정되어 있고 양수의 사용량이 남은 과거 풀을 식별하지 못하면, 추론 요청은 -제공자에 접속하기 전에 로컬 HTTP 429와 -`x-opencodex-local-refusal: workflow_pool_history_unresolved`로 거부됩니다. -워크플로 루트가 없는 요청을 포함해 원래 유효한 요청도 일시적으로 차단될 수 있습니다. -라우팅 후 사전 검사에서도 합산 사용량이 이미 한도에 도달한 정규 풀을 거부합니다. +`spend.pool.maxTokens`가 설정되어 있고 양수인 과거 잔액의 소유자를 알 수 없으면, +각 잔액은 원래 스코프에 보존되며 후보 제공자 풀마다 한 번씩 더해져 확인된 그룹과 합산됩니다. +합계와 예약량이 한도에 들어오는 경우에만 요청을 허용합니다. 이 방식은 사용하지 않는 풀도 일시적으로 제한할 수 있습니다. +로컬 거부 메시지는 별칭이나 저널 내용을 공개하지 않고 이러한 보수적 계산을 설명합니다. +라우트를 알기 전에는 불명확한 이력만으로 HTTP admission을 거부하지 않습니다. +라우팅 후 사전 검사에서도 보수적으로 합산한 사용량이 이미 한도에 도달한 정규 풀을 거부합니다. 예약을 통해 이미 허용된 복구 또는 콤보 전송에서는 그 예약을 자신에게 다시 계산하지 않습니다. 다른 예약은 모두 계산합니다. 이 검사가 사용량을 사후 보고하는 패스스루 전송을 원자적 예약으로 바꾸지는 않습니다. 한도를 넘기는 전송, 동시 요청 허용, 사후 보고되는 재시도에는 기존 제약이 그대로 남습니다. @@ -277,8 +278,9 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and `config.json`의 최상위에 `spendPoolAliases` 객체를 추가하세요. 각 키는 동일 설치의 저널에 있는 32자리 소문자 16진수 풀 별칭과 정확히 일치해야 하고, 각 값은 확인된 정규 제공자 ID여야 합니다. 이전 별칭이 이미 정규 제공자를 나타내더라도 ID 메타데이터가 없으면 명시적 항목이 필요합니다. -식별할 수 없는 양수 사용량의 별칭은 계속 차단됩니다. 비슷한 이름, 계정 삭제, 축약 레이블 충돌로 -매핑을 추측하지 말고, 저널이나 솔트를 공개하지 마세요. +식별할 수 없는 양수 사용량의 별칭은 모든 후보 풀에 보수적으로 계산됩니다. +비슷한 이름, 계정 삭제, 현재 제공자 이름이나 해시, 축약 레이블 충돌로 매핑을 추측하지 말고, +저널이나 솔트를 공개하지 마세요. `spendPoolAliases`는 `spend` 밖에 두세요. 이전 버전은 `spend` 안의 알 수 없는 키를 거부하며 섹션 전체를 비활성화할 수 있습니다. 잘못된 최상위 매핑은 설정 쓰기 시 거부됩니다. @@ -306,7 +308,7 @@ ID 식별 근거의 저장 용량에는 한도가 있으며, 용량을 소진하 이전 바이너리는 v1 원시 사용량을 읽을 수 있지만 별칭 간 제공자 합계에 상한을 적용하지 않으며, 새 계정 레이블 풀을 만들거나 압축할 때 선택적 ID 메타데이터를 버릴 수 있습니다. 다운그레이드를 자동으로 막는 장치는 없습니다. 이런 이전 쓰기 처리가 실행되었다면, 새 읽기 처리는 -운영자가 남아 있는 모든 매핑을 다시 명시적으로 확인할 때까지 식별되지 않은 양수 사용 이력을 거부합니다. +운영자가 남은 매핑을 확인할 때까지 식별되지 않은 양수 사용 이력을 모든 후보 제공자 풀에 보수적으로 계산합니다. 스토리지 또는 저널 무결성에 따른 거부는 별칭 문제와 별개이며 `workflow_spend_undurable`을 유지합니다. 안전하지 않은 파일이나 소유권 문제는 대신 스토리지 오류로 전파될 수 있으며, 이 경우에도 요청을 허용하지 않습니다. diff --git a/docs-site/src/content/docs/reference/configuration/server.md b/docs-site/src/content/docs/reference/configuration/server.md index bcecb76abfb..c9d03f06235 100644 --- a/docs-site/src/content/docs/reference/configuration/server.md +++ b/docs-site/src/content/docs/reference/configuration/server.md @@ -949,23 +949,27 @@ aliases, not the original provider/account names, so OpenCodex does not guess a current account roster, a label prefix, or a shortened account ID. If `spend.pool.maxTokens` is configured and positive historical pool balances remain unidentified, -inference admission returns local HTTP 429 with -`x-opencodex-local-refusal: workflow_pool_history_unresolved` before contacting a provider. -This can temporarily block otherwise valid requests, including requests without a workflow root. -After routing, preflight also refuses a canonical pool that is already exhausted by its combined -balances. A recovery or combo send already admitted by a reservation does not count that same -reservation against itself a second time; all other reservations remain counted. +the ledger keeps each balance in its original scope and conservatively counts it once against every +candidate provider pool, in addition to that provider's verified group. A request is admitted only +when that total plus its reservation fits the ceiling. This may restrict an otherwise unused pool +until the operator verifies mappings; the request-local refusal explains this overrestriction +without exposing aliases or journal contents. Unidentified history alone does not block admission +before a route is known. After routing, preflight also refuses a canonical pool whose conservative +total is already exhausted. A recovery or combo send already admitted by a reservation does not +count that same reservation against itself a second time; all other reservations remain counted. This check does not turn post-reported passthrough sends into atomic reservations: a crossing send, concurrent admissions or retries reported afterwards retain their existing limits. Observe-only installs remain observe-only. Root and identity ceilings remain in force independently. To resolve it, an operator must verify which canonical provider each historical salted pool alias -belongs to, using their own retained evidence. Add a top-level `spendPoolAliases` object in +belongs to, using their own retained evidence. Verified mappings keep an old balance from being +charged to every candidate pool. Add a top-level `spendPoolAliases` object in `config.json`: each key is the exact 32-character lowercase hexadecimal pool alias from that same installation's journal; each value is its verified canonical provider ID. An old alias that already represents the canonical provider still needs an explicit entry when it lacks identity metadata. -A positive alias that cannot be identified stays blocked. Do not infer a match from similar names, -account deletion, or a short-label collision, and do not share the journal or salt publicly. +A positive alias that cannot be identified remains conservatively charged to every candidate pool. +Do not infer a match from similar names, account deletion, a current provider name/hash, or a +short-label collision, and do not share the journal or salt publicly. Keep `spendPoolAliases` outside `spend`: older versions reject unknown keys inside `spend` and can disable the entire section. Invalid top-level mappings are rejected on configuration writes; @@ -993,7 +997,8 @@ change the salt, raise ceilings, or disable enforcement to make a downgrade appe An unmodified older binary is **not a supported rollback**. It can read the v1 raw balances but does not enforce the cross-alias provider total, may create a new account-label pool, and may discard optional identity metadata when compacting. There is no automatic downgrade barrier. If such an old -writer has run, the new reader refuses unidentified positive history until the operator explicitly -verifies all remaining mappings again. Storage or journal-integrity denials are separate from an -alias problem and keep `workflow_spend_undurable`; unsafe-file/ownership failures may instead -propagate as storage errors, without admitting the request. +writer has run, the new reader conservatively counts unidentified positive history against every +candidate provider pool until the operator verifies the remaining mappings. Storage or +journal-integrity denials are separate from an alias problem and keep `workflow_spend_undurable`; +unsafe-file/ownership failures may instead propagate as storage errors, without admitting the +request. diff --git a/docs-site/src/content/docs/ru/reference/configuration/server.md b/docs-site/src/content/docs/ru/reference/configuration/server.md index 3d8ec782188..b65b39c977f 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/server.md +++ b/docs-site/src/content/docs/ru/reference/configuration/server.md @@ -255,12 +255,13 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and под этими метками. Журнал хранит псевдонимы с солью, а не исходные имена провайдеров и аккаунтов, поэтому OpenCodex не выводит соответствия из текущего списка аккаунтов, префикса метки или сокращённого ID аккаунта. -Если задан `spend.pool.maxTokens` и остаётся положительный расход в неопознанных исторических пулах, -допуск запроса на инференс завершается локальным HTTP 429 с -`x-opencodex-local-refusal: workflow_pool_history_unresolved` до обращения к провайдеру. -Это может временно блокировать корректные запросы, в том числе запросы без корня workflow. -После маршрутизации предварительная проверка также отклоняет канонический пул, суммарный расход которого -уже исчерпал лимит. Отправка при восстановлении или через combo, уже допущенная по резервированию, +Если задан `spend.pool.maxTokens`, положительные исторические балансы с неизвестным владельцем +сохраняются в исходных областях и один раз учитываются для каждого возможного пула провайдера, +дополнительно к подтверждённой группе этого провайдера. Запрос допускается, только если сумма и резерв +укладываются в лимит. Это может временно ограничить даже неиспользуемые пулы. Локальный отказ объясняет +такой консервативный учёт, не раскрывая псевдонимы или содержимое журнала. Неизвестная история сама по себе +не блокирует HTTP-допуск до определения маршрута. После маршрутизации предварительная проверка отклоняет +канонический пул, консервативная сумма которого уже исчерпала лимит. Отправка при восстановлении или через combo, уже допущенная по резервированию, не учитывает то же резервирование повторно против самой себя; все остальные резервирования учитываются. Эта проверка не превращает passthrough-отправки с последующим отчётом об использовании в атомарные резервирования: отправка, пересекающая лимит, одновременные допуски и повторы с последующим отчётом сохраняют существующие ограничения. @@ -272,8 +273,9 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and с 32-символьным шестнадцатеричным псевдонимом пула в нижнем регистре из журнала той же установки, а каждое значение должно быть подтверждённым каноническим ID провайдера. Даже старый псевдоним, уже обозначающий канонического провайдера, требует явной записи, если у него нет метаданных идентичности. -Неопознанный псевдоним с положительным расходом остаётся заблокированным. Не выводите соответствие -из похожих имён, удаления аккаунта или совпадения коротких меток и не публикуйте журнал или соль. +Неопознанный псевдоним с положительным расходом консервативно учитывается для каждого возможного пула. +Не выводите соответствие из похожих имён, удаления аккаунта, текущего имени или хэша провайдера, +либо совпадения коротких меток; не публикуйте журнал или соль. Размещайте `spendPoolAliases` вне `spend`: старые версии отклоняют неизвестные ключи внутри `spend` и могут отключить весь раздел. Некорректные соответствия верхнего уровня отклоняются при записи конфигурации; @@ -306,8 +308,8 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and исходные величины расхода v1, но не применяет лимит к сумме по всем псевдонимам провайдера, может создать новый пул с меткой аккаунта и отбросить необязательные метаданные идентичности при уплотнении. Автоматического запрета понижения версии нет. Если такая старая версия выполняла запись, новая версия -при чтении отклоняет неопознанную историю с положительным расходом, пока оператор явно не подтвердит -все оставшиеся соответствия заново. Отказы из-за хранилища или целостности журнала не связаны с проблемой +при чтении консервативно учитывает неопознанную историю с положительным расходом для каждого возможного +пула провайдера, пока оператор не подтвердит оставшиеся соответствия. Отказы из-за хранилища или целостности журнала не связаны с проблемой псевдонимов и сохраняют `workflow_spend_undurable`; проблемы небезопасных файлов или прав владения могут вместо этого передаваться как ошибки хранилища, при этом запрос не допускается. diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/server.md b/docs-site/src/content/docs/zh-cn/reference/configuration/server.md index 21966a05af8..190a8fe6580 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/server.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/server.md @@ -220,11 +220,11 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and 旧日志可能将用量记在这些显示标签下。日志保存的是加盐别名，而不是原始提供方或账户名称， 因此 OpenCodex 不会根据当前账户列表、标签前缀或缩短的账户 ID 推测映射。 -如果配置了 `spend.pool.maxTokens`，并且仍有已记账用量大于零的历史池无法识别， -推理请求会在联系提供方之前被本地 HTTP 429 拒绝，并返回 -`x-opencodex-local-refusal: workflow_pool_history_unresolved`。 -这可能暂时阻止原本有效的请求，包括没有工作流根节点的请求。 -路由后的预检也会拒绝合计用量已耗尽额度的规范池。 +如果配置了 `spend.pool.maxTokens`，无法确认所有者的正历史余额会保留在原始范围内， +并且会对每个候选提供方池各计入一次，再与该提供方已验证的组相加。 +只有合计用量加上本次预留不超过额度时才会准入。这可能暂时限制原本未使用的池。 +本地拒绝会说明这种保守计算，不会公开别名或日志内容。路由确定前，仅有未识别历史不会阻止 HTTP 准入。 +路由后的预检也会拒绝保守合计已耗尽额度的规范池。 已通过预留获准的恢复或组合发送，不会再次将同一预留计入自身；其他所有预留仍会计入。 这项检查不会将事后报告用量的透传发送变成原子预留：跨越上限的发送、并发准入或事后报告的重试， 仍受现有机制的限制。仅观察模式的安装仍保持仅观察模式。根节点和身份上限继续独立生效。 @@ -233,8 +233,8 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and 在 `config.json` 顶层添加 `spendPoolAliases` 对象：每个键必须与同一安装实例日志中的 32 位小写十六进制池别名完全一致，每个值则是已经核实的规范提供方 ID。 旧别名即使已代表规范提供方，只要缺少身份元数据，仍然需要显式条目。 -用量大于零且无法识别的别名会继续被阻止。不要根据名称相似、账户删除或短标签冲突推测映射， -也不要公开日志或盐值。 +用量大于零且无法识别的别名会保守地计入每个候选池。不要根据名称相似、账户删除、 +当前提供方名称或哈希、短标签冲突推测映射，也不要公开日志或盐值。 请将 `spendPoolAliases` 放在 `spend` 之外：旧版本会拒绝 `spend` 内的未知键，并可能禁用整个部分。 无效的顶层映射会在写入配置时被拒绝；手动编辑导致 `spendPoolAliases` 格式错误时， @@ -257,8 +257,8 @@ The dashboard warns about old or unknown CLI versions, unavailable targets, and 未经修改的旧二进制文件**不是受支持的回滚方式**。它可以读取 v1 原始用量，但不会对提供方的跨别名合计用量实施限制， 可能创建新的账户标签池，也可能在压缩时丢弃可选身份元数据。没有自动阻止降级的机制。 -如果这样的旧写入程序已经运行，新读取程序会拒绝无法识别且用量大于零的历史记录， -直到操作员再次显式核实所有剩余映射。存储或日志完整性导致的拒绝与别名问题分开，仍使用 +如果这样的旧写入程序已经运行，新读取程序会将无法识别且用量大于零的历史记录保守地计入 +每个候选提供方池，直到操作员核实剩余映射。存储或日志完整性导致的拒绝与别名问题分开，仍使用 `workflow_spend_undurable`；不安全文件或所有权问题可能改为作为存储错误向上传递，同样不会允许请求通过。 ## 令牌预留与额度 diff --git a/src/lib/spend-reservation-ledger.ts b/src/lib/spend-reservation-ledger.ts index ac100b30491..e513d7bf2df 100644 --- a/src/lib/spend-reservation-ledger.ts +++ b/src/lib/spend-reservation-ledger.ts @@ -191,6 +191,8 @@ export type SpendDenial = readonly scopeId: string; readonly limit: number; readonly projected: number; + /** Pool checks include all unbound positive history until an operator verifies its owner. */ + readonly includesUnboundPoolHistory?: boolean; } /** This send id is already known -- open, settled, lost or abandoned. */ | { readonly reason: "duplicate-send-id"; readonly sendId: string } @@ -623,6 +625,8 @@ export interface SpendReservationLedger { reserve(request: SpendReservationRequest): SpendReservationDecision; /** Pre-dispatch guard, including transports which report their sends after dispatch. */ checkPoolContinuity(): SpendDenial | undefined; + /** Whether any positive pool balance is still unbound and therefore charged to each candidate pool. */ + hasUnboundPositivePoolHistory(): boolean; /** * The send left for upstream. Until this is called the reservation may be abandoned for * free; after it, a missing usage frame becomes unresolved spend. Returns false when the @@ -727,10 +731,24 @@ export function createSpendReservationLedger(options: { * credential id, a pool name -- never leaves this function, so nothing identifying is * written to disk or held in a map key. */ - const aliasFor = (kind: SpendScope | "send", id: string): string => + const aliasFor = (kind: SpendScope | "send" | "pool-current", id: string): string => createHash("sha256").update(salt).update("\u0000").update(kind).update("\u0000").update(id) .digest("hex").slice(0, 32); + // An old unbound label may hash exactly like today's provider ID. Only in that ambiguous + // case, put new routed spend in a separate hash domain so it belongs to this candidate + // without treating the old balance as verified ownership. Existing proven groups keep + // their alias, and explicit mappings of old aliases target the same current alias. + const poolAliasFor = (provider: string): string => { + const legacyAlias = aliasFor("pool", provider); + const legacy = scopes.get(scopeKey("pool", legacyAlias)); + const hasPositiveHistory = legacy !== undefined + && legacy.settled + legacy.reserved + legacy.unresolved > 0; + const ambiguous = hasPositiveHistory + && (!poolContinuity.known(legacyAlias) || poolContinuity.resolve(legacyAlias) !== legacyAlias); + return ambiguous ? aliasFor("pool-current", provider) : legacyAlias; + }; + const scopeState = (scope: SpendScope, alias: string): ScopeState => { const key = scopeKey(scope, alias); let state = scopes.get(key); @@ -744,22 +762,34 @@ export function createSpendReservationLedger(options: { const historicalPools = (): Set => new Set([...scopes].filter(([key, state]) => key.startsWith("pool\0") && state.settled + state.reserved + state.unresolved > 0, ).map(([key]) => key.slice(5))); - const unknownPoolHistory = (): boolean => [...historicalPools()].some(alias => !poolContinuity.known(alias)); - const poolState = (alias: string): ScopeState | undefined => { + const hasUnboundPositivePoolHistory = (): boolean => [...historicalPools()].some(alias => !poolContinuity.known(alias)); + const poolState = (alias: string): { totals: ScopeState; includesUnboundHistory: boolean } | undefined => { const group = poolContinuity.resolve(alias); let total: ScopeState | undefined; + let includesUnboundHistory = false; for (const [key, state] of scopes) { - if (!key.startsWith("pool\0") || poolContinuity.resolve(key.slice(5)) !== group) continue; + if (!key.startsWith("pool\0")) continue; + const member = key.slice(5); + const unbound = !poolContinuity.known(member); + if (unbound) { + // Ownership cannot be inferred because an old alias hashes to a current provider ID. + // Keep that original balance out of the proven group, then charge it once through the + // conservative overlay below for every candidate provider. + if (state.settled + state.reserved + state.unresolved === 0) continue; + includesUnboundHistory = true; + } else if (poolContinuity.resolve(member) !== group) { + continue; + } total ??= { settled: 0, reserved: 0, unresolved: 0, lastSeenAt: 0 }; total.settled += state.settled; total.reserved += state.reserved; total.unresolved += state.unresolved; total.lastSeenAt = Math.max(total.lastSeenAt, state.lastSeenAt); } - return total; + return total ? { totals: total, includesUnboundHistory } : undefined; }; const stateFor = (scope: SpendScope, alias: string): ScopeState | undefined => - scope === "pool" ? poolState(alias) : scopes.get(scopeKey(scope, alias)); + scope === "pool" ? poolState(alias)?.totals : scopes.get(scopeKey(scope, alias)); const limitFor = (scope: SpendScope): number | undefined => policy[scope].maxTokens; @@ -773,7 +803,7 @@ export function createSpendReservationLedger(options: { const refs: ScopeRef[] = []; if (targets.rootId !== undefined) refs.push({ scope: "root", alias: aliasFor("root", targets.rootId) }); if (targets.identityId !== undefined) refs.push({ scope: "identity", alias: aliasFor("identity", targets.identityId) }); - if (targets.poolId !== undefined) refs.push({ scope: "pool", alias: aliasFor("pool", targets.poolId) }); + if (targets.poolId !== undefined) refs.push({ scope: "pool", alias: poolAliasFor(targets.poolId) }); return refs; }; @@ -939,12 +969,23 @@ export function createSpendReservationLedger(options: { */ const evictScopes = (at: number, force: boolean): number => { const cutoff = at - policy.retentionMs; - // Candidate selection uses one snapshot of each pool group for the whole pass. - // Re-scanning all scopes per pool member repeats work on every reservation. + // Candidate selection uses one snapshot of each proven pool group plus the shared + // unbound overlay for the whole pass. Re-scanning all scopes per pool member repeats + // work on every reservation. const pools = new Map(); + const unboundPoolTotal: ScopeState = { settled: 0, reserved: 0, unresolved: 0, lastSeenAt: 0 }; for (const [key, state] of scopes) { if (!key.startsWith("pool\0")) continue; - const group = poolContinuity.resolve(key.slice(5)); + const alias = key.slice(5); + if (!poolContinuity.known(alias)) { + if (state.settled + state.reserved + state.unresolved > 0) { + unboundPoolTotal.settled += state.settled; + unboundPoolTotal.reserved += state.reserved; + unboundPoolTotal.unresolved += state.unresolved; + } + continue; + } + const group = poolContinuity.resolve(alias); let total = pools.get(group); if (!total) pools.set(group, total = { settled: 0, reserved: 0, unresolved: 0, lastSeenAt: 0 }); total.settled += state.settled; @@ -956,7 +997,19 @@ export function createSpendReservationLedger(options: { for (const [key, state] of scopes) { const separator = key.indexOf("\0"); const scope = key.slice(0, separator) as SpendScope; - const effective = scope === "pool" ? pools.get(poolContinuity.resolve(key.slice(separator + 1)))! : state; + const alias = key.slice(separator + 1); + const unboundZero = scope === "pool" && !poolContinuity.known(alias) + && state.settled + state.reserved + state.unresolved === 0; + const proven = scope === "pool" ? pools.get(poolContinuity.resolve(alias)) : undefined; + const effective = scope === "pool" && !unboundZero + ? { + settled: (proven?.settled ?? 0) + unboundPoolTotal.settled, + reserved: (proven?.reserved ?? 0) + unboundPoolTotal.reserved, + unresolved: (proven?.unresolved ?? 0) + unboundPoolTotal.unresolved, + // The overlay affects exhaustion but does not merge retention age across identities. + lastSeenAt: proven?.lastSeenAt ?? 0, + } + : state; if (effective.reserved > 0) continue; if (scope === "pool" && state.settled + state.unresolved > 0 && !poolContinuity.known(key.slice(separator + 1))) continue; @@ -1046,7 +1099,7 @@ export function createSpendReservationLedger(options: { const preparePoolContinuity = (at: number, requested?: string): SpendDenial | undefined => { const evidence = poolContinuity.prepare(policy.poolAliases, requested, historicalPools(), - provider => aliasFor("pool", provider), at, maxTrackedScopes()); + poolAliasFor, at, maxTrackedScopes()); if (evidence === false) return { reason: "pool-history-unresolved" }; if (evidence) { // A v1 checkpoint atomically carries the unchanged balances AND salted identity @@ -1054,7 +1107,9 @@ export function createSpendReservationLedger(options: { if (!append(checkpointRecord(at, evidence))) return { reason: "reserve-not-durable", sendId: "" }; if (!poolContinuity.restore(evidence)) return { reason: "pool-history-unresolved" }; } - return unknownPoolHistory() ? { reason: "pool-history-unresolved" } : undefined; + // Unbound positive history is accounted conservatively in every candidate pool view. + // Only invalid/conflicting identity evidence or a failed durable write refuses here. + return undefined; }; /** The denial when tracking cannot fit this request, or undefined when it can. */ @@ -1089,6 +1144,11 @@ export function createSpendReservationLedger(options: { return preparePoolContinuity(now()); }, + hasUnboundPositivePoolHistory(): boolean { + assertOwnedAccounting?.(); + return hasUnboundPositivePoolHistory(); + }, + reserve(request: SpendReservationRequest): SpendReservationDecision { assertOwnedAccounting?.(); const tokens = sanitizeTokens(request.inputTokens) + sanitizeTokens(request.outputCeilingTokens); @@ -1124,7 +1184,8 @@ export function createSpendReservationLedger(options: { // it is to let the total go OVER the ceiling so the next request can be refused. const limit = request.alreadySent === true ? undefined : limitFor(ref.scope); if (limit === undefined) continue; - const state = stateFor(ref.scope, ref.alias); + const poolView = ref.scope === "pool" ? poolState(ref.alias) : undefined; + const state = ref.scope === "pool" ? poolView?.totals : scopes.get(scopeKey(ref.scope, ref.alias)); const projected = (state ? state.settled + state.reserved + state.unresolved : 0) + tokens; if (projected > limit) { const scopeId = ref.scope === "root" @@ -1132,7 +1193,10 @@ export function createSpendReservationLedger(options: { : ref.scope === "identity" ? request.scopes.identityId : request.scopes.poolId; return { reserved: false, - denial: { reason: "spend-limit-exceeded", scope: ref.scope, scopeId: scopeId ?? "", limit, projected }, + denial: { + reason: "spend-limit-exceeded", scope: ref.scope, scopeId: scopeId ?? "", limit, projected, + ...(poolView?.includesUnboundHistory ? { includesUnboundPoolHistory: true } : {}), + }, }; } } @@ -1205,7 +1269,8 @@ export function createSpendReservationLedger(options: { // Reading accounting from a handle whose ownership has ended is as wrong as writing it: // the figures describe a journal this process no longer owns. assertOwnedAccounting?.(); - const state = stateFor(scope, aliasFor(scope, scopeId)); + const alias = scope === "pool" ? poolAliasFor(scopeId) : aliasFor(scope, scopeId); + const state = stateFor(scope, alias); if (!state) return undefined; return { settled: state.settled, @@ -1217,7 +1282,7 @@ export function createSpendReservationLedger(options: { exhausted(scope: SpendScope, scopeId: string, excludingSendId?: string): boolean { assertOwnedAccounting?.(); - const alias = aliasFor(scope, scopeId); + const alias = scope === "pool" ? poolAliasFor(scopeId) : aliasFor(scope, scopeId); const state = stateFor(scope, alias); if (!state) return false; const own = excludingSendId === undefined ? undefined : reservations.get(aliasFor("send", excludingSendId)); diff --git a/src/lib/workflow-budget.ts b/src/lib/workflow-budget.ts index 0c2e518045d..b491bd0f6fa 100644 --- a/src/lib/workflow-budget.ts +++ b/src/lib/workflow-budget.ts @@ -43,6 +43,8 @@ export interface WorkflowSpendDenialDetail { readonly limit: number; /** Tokens the refused reservation would have taken the scope to, where that is known. */ readonly projected?: number; + /** A pool ceiling also includes unknown-owner history conservatively charged to every pool. */ + readonly includesUnboundPoolHistory?: boolean; } /** Operator-facing name for each scope. What an operator calls it, not what the type calls it. */ @@ -207,8 +209,8 @@ export type WorkflowDenial = * accurate for exactly one of them. Worse, the wire cannot carry the distinction on its own: * `classifyError` rewrites every 429 to `rate_limit_error` / `rate_limit_exceeded`, so the body * of a refusal this proxy made is shaped exactly like a provider rate limit. Each sentence - * therefore says which ceiling fired AND that no provider was contacted, because that is the - * first thing an operator needs and the only place left to put it. + * therefore names which ceiling fired; send refusals describe the dispatch that was stopped + * without making claims about earlier attempts in the same request. */ export function workflowDenialSummary( reason: WorkflowDenial, @@ -248,7 +250,10 @@ export function workflowDenialSummary( + (spend.projected !== undefined ? " (this send would have taken it to " + formatTokenCount(spend.projected) + ")" : "") - + ", so no provider was contacted. Spend is durable, so it does not roll forward" + + (spend.includesUnboundPoolHistory + ? ". The total includes unassigned historical provider-pool balances, conservatively counted against every provider pool until verified mappings assign them; this can overrestrict otherwise unused pools" + : "") + + ", so this send was refused before contacting a provider. Spend is durable, so it does not roll forward" + " with the send window: raise or remove spend." + spend.scope + ".maxTokens in config.json to grant more." : "This proxy refused the request locally: the task reached a configured token" @@ -301,6 +306,8 @@ export interface WorkflowBudgetEvent { readonly spendScope?: SpendScope; /** That scope's ceiling, so the event is readable without the config open beside it. */ readonly spendLimit?: number; + /** The pool ceiling conservatively included unidentified historical pool balances. */ + readonly spendIncludesUnboundPoolHistory?: boolean; /** Windowed sends at the moment of the event. */ readonly sends: number; /** Windowed distinct children at the moment of the event. */ @@ -359,6 +366,7 @@ export function recordWorkflowRefusalEvent( rootId, reason, ...(spend ? { spendScope: spend.scope, spendLimit: spend.limit } : {}), + ...(spend?.includesUnboundPoolHistory ? { spendIncludesUnboundPoolHistory: true } : {}), sends: state ? windowedSends(state, now) : 0, children: state ? windowedChildren(state, now) : 0, }); @@ -389,6 +397,8 @@ export type WorkflowDecision = spendLimit?: number; /** Tokens the refused reservation would have taken the scope to, where known. */ spendProjected?: number; + /** A pool ceiling includes unbound historical balances for every candidate pool. */ + spendIncludesUnboundPoolHistory?: boolean; }; /** @@ -521,6 +531,7 @@ export function admitWorkflowTurn( spendScope: denial.scope, spendLimit: denial.limit, ...(denial.projected !== undefined ? { spendProjected: denial.projected } : {}), + ...(denial.includesUnboundPoolHistory ? { spendIncludesUnboundPoolHistory: true } : {}), } : {}), }; @@ -595,7 +606,12 @@ export function admitWorkflowTurn( return refuse( reason, denial.reason === "spend-limit-exceeded" - ? { scope: denial.scope, limit: denial.limit, projected: denial.projected } + ? { + scope: denial.scope, + limit: denial.limit, + projected: denial.projected, + ...(denial.includesUnboundPoolHistory ? { includesUnboundPoolHistory: true } : {}), + } : undefined, ); } @@ -747,7 +763,13 @@ export function workflowSpendCeilingReached( const root = rootId ? spentRootCeiling(rootId, ledger, excludingSendId) : undefined; if (root) return root; const limit = ledger.policy.pool.maxTokens; - return poolId && limit !== undefined && ledger.exhausted("pool", poolId, excludingSendId) ? { scope: "pool", limit } : undefined; + return poolId && limit !== undefined && ledger.exhausted("pool", poolId, excludingSendId) + ? { + scope: "pool", + limit, + ...(ledger.hasUnboundPositivePoolHistory() ? { includesUnboundPoolHistory: true } : {}), + } + : undefined; } export interface WorkflowBudgetSnapshot { diff --git a/src/server/request-log.ts b/src/server/request-log.ts index 87a2bf5fbb4..341fa962bdc 100644 --- a/src/server/request-log.ts +++ b/src/server/request-log.ts @@ -63,6 +63,7 @@ import { type UsageStatus, } from "../usage/log"; import type { RequestExecutionBudget } from "../lib/request-execution-budget"; +import type { WorkflowSpendDenialDetail } from "../lib/workflow-budget"; import { attributeFinalRequest, attributeSealedAttempt } from "./request-log-failure-attribution"; import { debugAttemptDeliverySummary } from "../lib/debug"; import { @@ -205,6 +206,8 @@ export interface RequestLogContext { spendInputEstimateTokens?: number; /** Canonical provider-pool spend identity; independent of mutable, account-specific log labels. */ spendPoolId?: string; + /** Internal safe refusal detail for a request whose ceiling includes unidentified old pool spend. */ + spendRefusalDetail?: WorkflowSpendDenialDetail; /** Settles this request's durable spend entries from `addFinalRequestLog`. */ spendTracker?: RequestSpendSettlement; attempts?: PersistedUsageAttempt[]; diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index 6deb79f1544..4b188397c58 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -98,6 +98,7 @@ import { resolveClientRetryAfter } from "../../lib/retry-after"; import { cancelBodyOnAbort } from "../../lib/abort"; import { chargeWorkflowSends } from "../../lib/workflow-budget"; import { isAntigravityValidationRefusal } from "./antigravity-validation-refusal"; +import { unboundPoolSpendRefusalResponse } from "../workflow-refusal"; /** One responsibility of the Responses request pipeline; state owners are explicit. */ export async function prepareAdapterExchange( @@ -406,7 +407,8 @@ export async function prepareAdapterExchange( // blaming the provider makes the caller send the whole turn again -- the amplification this // budget exists to stop. The passthrough path has answered 429 here since #4546. if (err instanceof SendBudgetExhaustedError) { - return formatErrorResponse(429, SEND_BUDGET_EXHAUSTED_CODE, err.message); + return unboundPoolSpendRefusalResponse(logCtx) + ?? formatErrorResponse(429, SEND_BUDGET_EXHAUSTED_CODE, err.message); } const msg = describeUpstreamConnectFailure(err, connectMs); return formatErrorResponse(502, "upstream_error", msg); @@ -621,7 +623,8 @@ export async function prepareAdapterExchange( // Same rule on the recovery leg: the ladder refused to send again, so the answer names // this proxy rather than the provider it never reached. if (err instanceof SendBudgetExhaustedError) { - return { failed: formatErrorResponse(429, SEND_BUDGET_EXHAUSTED_CODE, err.message) }; + return { failed: unboundPoolSpendRefusalResponse(logCtx) + ?? formatErrorResponse(429, SEND_BUDGET_EXHAUSTED_CODE, err.message) }; } const msg = describeUpstreamConnectFailure(err, connectMs); return { failed: formatErrorResponse(502, "upstream_error", msg) }; diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index bdda62fdfb1..0300064b4d8 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -164,6 +164,7 @@ import { ambiguousResendAllowanceFor, selfContainedResponsesBody } from "./reset import { upstreamErrorMessageFromPayload, ENCRYPTED_FUNCTION_OUTPUT_REJECTION } from "../../lib/errors"; import { isTransientConsoleGoUploadRejection } from "../../providers/opencode-zen-rate-limit"; import { planReasoningEffortDowngrade } from "../../providers/reasoning-metadata"; +import { unboundPoolSpendRefusalResponse } from "../workflow-refusal"; /** Prepares and recovers one native Responses exchange before client commitment. */ export async function preparePassthroughExchange( @@ -894,7 +895,8 @@ export async function preparePassthroughExchange( releaseUpstreamHostAdmission(nativeHostState.lease); nativeHostState.lease = null; releaseCodexAuthContextProbeLease(admissionState.authCtx); - return formatErrorResponse(429, "request_send_budget_exhausted", err.message); + return unboundPoolSpendRefusalResponse(logCtx) + ?? formatErrorResponse(429, "request_send_budget_exhausted", err.message); } const refusal = unwrapUpstreamRetryEvidenceError(err); // Pacing may outlive the selected account's admission. No fetch occurred, so do diff --git a/src/server/responses/request-send-budget.ts b/src/server/responses/request-send-budget.ts index 5b4d3e26eaa..3cdfe89916b 100644 --- a/src/server/responses/request-send-budget.ts +++ b/src/server/responses/request-send-budget.ts @@ -84,8 +84,9 @@ export function createResponsesSendBudget( // returns -- a refusal an operator cannot tell from an ordinary budget exhaustion, on a // ceiling they configured themselves. Asked before dispatch, it names the scope and the // number instead. Returns undefined and touches no ledger when no ceiling is configured. - // Passthrough reports sends after they leave. Historical identity uncertainty must - // refuse here as well as in reserve(), including requests with no workflow root. + // Passthrough reports sends after they leave. Invalid or conflicting identity evidence + // still refuses here as well as in reserve(), including requests with no workflow root. + // Positive unbound balances alone are instead overlaid on each candidate pool below. const continuityRefusal = poolContinuityRefusalReason(); if (continuityRefusal) { return workflowRefusalResponse(continuityRefusal, logCtx, undefined, workflowRootId); diff --git a/src/server/responses/request-spend.ts b/src/server/responses/request-spend.ts index faab6b84099..e24490d2744 100644 --- a/src/server/responses/request-spend.ts +++ b/src/server/responses/request-spend.ts @@ -44,7 +44,7 @@ export function createRequestSpendTracker( logCtx: Pick< RequestLogContext, "provider" | "accountLogLabel" | "usageLogInputTokens" | "spendOutputCeilingTokens" | "spendInputEstimateTokens" | "spendPoolId" - > & Partial>, + > & Partial>, rootId: string | undefined, injected?: SpendReservationLedger, ): RequestSpendTracker { @@ -127,10 +127,17 @@ export function createRequestSpendTracker( ? "workflow-tracking-exhausted" : "workflow-spend-exhausted"; const detail = denial.reason === "spend-limit-exceeded" - ? { scope: denial.scope, limit: denial.limit, projected: denial.projected } : undefined; + ? { + scope: denial.scope, + limit: denial.limit, + projected: denial.projected, + ...(denial.includesUnboundPoolHistory ? { includesUnboundPoolHistory: true } : {}), + } + : undefined; const summary = workflowDenialSummary(reason, detail); markLocalRequestLogRefusal(logCtx, summary.code); logCtx.errorCode = summary.code; + if (detail?.includesUnboundPoolHistory) logCtx.spendRefusalDetail = detail; recordWorkflowRefusalEvent(rootId, reason, Date.now(), detail); return false; } diff --git a/src/server/responses/run-turn-execution.ts b/src/server/responses/run-turn-execution.ts index 463c250c4a7..2f6b8e2c77f 100644 --- a/src/server/responses/run-turn-execution.ts +++ b/src/server/responses/run-turn-execution.ts @@ -51,6 +51,7 @@ import { planWebSearch } from "../../web-search"; import { runTurnWebSearchInitialParsed, runTurnWebSearchLoop } from "../../web-search/run-turn-loop"; import { WEB_SEARCH_TOOL_NAME } from "../../web-search/synthetic-tool"; import { orderDevinMessagesOutput } from "../../claude/devin-output-order"; +import { unboundPoolSpendRefusalMessage } from "../workflow-refusal"; // LOCAL PATCH (runturn-websearch): top-level fields route binding or the // adapter itself may write during a turn. Iteration-local `turnParsed` objects @@ -330,7 +331,7 @@ export async function executeResponsesRunTurn( status: 429, errorType: "rate_limit_error", code: SEND_BUDGET_EXHAUSTED_CODE, - message: err.message, + message: unboundPoolSpendRefusalMessage(logCtx) ?? err.message, } : { type: "error", diff --git a/src/server/workflow-refusal.ts b/src/server/workflow-refusal.ts index a1457fed130..8fe6696ba86 100644 --- a/src/server/workflow-refusal.ts +++ b/src/server/workflow-refusal.ts @@ -97,6 +97,22 @@ export function workflowRefusalResponse( return refusal; } +/** Render the approved conservative-overlay explanation after a dispatch reservation refuses. */ +export function unboundPoolSpendRefusalMessage( + logCtx: Pick, +): string | undefined { + const detail = logCtx.spendRefusalDetail; + if (!detail?.includesUnboundPoolHistory) return undefined; + return workflowDenialSummary("workflow-spend-exhausted", detail).message; +} + +/** HTTP form for pre-commit send-budget catches; the tracker already recorded the event. */ +export function unboundPoolSpendRefusalResponse(logCtx: RequestLogContext): Response | undefined { + const detail = logCtx.spendRefusalDetail; + if (!detail?.includesUnboundPoolHistory) return undefined; + return workflowRefusalResponse("workflow-spend-exhausted", logCtx, undefined, undefined, detail); +} + /** * Admit one HTTP turn against its root workflow budget. * @@ -145,6 +161,7 @@ export function workflowDecisionRefusalResponse( scope: decision.spendScope, limit: decision.spendLimit, ...(decision.spendProjected !== undefined ? { projected: decision.spendProjected } : {}), + ...(decision.spendIncludesUnboundPoolHistory ? { includesUnboundPoolHistory: true } : {}), } : undefined; return workflowRefusalResponse(decision.reason, logCtx, refusalLog, undefined, spend); diff --git a/structure/transports/responses-spend.md b/structure/transports/responses-spend.md index 26bcb35d84a..666e35e593f 100644 --- a/structure/transports/responses-spend.md +++ b/structure/transports/responses-spend.md @@ -137,13 +137,18 @@ for automatically assigning old balances. Old canonical-looking aliases also nee mapping when their positive history predates identity metadata. Zero-balance history needs none. The ledger retains original scope balances and reservation targets. The canonical view adds each -member once, keeping settled, reserved and unresolved buckets separate. Explicit links can join -previously canonical groups for a verified rename; an already redirected alias cannot be assigned -to a different group. The complete proposed graph is validated atomically, so a verified merge -that repeats existing member aliases is independent of salted-key order. A removed config entry -never removes a journaled link. Group activity, -last-seen time and exhaustion govern retention; unidentified positive balances cannot be evicted. -Identity evidence is bounded and retained even when dormant under-limit scopes are evicted. +proven member once, keeping settled, reserved and unresolved buckets separate. Until ownership is +verified, every positive unbound pool balance is also counted once against each candidate provider +pool. The unbound balance stays in its original scope; a current provider name or matching salted +alias does not establish ownership. This conservative overlay may restrict an otherwise unused +provider pool until an operator supplies a verified mapping. Applying a mapping moves that balance +from the overlay into its proven group without copying it. Explicit links can join previously +canonical groups for a verified rename; an already redirected alias cannot be assigned to a +different group. The complete proposed graph is validated atomically, so a verified merge that +repeats existing member aliases is independent of salted-key order. A removed config entry never +removes a journaled link. Group activity, last-seen time and exhaustion govern retention; unbound +positive balances cannot be evicted. Identity evidence is bounded and retained even when dormant +under-limit scopes are evicted. Each eviction pass aggregates pool groups once before choosing candidates. Retention uses the group's newest activity; capacity pressure still removes only the oldest eligible individual scope per pass, and every removal journals its original scope alias. @@ -154,16 +159,18 @@ journal. New reservations still use the routed canonical pool ID. Replays and co the links; complete invalid metadata fails closed, including at the final line, and corruption is never compacted away. Unparseable torn final JSON keeps the existing conservative replay rule. -With a configured pool ceiling, any remaining unidentified positive pool history or invalid/ -conflicting mapping refuses admission. HTTP workflow admission and the Responses pre-dispatch -seam check this even without a root ID, before passthrough transports that report sends afterwards. -After routing, that seam also refuses an already-exhausted canonical pool, including mapped historical totals. -A prepaid child excludes only its own open reservation, proven by its exact permit, shared send -ledger and still-pending receipt. Proof is single-use; unrelated reservations and other pool groups -remain counted. HTTP continuity refusals record one rooted workflow event; rootless ones record none. -It is a snapshot check, not a new atomic reservation for report-only transports: crossing sends, -concurrent preflight admissions and retries reported afterwards retain their existing limitations. -`workflow_pool_history_unresolved` identifies the local 429 without disclosing aliases; storage +With a configured pool ceiling, an invalid or conflicting mapping refuses admission. Unbound +positive history alone does not block HTTP admission before a route is known; once a candidate +pool is known, its reservation check includes the proven group and every unbound positive balance. +The Responses pre-dispatch seam checks this even without a root ID, before passthrough transports +that report sends afterwards. Its local refusal explains when unknown-owner history contributed, +without exposing aliases or journal contents. A prepaid child excludes only its own open +reservation, proven by its exact permit, shared send ledger and still-pending receipt. Proof is +single-use; unrelated reservations and other proven pool groups remain counted. HTTP continuity +refusals still record one rooted workflow event; rootless ones record none. It is a snapshot check, +not a new atomic reservation for report-only transports: crossing sends, concurrent preflight +admissions and retries reported afterwards retain their existing limitations. `workflow_pool_history_unresolved` +identifies an invalid or conflicting identity-metadata refusal without disclosing aliases; storage or replay failures keep `workflow_spend_undurable`. Already-sent reports still book actual spend. Observe-only mode has no new token refusal, and root/identity accounting remains independent. @@ -173,8 +180,9 @@ salt loses newer spend and is not a supported rollback. Unmodified older binarie they can read v1 counters but do not enforce the canonical aggregate, can introduce fresh account label pools, and can discard optional identity metadata during compaction. There is no automatic downgrade barrier and no unknown-record fence. If that unsupported write has happened, the current -reader requires explicit mappings for the remaining unidentified balances rather than assuming -zero. `tests/lib/spend-pool-continuity.test.ts` exercises this using the frozen pre-change reader, +reader conservatively counts remaining unidentified positive balances against every candidate +pool until the operator verifies their mappings. `tests/lib/spend-pool-continuity.test.ts` exercises +this using the frozen pre-change reader, as well as exact aggregation, active/unresolved sends, retention, failures and rootless preflight. Budget reservations retain their exact durable proof until their own dispatch/report confirms diff --git a/tests/lib/spend-ceiling-enforcement.test.ts b/tests/lib/spend-ceiling-enforcement.test.ts index 91acd121efe..95f04f078f3 100644 --- a/tests/lib/spend-ceiling-enforcement.test.ts +++ b/tests/lib/spend-ceiling-enforcement.test.ts @@ -70,6 +70,7 @@ const watched = (inner: SpendReservationLedger, asked: string[]): SpendReservati markLost: (sendId) => inner.markLost(sendId), snapshot: (scope, scopeId) => inner.snapshot(scope, scopeId), exhausted: (scope, scopeId) => { asked.push("exhausted"); return inner.exhausted(scope, scopeId); }, + hasUnboundPositivePoolHistory: () => inner.hasUnboundPositivePoolHistory(), prune: (at) => inner.prune(at), knows: (sendId) => inner.knows(sendId), reconfigure: (next) => inner.reconfigure(next), @@ -261,7 +262,7 @@ describe("a token refusal is legible on the wire", () => { expect(summary.message).toContain("account token ceiling of 100,000"); expect(summary.message).toContain("112,500"); expect(summary.message).toContain("spend.identity.maxTokens"); - expect(summary.message).toContain("no provider was contacted"); + expect(summary.message).toContain("this send was refused before contacting a provider"); }); test("without a denial in hand the sentence is the one it always was", () => { diff --git a/tests/lib/spend-pool-continuity.test.ts b/tests/lib/spend-pool-continuity.test.ts index 1973b053b26..55e341152a8 100644 --- a/tests/lib/spend-pool-continuity.test.ts +++ b/tests/lib/spend-pool-continuity.test.ts @@ -14,17 +14,18 @@ import { } from "../../src/lib/spend-reservation-ledger"; import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; -import { admitHttpWorkflowTurn, workflowDecisionRefusalResponse } from "../../src/server/workflow-refusal"; +import { admitHttpWorkflowTurn, unboundPoolSpendRefusalMessage, unboundPoolSpendRefusalResponse } from "../../src/server/workflow-refusal"; import { createResponsesSendBudget } from "../../src/server/responses/request-send-budget"; import { claimDispatchSpendProof, createRequestExecutionBudget, deriveRequestExecutionBudget, reportDispatchSends } from "../../src/lib/request-execution-budget"; import { createPoolContinuity } from "../../src/lib/spend-pool-continuity"; -import { listWorkflowBudgetEvents, resetWorkflowBudgetsForTest, workflowSpendCeilingReached } from "../../src/lib/workflow-budget"; +import { admitWorkflowTurn, listWorkflowBudgetEvents, resetWorkflowBudgetsForTest, workflowDenialSummary, workflowSpendCeilingReached } from "../../src/lib/workflow-budget"; import { createRequestSpendTracker } from "../../src/server/responses/request-spend"; import type { RequestLogContext } from "../../src/server/request-log"; const salt = "5".repeat(64); const alias = (kind: string, id: string) => createHash("sha256").update(salt).update("\0").update(kind).update("\0").update(id).digest("hex").slice(0, 32); const pool = (id: string) => alias("pool", id); +const currentPool = (id: string) => alias("pool-current", id); const policy = (poolAliases?: unknown, overrides: Partial = {}): SpendReservationPolicy => ({ ...DEFAULT_SPEND_RESERVATION_POLICY, pool: { maxTokens: 100 }, poolAliases, ...overrides, }); @@ -65,18 +66,136 @@ describe("historical pool identity continuity", () => { } }); - test("unmapped historical debt fails closed across labels, providers, pruning and restart", () => { + test("unbound positive history is charged once against every candidate pool across pruning and restart", () => { const disk = journal([checkpoint([["provider-old-label", 100, 0]])]); for (let restart = 0; restart < 2; restart += 1) { - const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 100_000 }); + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 1_000_000_000 }); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + expect(ledger.hasUnboundPositivePoolHistory()).toBe(true); ledger.prune(); for (const id of ["provider", "provider-old-label", "unrelated-provider"]) { - expect(reserve(ledger, `${restart}-${id}`, id)).toMatchObject({ reserved: false, denial: { reason: "pool-history-unresolved" } }); + expect(reserve(ledger, `${restart}-${id}`, id)).toMatchObject({ + reserved: false, + denial: { reason: "spend-limit-exceeded", scope: "pool", limit: 100, projected: 101, includesUnboundPoolHistory: true }, + }); } expect(ledger.snapshot("pool", "provider-old-label")?.settled).toBe(100); } }); + test("tracking capacity pressure cannot evict unbound positive pool history", () => { + const disk = journal([checkpoint([["old-label", 10, 0]])]); + const ledger = createSpendReservationLedger({ journal: disk, salt, + policy: policy(undefined, { maxTrackedScopes: 1 }), now: () => 2 }); + expect(reserve(ledger, "cannot-forget-history", "provider", 1)).toMatchObject({ + reserved: false, denial: { reason: "tracking-capacity-exhausted", scope: "pool" }, + }); + expect(ledger.snapshot("pool", "old-label")?.settled).toBe(10); + expect(disk.lines.map(line => JSON.parse(line)).filter(record => record.kind === "drop")).toEqual([]); + }); + + test("an exhausted candidate overlay protects its under-limit proven group from retention", () => { + const record = checkpoint([["verified-label", 60, 0], ["unbound-label", 40, 0], ["empty-old-label", 0, 0]]); + const disk = journal([record]); + const ledger = createSpendReservationLedger({ journal: disk, salt, + policy: policy({ [pool("verified-label")]: "provider" }, { retentionMs: 5 }), now: () => 100 }); + expect(ledger.checkPoolContinuity()).toBeUndefined(); + expect(ledger.snapshot("pool", "provider")?.settled).toBe(100); + ledger.prune(100); // All entries are older than the retention cutoff. + expect(ledger.snapshot("pool", "provider")?.settled).toBe(100); + expect(disk.lines.map(line => JSON.parse(line)).filter(record => record.kind === "drop")).toEqual([ + { v: 1, kind: "drop", scope: "pool", alias: pool("empty-old-label"), at: 100 }, + ]); + expect(reserve(ledger, "still-exhausted", "provider", 1)).toMatchObject({ reserved: false, + denial: { reason: "spend-limit-exceeded", projected: 101, includesUnboundPoolHistory: true } }); + }); + + test("proven group plus each unbound balance admits under/exact totals and refuses over", () => { + const disk = journal([checkpoint([["verified-label", 25, 0], ["unbound-label", 10, 5]])]); + const ledger = createSpendReservationLedger({ journal: disk, salt, + policy: policy({ [pool("verified-label")]: "provider" }), now: () => 2 }); + + // Proven group: 25. Unbound history: 10 settled + 5 unresolved. Neither bucket is copied. + expect(ledger.snapshot("pool", "provider")).toEqual({ settled: 35, reserved: 0, unresolved: 5, exhausted: false }); + expect(reserve(ledger, "under", "provider", 59).reserved).toBe(true); // 99 total + expect(ledger.abandon("under")).toBe(true); + expect(reserve(ledger, "exact", "provider", 60).reserved).toBe(true); // 100 total + expect(reserve(ledger, "over", "provider", 1)).toMatchObject({ + reserved: false, + denial: { reason: "spend-limit-exceeded", projected: 101, includesUnboundPoolHistory: true }, + }); + + // The same unbound 15 is also counted once for a different candidate provider. + expect(reserve(ledger, "other-exact", "unrelated-provider", 85).reserved).toBe(true); + expect(ledger.snapshot("pool", "unrelated-provider")).toEqual({ settled: 10, reserved: 85, unresolved: 5, exhausted: true }); + expect(ledger.hasUnboundPositivePoolHistory()).toBe(true); + }); + + test("same-spelling unknown history stays unbound and is not counted twice", () => { + const disk = journal([checkpoint([["provider", 40, 0]])]); + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 2 }); + + // Matching the current provider's salted alias is not identity evidence by itself. + expect(ledger.checkPoolContinuity()).toBeUndefined(); + expect(ledger.hasUnboundPositivePoolHistory()).toBe(true); + expect(reserve(ledger, "same-spelling-exact", "provider", 60).reserved).toBe(true); + expect(ledger.snapshot("pool", "provider")).toEqual({ settled: 40, reserved: 60, unresolved: 0, exhausted: true }); + // The legacy alias stays unknown and overlays every candidate, while the new send is + // stored under the canonical-current domain and belongs only to the routed provider. + expect(JSON.parse(disk.lines.at(-1)!).targets).toEqual([{ scope: "pool", alias: currentPool("provider") }]); + expect(reserve(ledger, "unrelated-under-overlay", "unrelated-provider", 60).reserved).toBe(true); + expect(ledger.snapshot("pool", "unrelated-provider")).toEqual({ + settled: 40, reserved: 60, unresolved: 0, exhausted: true, + }); + expect(reserve(ledger, "unrelated-over", "third-provider", 61)).toMatchObject({ + reserved: false, + denial: { reason: "spend-limit-exceeded", projected: 101, includesUnboundPoolHistory: true }, + }); + + // An explicit operator mapping on restart moves only the legacy 40-token balance into + // this provider's current group; it does not copy it or assign it by spelling. + const mapped = createSpendReservationLedger({ journal: disk, salt, + policy: policy({ [pool("provider")]: "provider" }), now: () => 3 }); + expect(mapped.checkPoolContinuity()).toBeUndefined(); + expect(mapped.hasUnboundPositivePoolHistory()).toBe(false); + expect(mapped.snapshot("pool", "provider")).toMatchObject({ settled: 40, unresolved: 60 }); + expect(reserve(mapped, "unrelated-after-mapping", "unrelated-provider", 41)).toMatchObject({ + reserved: false, + denial: { reason: "spend-limit-exceeded", projected: 101 }, + }); + }); + + test("unbound history is overlaid when a pool ceiling is enabled later", () => { + const disk = journal([checkpoint([["old-label", 40, 0]])]); + const ledger = createSpendReservationLedger({ journal: disk, salt, + policy: policy(undefined, { pool: {} }), now: () => 2 }); + + expect(reserve(ledger, "observe-only", "provider", 10).reserved).toBe(true); + expect(ledger.settle("observe-only", { inputTokens: 10, outputTokens: 0 })).toBe(true); + expect(ledger.hasUnboundPositivePoolHistory()).toBe(true); + ledger.reconfigure(policy()); + + expect(reserve(ledger, "exact-after-enable", "provider", 50).reserved).toBe(true); + expect(reserve(ledger, "over-after-enable", "provider", 1)).toMatchObject({ + reserved: false, + denial: { reason: "spend-limit-exceeded", projected: 101, includesUnboundPoolHistory: true }, + }); + }); + + test("competing same-process admissions include the shared unbound balance", async () => { + const ledger = createSpendReservationLedger({ + journal: journal([checkpoint([["old-label", 40, 0]])]), salt, policy: policy(), now: () => 2, + }); + const attempts = await Promise.all([ + Promise.resolve().then(() => reserve(ledger, "contender-a", "provider", 35)), + Promise.resolve().then(() => reserve(ledger, "contender-b", "provider", 35)), + ]); + expect(attempts[0]?.reserved).toBe(true); + expect(attempts[1]).toMatchObject({ reserved: false, + denial: { reason: "spend-limit-exceeded", scope: "pool", projected: 110, includesUnboundPoolHistory: true } }); + expect(ledger.snapshot("pool", "provider")).toMatchObject({ settled: 40, reserved: 35 }); + }); + test("explicit aliases aggregate each original balance once, including canonical history", () => { const disk = journal([checkpoint([["label-a", 40, 0], ["label-b", 0, 30], ["provider", 20, 0]])]); const aliases = { [pool("label-a")]: "provider", [pool("label-b")]: "provider", [pool("provider")]: "provider" }; @@ -186,27 +305,67 @@ describe("historical pool identity continuity", () => { { v: 1, kind: "drop", scope: "pool", alias: pool("oldest"), at: 50 }, ]); expect(ledger.snapshot("pool", "provider")?.settled).toBe(20); - expect(ledger.snapshot("pool", "other")?.settled).toBe(0); + expect(ledger.snapshot("pool", "other")).toBeUndefined(); expect(ledger.snapshot("root", "new-root")?.reserved).toBe(1); const restarted = createSpendReservationLedger({ journal: disk, salt, policy: config, now: () => 50 }); expect(restarted.snapshot("pool", "provider")?.settled).toBe(20); - expect(restarted.snapshot("pool", "other")?.settled).toBe(0); + expect(restarted.snapshot("pool", "other")).toBeUndefined(); expect(restarted.snapshot("root", "new-root")?.unresolved).toBe(1); }); - test("observe-only records already-sent requests while ambiguity still blocks new dispatches", () => { + test("observe-only charges already-sent requests while unbound history still constrains admission", () => { const disk = journal([checkpoint([["unknown-label", 40, 0]])]); const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 2 }); const context: RequestLogContext = { model: "fixture", provider: "provider-display", spendPoolId: "provider", usageLogInputTokens: 10 }; const tracker = createRequestSpendTracker(context, undefined, ledger); - expect(tracker.charge()).toBe(false); - expect(tracker.refusals).toBe(1); - expect(context.errorCode).toBe("workflow_pool_history_unresolved"); expect(tracker.charge({ alreadySent: true })).toBe(true); + expect(tracker.refusals).toBe(0); + expect(context.errorCode).toBeUndefined(); tracker.settle({ inputTokens: 8, outputTokens: 0 }); - expect(ledger.snapshot("pool", "provider")?.settled).toBe(8); + expect(ledger.snapshot("pool", "provider")?.settled).toBe(48); // current settled 8 plus shared unknown 40 expect(ledger.snapshot("pool", "unknown-label")?.settled).toBe(40); - expect(reserve(ledger, "new").reserved).toBe(false); + expect(reserve(ledger, "exact", "provider", 52).reserved).toBe(true); + expect(ledger.abandon("exact")).toBe(true); + const refused = reserve(ledger, "over", "provider", 53); + expect(refused).toMatchObject({ reserved: false, + denial: { reason: "spend-limit-exceeded", projected: 101, includesUnboundPoolHistory: true } }); + const rootRefusal = admitWorkflowTurn("root-with-unbound-history", "interactive", undefined, undefined, 2, + { sendId: "root-over", poolId: "provider", inputTokens: 53, outputCeilingTokens: 0 }, ledger); + expect(rootRefusal).toMatchObject({ admitted: false, reason: "workflow-spend-exhausted", + spendScope: "pool", spendIncludesUnboundPoolHistory: true }); + const message = workflowDenialSummary("workflow-spend-exhausted", { + scope: "pool", limit: 100, projected: 101, includesUnboundPoolHistory: true, + }).message; + expect(message).toContain("unassigned historical provider-pool balances"); + expect(message).not.toContain("unknown-label"); + expect(message).not.toContain(alias("pool", "unknown-label")); + expect(message).not.toContain(salt); + }); + + test("reservation crossing explains unbound-history refusal without exposing its alias", async () => { + resetWorkflowBudgetsForTest(); + const disk = journal([checkpoint([["private-old-label", 40, 0]])]); + const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 2 }); + const context: RequestLogContext = { + model: "fixture", provider: "provider", spendPoolId: "provider", usageLogInputTokens: 10, + }; + const tracker = createRequestSpendTracker(context, "root-unbound", ledger); + + expect(tracker.charge({ alreadySent: true })).toBe(true); + context.usageLogInputTokens = 51; // the next physical send would cross the cap after that upstream contact + expect(tracker.charge()).toBe(false); + const message = unboundPoolSpendRefusalMessage(context); + expect(message).toContain("unassigned historical provider-pool balances"); + expect(message).toContain("this send was refused before contacting a provider"); + expect(message).not.toContain("private-old-label"); + expect(message).not.toContain(alias("pool", "private-old-label")); + expect(message).not.toContain(salt); + const response = unboundPoolSpendRefusalResponse(context); + expect(response?.status).toBe(429); + expect(await response!.text()).toContain(message!); + expect(listWorkflowBudgetEvents(1)[0]).toMatchObject({ + reason: "workflow-spend-exhausted", spendIncludesUnboundPoolHistory: true, + }); }); test("malformed or conflicting maps fail closed without clearing ceilings or saved links", () => { @@ -229,7 +388,7 @@ describe("historical pool identity continuity", () => { disk.append = () => { throw new Error("synthetic write failure"); }; const ledger = createSpendReservationLedger({ journal: disk, salt, policy: policy({ [pool("label")]: "provider" }), now: () => 2 }); expect(reserve(ledger, "new")).toMatchObject({ reserved: false, denial: { reason: "reserve-not-durable" } }); - expect(ledger.snapshot("pool", "provider")).toBeUndefined(); + expect(ledger.snapshot("pool", "provider")?.settled).toBe(40); // the unbound view is visible without a committed link expect(ledger.snapshot("pool", "label")?.settled).toBe(40); expect(disk.lines).toHaveLength(1); }); @@ -263,7 +422,7 @@ describe("historical pool identity continuity", () => { }); -test("rootless HTTP and passthrough preflight refuse before any synthetic fetch", () => { +test("unbound history waits for routing, then limits each candidate before any synthetic fetch", async () => { const previous = process.env.OPENCODEX_HOME; const home = mkdtempSync(join(tmpdir(), "ocx-pool-history-")); process.env.OPENCODEX_HOME = home; @@ -274,24 +433,23 @@ test("rootless HTTP and passthrough preflight refuse before any synthetic fetch" configureSharedSpendLedger(policy()); resetWorkflowBudgetsForTest(); const rooted = admitHttpWorkflowTurn(new Headers({ "x-codex-parent-thread-id": "synthetic-root" })); - expect(rooted).toMatchObject({ admitted: false, reason: "workflow-pool-history-unresolved" }); - expect(listWorkflowBudgetEvents()).toMatchObject([{ rootId: "synthetic-root", reason: "workflow-pool-history-unresolved" }]); - if (rooted && !rooted.admitted) workflowDecisionRefusalResponse(rooted); - expect(listWorkflowBudgetEvents()).toHaveLength(1); + expect(rooted).toMatchObject({ admitted: true, lease: { rootId: "synthetic-root" } }); + if (rooted?.admitted) rooted.lease.release(); + expect(listWorkflowBudgetEvents()).toHaveLength(0); let syntheticFetches = 0; const decision = admitHttpWorkflowTurn(new Headers()); - expect(decision).toMatchObject({ admitted: false, reason: "workflow-pool-history-unresolved" }); - if (!decision || decision.admitted) syntheticFetches += 1; - else { - const response = workflowDecisionRefusalResponse(decision); - expect(response.status).toBe(429); - expect(response.headers.get("x-opencodex-local-refusal")).toBe("workflow_pool_history_unresolved"); - } + expect(decision).toBeUndefined(); const budget = createResponsesSendBudget({ req: new Request("https://fixture.example.test/v1/responses"), options: {}, logCtx: { model: "fixture", provider: "provider" } }); expect(budget).toBeInstanceOf(Response); - if (!(budget instanceof Response)) syntheticFetches += 1; + if (budget instanceof Response) { + expect(budget.status).toBe(429); + expect(budget.headers.get("x-opencodex-local-refusal")).toBe("workflow_spend_exhausted"); + const body = await budget.text(); + expect(body).toContain("unassigned historical provider-pool balances"); + expect(body).not.toContain("old-label"); + } else syntheticFetches += 1; expect(syntheticFetches).toBe(0); - expect(listWorkflowBudgetEvents()).toHaveLength(1); // rootless refusals add no event + expect(listWorkflowBudgetEvents()).toHaveLength(0); // rootless refusals add no event configureSharedSpendLedger(policy({ [pool("old-label")]: "provider" })); expect(admitHttpWorkflowTurn(new Headers())).toBeUndefined(); const mappedBudget = createResponsesSendBudget({ req: new Request("https://fixture.example.test/v1/responses"), options: {}, logCtx: { model: "fixture", provider: "provider-display", spendPoolId: "provider" } }); @@ -328,11 +486,18 @@ test("actual old-reader compaction preserves raw spend; compatible rollback reso old.settle("old-again", { inputTokens: 12, outputTokens: 0 }); const returned = createSpendReservationLedger({ journal: disk, salt, policy: policy({ [pool("old-label")]: "provider" }), now: () => 4 }); - expect(returned.checkPoolContinuity()?.reason).toBe("pool-history-unresolved"); - expect(returned.snapshot("pool", "new-old-label")?.settled).toBe(12); + expect(returned.checkPoolContinuity()).toBeUndefined(); + expect(returned.snapshot("pool", "new-old-label")?.settled).toBe(20); // unbound 12 plus canonical-looking 8, both still unassigned + expect(returned.snapshot("pool", "provider")?.settled).toBe(60); + expect(returned.snapshot("pool", "unrelated-provider")?.settled).toBe(20); + expect(reserve(returned, "candidate-after-old-writer", "unrelated-provider", 80).reserved).toBe(true); + expect(returned.abandon("candidate-after-old-writer")).toBe(true); returned.reconfigure(policy({ [pool("old-label")]: "provider", [pool("provider")]: "provider", [pool("new-old-label")]: "provider" })); expect(returned.checkPoolContinuity()).toBeUndefined(); expect(returned.snapshot("pool", "provider")?.settled).toBe(60); + const restarted = createSpendReservationLedger({ journal: disk, salt, policy: policy(), now: () => 5 }); + expect(restarted.checkPoolContinuity()).toBeUndefined(); + expect(restarted.snapshot("pool", "provider")?.settled).toBe(60); }); for (const kind of ["compaction", "combo"] as const) { @@ -505,19 +670,19 @@ test("preflight retains unrelated reservations, debt and mismatched scope or led }); test("prepaid exclusion follows only its canonical pool and retains settled/unresolved history", () => { - const disk = journal([checkpoint([["historical", 30, 10]])]); + const disk = journal([checkpoint([["historical", 30, 10], ["unbound", 10, 5]])]); const ledger = createSpendReservationLedger({ salt, journal: disk, policy: policy({ [pool("historical")]: "provider" }), now: () => 2 }); - const tracker = createRequestSpendTracker({ provider: "provider", usageLogInputTokens: 60 }, undefined, ledger); + const tracker = createRequestSpendTracker({ provider: "provider", usageLogInputTokens: 45 }, undefined, ledger); const budget = createRequestExecutionBudget(undefined, undefined, tracker); const decision = budget.reserveDispatch({ sendClass: "initial", targetKey: "provider", countedExternally: true }); if (!decision.allowed) throw new Error("synthetic permit refused"); const proof = claimDispatchSpendProof(budget, decision.permit)!; - expect(ledger.snapshot("pool", "provider")).toMatchObject({ settled: 30, unresolved: 10, reserved: 60 }); + expect(ledger.snapshot("pool", "provider")).toMatchObject({ settled: 40, unresolved: 15, reserved: 45 }); expect(workflowSpendCeilingReached(undefined, ledger, "provider", proof)).toBeUndefined(); ledger.reconfigure(policy({ [pool("historical")]: "renamed", [pool("provider")]: "renamed" })); expect(ledger.checkPoolContinuity()).toBeUndefined(); expect(workflowSpendCeilingReached(undefined, ledger, "renamed", proof)).toBeUndefined(); - expect(reserve(ledger, "other", "unrelated", 100).reserved).toBe(true); + expect(reserve(ledger, "other", "unrelated", 85).reserved).toBe(true); expect(workflowSpendCeilingReached(undefined, ledger, "unrelated", proof)).toMatchObject({ scope: "pool" }); ledger.reconfigure(policy(undefined, { pool: { maxTokens: 40 } })); expect(workflowSpendCeilingReached(undefined, ledger, "renamed", proof)).toMatchObject({ scope: "pool" }); diff --git a/tests/responses/responses-send-budget-errors.test.ts b/tests/responses/responses-send-budget-errors.test.ts index 6e1306980b8..dc53fa9efe9 100644 --- a/tests/responses/responses-send-budget-errors.test.ts +++ b/tests/responses/responses-send-budget-errors.test.ts @@ -78,6 +78,15 @@ describe("a spent send budget is reported as this proxy's refusal", () => { } }); + test("reservation denials preserve unbound-history detail on HTTP and SSE response paths", () => { + const adapter = source("src/server/responses/adapter-dispatch.ts"); + const passthrough = source("src/server/responses/passthrough-dispatch.ts"); + const runTurn = source("src/server/responses/run-turn-execution.ts"); + expect(adapter.match(/unboundPoolSpendRefusalResponse\(logCtx\)/g)).toHaveLength(2); + expect(passthrough).toContain("unboundPoolSpendRefusalResponse(logCtx)"); + expect(runTurn).toContain("unboundPoolSpendRefusalMessage(logCtx) ?? err.message"); + }); + test("a local 429 never rotates a credential or writes a cooldown", () => { const runTurn = source("src/server/responses/run-turn-execution.ts"); const rotate = runTurn.indexOf("const rotateRunTurnAdapterOnPreflight429");