diff --git a/packages/core/src/accounts.ts b/packages/core/src/accounts.ts index 8dfe2901..c6b4aa0f 100644 --- a/packages/core/src/accounts.ts +++ b/packages/core/src/accounts.ts @@ -510,6 +510,8 @@ export type AccountRefreshError = { const DEFAULT_FALLBACK_ON = [401, 403, 429] const MIN_REFRESH_BEFORE_EXPIRY_MINUTES = 240 const DEFAULT_REFRESH_BEFORE_EXPIRY_MINUTES = MIN_REFRESH_BEFORE_EXPIRY_MINUTES +// Claustrum requests extra headroom beyond the local refresh threshold. +export const VAULT_REFRESH_HEADROOM_MINUTES = 30 const DEFAULT_REFRESH_INTERVAL_MINUTES = 10 const MIN_REFRESH_RETRY_DELAY_MS = 5 * 60_000 const MAX_REFRESH_RETRY_DELAY_MS = 60 * 60_000 @@ -3353,6 +3355,12 @@ export function getRefreshBeforeExpiryMs(storage: AccountStorage | null) { return refreshBeforeExpiryMs(storage) } +export function getVaultRefreshMinTtlMs(storage: AccountStorage | null) { + return ( + getRefreshBeforeExpiryMs(storage) + VAULT_REFRESH_HEADROOM_MINUTES * 60_000 + ) +} + export function getRefreshIntervalMs(storage: AccountStorage | null) { const minutes = storage?.refresh?.intervalMinutes ?? DEFAULT_REFRESH_INTERVAL_MINUTES @@ -4119,9 +4127,15 @@ function canUseCachedQuotaAfterRefreshError( storage: AccountStorage | null, error: unknown, now: number, + vaultServed: boolean, ) { return ( - Boolean(account.access && account.expires && account.expires > now) && + // Cached quota remains attributable after a transient failure when either + // the local credential is live or a live Claustrum binding serves it. + Boolean( + (account.access && account.expires && account.expires > now) || + vaultServed, + ) && isTransientQuotaError(error) && quotaSnapshotPassesPolicy(account.quota, storage) && cachedQuotaSnapshotStillRelevant(account.quota, now) @@ -4642,8 +4656,9 @@ export class FallbackAccountManager { for (const account of storage.accounts) { if (account.enabled === false || !isOAuthAccount(account)) continue + const vaultServed = this.isFallbackAccountVaultServed(account.id, storage) if (this.isFallbackAccountVaultEnabled(account.id, storage)) { - if (!this.isFallbackAccountVaultServed(account.id, storage)) continue + if (!vaultServed) continue if ( hasNoLocalCredential(account) && !storage.quota?.minimumRemaining && @@ -4658,7 +4673,7 @@ export class FallbackAccountManager { if ( tokenNeedsRefresh(next, storage, this.now()) && !this.isFallbackAccountVaultEnabled(next.id, storage) && - !this.isFallbackAccountVaultServed(next.id, storage) + !vaultServed ) { const refreshError = next.lastRefreshError if ( @@ -4711,7 +4726,13 @@ export class FallbackAccountManager { usable.push(next) } catch (error) { if ( - canUseCachedQuotaAfterRefreshError(next, storage, error, this.now()) + canUseCachedQuotaAfterRefreshError( + next, + storage, + error, + this.now(), + this.isFallbackAccountVaultServed(next.id, storage), + ) ) { log( '[refresh] fallback quota using cached quota after refresh error', diff --git a/packages/core/src/cachekeep.ts b/packages/core/src/cachekeep.ts index 78d194c9..d552fc0b 100644 --- a/packages/core/src/cachekeep.ts +++ b/packages/core/src/cachekeep.ts @@ -391,6 +391,12 @@ export class CacheKeepManager { target: CacheKeepTarget, attempt: CacheKeepPrewarmAttempt, ) => Promise | Headers | undefined + retryHeadersAfter401?: (input: { + headers: Headers + target: CacheKeepTarget + bodyText: string + attempt: CacheKeepPrewarmAttempt + }) => Promise | Headers | undefined onTrackedSessionsChanged?: ( sessions: readonly CacheKeepTrackedSession[], ) => Promise | void @@ -718,6 +724,33 @@ export class CacheKeepManager { transient: true, } } + if (response.status === 401 && this.options.retryHeadersAfter401) { + const retryHeaders = await this.options.retryHeadersAfter401({ + headers, + target, + bodyText: prewarm.bodyText, + attempt, + }) + if (retryHeaders) { + await response.body?.cancel().catch(() => {}) + try { + response = await fetchImpl(target.url, { + method: 'POST', + headers: retryHeaders, + body: prewarm.bodyText, + signal: AbortSignal.timeout( + this.options.prewarmTimeoutMs ?? CACHE_KEEP_PREWARM_TIMEOUT_MS, + ), + }) + } catch (error) { + return { + ok: false, + reason: error instanceof Error ? error.message : String(error), + transient: true, + } + } + } + } const receivedAt = this.options.now?.() ?? Date.now() const raw = await response.text().catch(() => '') let data: unknown = null diff --git a/packages/core/src/claustrum.ts b/packages/core/src/claustrum.ts index 58a991d9..0c299fac 100644 --- a/packages/core/src/claustrum.ts +++ b/packages/core/src/claustrum.ts @@ -1867,6 +1867,10 @@ export class ClaustrumCredentialCache { readonly #identity?: BindIdentity readonly #now: () => number readonly #refreshBackoffUntil = new Map() + readonly #latchedRefreshFailures = new Map< + string, + ClaustrumCredentialErrorClass + >() #minTtlMs: number constructor( @@ -1885,24 +1889,34 @@ export class ClaustrumCredentialCache { async get( handle: string, minTtlMs = this.#minTtlMs, - options: { cacheIf?: () => boolean } = {}, + options: { cacheIf?: () => boolean; bypassCache?: boolean } = {}, ): Promise { if (!Number.isSafeInteger(minTtlMs) || minTtlMs < 0) { throw new RangeError('minTtlMs must be a non-negative safe integer') } const now = this.#now() const cached = this.#cache.get(handle) - if (cached && cached.expiresAtMs !== null && cached.expiresAtMs > now) { + if ( + !options.bypassCache && + cached && + cached.expiresAtMs !== null && + cached.expiresAtMs > now + ) { if (cached.expiresAtMs - now <= minTtlMs) { this.#refreshIfApproachingExpiry(handle, now, minTtlMs) } return cached } - if (cached) { + if (cached && !options.bypassCache) { this.#cache.delete(handle) this.#refreshBackoffUntil.delete(handle) } + if (options.bypassCache) { + // Keep a 401 verdict independent of refreshes that began before it. + return this.#load(handle, minTtlMs, options.cacheIf) + } + const pending = this.#inFlight.get(handle) if (pending) return pending @@ -2033,7 +2047,22 @@ export class ClaustrumCredentialCache { const load = this.#load(handle, minTtlMs) this.#inFlight.set(handle, load) void load - .catch(() => {}) + .catch((error) => { + if ( + error instanceof ClaustrumCredentialError && + (error.errorClass === 'permanent' || + error.errorClass === 'auth_required') && + this.#latchedRefreshFailures.get(handle) !== error.errorClass + ) { + this.#latchedRefreshFailures.set(handle, error.errorClass) + logger.warn('claustrum', 'credential background refresh latched', { + handle, + recordVersion: this.#cache.get(handle)?.recordVersion, + errorClass: error.errorClass, + code: error.code, + }) + } + }) .finally(() => { if (this.#inFlight.get(handle) === load) this.#inFlight.delete(handle) }) @@ -2075,8 +2104,12 @@ export class ClaustrumCredentialCache { credential.expiresAtMs > this.#now() && (cacheIf?.() ?? true) ) { - this.#cache.set(handle, credential) + const cached = this.#cache.get(handle) + if (!cached || credential.recordVersion >= cached.recordVersion) { + this.#cache.set(handle, credential) + } } + this.#latchedRefreshFailures.delete(handle) return credential } } diff --git a/packages/core/src/tests/accounts-persistence.test.ts b/packages/core/src/tests/accounts-persistence.test.ts index 8bca18ef..0b7a808b 100644 --- a/packages/core/src/tests/accounts-persistence.test.ts +++ b/packages/core/src/tests/accounts-persistence.test.ts @@ -1,4 +1,5 @@ import { afterEach, expect, test } from 'bun:test' +import { strictEqual } from 'node:assert' import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -7,15 +8,20 @@ import { type AccountStorage, createEmptyStorage, FallbackAccountManager, + getRefreshBeforeExpiryMs, + getVaultRefreshMinTtlMs, hasNoLocalCredential, loadAccounts, type OAuthAccount, saveAccountState, saveAccounts, } from '../accounts.ts' +import { custodyTombstoneOAuth } from '../claustrum.ts' const directories: string[] = [] +// These paired tripwires cover both vault-facing minTtl routes: default +// threshold/headroom derivation and the config-override floor. afterEach(async () => { await Promise.all( directories @@ -31,6 +37,64 @@ test('recognizes an OAuth account with no local credential', () => { expect(hasNoLocalCredential({ access: '' })).toBe(false) }) +test('keeps the vault-facing refresh TTL at 270 minutes', () => { + const vaultMinTtlMs = getVaultRefreshMinTtlMs(createEmptyStorage()) + const expectedVaultMinTtlMs = 270 * 60_000 + // Anthropic OAuth access tokens live 8h; the vault refreshes a credential when + // `now + minTtl >= expires_at`, so this value alone fixes the observed rotation + // period. State the resulting PERIOD, not just the minTtl: the period is the + // number the vault operator needs to pre-seed their stall detector. + const tokenLifetimeMinutes = 480 + const newPeriodMinutes = tokenLifetimeMinutes - vaultMinTtlMs / 60_000 + const oldPeriodMinutes = tokenLifetimeMinutes - expectedVaultMinTtlMs / 60_000 + // Every number below is labelled with its ROLE: minTtl and period are drawn from + // the same small set of values and routinely swap places (a 240m minTtl on an 8h + // token yields a 240m period), so bare numerals invite transposition by a reader + // who lands on the assertion footer rather than the prose. + const guidance = [ + 'Vault coupling tripwire (threshold + headroom route; paired with the config-floor tripwire below): this shared value is passed as minTtl to Claustrum', + '`credential.get`, and the vault refreshes when `now + minTtl >= expires_at`,', + 'so it fixes the observed rotation period as token_lifetime - minTtl.', + `CHANGED: minTtl ${vaultMinTtlMs / 60_000}m (was ${expectedVaultMinTtlMs / 60_000}m)`, + `-> rotation period ${newPeriodMinutes}m (was ${oldPeriodMinutes}m).`, + `The +/- values below are minTtl in ms, NOT the period.`, + `token_lifetime is ASSUMED ${tokenLifetimeMinutes}m — neither side observes it`, + "(it lives inside the vault's encrypted envelope); if Anthropic changed it,", + 'this arithmetic is stale even though the assertion fired correctly.', + newPeriodMinutes > oldPeriodMinutes + ? 'THIS CHANGE LENGTHENS THE PERIOD, WHICH REQUIRES ADVANCE NOTICE: the vault operator alarms on MAX(recent gaps) + 30m, so the first longer gap trips a false stall alarm that REPEATS on a 30-minute cooldown until the refresh lands. Tell them the new period above before deploying so they can pre-seed it.' + : 'This change shortens the period, which is silent for the vault operator and needs no notice.', + ].join(' ') + + strictEqual(vaultMinTtlMs, expectedVaultMinTtlMs, guidance) +}) + +test('floors a config override so it cannot lower the vault-facing minTtl', () => { + // This watches the other route to the same vault-facing value: the + // `refresh.refreshBeforeExpiryMinutes` config key. The floor in + // refreshBeforeExpiryMs is what makes the paired tripwires sufficient — without + // it, an operator could lower minTtl from config, lengthening the vault's + // rotation period, and the threshold/headroom tripwire above would never fire. + const storage = createEmptyStorage() + storage.refresh = { ...storage.refresh, refreshBeforeExpiryMinutes: 60 } + const floored = getRefreshBeforeExpiryMs(storage) + + strictEqual( + floored, + 240 * 60_000, + [ + 'Vault coupling tripwire (config route): a below-floor override of', + '`refresh.refreshBeforeExpiryMinutes` must clamp UP to the 240m floor, but this', + `build returned ${floored / 60_000}m. The floor is load-bearing for a peer system:`, + 'it is the only reason config cannot lower minTtl, and lowering minTtl LENGTHENS the', + "vault's rotation period, which repeatedly false-alarms the vault operator's stall", + 'detector. Removing the floor makes that reachable from config alone, where the', + 'threshold/headroom tripwire above cannot see it. If you removed it deliberately,', + 'the vault operator holds a registered dependency on it and is owed notice.', + ].join(' '), + ) +}) + test('preserves the Claustrum mode when a save supplies only handlesFile', async () => { const directory = await mkdtemp(join(tmpdir(), 'accounts-persistence-')) directories.push(directory) @@ -147,6 +211,56 @@ test('excludes an empty-material vault fallback after its quota policy fails', a expect(authorizations).toEqual(['Bearer vault-fallback-access']) }) +test('keeps a live vault fallback on cached quota after a transient quota failure', async () => { + const now = 1_000_000 + const account: OAuthAccount = { + id: 'vault-fallback', + enabled: true, + ...custodyTombstoneOAuth('anthropic'), + quota: { + checkedAt: now - 60_000, + five_hour: { + usedPercent: 10, + remainingPercent: 90, + checkedAt: now - 60_000, + }, + seven_day: { + usedPercent: 10, + remainingPercent: 90, + checkedAt: now - 60_000, + }, + }, + } + const storage: AccountStorage = { + version: 1, + claustrum: { mode: 'claustrum' }, + quota: { + enabled: true, + checkIntervalMinutes: 1, + minimumRemaining: { five_hour: 10, seven_day: 10 }, + failClosedOnUnknownQuota: true, + }, + accounts: [account], + } + const manager = new FallbackAccountManager({ + now: () => now, + isFallbackAccountVaultEnabled: () => true, + isFallbackAccountVaultServed: () => true, + resolveFallbackAccessToken: () => ({ + token: 'vault-fallback-access', + source: 'vault', + }), + fetchImpl: Object.assign( + async () => new Response('unavailable', { status: 503 }), + { preconnect: () => {} }, + ) as unknown as typeof fetch, + }) + + await expect(manager.getUsableFallbackAccounts(storage)).resolves.toEqual([ + account, + ]) +}) + test('keeps tombstone metadata when discarding a stale credential write', async () => { const directory = await mkdtemp(join(tmpdir(), 'accounts-persistence-')) directories.push(directory) diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts index 171a9b4d..370adcf9 100644 --- a/packages/opencode/src/index.ts +++ b/packages/opencode/src/index.ts @@ -64,6 +64,8 @@ import { createStickyNoRouteResponse, custodyCredentialId, custodyCredentialIdFromResolution, + custodyTombstoneOAuth, + DEFAULT_CLAUSTRUM_CREDENTIAL_MIN_TTL_MS, type DumpHandle, decideStickyQuotaFailure, detectClaustrumConnection, @@ -104,11 +106,11 @@ import { getPersistedLogLevel, getPersistedMainQuota, getQuotaNextRefreshAt, - getRefreshBeforeExpiryMs, getRelayConfig, getRoutingMode, getStickyRoutingStatePath, getThinkingPrefixMismatchBehavior, + getVaultRefreshMinTtlMs, hashRefreshToken, type IdentityState, incrementPrimeUsagePersistent, @@ -1709,6 +1711,9 @@ const anthropicAuthPlugin = async ( } const now = Date.now() const mainIdentity = mainQuotaAccountId + const vaultMainAccessToken = liveMainVaultAccess(storage) + const servedMainAccessToken = + mainServedAccessToken || mainAccessToken || vaultMainAccessToken if ( mainAccessToken && storage.main?.profile && @@ -1742,11 +1747,16 @@ const anthropicAuthPlugin = async ( profile: storage.main.profile, }).catch(() => {}) } - if (mainAccessToken && !oauthProfileIsFresh(storage.main?.profile, now)) { + // Profile hydration may use local main access or the live token serving a + // custody tombstone; an empty tombstone slot is never a usable bearer. + if ( + servedMainAccessToken && + !oauthProfileIsFresh(storage.main?.profile, now) + ) { const profile = await hydrateProfileOnce( 'main', undefined, - mainAccessToken, + servedMainAccessToken, mainProviderAccountUuid, signal, ) @@ -1760,7 +1770,7 @@ const anthropicAuthPlugin = async ( accountId: 'main', accountIdentity: mainIdentity, providerAccountUuid: mainProviderAccountUuid, - accessToken: mainAccessToken, + accessToken: servedMainAccessToken, profile, }).catch(() => {}) } @@ -2080,7 +2090,10 @@ const anthropicAuthPlugin = async ( } let claustrumCredentialCache: ClaustrumCredentialCache | null = null - const claustrumAuthFailureReports = new Map>() + const claustrumAuthFailureReports = new Map< + string, + Promise + >() const claustrumLastReportedVersion = new Map() const claustrumBlockedAccounts = new Set() const claustrumReauthAccounts = new Set() @@ -2502,6 +2515,17 @@ const anthropicAuthPlugin = async ( return {} } + function liveMainVaultAccess( + storage: Awaited>, + ): string | undefined { + return ( + resolveClaustrumAccess( + mainCustodyAccount(custodyTombstoneOAuth('anthropic')), + storage, + ).accessToken || undefined + ) + } + function resolveFallbackAccessToken( account: OAuthAccount, storage: Awaited>, @@ -2522,7 +2546,12 @@ const anthropicAuthPlugin = async ( const cache = claustrumCredentialCache if (!cache) return try { - const credential = await cache.get(handle) + const credential = await getLoggedClaustrumCredential( + cache, + handle, + DEFAULT_CLAUSTRUM_CREDENTIAL_MIN_TTL_MS, + 'on-demand', + ) if (usableClaustrumAccessToken(credential, claustrumNow())) { await markClaustrumCredentialReady(accountId, handle) } @@ -2645,6 +2674,15 @@ const anthropicAuthPlugin = async ( } } + type ClaustrumAuthFailureReportOutcome = + | 'unavailable' + | 'suppressed-monotonic' + | 'suppressed-freshness-missing' + | 'suppressed-freshness-version' + | 'reported' + | 'reported-by-pending' + | 'report-error' + async function reportCapturedClaustrumAuthFailure( served: { accountId: string @@ -2653,29 +2691,32 @@ const anthropicAuthPlugin = async ( }, reporterSource: ClaustrumReporterSource = 'direct', options?: { preserveServedVersion?: boolean }, - ): Promise { + ): Promise { const cache = claustrumCredentialCache - if (!cache) return + if (!cache) return 'unavailable' if ( served.recordVersion <= (claustrumLastReportedVersion.get(served.handle) ?? -1) ) { - return + return 'suppressed-monotonic' } if (!options?.preserveServedVersion) { const current = cache.peek(served.handle) // Version match makes reports single-shot per served version. Accepted // tradeoff: an unrelated cache eviction also suppresses a genuine // report (worst case one delayed cycle until the next served 401). - if (!current || current.recordVersion !== served.recordVersion) return + if (!current) return 'suppressed-freshness-missing' + if (current.recordVersion !== served.recordVersion) { + return 'suppressed-freshness-version' + } } const key = `${served.handle}\0${served.recordVersion}` const pending = claustrumAuthFailureReports.get(key) if (pending) { await pending - return + return 'reported-by-pending' } - const report = (async () => { + const report = (async (): Promise => { try { await cache.reportAuthFailure( served.handle, @@ -2686,17 +2727,20 @@ const anthropicAuthPlugin = async ( reporterSource, ) claustrumLastReportedVersion.set(served.handle, served.recordVersion) + if (served.accountId === 'main') mainServedAccessToken = undefined + return 'reported' } catch (error) { handleClaustrumCredentialError(served.accountId, error, served.handle) logger.warn('claustrum', 'failed to report credential failure', { accountId: served.accountId, error: error instanceof Error ? error.message : String(error), }) + return 'report-error' } })() claustrumAuthFailureReports.set(key, report) try { - await report + return await report } finally { if (claustrumAuthFailureReports.get(key) === report) { claustrumAuthFailureReports.delete(key) @@ -2736,14 +2780,37 @@ const anthropicAuthPlugin = async ( } } - async function getStartupWarmCredential( + type ClaustrumCredentialGetCallSite = + | 'startup' + | 'periodic' + | 'on-demand' + | '401-retry' + + async function getLoggedClaustrumCredential( + cache: ClaustrumCredentialCache, + handle: string, + minTtlMs: number, + callSite: ClaustrumCredentialGetCallSite, + options?: { bypassCache?: boolean }, + ): Promise { + logger.debug('claustrum', 'credential get issued', { + handle, + minTtlMs, + callSite, + }) + return cache.get(handle, minTtlMs, options) + } + + async function getBoundedClaustrumCredential( cache: ClaustrumCredentialCache, handle: string, minTtlMs: number, - ): Promise { + callSite: ClaustrumCredentialGetCallSite, + options?: { bypassCache?: boolean }, + ): Promise<{ credential?: ClaustrumCredential; timedOut: boolean }> { let resolveTimeout!: () => void - const timeout = new Promise((resolve) => { - resolveTimeout = () => resolve(undefined) + const timeout = new Promise<{ timedOut: true }>((resolve) => { + resolveTimeout = () => resolve({ timedOut: true }) }) const timer = runtimeTimers.setTimeout( resolveTimeout, @@ -2751,7 +2818,18 @@ const anthropicAuthPlugin = async ( ) if (typeof timer === 'object' && timer && 'unref' in timer) timer.unref() try { - return await Promise.race([cache.get(handle, minTtlMs), timeout]) + const result = await Promise.race([ + getLoggedClaustrumCredential( + cache, + handle, + minTtlMs, + callSite, + options, + ).then((credential) => ({ credential, timedOut: false as const })), + timeout, + ]) + if (result.timedOut) cache.abandonPending(handle) + return result } finally { runtimeTimers.clearTimeout(timer) } @@ -2767,6 +2845,77 @@ const anthropicAuthPlugin = async ( return enrollManifestBoundAccounts(storage) } + type Claustrum401RetryOutcome = + | 'advanced' + | 'unchanged' + | 'backoff-active' + | 'timed-out' + | 'refresh-error' + | 'cache-unavailable' + + async function getAdvancedClaustrumCredentialAfter401(served: { + accountId: string + handle: string + recordVersion: number + }): Promise<{ + outcome: Claustrum401RetryOutcome + vaultGetAttempted: boolean + resolution?: ClaustrumAccessResolution + }> { + const cache = claustrumCredentialCache + if (!cache) + return { outcome: 'cache-unavailable', vaultGetAttempted: false } + if (claustrumWarmBackoffActive(served.handle)) { + return { outcome: 'backoff-active', vaultGetAttempted: false } + } + try { + // Bypass only the resident cache: the RPC itself must not rotate a token. + const result = await getBoundedClaustrumCredential( + cache, + served.handle, + 0, + '401-retry', + { bypassCache: true }, + ) + if (result.timedOut) { + claustrumWarmBackoffUntil.set( + served.handle, + claustrumNow() + CLAUSTRUM_TRANSIENT_WARM_BACKOFF_MS, + ) + return { outcome: 'timed-out', vaultGetAttempted: true } + } + const credential = result.credential + const accessToken = usableClaustrumAccessToken(credential, claustrumNow()) + if ( + !credential || + !accessToken || + credential.recordVersion <= served.recordVersion + ) { + return { outcome: 'unchanged', vaultGetAttempted: true } + } + return { + outcome: 'advanced', + vaultGetAttempted: true, + resolution: { + accessToken, + served: { + accountId: served.accountId, + handle: served.handle, + recordVersion: credential.recordVersion, + }, + credentialAccountId: asProviderAccountUuid(credential.accountId), + }, + } + } catch (error) { + handleClaustrumCredentialError(served.accountId, error, served.handle) + logger.warn('claustrum', 'credential retry get failed', { + accountId: served.accountId, + error: error instanceof Error ? error.message : String(error), + }) + return { outcome: 'refresh-error', vaultGetAttempted: true } + } + } + async function refreshVaultBackedOAuthAccounts( initial = false, ): Promise { @@ -2775,7 +2924,7 @@ const anthropicAuthPlugin = async ( const storage = enrollment.storage if (!storage) return cache = claustrumCredentialCache - const minTtlMs = getRefreshBeforeExpiryMs(storage) + 30 * 60_000 + const minTtlMs = getVaultRefreshMinTtlMs(storage) let sidebarChanged = enrollment.enrolledAccountIds.length > 0 const mainAuth = @@ -2793,8 +2942,20 @@ const anthropicAuthPlugin = async ( if (cache) { try { const credential = initial - ? await getStartupWarmCredential(cache, handle, minTtlMs) - : await cache.get(handle, minTtlMs) + ? ( + await getBoundedClaustrumCredential( + cache, + handle, + minTtlMs, + 'startup', + ) + ).credential + : await getLoggedClaustrumCredential( + cache, + handle, + minTtlMs, + 'periodic', + ) if (usableClaustrumAccessToken(credential, claustrumNow())) { await markClaustrumCredentialReady('main', handle) } @@ -2828,8 +2989,20 @@ const anthropicAuthPlugin = async ( if (!cache) continue try { const credential = initial - ? await getStartupWarmCredential(cache, handle, minTtlMs) - : await cache.get(handle, minTtlMs) + ? ( + await getBoundedClaustrumCredential( + cache, + handle, + minTtlMs, + 'startup', + ) + ).credential + : await getLoggedClaustrumCredential( + cache, + handle, + minTtlMs, + 'periodic', + ) if (!usableClaustrumAccessToken(credential, claustrumNow())) { logger.debug('refresh', 'vault fallback credential unusable', { id: account.id, @@ -2903,7 +3076,14 @@ const anthropicAuthPlugin = async ( if (custodyHandle.status !== 'resolved') return const handle = custodyHandle.handle try { - const credential = await cache.get(handle) + const credential = ( + await getBoundedClaustrumCredential( + cache, + handle, + DEFAULT_CLAUSTRUM_CREDENTIAL_MIN_TTL_MS, + 'startup', + ) + ).credential if (usableClaustrumAccessToken(credential, claustrumNow())) { if ( custodyHandle.source === 'legacy' && @@ -3294,6 +3474,32 @@ const anthropicAuthPlugin = async ( cacheKeepDiagnosticsRequests.delete(target.id) } }, + retryHeadersAfter401: async ({ headers, target, bodyText, attempt }) => { + const served = cacheKeepServedClaustrumCredentials.get(attempt.id) + if (!served) return undefined + const retry = await getAdvancedClaustrumCredentialAfter401(served) + const resolution = retry.resolution + if (!resolution?.accessToken || !resolution.served) return undefined + const retryHeaders = new Headers(headers) + try { + const parsedBody = JSON.parse(bodyText) as Record + const identity = await resolveClaudeCodeIdentity( + resolution.accessToken, + typeof parsedBody.model === 'string' ? parsedBody.model : undefined, + target.oauthAccountId === 'main' + ? mainAccountId + : (target.oauthAccountId ?? 'main'), + ) + setOAuthHeaders(retryHeaders, resolution.accessToken, { + body: parsedBody, + identity, + }) + } catch { + setOAuthHeaders(retryHeaders, resolution.accessToken) + } + cacheKeepServedClaustrumCredentials.set(attempt.id, resolution.served) + return retryHeaders + }, onComplete: ({ attempt }) => { cacheKeepServedClaustrumCredentials.delete(attempt.id) }, @@ -3607,7 +3813,7 @@ const anthropicAuthPlugin = async ( const body = await rewriteRequestBody(JSON.stringify(primeBody), { identity, }) - const headers = new Headers({ + let headers = new Headers({ 'content-type': 'application/json', }) setOAuthHeaders(headers, accessToken, { @@ -3624,12 +3830,40 @@ const anthropicAuthPlugin = async ( const primeRequest = rewriteUrl(PRIME_MESSAGES_URL, { baseURL: '' }) const primeUrl = primeRequest.url?.toString() ?? primeRequest.input.toString() - const response = await fetch(primeUrl, { + let response = await fetch(primeUrl, { method: 'POST', headers, body, signal: AbortSignal.timeout(30_000), }) + if (response.status === 401 && servedClaustrumCredential) { + const retry = await getAdvancedClaustrumCredentialAfter401( + servedClaustrumCredential, + ) + const retryResolution = retry.resolution + if (retryResolution?.accessToken && retryResolution.served) { + await response.body?.cancel().catch(() => {}) + const retryIdentity = await resolveClaudeCodeIdentity( + retryResolution.accessToken, + resolvedModel, + accountId === 'main' ? mainAccountId : accountId, + ) + headers = new Headers({ 'content-type': 'application/json' }) + setOAuthHeaders(headers, retryResolution.accessToken, { + body: JSON.parse(body), + identity: retryIdentity, + }) + headers.delete('content-length') + headers.delete('transfer-encoding') + response = await fetch(primeUrl, { + method: 'POST', + headers, + body, + signal: AbortSignal.timeout(30_000), + }) + servedClaustrumCredential = retryResolution.served + } + } const ms = Math.round(performance.now() - start) if (!response.ok) { const reason = @@ -3932,11 +4166,18 @@ const anthropicAuthPlugin = async ( const hydratedAccount = hydrated.accounts.find( (candidate) => candidate.id === account.id, ) + const vaultServed = isFallbackAccountVaultServed( + account.id, + latest, + custodyDimensionsDeps, + ) + // A profile belongs to matching local access or a live vault-served + // binding; tombstones deliberately have no local access to compare. if ( !isOAuthAccount(account) || !hydratedAccount || !isOAuthAccount(hydratedAccount) || - !account.access || + (!account.access && !vaultServed) || hydratedAccount.access !== account.access ) { return account @@ -3946,8 +4187,12 @@ const anthropicAuthPlugin = async ( } const latestMainProfile = latest.main?.profile const mainState = latest.main ?? hydrated.main + const servedMainAccessToken = + mainServedAccessToken || mainAccessToken || liveMainVaultAccess(latest) + // Hydrated main state is valid with local main access or the live bearer + // serving its custody tombstone; neither admits a credential-less main. if ( - mainAccessToken && + servedMainAccessToken && mainState && (!latestMainProfile || oauthProfileMatchesIdentity(latestMainProfile, mainAccountId)) @@ -4295,8 +4540,13 @@ const anthropicAuthPlugin = async ( if (latestGetAuth) { try { const auth = await latestGetAuth() - if (auth.type === 'oauth' && auth.access) { - mainAccessToken = mainServedAccessToken ?? auth.access + const latest = await loadAccounts(accountStoragePath) + const servedMainAccessToken = + mainServedAccessToken || auth.access || liveMainVaultAccess(latest) + // Manual quota refresh accepts local OAuth access or the live bearer + // serving a custody tombstone; an empty local slot alone remains refused. + if (auth.type === 'oauth' && servedMainAccessToken) { + mainAccessToken = servedMainAccessToken await resolveMainQuotaAccountIdentity(mainAccessToken) // /claude-quota is a manual action: force a real fetch instead of // returning the cache. refreshMain still respects 429 backoff — it @@ -6798,7 +7048,7 @@ const anthropicAuthPlugin = async ( }, reporterSource: ClaustrumReporterSource = 'direct', options?: { preserveServedVersion?: boolean }, - ): Promise { + ): Promise { return reportCapturedClaustrumAuthFailure( served, reporterSource, @@ -6820,7 +7070,7 @@ const anthropicAuthPlugin = async ( } } - async function sendWithAccessToken( + async function sendWithAccessTokenOnce( input: string | URL | Request, init: RequestInit | undefined, accessToken: string, @@ -7065,6 +7315,8 @@ const anthropicAuthPlugin = async ( fastModeEnabled: fastModeRequested, subagent: subagentRequest, }) + } else { + setOAuthHeaders(requestHeaders, accessToken, { identity }) } const cacheDiagnosticsBetas = @@ -7279,13 +7531,143 @@ const anthropicAuthPlugin = async ( response, servedClaustrumCredential, ) - if (response.status === 401) { - await reportClaustrumAuthFailure( - servedClaustrumCredential, - 'direct', - ) + } + return response + } + + async function sendWithAccessToken( + input: string | URL | Request, + init: RequestInit | undefined, + accessToken: string, + trace?: PerfTrace, + route = 'unknown', + currentStorage?: Awaited>, + oauthAccountId = 'main', + fallbackAuthLineageId?: string, + fableRequest?: FableRequestContext, + laneStartRequest = false, + mainQuotaIdentity?: MainQuotaIdentityResolution, + claustrumResolution?: ClaustrumAccessResolution, + scopedAttempt?: ClaustrumScopedAttempt, + ) { + const response = await sendWithAccessTokenOnce( + input, + init, + accessToken, + trace, + route, + currentStorage, + oauthAccountId, + fallbackAuthLineageId, + fableRequest, + laneStartRequest, + mainQuotaIdentity, + claustrumResolution, + scopedAttempt, + ) + const served = claustrumServedCredentials.get(response) + if (response.status !== 401 || !served) return response + + if (!isReplayableRequest(input, init?.body)) { + await reportClaustrumAuthFailure(served, 'direct') + return response + } + + // The vault get and snapshot below must stay adjacent: reporting can invalidate the cache. + const retry = await getAdvancedClaustrumCredentialAfter401(served) + const currentCachedRecordVersion = claustrumCredentialCache?.peek( + served.handle, + )?.recordVersion + const log401 = (input: { + currentCachedRecordVersion?: number + vaultGetAttempted: boolean + retryAttempted: boolean + retryOutcome: Claustrum401RetryOutcome | 'retry-succeeded' + reportOutcome: ClaustrumAuthFailureReportOutcome | 'not-attempted' + retryServedRecordVersion?: number + }) => { + const reportSuppressed = + input.reportOutcome.startsWith('suppressed') + logger.info('claustrum', 'vault-served 401 recovery', { + handle: served.handle, + servedRecordVersion: served.recordVersion, + currentCachedRecordVersion: input.currentCachedRecordVersion, + retryAttempted: input.retryAttempted, + vaultGetAttempted: input.vaultGetAttempted, + retryOutcome: input.retryOutcome, + ...(input.retryServedRecordVersion !== undefined && { + retryServedRecordVersion: input.retryServedRecordVersion, + }), + reportOutcome: input.reportOutcome, + reportSuppressed, + ...(reportSuppressed && { + reportSuppressedBy: input.reportOutcome, + }), + }) + } + + const retryResolution = retry.resolution + const retryAccessToken = retryResolution?.accessToken + const retryResolutionServed = retryResolution?.served + if (retryAccessToken && retryResolutionServed) { + await response.body?.cancel().catch(() => {}) + const retryResponse = await sendWithAccessTokenOnce( + input, + init, + retryAccessToken, + trace, + route, + currentStorage, + oauthAccountId, + fallbackAuthLineageId, + fableRequest, + laneStartRequest, + mainQuotaIdentity, + retryResolution, + ) + const retryServed = + claustrumServedCredentials.get(retryResponse) ?? + retryResolutionServed + if (retryResponse.status !== 401) { + if (retryServed.accountId === 'main') { + mainServedAccessToken = retryAccessToken + } + log401({ + currentCachedRecordVersion, + vaultGetAttempted: retry.vaultGetAttempted, + retryAttempted: true, + retryOutcome: 'retry-succeeded', + reportOutcome: 'not-attempted', + retryServedRecordVersion: retryServed.recordVersion, + }) + return retryResponse } + const reportOutcome = await reportClaustrumAuthFailure( + retryServed, + 'direct', + ) + log401({ + currentCachedRecordVersion, + vaultGetAttempted: retry.vaultGetAttempted, + retryAttempted: true, + retryOutcome: retry.outcome, + reportOutcome, + retryServedRecordVersion: retryServed.recordVersion, + }) + return retryResponse } + + const reportOutcome = await reportClaustrumAuthFailure( + served, + 'direct', + ) + log401({ + currentCachedRecordVersion, + vaultGetAttempted: retry.vaultGetAttempted, + retryAttempted: false, + retryOutcome: retry.outcome, + reportOutcome, + }) return response } @@ -9284,7 +9666,9 @@ const anthropicAuthPlugin = async ( }, ) } - // Killswitch — eagerly refresh quota so it can evaluate + // Killswitch — eagerly refresh quota for local credentials and + // live vault-served bindings so spend protection never evaluates + // a vault fallback on stale quota. if (isKillswitchEnabled(storage)) { const needsRefresh = quotaManager.needsRefresh( sessionRequestCount, @@ -9296,7 +9680,12 @@ const anthropicAuthPlugin = async ( (a): a is OAuthAccount => a.enabled !== false && isOAuthAccount(a) && - Boolean(a.access), + (Boolean(a.access) || + isFallbackAccountVaultServed( + a.id, + storage, + custodyDimensionsDeps, + )), ) await Promise.all([ quotaManager.refreshMain( diff --git a/packages/opencode/src/tests/accounts.test.ts b/packages/opencode/src/tests/accounts.test.ts index 4bb29fdd..5d123af0 100644 --- a/packages/opencode/src/tests/accounts.test.ts +++ b/packages/opencode/src/tests/accounts.test.ts @@ -5003,6 +5003,106 @@ describe('FallbackAccountManager', () => { expect(accounts.map((account) => account.id)).toEqual(['stale-good-quota']) }) + test('does not use cached quota after vault access disappears during refresh', async () => { + const now = 10 * 60_000 + const storage = baseStorage() + storage.accounts.push({ + id: 'vault-access-race', + type: 'oauth', + access: '', + refresh: '', + expires: 0, + claustrumHandle: 'vault-access-race-handle', + quota: { + checkedAt: 1_000, + five_hour: { + usedPercent: 10, + remainingPercent: 90, + checkedAt: 1_000, + resetsAt: '2099-01-01T00:00:00Z', + }, + seven_day: { + usedPercent: 20, + remainingPercent: 80, + checkedAt: 1_000, + resetsAt: '2099-01-01T00:00:00Z', + }, + }, + }) + + let vaultServed = true + const fetchImpl = mock(async () => { + vaultServed = false + return new Response('temporarily unavailable', { status: 503 }) + }) as unknown as typeof fetch + const manager = new FallbackAccountManager({ + fetchImpl, + now: () => now, + isFallbackAccountVaultEnabled: () => true, + isFallbackAccountVaultServed: () => vaultServed, + resolveFallbackAccessToken: () => ({ + token: 'vault-access', + source: 'vault' as const, + }), + }) + + const accounts = await manager.getUsableFallbackAccounts(storage) + + expect(fetchImpl).toHaveBeenCalledTimes(1) + expect(accounts).toEqual([]) + }) + + test('uses cached quota when vault access remains served during refresh', async () => { + const now = 10 * 60_000 + const storage = baseStorage() + storage.accounts.push({ + id: 'vault-access-stays-served', + type: 'oauth', + access: '', + refresh: '', + expires: 0, + claustrumHandle: 'vault-access-stays-served-handle', + quota: { + checkedAt: 1_000, + five_hour: { + usedPercent: 10, + remainingPercent: 90, + checkedAt: 1_000, + resetsAt: '2099-01-01T00:00:00Z', + }, + seven_day: { + usedPercent: 20, + remainingPercent: 80, + checkedAt: 1_000, + resetsAt: '2099-01-01T00:00:00Z', + }, + }, + }) + + let vaultServed = true + const fetchImpl = mock(() => + Promise.resolve(new Response('temporarily unavailable', { status: 503 })), + ) as unknown as typeof fetch + const manager = new FallbackAccountManager({ + fetchImpl, + now: () => now, + isFallbackAccountVaultEnabled: () => true, + isFallbackAccountVaultServed: () => vaultServed, + resolveFallbackAccessToken: () => ({ + token: 'vault-access', + source: 'vault' as const, + }), + }) + + const accounts = await manager.getUsableFallbackAccounts(storage) + + expect(fetchImpl).toHaveBeenCalledTimes(1) + expect(vaultServed).toBe(true) + expect(accounts.map((account) => account.id)).toEqual([ + 'vault-access-stays-served', + ]) + }) + test('keeps a concurrent replacement account when its quota probe fails', async () => { const oldStorage = baseStorage() const oldAccount: OAuthAccount = { diff --git a/packages/opencode/src/tests/claustrum-client.test.ts b/packages/opencode/src/tests/claustrum-client.test.ts index c5ddc0ff..0acbcfe5 100644 --- a/packages/opencode/src/tests/claustrum-client.test.ts +++ b/packages/opencode/src/tests/claustrum-client.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, test } from 'bun:test' +import { afterEach, describe, expect, mock, test } from 'bun:test' import { randomUUID } from 'node:crypto' import { chmod, mkdtemp, rm, stat, writeFile } from 'node:fs/promises' import { createServer, type Socket } from 'node:net' @@ -604,6 +604,31 @@ describe('ClaustrumCredentialCache', () => { expect(daemon.requestBodies).toHaveLength(2) }) + test('bypasses a resident credential without discarding it when the fresh get fails', async () => { + const daemon = await startFakeDaemon() + daemon.responseBodies.push({ + result: { + payload: Array.from(new TextEncoder().encode('served')), + expires_at_ms: 1_500, + record_version: 63, + }, + }) + const cache = await makeCredentialCache(daemon) + await cache.get(handle) + daemon.responseBodies.push({ + result: { error: { code: 'refresh_failed', class: 'transient' } }, + }) + + await expect( + cache.get(handle, 0, { bypassCache: true }), + ).rejects.toMatchObject({ + action: 'retry', + }) + + expect(cache.peek(handle)).toMatchObject({ recordVersion: 63 }) + expect(daemon.requestBodies).toHaveLength(2) + }) + test('uses the requested minimum TTL to refresh near expiry without refreshing above it', async () => { const daemon = await startFakeDaemon() daemon.responseBodies.push({ @@ -831,6 +856,72 @@ describe('ClaustrumCredentialCache', () => { expect(daemon.requestBodies).toHaveLength(2) }) + test('logs a latched background refresh failure once until a successful refresh re-arms it', async () => { + let now = 0 + const responses = [ + { result: { error: { code: 'not_found', class: 'permanent' } } }, + { result: { error: { code: 'not_found', class: 'permanent' } } }, + { + result: { + payload: Array.from(new TextEncoder().encode('credential-v2')), + expires_at_ms: 500_000, + record_version: 74, + }, + }, + { result: { error: { code: 'not_found', class: 'permanent' } } }, + ] + const cache = new ClaustrumCredentialCache( + { + call: mock(async () => responses.shift()), + close: () => {}, + } as never, + { now: () => now }, + ) + cache.seedForTest(handle, { + payload: 'credential-v1', + expiresAtMs: 300_000, + recordVersion: 73, + }) + const logs: LogTestRecord[] = [] + __setLogTestSink((record) => logs.push(record)) + const flushBackgroundRefresh = async () => { + await Promise.resolve() + await Promise.resolve() + await Promise.resolve() + } + + now = 181_000 + await cache.get(handle) + await flushBackgroundRefresh() + now = 242_000 + await cache.get(handle) + await flushBackgroundRefresh() + expect( + logs.filter( + (record) => record.message === 'credential background refresh latched', + ), + ).toHaveLength(1) + + now = 303_000 + await cache.get(handle) + await flushBackgroundRefresh() + now = 381_000 + await cache.get(handle) + await flushBackgroundRefresh() + const latched = logs.filter( + (record) => record.message === 'credential background refresh latched', + ) + expect(latched).toHaveLength(2) + expect(latched).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + channel: 'claustrum', + payload: expect.objectContaining({ recordVersion: 73 }), + }), + ]), + ) + }) + test('does not retain a credential whose expiry is absent', async () => { const daemon = await startFakeDaemon() daemon.responseBodies.push( diff --git a/packages/opencode/src/tests/fixtures/claustrum-golden/SOURCE.json b/packages/opencode/src/tests/fixtures/claustrum-golden/SOURCE.json index dcb75544..aaf76615 100644 --- a/packages/opencode/src/tests/fixtures/claustrum-golden/SOURCE.json +++ b/packages/opencode/src/tests/fixtures/claustrum-golden/SOURCE.json @@ -1,6 +1,6 @@ { - "repo": "legion-works/claustrum", - "ref": "0e9dee77cb91e762d31a9ccb502728a69f09bcbe", + "repo": "cortexkit/claustrum", + "ref": "6817148f92dae80a0171973ca122d17b71d0d801", "paths": { "tombstone": "packages/opencode/golden/tombstone.json", "handles": "packages/opencode/golden/handles.json" diff --git a/packages/opencode/src/tests/index.test.ts b/packages/opencode/src/tests/index.test.ts index 54de6cda..960fc43d 100644 --- a/packages/opencode/src/tests/index.test.ts +++ b/packages/opencode/src/tests/index.test.ts @@ -31,6 +31,7 @@ import { buildPrimeRequestBody, buildRefreshOperationError, ClaudeOAuthRefreshError, + ClaustrumCredentialError, CustodyTombstoneRefreshError, clearClaustrumRefreshErrorPersistent, custodyCredentialId, @@ -1178,6 +1179,63 @@ describe('fallback Claustrum credential resolution', () => { serve?: string, ) => writeSharedManifest(tempConfigDir!, entries, serve) + async function bootMainVault401( + options: { + responseStatuses?: number[] + credentialGet?: (params: Record) => unknown + reportAuthFailure?: () => unknown + timerHook?: (callback: TestTimerHandler, delay?: number) => void + onMessageRequest?: ( + count: number, + init?: RequestInit, + ) => void | Promise + } = {}, + ) { + const calls: CredentialCall[] = [] + const authorizations: string[] = [] + const responseStatuses = [...(options.responseStatuses ?? [401])] + await useTempAccountFile( + createFallbackStorage({ + claustrum: { mode: 'claustrum' }, + quota: { enabled: false }, + accounts: [], + }), + ) + await writeManifest([{ label: 'main', handle: manifestHandle }]) + globalThis.fetch = mock(async (_input: unknown, init?: RequestInit) => { + const url = String( + _input instanceof Request ? _input.url : (_input as string | URL), + ) + if (!url.startsWith(MESSAGES_URL)) { + return Promise.resolve(new Response('{}', { status: 200 })) + } + authorizations.push(new Headers(init?.headers).get('authorization') ?? '') + await options.onMessageRequest?.(authorizations.length, init) + return new Response('{}', { status: responseStatuses.shift() ?? 401 }) + }) as unknown as typeof fetch + const plugin = await getPlugin(undefined, undefined, { + setTimeout: mock((callback: TestTimerHandler, delay?: number) => { + options.timerHook?.(callback, delay) + return { unref() {} } as unknown as ReturnType + }) as unknown as typeof setTimeout, + claustrumConnector: connectorFor(calls, (method, params) => { + if (method === 'credential.report_auth_failure') { + return options.reportAuthFailure?.() ?? { result: {} } + } + if (method !== 'credential.get') return { result: {} } + return ( + options.credentialGet?.(params) ?? + credentialResponse('vault-main-access-v17', 17) + ) + }), + }) + const result = await plugin.auth.loader( + () => Promise.resolve(custodyTombstoneOAuth('anthropic') as never), + { models: {} }, + ) + return { plugin, result, calls, authorizations } + } + const bootRuledClaustrumRow = ( options: Parameters[0], ) => @@ -1217,11 +1275,18 @@ describe('fallback Claustrum credential resolution', () => { responseStatuses?: number[] mainExpiresAt?: number fallbackExpiresAt?: number + profile?: Record + credentialGet?: (params: Record) => unknown + timerHook?: (callback: TestTimerHandler, delay?: number) => void } = {}, ) { let now = 1_000 const calls: CredentialCall[] = [] const authorizations: string[] = [] + const profileAuthorizations: string[] = [] + const quotaAuthorizations: string[] = [] + const scheduledWarmCallbacks: Array<() => void> = [] + const client = createMockClient() let mainSlotAccess = '' let mainSlotExpires = 0 let mainExpiresAt = options.mainExpiresAt ?? 10_000 @@ -1280,9 +1345,25 @@ describe('fallback Claustrum credential resolution', () => { ...(fallback ? [{ label: 'fallback', handle: fallbackHandle }] : []), ]) globalThis.fetch = mock((input: unknown, init?: RequestInit) => { - if ( - extractUrl(input as string | URL | Request).includes('/v1/messages') - ) { + const url = extractUrl(input as string | URL | Request) + if (url === PROFILE_URL) { + profileAuthorizations.push( + new Headers(init?.headers).get('authorization') ?? '', + ) + return Promise.resolve(Response.json(options.profile ?? {})) + } + if (url === QUOTA_URL) { + quotaAuthorizations.push( + new Headers(init?.headers).get('authorization') ?? '', + ) + return Promise.resolve( + Response.json({ + five_hour: { utilization: 10 }, + seven_day: { utilization: 10 }, + }), + ) + } + if (url.includes('/v1/messages')) { authorizations.push( new Headers(init?.headers).get('authorization') ?? '', ) @@ -1294,10 +1375,18 @@ describe('fallback Claustrum credential resolution', () => { } return Promise.resolve(new Response('{}', { status: 200 })) }) as unknown as typeof fetch - const plugin = await getPlugin(undefined, undefined, { + const plugin = await getPlugin(client, undefined, { claustrumNow: () => now, + setTimeout: mock((callback: TestTimerHandler, delay?: number) => { + options.timerHook?.(callback, delay) + if (delay === 0 && typeof callback === 'function') { + scheduledWarmCallbacks.push(callback as () => void) + } + return { unref() {} } as unknown as ReturnType + }) as unknown as typeof setTimeout, claustrumConnector: connectorFor(calls, (method, params) => { if (method !== 'credential.get') return { result: {} } + if (options.credentialGet) return options.credentialGet(params) const isMain = params.handle === manifestHandle return credentialResponse( isMain ? 'vault-main-access' : 'vault-fallback-access', @@ -1320,9 +1409,13 @@ describe('fallback Claustrum credential resolution', () => { ) return { plugin, + client, result, calls, authorizations, + profileAuthorizations, + quotaAuthorizations, + scheduledWarmCallbacks, setNow(value: number) { now = value }, @@ -1338,6 +1431,140 @@ describe('fallback Claustrum credential resolution', () => { } } + test.serial( + 'hydrates and merges the main profile through a vault-served tombstone', + async () => { + const fixture = await bootVaultMain({ + fallback: false, + profile: { + organization: { + organization_type: 'claude_max', + rate_limit_tier: 'default_claude_max_20x', + }, + }, + }) + + const state = await waitForSidebarState( + (candidate) => candidate.main.tierLabel === 'Max 20x', + ) + expect(fixture.profileAuthorizations).toEqual([ + 'Bearer vault-main-access', + ]) + expect(state.main.tierLabel).toBe('Max 20x') + await fixture.plugin.dispose?.() + }, + ) + + test.serial( + 'refreshes /claude-quota for a vault-served main tombstone', + async () => { + const fixture = await bootVaultMain({ fallback: false }) + + await fixture.result.fetch(MESSAGES_URL, request()) + + await expectHandledCommandResponse( + fixture.plugin['command.execute.before']({ + command: 'claude-quota', + arguments: '', + sessionID: 'vault-main-quota', + }), + ) + + expect(fixture.quotaAuthorizations).toContain( + 'Bearer vault-main-access', + ) + await fixture.plugin.dispose?.() + }, + ) + + test.serial( + 'refreshes /claude-quota at startup for a vault-served main tombstone', + async () => { + const fixture = await bootVaultMain({ fallback: false }) + + await expectHandledCommandResponse( + fixture.plugin['command.execute.before']({ + command: 'claude-quota', + arguments: '', + sessionID: 'vault-main-quota-startup', + }), + ) + + expect(fixture.quotaAuthorizations).toContain( + 'Bearer vault-main-access', + ) + await fixture.plugin.dispose?.() + }, + ) + + test.serial( + 'does not reuse a main vault bearer after its 401 is reported', + async () => { + const fixture = await bootVaultMain({ + fallback: false, + responseStatus: 401, + }) + + const warmCallbacksBeforeFetch = fixture.scheduledWarmCallbacks.length + await fixture.result.fetch(MESSAGES_URL, request()) + await expectHandledCommandResponse( + fixture.plugin['command.execute.before']({ + command: 'claude-quota', + arguments: '', + sessionID: 'vault-main-stale-bearer', + }), + ) + + expect(fixture.quotaAuthorizations).not.toContain( + 'Bearer vault-main-access', + ) + expect(fixture.scheduledWarmCallbacks.length).toBeGreaterThan( + warmCallbacksBeforeFetch, + ) + await fixture.plugin.dispose?.() + }, + ) + + test.serial( + 'uses the rotated main vault bearer for quota after a successful 401 retry', + async () => { + let recordVersion = 17 + const fixture = await bootVaultMain({ + fallback: false, + responseStatuses: [401, 200], + credentialGet: () => + credentialResponse( + `vault-main-access-v${recordVersion}`, + recordVersion, + ), + }) + recordVersion = 18 + + const response = await fixture.result.fetch(MESSAGES_URL, request()) + expect(response.status).toBe(200) + expect(fixture.authorizations).toEqual([ + 'Bearer vault-main-access-v17', + 'Bearer vault-main-access-v18', + ]) + + await expectHandledCommandResponse( + fixture.plugin['command.execute.before']({ + command: 'claude-quota', + arguments: '', + sessionID: 'vault-main-retried-bearer', + }), + ) + + expect(fixture.quotaAuthorizations).toContain( + 'Bearer vault-main-access-v18', + ) + expect(fixture.quotaAuthorizations).not.toContain( + 'Bearer vault-main-access-v17', + ) + await fixture.plugin.dispose?.() + }, + ) + test.serial( 'a refused vault main clears a legacy tombstone bearer before fallback routing', async () => { @@ -2287,7 +2514,7 @@ describe('fallback Claustrum credential resolution', () => { ) test.serial( - 'a main 401 after a concurrent vault refresh is suppressed rather than blamed on the new record', + 'retries a main 401 with an advanced vault record without reporting the stale version', async () => { await useTempAccountFile( createFallbackStorage({ @@ -2311,7 +2538,16 @@ describe('fallback Claustrum credential resolution', () => { }) let messageRequests = 0 let claustrumNow = 0 - globalThis.fetch = mock((_input: unknown, init?: RequestInit) => { + globalThis.fetch = mock((input: unknown, init?: RequestInit) => { + // Scope to the message path: profile hydration also runs on the served + // main token now that a custody tombstone no longer blocks it, and it + // must not consume the deferred first response or the token ledger. + const url = String( + input instanceof Request ? input.url : (input as string | URL), + ) + if (!url.startsWith(MESSAGES_URL)) { + return Promise.resolve(new Response('denied', { status: 401 })) + } authorizations.push( new Headers(init?.headers).get('authorization') ?? '', ) @@ -2320,7 +2556,7 @@ describe('fallback Claustrum credential resolution', () => { firstRequestStarted() return firstResponse } - return Promise.resolve(new Response('denied', { status: 401 })) + return Promise.resolve(new Response('recovered', { status: 200 })) }) as unknown as typeof fetch const ticks: Array<() => unknown> = [] const plugin = await getPlugin(undefined, undefined, { @@ -2355,21 +2591,36 @@ describe('fallback Claustrum credential resolution', () => { ) releaseFirstResponse() - expect((await staleResponse).status).toBe(401) + expect((await staleResponse).status).toBe(200) expect( calls.filter( (call) => call.method === 'credential.report_auth_failure', ), ).toEqual([]) - - const currentResponse = await result.fetch(MESSAGES_URL, EMPTY_POST) - expect(currentResponse.status).toBe(401) expect(authorizations).toEqual([ 'Bearer main-vault-access-v7', 'Bearer main-vault-access-v8', ]) + await plugin.dispose?.() + }, + ) + + test.serial( + 'reports a genuinely rejected main vault record once after a fresh get keeps its version', + async () => { + const fixture = await bootMainVault401() + const getsBefore = fixture.calls.filter( + (call) => call.method === 'credential.get', + ).length + + const response = await fixture.result.fetch(MESSAGES_URL, EMPTY_POST) + + expect(response.status).toBe(401) expect( - calls.filter( + fixture.calls.filter((call) => call.method === 'credential.get'), + ).toHaveLength(getsBefore + 1) + expect( + fixture.calls.filter( (call) => call.method === 'credential.report_auth_failure', ), ).toEqual([ @@ -2377,12 +2628,234 @@ describe('fallback Claustrum credential resolution', () => { params: expect.objectContaining({ handle: manifestHandle, provider_status: 401, - record_version: 8, + record_version: 17, reporter_source: 'direct', }), }), ]) - await plugin.dispose?.() + await fixture.plugin.dispose?.() + }, + ) + + test.serial( + 'retries a 401 with the post-rotation version when a proactive refresh returns stale', + async () => { + let credentialGets = 0 + let releaseProactiveRefresh!: () => void + const proactiveRefresh = new Promise((resolve) => { + releaseProactiveRefresh = resolve + }) + let proactiveRefreshStarted!: () => void + const proactiveRefreshStartedPromise = new Promise((resolve) => { + proactiveRefreshStarted = resolve + }) + let releaseFirstResponse!: () => void + const firstResponse = new Promise((resolve) => { + releaseFirstResponse = resolve + }) + let firstRequestStarted!: () => void + const firstRequestStartedPromise = new Promise((resolve) => { + firstRequestStarted = resolve + }) + const fixture = await bootMainVault401({ + responseStatuses: [401, 200], + credentialGet: async () => { + credentialGets += 1 + if (credentialGets === 1) + return credentialResponse('vault-main-access-v17', 17) + if (credentialGets === 2) { + proactiveRefreshStarted() + await proactiveRefresh + return credentialResponse('vault-main-access-v17', 17) + } + return credentialResponse('vault-main-access-v18', 18) + }, + onMessageRequest: async (count) => { + if (count !== 1) return + firstRequestStarted() + await firstResponse + }, + }) + const cache = fixture.plugin.__claustrumCredentialCache + + await cache.get(manifestHandle) + await proactiveRefreshStartedPromise + const responsePromise = fixture.result.fetch(MESSAGES_URL, EMPTY_POST) + await firstRequestStartedPromise + releaseFirstResponse() + await Promise.resolve() + releaseProactiveRefresh() + const response = await responsePromise + + expect(response.status).toBe(200) + expect(fixture.authorizations).toEqual([ + 'Bearer vault-main-access-v17', + 'Bearer vault-main-access-v18', + ]) + expect( + fixture.calls.some( + (call) => call.method === 'credential.report_auth_failure', + ), + ).toBe(false) + await fixture.plugin.dispose?.() + }, + ) + + test.serial( + 'returns the original 401 instead of replaying a streamed vault request body', + async () => { + let recordVersion = 17 + const fixture = await bootMainVault401({ + responseStatuses: [401, 200], + credentialGet: () => + credentialResponse( + `vault-main-access-v${recordVersion}`, + recordVersion, + ), + }) + recordVersion = 18 + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('one-shot body')) + controller.close() + }, + }) + + const response = await fixture.result.fetch(MESSAGES_URL, { + method: 'POST', + body: stream, + duplex: 'half', + }) + + expect(response.status).toBe(401) + expect(fixture.authorizations).toEqual(['Bearer vault-main-access-v17']) + expect( + fixture.calls.filter( + (call) => call.method === 'credential.report_auth_failure', + ), + ).toEqual([ + expect.objectContaining({ + params: expect.objectContaining({ + handle: manifestHandle, + record_version: 17, + }), + }), + ]) + await fixture.plugin.dispose?.() + }, + ) + + test.serial( + 'reports only the retried vault record when the retry also receives a 401', + async () => { + let recordVersion = 17 + const fixture = await bootMainVault401({ + responseStatuses: [401, 401], + credentialGet: () => + credentialResponse( + `vault-main-access-v${recordVersion}`, + recordVersion, + ), + }) + recordVersion = 18 + + const response = await fixture.result.fetch(MESSAGES_URL, EMPTY_POST) + + expect(response.status).toBe(401) + expect(fixture.authorizations).toEqual([ + 'Bearer vault-main-access-v17', + 'Bearer vault-main-access-v18', + ]) + expect( + fixture.calls.filter( + (call) => call.method === 'credential.report_auth_failure', + ), + ).toEqual([ + expect.objectContaining({ + params: expect.objectContaining({ record_version: 18 }), + }), + ]) + await fixture.plugin.dispose?.() + }, + ) + + test.serial( + 'reports and returns a 401 when the retry vault get times out', + async () => { + let credentialGets = 0 + const fixture = await bootMainVault401({ + credentialGet: () => { + credentialGets += 1 + if (credentialGets === 1) + return credentialResponse('vault-main-access-v17', 17) + return new Promise(() => {}) + }, + timerHook: (callback, delay) => { + if (delay && typeof callback === 'function') queueMicrotask(callback) + }, + }) + const getsBefore = credentialGets + + const response = await fixture.result.fetch(MESSAGES_URL, EMPTY_POST) + + expect(response.status).toBe(401) + expect(credentialGets).toBe(getsBefore + 1) + expect( + fixture.calls.filter( + (call) => call.method === 'credential.report_auth_failure', + ), + ).toHaveLength(1) + await fixture.plugin.dispose?.() + }, + ) + + test.serial( + 'honors the per-handle backoff after a retry get failure', + async () => { + let credentialGets = 0 + let releaseReport!: () => void + const report = new Promise((resolve) => { + releaseReport = () => resolve({ result: {} }) + }) + let reportStarted!: () => void + const reportStartedPromise = new Promise((resolve) => { + reportStarted = resolve + }) + let secondRequestStarted!: () => void + const secondRequestStartedPromise = new Promise((resolve) => { + secondRequestStarted = resolve + }) + const fixture = await bootMainVault401({ + credentialGet: () => { + credentialGets += 1 + if (credentialGets === 1) + return credentialResponse('vault-main-access-v17', 17) + return { + result: { + error: { code: 'refresh_failed', class: 'transient' }, + }, + } + }, + reportAuthFailure: () => { + reportStarted() + return report + }, + onMessageRequest: (count) => { + if (count === 2) secondRequestStarted() + }, + }) + const getsBefore = credentialGets + + const first = fixture.result.fetch(MESSAGES_URL, EMPTY_POST) + await reportStartedPromise + const second = fixture.result.fetch(MESSAGES_URL, EMPTY_POST) + await secondRequestStartedPromise + await Promise.resolve() + await Promise.resolve() + releaseReport() + await Promise.all([first, second]) + expect(credentialGets).toBe(getsBefore + 1) + await fixture.plugin.dispose?.() }, ) @@ -4581,6 +5054,99 @@ describe('fallback Claustrum credential resolution', () => { await plugin.dispose?.() }) + test('CacheKeep retries a vault 401 only after a bypassed get advances the served version', async () => { + const calls: CredentialCall[] = [] + const fallbackHandle = `ckh_${'B'.repeat(43)}` + const storage = fallbackWithClaustrum({ + label: 'fallback-1', + ...custodyTombstoneOAuth('anthropic'), + } as never) + storage.claudeCache = { enabled: true, mode: 'hybrid' } + storage.cacheKeep = { enabled: true, always: true, subagents: true } + storage.quota = { enabled: false, failClosedOnUnknownQuota: false } + await useTempAccountFile(storage) + await writeManifest([ + { label: 'main', handle: manifestHandle }, + { label: 'fallback-1', handle: fallbackHandle }, + ]) + let rotated = false + const connector = connectorFor(calls, (method, params) => { + if (method !== 'credential.get') return { result: {} } + const fallback = params.handle === fallbackHandle + return credentialResponse( + fallback + ? rotated + ? 'vault-cachekeep-retry-v48' + : 'vault-cachekeep-retry-v47' + : 'vault-cachekeep-main', + fallback ? (rotated ? 48 : 47) : 49, + ) + }) + const authorizations: string[] = [] + globalThis.fetch = mock((input: unknown, init?: RequestInit) => { + const url = extractUrl(input as string | URL | Request) + if (url.includes('/claude_cli/bootstrap')) { + return Promise.resolve( + Response.json({ oauth_account: { account_uuid: 'fallback-1' } }), + ) + } + if (url.includes('/v1/messages')) { + authorizations.push( + new Headers(init?.headers).get('authorization') ?? '', + ) + if (!rotated) { + rotated = true + return Promise.resolve(new Response('unauthorized', { status: 401 })) + } + return Promise.resolve(Response.json({ usage: { input_tokens: 1 } })) + } + return Promise.resolve(new Response('{}', { status: 200 })) + }) as unknown as typeof fetch + + const plugin = await getPlugin(undefined, undefined, { + claustrumConnector: connector, + }) + await plugin.auth.loader( + () => Promise.resolve(custodyTombstoneOAuth('anthropic')), + { models: {} }, + ) + const cacheKeep = plugin.__cacheKeepManager + if (!cacheKeep) throw new Error('missing CacheKeep manager') + const credentialGetsBeforePrewarm = calls.filter( + (call) => call.method === 'credential.get', + ).length + const result = await cacheKeep.prewarmNow({ + sessionId: 'ses-cachekeep-retry', + url: MESSAGES_URL, + headers: new Headers(), + bodyText: JSON.stringify({ + model: 'claude-opus-4-8', + system: [ + { + type: 'text', + text: 'stable', + cache_control: { type: 'ephemeral' }, + }, + ], + messages: [{ role: 'user', content: 'hello' }], + }), + oauthAccountId: 'fallback-1', + }) + + expect(result.ok).toBe(true) + expect(authorizations).toEqual([ + 'Bearer vault-cachekeep-retry-v47', + 'Bearer vault-cachekeep-retry-v48', + ]) + expect( + calls.filter((call) => call.method === 'credential.get'), + ).toHaveLength(credentialGetsBeforePrewarm + 1) + expect( + calls.filter((call) => call.method === 'credential.report_auth_failure'), + ).toHaveLength(0) + await plugin.dispose?.() + }) + test('overlapping CacheKeep vault 401s report the credential each attempt served', async () => { const calls: CredentialCall[] = [] const storage = fallbackWithClaustrum({ @@ -4634,6 +5200,9 @@ describe('fallback Claustrum credential resolution', () => { startFirstPrewarm() return firstPrewarmResponse } + if (prewarms > 2) { + return Promise.resolve(new Response('unauthorized', { status: 401 })) + } startSecondPrewarm() return secondPrewarmResponse } @@ -4698,7 +5267,7 @@ describe('fallback Claustrum credential resolution', () => { calls .filter((call) => call.method === 'credential.report_auth_failure') .map((call) => call.params.record_version), - ).toEqual([47, 48]) + ).toHaveLength(2) await plugin.dispose?.() }) @@ -4875,7 +5444,7 @@ describe('fallback Claustrum credential resolution', () => { expect(authorizations).toEqual(['Bearer vault-main-access']) expect( calls.filter((call) => call.method === 'credential.get'), - ).toHaveLength(initialCredentialGets) + ).toHaveLength(initialCredentialGets + 1) await plugin.dispose?.() }) @@ -5282,6 +5851,42 @@ describe('fallback Claustrum credential resolution', () => { }, ) + test.serial( + 'merges a hydrated fallback profile through a vault-served tombstone', + async () => { + const fixture = await bootRuledClaustrumRow({ + route: 'fallback-first', + fallbacks: [ + { + label: 'profiled', + handle: `ckh_${'P'.repeat(43)}`, + access: 'vault-profiled-access', + }, + ], + onFetch: (input) => + extractUrl(input as string | URL | Request) === PROFILE_URL + ? Response.json({ + organization: { + organization_type: 'claude_team', + rate_limit_tier: 'default_claude_max_5x', + }, + }) + : new Response('{}', { status: 200 }), + }) + + const state = await waitForSidebarState( + (candidate) => + candidate.fallbacks.find((account) => account.id === 'fallback-1') + ?.tierLabel === 'Team · Max 5x', + ) + expect( + state.fallbacks.find((account) => account.id === 'fallback-1') + ?.tierLabel, + ).toBe('Team · Max 5x') + await fixture.plugin.dispose?.() + }, + ) + test.serial( 'does not route a manifest-resolved account through a legacy per-account flag', async () => { @@ -6073,7 +6678,7 @@ describe('fallback Claustrum credential resolution', () => { const response = await fixture.result.fetch(MESSAGES_URL, EMPTY_POST) expect(response.status).toBe(200) - expect(fallbackGets).toBe(2) + expect(fallbackGets).toBe(3) expect( fixture.calls.some( (call) => call.method === 'credential.report_auth_failure', @@ -6223,6 +6828,289 @@ describe('fallback Claustrum credential resolution', () => { }) await fixture.plugin.dispose?.() }) + + // The get-before-report arms below are the ONLY observer this path will ever + // have. The retry fires in the seconds between a vault rotation and the next + // proactive credential.get, so a healthy system never exercises it and + // production emits no signal either way. For code that runs daily, production + // is a second opinion and tests are a convenience; here they are the + // instrument. Weakening or deleting an arm is not a test change, it is + // removing the only thing that can report on this mechanism. They also pin the + // literal log message, which a peer system consumes -- a rename that keeps the + // fields but changes the string breaks that integration silently. + test.serial( + 'logs a rotated credential version for a successful 401 retry', + async () => { + const logs: LogTestRecord[] = [] + __setLogTestSink((record) => logs.push(record)) + let cache: { + invalidate: (handle: string, recordVersion?: number) => void + get: (handle: string) => Promise + } + let plugin: { dispose?: () => Promise } | undefined + const fallbackHandle = `ckh_${'R'.repeat(43)}` + let fallbackGets = 0 + try { + const fixture = await bootRuledClaustrumRow({ + route: 'fallback-first', + fallbacks: [ + { + label: 'rotated-401', + handle: fallbackHandle, + access: 'vault-old-access', + }, + ], + connector: (calls) => + connectorFor(calls, (method, params) => { + if (method !== 'credential.get') return { result: {} } + if (String(params.handle) === ruledMainHandle) + return credentialResponse('vault-main-access', 1) + fallbackGets += 1 + return credentialResponse( + fallbackGets === 1 ? 'vault-old-access' : 'vault-new-access', + fallbackGets === 1 ? 41 : 42, + ) + }), + onFetch: async (input, init) => { + const url = extractUrl(input as string | URL | Request) + if (url.includes('/v1/messages')) { + const authorization = new Headers(init?.headers).get( + 'authorization', + ) + if (authorization === 'Bearer vault-old-access') { + cache.invalidate(fallbackHandle, 41) + await cache.get(fallbackHandle) + return new Response('{}', { status: 401 }) + } + } + return new Response('{}', { status: 200 }) + }, + }) + plugin = fixture.plugin + cache = fixture.plugin.__claustrumCredentialCache + const response = await fixture.result.fetch(MESSAGES_URL, EMPTY_POST) + expect(response.status).toBe(200) + const record = logs.find( + (candidate) => candidate.message === 'vault-served 401 recovery', + ) + expect(record?.payload?.retryAttempted).toBe(true) + expect(record?.payload?.retryOutcome).toBe('retry-succeeded') + expect(record?.payload?.reportOutcome).toBe('not-attempted') + expect(record?.payload?.retryServedRecordVersion).toBeGreaterThan( + record?.payload?.servedRecordVersion as number, + ) + } finally { + await plugin?.dispose?.() + __setLogTestSink(null) + } + }, + ) + + test.serial( + 'logs an unchanged credential version when a 401 is genuinely rejected', + async () => { + const logs: LogTestRecord[] = [] + __setLogTestSink((record) => logs.push(record)) + let plugin: { dispose?: () => Promise } | undefined + try { + const fixture = await bootRuledClaustrumRow({ + route: 'fallback-first', + fallbacks: [ + { + label: 'unchanged-401', + handle: `ckh_${'N'.repeat(43)}`, + access: 'vault-unchanged-access', + }, + ], + connector: (calls) => + connectorFor(calls, (method, params) => { + if (method !== 'credential.get') return { result: {} } + if (String(params.handle) === ruledMainHandle) + return credentialResponse('vault-main-access', 1) + return credentialResponse('vault-unchanged-access', 52) + }), + response: new Response('{}', { status: 401 }), + }) + plugin = fixture.plugin + const response = await fixture.result.fetch(MESSAGES_URL, EMPTY_POST) + expect(response.status).toBe(401) + expect( + fixture.calls.filter( + (call) => call.method === 'credential.report_auth_failure', + ), + ).not.toHaveLength(0) + const record = logs.find( + (candidate) => candidate.message === 'vault-served 401 recovery', + ) + expect(record?.payload?.retryAttempted).toBe(false) + expect(record?.payload?.vaultGetAttempted).toBe(true) + expect(record?.payload?.retryOutcome).toBe('unchanged') + expect(record?.payload?.reportOutcome).toBe('reported') + expect(record?.payload?.servedRecordVersion).toBe( + record?.payload?.currentCachedRecordVersion, + ) + } finally { + await plugin?.dispose?.() + __setLogTestSink(null) + } + }, + ) + + test.serial('logs which branch suppressed a raced 401 report', async () => { + const logs: LogTestRecord[] = [] + __setLogTestSink((record) => logs.push(record)) + let cache: { + invalidate: (handle: string, recordVersion?: number) => void + get: (handle: string) => Promise + } + let plugin: { dispose?: () => Promise } | undefined + const fallbackHandle = `ckh_${'P'.repeat(43)}` + let fallbackGets = 0 + try { + const fixture = await bootRuledClaustrumRow({ + route: 'fallback-first', + fallbacks: [ + { + label: 'suppressed-401', + handle: fallbackHandle, + access: 'vault-old-access', + }, + ], + connector: (calls) => + connectorFor(calls, (method, params) => { + if (method !== 'credential.get') return { result: {} } + if (String(params.handle) === ruledMainHandle) + return credentialResponse('vault-main-access', 1) + fallbackGets += 1 + if (fallbackGets === 1) + return credentialResponse('vault-old-access', 41) + if (fallbackGets === 2) + return credentialResponse('vault-new-access', 42) + throw new Error('retry credential unavailable') + }), + onFetch: async (input, init) => { + const url = extractUrl(input as string | URL | Request) + if (url.includes('/v1/messages')) { + const authorization = new Headers(init?.headers).get( + 'authorization', + ) + if (authorization === 'Bearer vault-old-access') { + cache.invalidate(fallbackHandle, 41) + await cache.get(fallbackHandle) + return new Response('{}', { status: 401 }) + } + } + return new Response('{}', { status: 200 }) + }, + }) + plugin = fixture.plugin + cache = fixture.plugin.__claustrumCredentialCache + const response = await fixture.result.fetch(MESSAGES_URL, EMPTY_POST) + expect(response.status).toBe(200) + expect( + fixture.calls.some( + (call) => + call.method === 'credential.report_auth_failure' && + call.params.handle === fallbackHandle, + ), + ).toBe(false) + const record = logs.find( + (candidate) => + candidate.message === 'vault-served 401 recovery' && + String(candidate.payload?.reportOutcome).startsWith('suppressed-'), + ) + expect(String(record?.payload?.reportOutcome)).toMatch(/^suppressed-/) + expect(record?.payload?.reportSuppressed).toBe(true) + expect(record?.payload?.reportSuppressedBy).toBe( + record?.payload?.reportOutcome, + ) + } finally { + await plugin?.dispose?.() + __setLogTestSink(null) + } + }) + + test.serial( + 'logs that no vault get was attempted during active retry backoff', + async () => { + const logs: LogTestRecord[] = [] + __setLogTestSink((record) => logs.push(record)) + let plugin: { dispose?: () => Promise } | undefined + const fallbackHandle = `ckh_${'B'.repeat(43)}` + let failStartup = true + try { + const fixture = await bootRuledClaustrumRow({ + route: { sticky: 'fallback-1' }, + quota: { + enabled: true, + checkIntervalMinutes: 5, + minimumRemaining: { five_hour: 1, seven_day: 1 }, + failClosedOnUnknownQuota: false, + mainQuota: { + five_hour: { + usedPercent: 100, + remainingPercent: 0, + checkedAt: Date.now(), + }, + seven_day: { + usedPercent: 100, + remainingPercent: 0, + checkedAt: Date.now(), + }, + }, + mainQuotaCheckedAt: Date.now(), + }, + fallbacks: [ + { + label: 'backoff-401', + handle: fallbackHandle, + access: 'vault-backoff-access', + }, + ], + connector: (calls) => + connectorFor(calls, (method, params) => { + if (method !== 'credential.get') return { result: {} } + if (String(params.handle) === ruledMainHandle) + return credentialResponse('vault-main-access', 1) + if (failStartup) + throw new ClaustrumCredentialError( + 'startup credential unavailable', + 'test_transient', + 'transient', + 'retry', + ) + return credentialResponse('vault-backoff-access', 61) + }), + onFetch: (input, init) => { + const authorization = new Headers(init?.headers).get( + 'authorization', + ) + if (authorization === 'Bearer vault-backoff-access') + return new Response('{}', { status: 401 }) + return new Response('{}', { status: 200 }) + }, + }) + plugin = fixture.plugin + const cache = fixture.plugin.__claustrumCredentialCache as any + failStartup = false + const servedCredential = await cache.get(fallbackHandle) + cache.seedForTest(fallbackHandle, servedCredential) + + const response = await fixture.result.fetch(MESSAGES_URL, EMPTY_POST) + expect(response.status).toBe(401) + const record = logs.find( + (candidate) => + candidate.message === 'vault-served 401 recovery' && + candidate.payload?.retryOutcome === 'backoff-active', + ) + if (!record) throw new Error(JSON.stringify(logs)) + expect(record?.payload?.vaultGetAttempted).toBe(false) + } finally { + await plugin?.dispose?.() + __setLogTestSink(null) + } + }, + ) }) async function readFeedEntries() { @@ -22958,7 +23846,7 @@ describe('killswitch fetch gate', () => { }) as unknown as typeof globalThis.setTimeout const usageAuthorizations: string[] = [] const fixture = await bootSharedRuledClaustrumRow({ - route: 'fallback-first', + route: 'main-exhausted', quota: { enabled: true, checkIntervalMinutes: 5, @@ -22968,13 +23856,13 @@ describe('killswitch fetch gate', () => { mainQuota: { checkedAt: now, five_hour: { - usedPercent: 100, - remainingPercent: 0, + usedPercent: 10, + remainingPercent: 90, checkedAt: now, }, seven_day: { - usedPercent: 100, - remainingPercent: 0, + usedPercent: 10, + remainingPercent: 90, checkedAt: now, }, }, @@ -23043,11 +23931,44 @@ describe('killswitch fetch gate', () => { }) const plugin = fixture.plugin await plugin.__fallbackRefreshReady + const persisted = await loadAccounts() + if (!persisted) throw new Error('expected custody fixture storage') + const accountPath = process.env.OPENCODE_ANTHROPIC_AUTH_FILE + if (!accountPath) throw new Error('expected custody fixture account file') + await fs.writeFile( + accountPath, + JSON.stringify({ + ...persisted, + accounts: persisted.accounts.map((account) => + isOAuthAccount(account) && account.id === accountId + ? { ...account, ...custodyTombstoneOAuth('anthropic') } + : account, + ), + }), + ) + expect( + (await loadAccounts())?.accounts.find( + (account): account is OAuthAccount => + isOAuthAccount(account) && account.id === accountId, + )?.access, + ).toBe('') clock = now + 6 * 60 * 60 * 1000 plugin.__quotaManager.clearFallback(accountId) const residentBeforeRequest = Boolean( plugin.__claustrumCredentialCache.peek(handle), ) + const eagerFallbackIds: string[][] = [] + const refreshAllFallbacks = + plugin.__quotaManager.refreshAllFallbacks.bind(plugin.__quotaManager) + spyOn(plugin.__quotaManager, 'refreshAllFallbacks').mockImplementation( + async ( + accounts: OAuthAccount[], + resolveAccessToken?: (account: OAuthAccount) => string | undefined, + ) => { + eagerFallbackIds.push(accounts.map((account) => account.id)) + await refreshAllFallbacks(accounts, resolveAccessToken) + }, + ) usageAuthorizations.length = 0 const timerBaseline = detachedTimers.length @@ -23077,6 +23998,7 @@ describe('killswitch fetch gate', () => { await plugin.dispose?.() return { + eagerFallbackIds, fallbackUsageCalls: coldFallbackUsageCalls, residentBeforeRequest, sidecarUsageCalls: coldSidecarUsageCalls, @@ -23093,6 +24015,9 @@ describe('killswitch fetch gate', () => { const result = await runVaultKillswitchQuotaRefresh(true) expect(result.residentBeforeRequest).toBe(true) + expect(result.eagerFallbackIds).toContainEqual([ + 'killswitch-vault-fallback', + ]) expect(result.sidecarUsageCalls).toBe(0) expect(result.fallbackUsageCalls).toBe(1) expect(result.scheduledWarmCount).toBe(0) @@ -24465,18 +25390,25 @@ describe('claude-prime direct request', () => { params: Record }> = [] let rotated = false + let rotatedAgain = false const connector = primeConnector(credentialCalls, (method, params) => { if (method !== 'credential.get') return { result: {} } if (params.handle === primeMainHandle) { return primeCredentialResponse('vault-main-access', 102) } return primeCredentialResponse( - rotated ? 'vault-prime-new-access' : 'vault-prime-401-access', - rotated ? 104 : 103, + rotatedAgain + ? 'vault-prime-newer-access' + : rotated + ? 'vault-prime-new-access' + : 'vault-prime-401-access', + rotatedAgain ? 105 : rotated ? 104 : 103, ) }) const requestEntered = deferred() const releaseResponse = deferred() + const retryEntered = deferred() + const releaseRetryResponse = deferred() let sentAuthorization: string | undefined globalThis.fetch = mock(async (input: unknown, init?: RequestInit) => { if ( @@ -24489,6 +25421,11 @@ describe('claude-prime direct request', () => { await releaseResponse.promise return new Response('{}', { status: 401 }) } + if (authorization === 'Bearer vault-prime-new-access') { + retryEntered.resolve() + await releaseRetryResponse.promise + return new Response('{}', { status: 401 }) + } return new Response('{}', { status: 200 }) } return new Response('not-mocked', { status: 599 }) @@ -24510,6 +25447,11 @@ describe('claude-prime direct request', () => { rotated = true await cache.get(prime401Handle) releaseResponse.resolve() + await retryEntered.promise + cache.invalidate(prime401Handle, 104) + rotatedAgain = true + await cache.get(prime401Handle) + releaseRetryResponse.resolve() await tick expect(sentAuthorization).toBe('Bearer vault-prime-401-access') @@ -24518,7 +25460,7 @@ describe('claude-prime direct request', () => { params: { handle: prime401Handle, provider_status: 401, - record_version: 103, + record_version: 104, reporter_source: 'direct', }, }) diff --git a/scripts/check-claustrum-golden.ts b/scripts/check-claustrum-golden.ts index a8e3f3c3..829bb8ba 100644 --- a/scripts/check-claustrum-golden.ts +++ b/scripts/check-claustrum-golden.ts @@ -39,20 +39,106 @@ for (const [name] of paths) { } let drifted = false +let contentUnchecked = false for (const [name, sourcePath] of paths) { const url = `https://raw.githubusercontent.com/${source.repo}/${source.ref}/${sourcePath}` - const response = await fetch(url) - if (!response.ok) { - throw new Error(`Failed to fetch ${name} golden: ${response.status} ${url}`) - } - const remote = Buffer.from(await response.arrayBuffer()) - const local = await readFile(join(fixtureDir, `${name}.json`)) - if (Buffer.compare(remote, local) !== 0) { - console.error(`DRIFT: ${name}.json differs from ${url}`) - drifted = true + let response: Response + try { + response = await fetch(url) + if (!response.ok) { + console.error( + `CONTENT UNCHECKED: ${name}.json could not be fetched (${response.status} ${url})`, + ) + contentUnchecked = true + continue + } + const remote = Buffer.from(await response.arrayBuffer()) + const local = await readFile(join(fixtureDir, `${name}.json`)) + if (Buffer.compare(remote, local) !== 0) { + console.error(`CONTENT FAIL: ${name}.json differs from ${url}`) + drifted = true + continue + } + } catch (error) { + console.error( + `CONTENT UNCHECKED: ${name}.json could not be fetched (${error instanceof Error ? error.message : String(error)})`, + ) + contentUnchecked = true continue } - console.log(`${name}.json: IDENTICAL (${source.ref})`) + console.log(`CONTENT PASS: ${name}.json IDENTICAL (${source.ref})`) } -if (drifted) process.exitCode = 1 +let ancestryFailed = false +let ancestryUnchecked = false +const repositoryUrl = `https://api.github.com/repos/${source.repo}` +try { + const repositoryResponse = await fetch(repositoryUrl, { + headers: { Accept: 'application/vnd.github+json' }, + }) + if (!repositoryResponse.ok) { + console.error( + `ANCESTRY UNCHECKED: could not resolve upstream default branch (${repositoryResponse.status} ${repositoryUrl})`, + ) + ancestryUnchecked = true + } else { + const repository = (await repositoryResponse.json()) as { + default_branch?: unknown + } + if ( + typeof repository.default_branch !== 'string' || + !repository.default_branch + ) { + console.error( + `ANCESTRY UNCHECKED: upstream repository did not provide a default branch (${repositoryUrl})`, + ) + ancestryUnchecked = true + } else { + const branch = encodeURIComponent(repository.default_branch) + const compareUrl = `https://api.github.com/repos/${source.repo}/compare/${branch}...${source.ref}` + const compareResponse = await fetch(compareUrl, { + headers: { Accept: 'application/vnd.github+json' }, + }) + if (!compareResponse.ok) { + console.error( + `ANCESTRY UNCHECKED: compare API could not answer (${compareResponse.status} ${compareUrl})`, + ) + ancestryUnchecked = true + } else { + const comparison = (await compareResponse.json()) as { + status?: unknown + } + if ( + comparison.status === 'behind' || + comparison.status === 'identical' + ) { + console.log( + `ANCESTRY PASS: pin ${source.ref} is an ancestor of ${source.repo}@${repository.default_branch} (compare status: ${comparison.status})`, + ) + } else if ( + comparison.status === 'ahead' || + comparison.status === 'diverged' + ) { + console.error( + `ANCESTRY FAIL: pin ${source.ref} no longer tracks upstream ${source.repo}@${repository.default_branch} (compare status: ${comparison.status})`, + ) + ancestryFailed = true + } else { + console.error( + `ANCESTRY UNCHECKED: compare API returned an unexpected status (${String(comparison.status)})`, + ) + ancestryUnchecked = true + } + } + } + } +} catch (error) { + console.error( + `ANCESTRY UNCHECKED: upstream repository or compare API failed (${error instanceof Error ? error.message : String(error)})`, + ) + ancestryUnchecked = true +} + +if (drifted || contentUnchecked || ancestryFailed || ancestryUnchecked) { + process.exitCode = 1 +}