fix(claustrum): retry rotated vault credentials before reporting a 401

packages/core/src/accounts.ts

@@ -510,6 +510,8 @@ export type AccountRefreshError = {
 const DEFAULT_FALLBACK_ON = [401, 403, 429]
 const MIN_REFRESH_BEFORE_EXPIRY_MINUTES = 240
 const DEFAULT_REFRESH_BEFORE_EXPIRY_MINUTES = MIN_REFRESH_BEFORE_EXPIRY_MINUTES
+// Claustrum requests extra headroom beyond the local refresh threshold.
+export const VAULT_REFRESH_HEADROOM_MINUTES = 30
 const DEFAULT_REFRESH_INTERVAL_MINUTES = 10
 const MIN_REFRESH_RETRY_DELAY_MS = 5 * 60_000
 const MAX_REFRESH_RETRY_DELAY_MS = 60 * 60_000
@@ -3353,6 +3355,12 @@ export function getRefreshBeforeExpiryMs(storage: AccountStorage | null) {
   return refreshBeforeExpiryMs(storage)
 }
 
+export function getVaultRefreshMinTtlMs(storage: AccountStorage | null) {
+  return (
+    getRefreshBeforeExpiryMs(storage) + VAULT_REFRESH_HEADROOM_MINUTES * 60_000
+  )
+}
+
 export function getRefreshIntervalMs(storage: AccountStorage | null) {
   const minutes =
     storage?.refresh?.intervalMinutes ?? DEFAULT_REFRESH_INTERVAL_MINUTES
@@ -4119,9 +4127,15 @@ function canUseCachedQuotaAfterRefreshError(
   storage: AccountStorage | null,
   error: unknown,
   now: number,
+  vaultServed: boolean,
 ) {
   return (
-    Boolean(account.access && account.expires && account.expires > now) &&
+    // Cached quota remains attributable after a transient failure when either
+    // the local credential is live or a live Claustrum binding serves it.
+    Boolean(
+      (account.access && account.expires && account.expires > now) ||
+        vaultServed,
+    ) &&
     isTransientQuotaError(error) &&
     quotaSnapshotPassesPolicy(account.quota, storage) &&
     cachedQuotaSnapshotStillRelevant(account.quota, now)
@@ -4642,8 +4656,9 @@ export class FallbackAccountManager {
 
     for (const account of storage.accounts) {
       if (account.enabled === false || !isOAuthAccount(account)) continue
+      const vaultServed = this.isFallbackAccountVaultServed(account.id, storage)
       if (this.isFallbackAccountVaultEnabled(account.id, storage)) {
-        if (!this.isFallbackAccountVaultServed(account.id, storage)) continue
+        if (!vaultServed) continue
         if (
           hasNoLocalCredential(account) &&
           !storage.quota?.minimumRemaining &&
@@ -4658,7 +4673,7 @@ export class FallbackAccountManager {
         if (
           tokenNeedsRefresh(next, storage, this.now()) &&
           !this.isFallbackAccountVaultEnabled(next.id, storage) &&
-          !this.isFallbackAccountVaultServed(next.id, storage)
+          !vaultServed
         ) {
           const refreshError = next.lastRefreshError
           if (
@@ -4711,7 +4726,13 @@ export class FallbackAccountManager {
           usable.push(next)
       } catch (error) {
         if (
-          canUseCachedQuotaAfterRefreshError(next, storage, error, this.now())
+          canUseCachedQuotaAfterRefreshError(
+            next,
+            storage,
+            error,
+            this.now(),
+            this.isFallbackAccountVaultServed(next.id, storage),
+          )
         ) {
           log(
             '[refresh] fallback quota using cached quota after refresh error',

packages/core/src/cachekeep.ts

@@ -391,6 +391,12 @@ export class CacheKeepManager {
         target: CacheKeepTarget,
         attempt: CacheKeepPrewarmAttempt,
       ) => Promise<Headers | undefined> | Headers | undefined
+      retryHeadersAfter401?: (input: {
+        headers: Headers
+        target: CacheKeepTarget
+        bodyText: string
+        attempt: CacheKeepPrewarmAttempt
+      }) => Promise<Headers | undefined> | Headers | undefined
       onTrackedSessionsChanged?: (
         sessions: readonly CacheKeepTrackedSession[],
       ) => Promise<void> | void
@@ -718,6 +724,33 @@ export class CacheKeepManager {
           transient: true,
         }
       }
+      if (response.status === 401 && this.options.retryHeadersAfter401) {
+        const retryHeaders = await this.options.retryHeadersAfter401({
+          headers,
+          target,
+          bodyText: prewarm.bodyText,
+          attempt,
+        })
+        if (retryHeaders) {
+          await response.body?.cancel().catch(() => {})
+          try {
+            response = await fetchImpl(target.url, {
+              method: 'POST',
+              headers: retryHeaders,
+              body: prewarm.bodyText,
+              signal: AbortSignal.timeout(
+                this.options.prewarmTimeoutMs ?? CACHE_KEEP_PREWARM_TIMEOUT_MS,
+              ),
+            })
+          } catch (error) {
+            return {
+              ok: false,
+              reason: error instanceof Error ? error.message : String(error),
+              transient: true,
+            }
+          }
+        }
+      }
       const receivedAt = this.options.now?.() ?? Date.now()
       const raw = await response.text().catch(() => '')
       let data: unknown = null

packages/core/src/claustrum.ts

@@ -1867,6 +1867,10 @@ export class ClaustrumCredentialCache {
   readonly #identity?: BindIdentity
   readonly #now: () => number
   readonly #refreshBackoffUntil = new Map<string, number>()
+  readonly #latchedRefreshFailures = new Map<
+    string,
+    ClaustrumCredentialErrorClass
+  >()
   #minTtlMs: number
 
   constructor(
@@ -1885,24 +1889,34 @@ export class ClaustrumCredentialCache {
   async get(
     handle: string,
     minTtlMs = this.#minTtlMs,
-    options: { cacheIf?: () => boolean } = {},
+    options: { cacheIf?: () => boolean; bypassCache?: boolean } = {},
   ): Promise<ClaustrumCredential> {
     if (!Number.isSafeInteger(minTtlMs) || minTtlMs < 0) {
       throw new RangeError('minTtlMs must be a non-negative safe integer')
     }
     const now = this.#now()
     const cached = this.#cache.get(handle)
-    if (cached && cached.expiresAtMs !== null && cached.expiresAtMs > now) {
+    if (
+      !options.bypassCache &&
+      cached &&
+      cached.expiresAtMs !== null &&
+      cached.expiresAtMs > now
+    ) {
       if (cached.expiresAtMs - now <= minTtlMs) {
         this.#refreshIfApproachingExpiry(handle, now, minTtlMs)
       }
       return cached
     }
-    if (cached) {
+    if (cached && !options.bypassCache) {
       this.#cache.delete(handle)
       this.#refreshBackoffUntil.delete(handle)
     }
 
+    if (options.bypassCache) {
+      // Keep a 401 verdict independent of refreshes that began before it.
+      return this.#load(handle, minTtlMs, options.cacheIf)
+    }
+
     const pending = this.#inFlight.get(handle)
     if (pending) return pending
 
@@ -2033,7 +2047,22 @@ export class ClaustrumCredentialCache {
     const load = this.#load(handle, minTtlMs)
     this.#inFlight.set(handle, load)
     void load
-      .catch(() => {})
+      .catch((error) => {
+        if (
+          error instanceof ClaustrumCredentialError &&
+          (error.errorClass === 'permanent' ||
+            error.errorClass === 'auth_required') &&
+          this.#latchedRefreshFailures.get(handle) !== error.errorClass
+        ) {
+          this.#latchedRefreshFailures.set(handle, error.errorClass)
+          logger.warn('claustrum', 'credential background refresh latched', {
+            handle,
+            recordVersion: this.#cache.get(handle)?.recordVersion,
+            errorClass: error.errorClass,
+            code: error.code,
+          })
+        }
+      })
       .finally(() => {
         if (this.#inFlight.get(handle) === load) this.#inFlight.delete(handle)
       })
@@ -2075,8 +2104,12 @@ export class ClaustrumCredentialCache {
       credential.expiresAtMs > this.#now() &&
       (cacheIf?.() ?? true)
     ) {
-      this.#cache.set(handle, credential)
+      const cached = this.#cache.get(handle)
+      if (!cached || credential.recordVersion >= cached.recordVersion) {
+        this.#cache.set(handle, credential)
+      }
     }
+    this.#latchedRefreshFailures.delete(handle)
     return credential
   }
 }

packages/core/src/tests/accounts-persistence.test.ts

@@ -1,4 +1,5 @@
 import { afterEach, expect, test } from 'bun:test'
+import { strictEqual } from 'node:assert'
 import { mkdtemp, rm, writeFile } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
@@ -7,15 +8,20 @@ import {
   type AccountStorage,
   createEmptyStorage,
   FallbackAccountManager,
+  getRefreshBeforeExpiryMs,
+  getVaultRefreshMinTtlMs,
   hasNoLocalCredential,
   loadAccounts,
   type OAuthAccount,
   saveAccountState,
   saveAccounts,
 } from '../accounts.ts'
+import { custodyTombstoneOAuth } from '../claustrum.ts'
 
 const directories: string[] = []
 
+// These paired tripwires cover both vault-facing minTtl routes: default
+// threshold/headroom derivation and the config-override floor.
 afterEach(async () => {
   await Promise.all(
     directories
@@ -31,6 +37,64 @@ test('recognizes an OAuth account with no local credential', () => {
   expect(hasNoLocalCredential({ access: '' })).toBe(false)
 })
 
+test('keeps the vault-facing refresh TTL at 270 minutes', () => {
+  const vaultMinTtlMs = getVaultRefreshMinTtlMs(createEmptyStorage())
+  const expectedVaultMinTtlMs = 270 * 60_000
+  // Anthropic OAuth access tokens live 8h; the vault refreshes a credential when
+  // `now + minTtl >= expires_at`, so this value alone fixes the observed rotation
+  // period. State the resulting PERIOD, not just the minTtl: the period is the
+  // number the vault operator needs to pre-seed their stall detector.
+  const tokenLifetimeMinutes = 480
+  const newPeriodMinutes = tokenLifetimeMinutes - vaultMinTtlMs / 60_000
+  const oldPeriodMinutes = tokenLifetimeMinutes - expectedVaultMinTtlMs / 60_000
+  // Every number below is labelled with its ROLE: minTtl and period are drawn from
+  // the same small set of values and routinely swap places (a 240m minTtl on an 8h
+  // token yields a 240m period), so bare numerals invite transposition by a reader
+  // who lands on the assertion footer rather than the prose.
+  const guidance = [
+    'Vault coupling tripwire (threshold + headroom route; paired with the config-floor tripwire below): this shared value is passed as minTtl to Claustrum',
+    '`credential.get`, and the vault refreshes when `now + minTtl >= expires_at`,',
+    'so it fixes the observed rotation period as token_lifetime - minTtl.',
+    `CHANGED: minTtl ${vaultMinTtlMs / 60_000}m (was ${expectedVaultMinTtlMs / 60_000}m)`,
+    `-> rotation period ${newPeriodMinutes}m (was ${oldPeriodMinutes}m).`,
+    `The +/- values below are minTtl in ms, NOT the period.`,
+    `token_lifetime is ASSUMED ${tokenLifetimeMinutes}m — neither side observes it`,
+    "(it lives inside the vault's encrypted envelope); if Anthropic changed it,",
+    'this arithmetic is stale even though the assertion fired correctly.',
+    newPeriodMinutes > oldPeriodMinutes
+      ? 'THIS CHANGE LENGTHENS THE PERIOD, WHICH REQUIRES ADVANCE NOTICE: the vault operator alarms on MAX(recent gaps) + 30m, so the first longer gap trips a false stall alarm that REPEATS on a 30-minute cooldown until the refresh lands. Tell them the new period above before deploying so they can pre-seed it.'
+      : 'This change shortens the period, which is silent for the vault operator and needs no notice.',
+  ].join(' ')
+
+  strictEqual(vaultMinTtlMs, expectedVaultMinTtlMs, guidance)
+})
+
+test('floors a config override so it cannot lower the vault-facing minTtl', () => {
+  // This watches the other route to the same vault-facing value: the
+  // `refresh.refreshBeforeExpiryMinutes` config key. The floor in
+  // refreshBeforeExpiryMs is what makes the paired tripwires sufficient — without
+  // it, an operator could lower minTtl from config, lengthening the vault's
+  // rotation period, and the threshold/headroom tripwire above would never fire.
+  const storage = createEmptyStorage()
+  storage.refresh = { ...storage.refresh, refreshBeforeExpiryMinutes: 60 }
+  const floored = getRefreshBeforeExpiryMs(storage)
+
+  strictEqual(
+    floored,
+    240 * 60_000,
+    [
+      'Vault coupling tripwire (config route): a below-floor override of',
+      '`refresh.refreshBeforeExpiryMinutes` must clamp UP to the 240m floor, but this',
+      `build returned ${floored / 60_000}m. The floor is load-bearing for a peer system:`,
+      'it is the only reason config cannot lower minTtl, and lowering minTtl LENGTHENS the',
+      "vault's rotation period, which repeatedly false-alarms the vault operator's stall",
+      'detector. Removing the floor makes that reachable from config alone, where the',
+      'threshold/headroom tripwire above cannot see it. If you removed it deliberately,',
+      'the vault operator holds a registered dependency on it and is owed notice.',
+    ].join(' '),
+  )
+})
+
 test('preserves the Claustrum mode when a save supplies only handlesFile', async () => {
   const directory = await mkdtemp(join(tmpdir(), 'accounts-persistence-'))
   directories.push(directory)
@@ -147,6 +211,56 @@ test('excludes an empty-material vault fallback after its quota policy fails', a
   expect(authorizations).toEqual(['Bearer vault-fallback-access'])
 })
 
+test('keeps a live vault fallback on cached quota after a transient quota failure', async () => {
+  const now = 1_000_000
+  const account: OAuthAccount = {
+    id: 'vault-fallback',
+    enabled: true,
+    ...custodyTombstoneOAuth('anthropic'),
+    quota: {
+      checkedAt: now - 60_000,
+      five_hour: {
+        usedPercent: 10,
+        remainingPercent: 90,
+        checkedAt: now - 60_000,
+      },
+      seven_day: {
+        usedPercent: 10,
+        remainingPercent: 90,
+        checkedAt: now - 60_000,
+      },
+    },
+  }
+  const storage: AccountStorage = {
+    version: 1,
+    claustrum: { mode: 'claustrum' },
+    quota: {
+      enabled: true,
+      checkIntervalMinutes: 1,
+      minimumRemaining: { five_hour: 10, seven_day: 10 },
+      failClosedOnUnknownQuota: true,
+    },
+    accounts: [account],
+  }
+  const manager = new FallbackAccountManager({
+    now: () => now,
+    isFallbackAccountVaultEnabled: () => true,
+    isFallbackAccountVaultServed: () => true,
+    resolveFallbackAccessToken: () => ({
+      token: 'vault-fallback-access',
+      source: 'vault',
+    }),
+    fetchImpl: Object.assign(
+      async () => new Response('unavailable', { status: 503 }),
+      { preconnect: () => {} },
+    ) as unknown as typeof fetch,
+  })
+
+  await expect(manager.getUsableFallbackAccounts(storage)).resolves.toEqual([
+    account,
+  ])
+})
+
 test('keeps tombstone metadata when discarding a stale credential write', async () => {
   const directory = await mkdtemp(join(tmpdir(), 'accounts-persistence-'))
   directories.push(directory)