From b6c7de75f0dc6cfd3222b69c9a1543b2ec5c3cff Mon Sep 17 00:00:00 2001 From: "elliot.fouchy" Date: Sat, 31 Oct 2026 00:30:59 +0100 Subject: [PATCH] fix: retain Google usage on JSON errors --- src/handlers/streamHandler.test.ts | 56 ++++++------- .../google-vertex-ai/chatComplete.ts | 65 +++++++++------- src/providers/google/chatComplete.ts | 78 +++++++++++-------- 3 files changed, 112 insertions(+), 87 deletions(-) diff --git a/src/handlers/streamHandler.test.ts b/src/handlers/streamHandler.test.ts index 91348e0d1..4895a9111 100644 --- a/src/handlers/streamHandler.test.ts +++ b/src/handlers/streamHandler.test.ts @@ -87,7 +87,7 @@ describe.each([ expect(transform({ candidates: [] }).usage).toBeUndefined(); }); - it('rejects Google’s documented unspecified prompt block and retains its SSE usage', () => { + it('rejects Google’s documented unspecified prompt block and retains its usage', () => { const payload = { promptFeedback: { blockReason: 'BLOCKED_REASON_UNSPECIFIED' }, usageMetadata: { promptTokenCount: 7, totalTokenCount: 7 }, @@ -103,36 +103,40 @@ describe.each([ type: 'BLOCKED_REASON_UNSPECIFIED', }, }); + expect((result as any).usage).toMatchObject({ + prompt_tokens: 7, + completion_tokens: 0, + total_tokens: 7, + }); } - expect(transform(payload).usage).toMatchObject({ - prompt_tokens: 7, - completion_tokens: 0, - total_tokens: 7, - }); }); - it('keeps upstream errors carried inside a successful HTTP stream', () => { - expect( - transform({ - error: { - code: 429, - status: 'RESOURCE_EXHAUSTED', - message: 'Quota exhausted', - }, - usageMetadata: { - promptTokenCount: 7, - candidatesTokenCount: 2, - totalTokenCount: 9, - }, - }) - ).toMatchObject({ - provider, + it('retains JSON and SSE upstream errors with their usage', () => { + const payload = { error: { - type: 'RESOURCE_EXHAUSTED', - message: expect.stringContaining('Quota exhausted'), + code: 429, + status: 'RESOURCE_EXHAUSTED', + message: 'Quota exhausted', }, - usage: { prompt_tokens: 7, completion_tokens: 2, total_tokens: 9 }, - }); + usageMetadata: { + promptTokenCount: 7, + candidatesTokenCount: 2, + totalTokenCount: 9, + }, + }; + for (const result of [ + responseTransform(payload as any, 429, new Headers(), false), + transform(payload), + ]) { + expect(result).toMatchObject({ + provider, + error: { + type: 'RESOURCE_EXHAUSTED', + message: expect.stringContaining('Quota exhausted'), + }, + usage: { prompt_tokens: 7, completion_tokens: 2, total_tokens: 9 }, + }); + } }); it('preserves filtered candidates without content parts and their reported usage', () => { diff --git a/src/providers/google-vertex-ai/chatComplete.ts b/src/providers/google-vertex-ai/chatComplete.ts index 807cbef43..779389ffc 100644 --- a/src/providers/google-vertex-ai/chatComplete.ts +++ b/src/providers/google-vertex-ai/chatComplete.ts @@ -483,12 +483,14 @@ export const GoogleChatCompleteResponseTransform: ( ); } - if (!Array.isArray(response)) { - const errorResponse = GoogleErrorResponseTransform( - response, - GOOGLE_VERTEX_AI - ); - if (errorResponse) return errorResponse; + const errorResponse = !Array.isArray(response) + ? GoogleErrorResponseTransform(response, GOOGLE_VERTEX_AI) + : undefined; + if ( + errorResponse && + !('usageMetadata' in response && response.usageMetadata) + ) { + return errorResponse; } // sometimes vertex gemini returns usageMetadata without candidates @@ -513,6 +515,32 @@ export const GoogleChatCompleteResponseTransform: ( return acc; }, 0); + const usage = { + prompt_tokens: promptTokenCount, + completion_tokens: candidatesTokenCount + thoughtsTokenCount, + total_tokens: totalTokenCount, + completion_tokens_details: { + reasoning_tokens: thoughtsTokenCount, + audio_tokens: outputAudioTokens, + }, + prompt_tokens_details: { + // The request that created the cachedContents resource is + // billed the whole prefix at the standard input rate, so + // zero the cached discount for it (see cachedContents.ts). + cached_tokens: wasCacheCreated(gatewayRequest) + ? 0 + : cachedContentTokenCount, + audio_tokens: inputAudioTokens, + }, + // Report the written prefix on the creating request so + // downstream billing can price the create and its cache + // storage explicitly (Anthropic usage convention). + ...(wasCacheCreated(gatewayRequest) && cachedContentTokenCount > 0 + ? { cache_creation_input_tokens: cachedContentTokenCount } + : {}), + }; + if (errorResponse) return { ...errorResponse, usage }; + return { id: 'portkey-' + crypto.randomUUID(), object: 'chat.completion', @@ -608,30 +636,7 @@ export const GoogleChatCompleteResponseTransform: ( : {}), }; }) ?? [], - usage: { - prompt_tokens: promptTokenCount, - completion_tokens: candidatesTokenCount + thoughtsTokenCount, - total_tokens: totalTokenCount, - completion_tokens_details: { - reasoning_tokens: thoughtsTokenCount, - audio_tokens: outputAudioTokens, - }, - prompt_tokens_details: { - // The request that created the cachedContents resource is - // billed the whole prefix at the standard input rate, so - // zero the cached discount for it (see cachedContents.ts). - cached_tokens: wasCacheCreated(gatewayRequest) - ? 0 - : cachedContentTokenCount, - audio_tokens: inputAudioTokens, - }, - // Report the written prefix on the creating request so - // downstream billing can price the create and its cache - // storage explicitly (Anthropic usage convention). - ...(wasCacheCreated(gatewayRequest) && cachedContentTokenCount > 0 - ? { cache_creation_input_tokens: cachedContentTokenCount } - : {}), - }, + usage, }; } diff --git a/src/providers/google/chatComplete.ts b/src/providers/google/chatComplete.ts index 4c42a05d9..a79af8f45 100644 --- a/src/providers/google/chatComplete.ts +++ b/src/providers/google/chatComplete.ts @@ -507,6 +507,7 @@ export const GoogleChatCompleteConfig: ProviderConfig = { }; export interface GoogleErrorResponse { + usageMetadata?: GoogleGenerateContentResponse['usageMetadata']; error: { code: string | number; message: string; @@ -625,6 +626,42 @@ export const GoogleErrorResponseTransform: ( return undefined; }; +function googleJsonUsage( + metadata: GoogleGenerateContentResponse['usageMetadata'] +) { + const { + promptTokenCount = 0, + candidatesTokenCount = 0, + totalTokenCount = 0, + thoughtsTokenCount = 0, + cachedContentTokenCount = 0, + promptTokensDetails = [], + candidatesTokensDetails = [], + } = metadata; + const inputAudioTokens = promptTokensDetails.reduce((acc, curr) => { + if (curr.modality === VERTEX_MODALITY.AUDIO) return acc + curr.tokenCount; + return acc; + }, 0); + const outputAudioTokens = candidatesTokensDetails.reduce((acc, curr) => { + if (curr.modality === VERTEX_MODALITY.AUDIO) return acc + curr.tokenCount; + return acc; + }, 0); + + return { + prompt_tokens: promptTokenCount, + completion_tokens: candidatesTokenCount + thoughtsTokenCount, + total_tokens: totalTokenCount, + completion_tokens_details: { + reasoning_tokens: thoughtsTokenCount, + audio_tokens: outputAudioTokens, + }, + prompt_tokens_details: { + cached_tokens: cachedContentTokenCount, + audio_tokens: inputAudioTokens, + }, + }; +} + export const GoogleChatCompleteResponseTransform: ( response: GoogleGenerateContentResponse | GoogleErrorResponse, responseStatus: number, @@ -637,26 +674,17 @@ export const GoogleChatCompleteResponseTransform: ( strictOpenAiCompliance ) => { const errorResponse = GoogleErrorResponseTransform(response); - if (errorResponse) return errorResponse; + if (errorResponse) { + return { + ...errorResponse, + ...(response.usageMetadata && { + usage: googleJsonUsage(response.usageMetadata), + }), + }; + } if ('candidates' in response) { - const { - promptTokenCount = 0, - candidatesTokenCount = 0, - totalTokenCount = 0, - thoughtsTokenCount = 0, - cachedContentTokenCount = 0, - promptTokensDetails = [], - candidatesTokensDetails = [], - } = response.usageMetadata; - const inputAudioTokens = promptTokensDetails.reduce((acc, curr) => { - if (curr.modality === VERTEX_MODALITY.AUDIO) return acc + curr.tokenCount; - return acc; - }, 0); - const outputAudioTokens = candidatesTokensDetails.reduce((acc, curr) => { - if (curr.modality === VERTEX_MODALITY.AUDIO) return acc + curr.tokenCount; - return acc; - }, 0); + const usage = googleJsonUsage(response.usageMetadata); return { id: 'portkey-' + crypto.randomUUID(), @@ -729,19 +757,7 @@ export const GoogleChatCompleteResponseTransform: ( : {}), }; }) ?? [], - usage: { - prompt_tokens: promptTokenCount, - completion_tokens: candidatesTokenCount + thoughtsTokenCount, - total_tokens: totalTokenCount, - completion_tokens_details: { - reasoning_tokens: thoughtsTokenCount, - audio_tokens: outputAudioTokens, - }, - prompt_tokens_details: { - cached_tokens: cachedContentTokenCount, - audio_tokens: inputAudioTokens, - }, - }, + usage, }; }