Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 30 additions & 26 deletions src/handlers/streamHandler.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ describe.each([
expect(transform({ candidates: [] }).usage).toBeUndefined();
});

it('rejects Google’s documented unspecified prompt block and retains its SSE usage', () => {
it('rejects Google’s documented unspecified prompt block and retains its usage', () => {
const payload = {
promptFeedback: { blockReason: 'BLOCKED_REASON_UNSPECIFIED' },
usageMetadata: { promptTokenCount: 7, totalTokenCount: 7 },
Expand All @@ -103,36 +103,40 @@ describe.each([
type: 'BLOCKED_REASON_UNSPECIFIED',
},
});
expect((result as any).usage).toMatchObject({
prompt_tokens: 7,
completion_tokens: 0,
total_tokens: 7,
});
}
expect(transform(payload).usage).toMatchObject({
prompt_tokens: 7,
completion_tokens: 0,
total_tokens: 7,
});
});

it('keeps upstream errors carried inside a successful HTTP stream', () => {
expect(
transform({
error: {
code: 429,
status: 'RESOURCE_EXHAUSTED',
message: 'Quota exhausted',
},
usageMetadata: {
promptTokenCount: 7,
candidatesTokenCount: 2,
totalTokenCount: 9,
},
})
).toMatchObject({
provider,
it('retains JSON and SSE upstream errors with their usage', () => {
const payload = {
error: {
type: 'RESOURCE_EXHAUSTED',
message: expect.stringContaining('Quota exhausted'),
code: 429,
status: 'RESOURCE_EXHAUSTED',
message: 'Quota exhausted',
},
usage: { prompt_tokens: 7, completion_tokens: 2, total_tokens: 9 },
});
usageMetadata: {
promptTokenCount: 7,
candidatesTokenCount: 2,
totalTokenCount: 9,
},
};
for (const result of [
responseTransform(payload as any, 429, new Headers(), false),
transform(payload),
]) {
expect(result).toMatchObject({
provider,
error: {
type: 'RESOURCE_EXHAUSTED',
message: expect.stringContaining('Quota exhausted'),
},
usage: { prompt_tokens: 7, completion_tokens: 2, total_tokens: 9 },
});
}
});

it('preserves filtered candidates without content parts and their reported usage', () => {
Expand Down
65 changes: 35 additions & 30 deletions src/providers/google-vertex-ai/chatComplete.ts
Original file line number Diff line number Diff line change
Expand Up @@ -483,12 +483,14 @@ export const GoogleChatCompleteResponseTransform: (
);
}

if (!Array.isArray(response)) {
const errorResponse = GoogleErrorResponseTransform(
response,
GOOGLE_VERTEX_AI
);
if (errorResponse) return errorResponse;
const errorResponse = !Array.isArray(response)
? GoogleErrorResponseTransform(response, GOOGLE_VERTEX_AI)
: undefined;
if (
errorResponse &&
!('usageMetadata' in response && response.usageMetadata)
) {
return errorResponse;
}

// sometimes vertex gemini returns usageMetadata without candidates
Expand All @@ -513,6 +515,32 @@ export const GoogleChatCompleteResponseTransform: (
return acc;
}, 0);

const usage = {
prompt_tokens: promptTokenCount,
completion_tokens: candidatesTokenCount + thoughtsTokenCount,
total_tokens: totalTokenCount,
completion_tokens_details: {
reasoning_tokens: thoughtsTokenCount,
audio_tokens: outputAudioTokens,
},
prompt_tokens_details: {
// The request that created the cachedContents resource is
// billed the whole prefix at the standard input rate, so
// zero the cached discount for it (see cachedContents.ts).
cached_tokens: wasCacheCreated(gatewayRequest)
? 0
: cachedContentTokenCount,
audio_tokens: inputAudioTokens,
},
// Report the written prefix on the creating request so
// downstream billing can price the create and its cache
// storage explicitly (Anthropic usage convention).
...(wasCacheCreated(gatewayRequest) && cachedContentTokenCount > 0
? { cache_creation_input_tokens: cachedContentTokenCount }
: {}),
};
if (errorResponse) return { ...errorResponse, usage };

return {
id: 'portkey-' + crypto.randomUUID(),
object: 'chat.completion',
Expand Down Expand Up @@ -608,30 +636,7 @@ export const GoogleChatCompleteResponseTransform: (
: {}),
};
}) ?? [],
usage: {
prompt_tokens: promptTokenCount,
completion_tokens: candidatesTokenCount + thoughtsTokenCount,
total_tokens: totalTokenCount,
completion_tokens_details: {
reasoning_tokens: thoughtsTokenCount,
audio_tokens: outputAudioTokens,
},
prompt_tokens_details: {
// The request that created the cachedContents resource is
// billed the whole prefix at the standard input rate, so
// zero the cached discount for it (see cachedContents.ts).
cached_tokens: wasCacheCreated(gatewayRequest)
? 0
: cachedContentTokenCount,
audio_tokens: inputAudioTokens,
},
// Report the written prefix on the creating request so
// downstream billing can price the create and its cache
// storage explicitly (Anthropic usage convention).
...(wasCacheCreated(gatewayRequest) && cachedContentTokenCount > 0
? { cache_creation_input_tokens: cachedContentTokenCount }
: {}),
},
usage,
};
}

Expand Down
78 changes: 47 additions & 31 deletions src/providers/google/chatComplete.ts
Original file line number Diff line number Diff line change
Expand Up @@ -507,6 +507,7 @@ export const GoogleChatCompleteConfig: ProviderConfig = {
};

export interface GoogleErrorResponse {
usageMetadata?: GoogleGenerateContentResponse['usageMetadata'];
error: {
code: string | number;
message: string;
Expand Down Expand Up @@ -625,6 +626,42 @@ export const GoogleErrorResponseTransform: (
return undefined;
};

function googleJsonUsage(
metadata: GoogleGenerateContentResponse['usageMetadata']
) {
const {
promptTokenCount = 0,
candidatesTokenCount = 0,
totalTokenCount = 0,
thoughtsTokenCount = 0,
cachedContentTokenCount = 0,
promptTokensDetails = [],
candidatesTokensDetails = [],
} = metadata;
const inputAudioTokens = promptTokensDetails.reduce((acc, curr) => {
if (curr.modality === VERTEX_MODALITY.AUDIO) return acc + curr.tokenCount;
return acc;
}, 0);
const outputAudioTokens = candidatesTokensDetails.reduce((acc, curr) => {
if (curr.modality === VERTEX_MODALITY.AUDIO) return acc + curr.tokenCount;
return acc;
}, 0);

return {
prompt_tokens: promptTokenCount,
completion_tokens: candidatesTokenCount + thoughtsTokenCount,
total_tokens: totalTokenCount,
completion_tokens_details: {
reasoning_tokens: thoughtsTokenCount,
audio_tokens: outputAudioTokens,
},
prompt_tokens_details: {
cached_tokens: cachedContentTokenCount,
audio_tokens: inputAudioTokens,
},
};
}

export const GoogleChatCompleteResponseTransform: (
response: GoogleGenerateContentResponse | GoogleErrorResponse,
responseStatus: number,
Expand All @@ -637,26 +674,17 @@ export const GoogleChatCompleteResponseTransform: (
strictOpenAiCompliance
) => {
const errorResponse = GoogleErrorResponseTransform(response);
if (errorResponse) return errorResponse;
if (errorResponse) {
return {
...errorResponse,
...(response.usageMetadata && {
usage: googleJsonUsage(response.usageMetadata),
}),
};
}

if ('candidates' in response) {
const {
promptTokenCount = 0,
candidatesTokenCount = 0,
totalTokenCount = 0,
thoughtsTokenCount = 0,
cachedContentTokenCount = 0,
promptTokensDetails = [],
candidatesTokensDetails = [],
} = response.usageMetadata;
const inputAudioTokens = promptTokensDetails.reduce((acc, curr) => {
if (curr.modality === VERTEX_MODALITY.AUDIO) return acc + curr.tokenCount;
return acc;
}, 0);
const outputAudioTokens = candidatesTokensDetails.reduce((acc, curr) => {
if (curr.modality === VERTEX_MODALITY.AUDIO) return acc + curr.tokenCount;
return acc;
}, 0);
const usage = googleJsonUsage(response.usageMetadata);

return {
id: 'portkey-' + crypto.randomUUID(),
Expand Down Expand Up @@ -729,19 +757,7 @@ export const GoogleChatCompleteResponseTransform: (
: {}),
};
}) ?? [],
usage: {
prompt_tokens: promptTokenCount,
completion_tokens: candidatesTokenCount + thoughtsTokenCount,
total_tokens: totalTokenCount,
completion_tokens_details: {
reasoning_tokens: thoughtsTokenCount,
audio_tokens: outputAudioTokens,
},
prompt_tokens_details: {
cached_tokens: cachedContentTokenCount,
audio_tokens: inputAudioTokens,
},
},
usage,
};
}

Expand Down
Loading