Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
189 changes: 188 additions & 1 deletion src/handlers/streamHandler.test.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,196 @@
import { BEDROCK, OPEN_AI, PERPLEXITY_AI } from '../globals';
import {
BEDROCK,
GOOGLE,
GOOGLE_VERTEX_AI,
OPEN_AI,
PERPLEXITY_AI,
} from '../globals';
import {
GoogleChatCompleteResponseTransform,
GoogleChatCompleteStreamChunkTransform,
} from '../providers/google/chatComplete';
import {
GoogleChatCompleteResponseTransform as vertexResponse,
GoogleChatCompleteStreamChunkTransform as vertexStream,
} from '../providers/google-vertex-ai/chatComplete';
import { PerplexityAIChatCompleteStreamChunkTransform } from '../providers/perplexity-ai/chatComplete';
import { BedrockChatCompleteStreamChunkTransform } from '../providers/bedrock/chatComplete';
import { getStreamModeSplitPattern } from '../utils';
import { handleStreamingMode, readStream } from './streamHandler';

describe.each([
[
GOOGLE,
GoogleChatCompleteResponseTransform,
GoogleChatCompleteStreamChunkTransform,
],
[GOOGLE_VERTEX_AI, vertexResponse, vertexStream],
] as const)(
'%s Gemini response metadata',
(provider, responseTransform, streamTransform) => {
const transform = (payload: object) =>
JSON.parse(
streamTransform(JSON.stringify(payload), 'test', {}, false).slice(6)
);
const blocked = {
modelVersion: 'gemini-3-flash-preview',
promptFeedback: {
blockReason: 'SAFETY',
blockReasonMessage: 'Test prompt blocked',
},
};

it('preserves prompt blocks in JSON and SSE instead of returning an empty completion', () => {
for (const result of [
responseTransform(blocked as any, 200, new Headers(), false),
transform(blocked),
]) {
expect(result).toMatchObject({
provider,
error: { code: 'content_policy_violation', type: 'SAFETY' },
});
expect((result as any).error.message).toContain('Test prompt blocked');
expect((result as any).usage).toBeUndefined();
}
});

it.each([0, undefined])(
'preserves usage when candidatesTokenCount is %s',
(candidatesTokenCount) => {
expect(
transform({
usageMetadata: {
promptTokenCount: 10,
candidatesTokenCount,
totalTokenCount: 15,
thoughtsTokenCount: 5,
},
}).usage
).toMatchObject({
prompt_tokens: 10,
completion_tokens: 5,
total_tokens: 15,
});
}
);

it('preserves prompt-only usage and never invents a missing usage block', () => {
expect(
transform({
usageMetadata: { promptTokenCount: 10, totalTokenCount: 10 },
}).usage
).toMatchObject({
prompt_tokens: 10,
completion_tokens: 0,
total_tokens: 10,
});
expect(transform({ candidates: [] }).usage).toBeUndefined();
});

it('rejects Google’s documented unspecified prompt block and retains its SSE usage', () => {
const payload = {
promptFeedback: { blockReason: 'BLOCKED_REASON_UNSPECIFIED' },
usageMetadata: { promptTokenCount: 7, totalTokenCount: 7 },
};
for (const result of [
responseTransform(payload as any, 200, new Headers(), false),
transform(payload),
]) {
expect(result).toMatchObject({
provider,
error: {
code: 'content_policy_violation',
type: 'BLOCKED_REASON_UNSPECIFIED',
},
});
}
expect(transform(payload).usage).toMatchObject({
prompt_tokens: 7,
completion_tokens: 0,
total_tokens: 7,
});
});

it('keeps upstream errors carried inside a successful HTTP stream', () => {
expect(
transform({
error: {
code: 429,
status: 'RESOURCE_EXHAUSTED',
message: 'Quota exhausted',
},
usageMetadata: {
promptTokenCount: 7,
candidatesTokenCount: 2,
totalTokenCount: 9,
},
})
).toMatchObject({
provider,
error: {
type: 'RESOURCE_EXHAUSTED',
message: expect.stringContaining('Quota exhausted'),
},
usage: { prompt_tokens: 7, completion_tokens: 2, total_tokens: 9 },
});
});

it('preserves filtered candidates without content parts and their reported usage', () => {
// Live Vertex safety-filter response to our synthetic test prompt.
const result = transform({
candidates: [{ content: { role: 'model' }, finishReason: 'SAFETY' }],
usageMetadata: {
promptTokenCount: 12,
candidatesTokenCount: 3,
thoughtsTokenCount: 121,
totalTokenCount: 136,
},
});
expect(result.error).toBeUndefined();
expect(result.choices[0].finish_reason).toBe('content_filter');
expect(result.usage).toMatchObject({
prompt_tokens: 12,
completion_tokens: 124,
total_tokens: 136,
});
});
}
);

it('preserves Vertex prompt-block errors across fragmented SSE transport', async () => {
const bytes = new TextEncoder().encode(
'data: ' +
JSON.stringify({
promptFeedback: { blockReason: 'PROHIBITED_CONTENT' },
modelVersion: 'gemini-3-flash-preview',
}) +
'\r\n\r\n'
);
const response = handleStreamingMode(
new Response(
new ReadableStream({
start(controller) {
for (const byte of bytes) controller.enqueue(Uint8Array.of(byte));
controller.close();
},
}),
{ headers: { 'content-type': 'text/event-stream' } }
),
GOOGLE_VERTEX_AI,
vertexStream,
'/publishers/google/models/gemini-3-flash-preview:streamGenerateContent',
false,
{ model: 'gemini-3-flash-preview' },
'chatComplete',
{ beforeRequestHooksResult: [], afterRequestHooksResult: [] }
);
const events = (await response.text()).trim().split('\n\n');
expect(events).toHaveLength(1);
expect(JSON.parse(events[0].slice(6))).toMatchObject({
error: { code: 'content_policy_violation', type: 'PROHIBITED_CONTENT' },
});
});

describe('Bedrock Converse stream errors', () => {
// AWS EventStream frame: prelude, headers, JSON payload, message CRC.
// The reader ignores headers and CRCs; exercise its actual binary framing.
Expand Down
39 changes: 24 additions & 15 deletions src/providers/google-vertex-ai/chatComplete.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import {
SYSTEM_INSTRUCTION_DISABLED_MODELS,
transformOpenAIRoleToGoogleRole,
transformToolChoiceForGemini,
GoogleErrorResponseTransform,
} from '../google/chatComplete';
import { GOOGLE_GENERATE_CONTENT_FINISH_REASON } from '../google/types';
import {
Expand Down Expand Up @@ -482,17 +483,12 @@ export const GoogleChatCompleteResponseTransform: (
);
}

if (responseStatus !== 200 && 'error' in response) {
const { error } = response;
return generateErrorResponse(
{
message: error.message,
type: error.status,
param: null,
code: String(error.code),
},
if (!Array.isArray(response)) {
const errorResponse = GoogleErrorResponseTransform(
response,
GOOGLE_VERTEX_AI
);
if (errorResponse) return errorResponse;
}

// sometimes vertex gemini returns usageMetadata without candidates
Expand Down Expand Up @@ -723,14 +719,19 @@ export const GoogleChatCompleteStreamChunkTransform: (
return `data: ${chunk}\n\n`;
}

let parsedChunk: GoogleGenerateContentResponse = JSON.parse(chunk);
const parsedChunk: GoogleGenerateContentResponse = JSON.parse(chunk);

const errorResponse = GoogleErrorResponseTransform(
parsedChunk,
GOOGLE_VERTEX_AI
);

let usageMetadata;
if (parsedChunk.usageMetadata) {
usageMetadata = {
prompt_tokens: parsedChunk.usageMetadata.promptTokenCount,
completion_tokens:
parsedChunk.usageMetadata.candidatesTokenCount +
(parsedChunk.usageMetadata.candidatesTokenCount ?? 0) +
(parsedChunk.usageMetadata.thoughtsTokenCount ?? 0),
total_tokens: parsedChunk.usageMetadata.totalTokenCount,
completion_tokens_details: {
Expand Down Expand Up @@ -774,6 +775,14 @@ export const GoogleChatCompleteStreamChunkTransform: (
};
}

if (errorResponse) {
return `data: ${JSON.stringify({
...errorResponse,
model: parsedChunk.modelVersion,
usage: usageMetadata,
})}\n\n`;
}

const dataChunk = {
id: fallbackId,
object: 'chat.completion.chunk',
Expand All @@ -790,7 +799,7 @@ export const GoogleChatCompleteStreamChunkTransform: (
)
: null;
let message: any = { role: 'assistant', content: '' };
if (generation.content?.parts[0]?.text) {
if (generation.content?.parts?.[0]?.text) {
const contentBlocks = [];
let content = '';
for (const part of generation.content.parts) {
Expand All @@ -816,7 +825,7 @@ export const GoogleChatCompleteStreamChunkTransform: (
content_blocks: contentBlocks,
}),
};
} else if (generation.content?.parts[0]?.functionCall) {
} else if (generation.content?.parts?.[0]?.functionCall) {
message = {
role: 'assistant',
tool_calls: generation.content.parts.map((part, idx) => {
Expand All @@ -837,7 +846,7 @@ export const GoogleChatCompleteStreamChunkTransform: (
}
}),
};
} else if (generation.content?.parts[0]?.inlineData) {
} else if (generation.content?.parts?.[0]?.inlineData) {
const part = generation.content.parts[0];
const contentBlocks = [
{
Expand Down Expand Up @@ -867,7 +876,7 @@ export const GoogleChatCompleteStreamChunkTransform: (
: {}),
};
}) ?? [],
...(parsedChunk.usageMetadata?.candidatesTokenCount && {
...(usageMetadata && {
usage: usageMetadata,
}),
};
Expand Down
4 changes: 3 additions & 1 deletion src/providers/google-vertex-ai/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,9 @@ export interface GoogleResponseCandidate {
export interface GoogleGenerateContentResponse {
modelVersion: string;
candidates: GoogleResponseCandidate[];
promptFeedback: {
promptFeedback?: {
blockReason?: string;
blockReasonMessage?: string;
safetyRatings: {
category: string;
probability: string;
Expand Down
Loading
Loading