diff --git a/.github/workflows/link-checker.yml b/.github/workflows/link-checker.yml index 61b585a88..545f763aa 100644 --- a/.github/workflows/link-checker.yml +++ b/.github/workflows/link-checker.yml @@ -17,16 +17,27 @@ jobs: steps: - uses: actions/checkout@v3 + - name: List Markdown files + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.number }} + run: | + if [ "$GITHUB_EVENT_NAME" = pull_request ]; then + # PRs check only the Markdown they add, change or rename. + gh api --paginate "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/files" \ + --jq '.[] | select(.status != "removed" and (.filename | endswith(".md"))) | .filename' + else + git ls-files '*.md' + fi > md-files.txt + - name: Link Checker - uses: lycheeverse/lychee-action@v1.8.0 + uses: lycheeverse/lychee-action@v2.9.0 with: - args: --verbose --no-progress './**/*.md' - fail: true - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + args: --verbose --no-progress --root-dir ${{ github.workspace }} --files-from md-files.txt + failIfEmpty: false - name: Create Issue If Failed - if: failure() + if: failure() && github.event_name != 'pull_request' uses: actions/github-script@v6 with: script: | diff --git a/src/providers/azure-ai-inference/index.ts b/src/providers/azure-ai-inference/index.ts index 4127cce2f..1bba0fce8 100644 --- a/src/providers/azure-ai-inference/index.ts +++ b/src/providers/azure-ai-inference/index.ts @@ -1,117 +1,111 @@ -import { ProviderConfigs } from "../types"; +import { ProviderConfigs } from '../types'; import { - AzureAIInferenceCompleteConfig, - AzureAIInferenceCompleteResponseTransform, -} from "./complete"; + AzureAIInferenceCompleteConfig, + AzureAIInferenceCompleteResponseTransform, +} from './complete'; import { - AzureAIInferenceEmbedConfig, - AzureAIInferenceEmbedResponseTransform, -} from "./embed"; -import AzureAIInferenceAPI from "./api"; + AzureAIInferenceEmbedConfig, + AzureAIInferenceEmbedResponseTransform, +} from './embed'; +import AzureAIInferenceAPI from './api'; import { - AzureAIInferenceChatCompleteConfig, - AzureAIInferenceChatCompleteResponseTransform, -} from "./chatComplete"; -import { AZURE_AI_INFERENCE, GITHUB } from "../../globals"; -import { AzureOpenAIImageGenerateConfig } from "../azure-openai/imageGenerate"; -import { AzureOpenAICreateSpeechConfig } from "../azure-openai/createSpeech"; -import { OpenAICreateFinetuneConfig } from "../openai/createFinetune"; -import { AzureOpenAICreateBatchConfig } from "../azure-openai/createBatch"; -import { AzureAIInferenceGetBatchOutputRequestHandler } from "./getBatchOutput"; -import { OpenAIFileUploadRequestTransform } from "../openai/uploadFile"; + AzureAIInferenceChatCompleteConfig, + AzureAIInferenceChatCompleteResponseTransform, +} from './chatComplete'; +import { AZURE_AI_INFERENCE, GITHUB } from '../../globals'; +import { AzureOpenAIImageGenerateConfig } from '../azure-openai/imageGenerate'; +import { AzureOpenAICreateSpeechConfig } from '../azure-openai/createSpeech'; +import { OpenAICreateFinetuneConfig } from '../openai/createFinetune'; +import { AzureOpenAICreateBatchConfig } from '../azure-openai/createBatch'; +import { AzureAIInferenceGetBatchOutputRequestHandler } from './getBatchOutput'; +import { OpenAIFileUploadRequestTransform } from '../openai/uploadFile'; import { - AzureAIInferenceCreateSpeechResponseTransform, - AzureAIInferenceCreateTranscriptionResponseTransform, - AzureAIInferenceCreateTranslationResponseTransform, - AzureAIInferenceResponseTransform, -} from "./utils"; + AzureAIInferenceCreateSpeechResponseTransform, + AzureAIInferenceCreateTranscriptionResponseTransform, + AzureAIInferenceCreateTranslationResponseTransform, + AzureAIInferenceResponseTransform, +} from './utils'; import { - AnthropicChatCompleteConfig, - getAnthropicChatCompleteResponseTransform, - getAnthropicStreamChunkTransform, -} from "../anthropic/chatComplete"; + AnthropicChatCompleteConfig, + getAnthropicChatCompleteResponseTransform, + getAnthropicStreamChunkTransform, +} from '../anthropic/chatComplete'; import { - AzureAIInferenceMessagesConfig, - AzureAIInferenceMessagesResponseTransform, -} from "./messages"; + AzureAIInferenceMessagesConfig, + AzureAIInferenceMessagesResponseTransform, +} from './messages'; const AzureAIInferenceAPIConfig: ProviderConfigs = { - api: AzureAIInferenceAPI, - getConfig: ({ providerOptions }) => { - const { azureFoundryUrl } = providerOptions || {}; - const isAnthropicModel = azureFoundryUrl?.includes("anthropic"); - const chatCompleteConfig = isAnthropicModel - ? AnthropicChatCompleteConfig - : AzureAIInferenceChatCompleteConfig; - const chatCompleteResponseTransform = isAnthropicModel - ? getAnthropicChatCompleteResponseTransform(AZURE_AI_INFERENCE) - : AzureAIInferenceChatCompleteResponseTransform(AZURE_AI_INFERENCE); - return { - complete: AzureAIInferenceCompleteConfig, - embed: AzureAIInferenceEmbedConfig, - chatComplete: chatCompleteConfig, - messages: AzureAIInferenceMessagesConfig, - imageGenerate: AzureOpenAIImageGenerateConfig, - imageEdit: {}, - createSpeech: AzureOpenAICreateSpeechConfig, - createFinetune: OpenAICreateFinetuneConfig, - createTranscription: {}, - createTranslation: {}, - realtime: {}, - cancelBatch: {}, - createBatch: AzureOpenAICreateBatchConfig, - cancelFinetune: {}, - requestHandlers: { - getBatchOutput: AzureAIInferenceGetBatchOutputRequestHandler, - }, - requestTransforms: { - uploadFile: OpenAIFileUploadRequestTransform, - }, - responseTransforms: { - complete: - AzureAIInferenceCompleteResponseTransform( - AZURE_AI_INFERENCE, - ), - ...(isAnthropicModel && { - "stream-chatComplete": - getAnthropicStreamChunkTransform(AZURE_AI_INFERENCE), - }), - chatComplete: chatCompleteResponseTransform, - messages: AzureAIInferenceMessagesResponseTransform, - embed: AzureAIInferenceEmbedResponseTransform( - AZURE_AI_INFERENCE, - ), - imageGenerate: AzureAIInferenceResponseTransform, - createSpeech: AzureAIInferenceCreateSpeechResponseTransform, - createTranscription: - AzureAIInferenceCreateTranscriptionResponseTransform, - createTranslation: - AzureAIInferenceCreateTranslationResponseTransform, - realtime: {}, - createBatch: AzureAIInferenceResponseTransform, - retrieveBatch: AzureAIInferenceResponseTransform, - cancelBatch: AzureAIInferenceResponseTransform, - listBatches: AzureAIInferenceResponseTransform, - uploadFile: AzureAIInferenceResponseTransform, - listFiles: AzureAIInferenceResponseTransform, - retrieveFile: AzureAIInferenceResponseTransform, - deleteFile: AzureAIInferenceResponseTransform, - retrieveFileContent: AzureAIInferenceResponseTransform, - }, - }; - }, + api: AzureAIInferenceAPI, + getConfig: ({ providerOptions }) => { + const { azureFoundryUrl } = providerOptions || {}; + const isAnthropicModel = azureFoundryUrl?.includes('anthropic'); + const chatCompleteConfig = isAnthropicModel + ? AnthropicChatCompleteConfig + : AzureAIInferenceChatCompleteConfig; + const chatCompleteResponseTransform = isAnthropicModel + ? getAnthropicChatCompleteResponseTransform(AZURE_AI_INFERENCE) + : AzureAIInferenceChatCompleteResponseTransform(AZURE_AI_INFERENCE); + return { + complete: AzureAIInferenceCompleteConfig, + embed: AzureAIInferenceEmbedConfig, + chatComplete: chatCompleteConfig, + messages: AzureAIInferenceMessagesConfig, + imageGenerate: AzureOpenAIImageGenerateConfig, + imageEdit: {}, + createSpeech: AzureOpenAICreateSpeechConfig, + createFinetune: OpenAICreateFinetuneConfig, + createTranscription: {}, + createTranslation: {}, + realtime: {}, + cancelBatch: {}, + createBatch: AzureOpenAICreateBatchConfig, + cancelFinetune: {}, + requestHandlers: { + getBatchOutput: AzureAIInferenceGetBatchOutputRequestHandler, + }, + requestTransforms: { + uploadFile: OpenAIFileUploadRequestTransform, + }, + responseTransforms: { + complete: AzureAIInferenceCompleteResponseTransform(AZURE_AI_INFERENCE), + ...(isAnthropicModel && { + 'stream-chatComplete': + getAnthropicStreamChunkTransform(AZURE_AI_INFERENCE), + }), + chatComplete: chatCompleteResponseTransform, + messages: AzureAIInferenceMessagesResponseTransform, + embed: AzureAIInferenceEmbedResponseTransform(AZURE_AI_INFERENCE), + imageGenerate: AzureAIInferenceResponseTransform, + createSpeech: AzureAIInferenceCreateSpeechResponseTransform, + createTranscription: + AzureAIInferenceCreateTranscriptionResponseTransform, + createTranslation: AzureAIInferenceCreateTranslationResponseTransform, + realtime: {}, + createBatch: AzureAIInferenceResponseTransform, + retrieveBatch: AzureAIInferenceResponseTransform, + cancelBatch: AzureAIInferenceResponseTransform, + listBatches: AzureAIInferenceResponseTransform, + uploadFile: AzureAIInferenceResponseTransform, + listFiles: AzureAIInferenceResponseTransform, + retrieveFile: AzureAIInferenceResponseTransform, + deleteFile: AzureAIInferenceResponseTransform, + retrieveFileContent: AzureAIInferenceResponseTransform, + }, + }; + }, }; const GithubModelAPiConfig: ProviderConfigs = { - complete: AzureAIInferenceCompleteConfig, - embed: AzureAIInferenceEmbedConfig, - api: AzureAIInferenceAPI, - chatComplete: AzureAIInferenceChatCompleteConfig, - responseTransforms: { - complete: AzureAIInferenceCompleteResponseTransform(GITHUB), - chatComplete: AzureAIInferenceChatCompleteResponseTransform(GITHUB), - embed: AzureAIInferenceEmbedResponseTransform(GITHUB), - }, + complete: AzureAIInferenceCompleteConfig, + embed: AzureAIInferenceEmbedConfig, + api: AzureAIInferenceAPI, + chatComplete: AzureAIInferenceChatCompleteConfig, + responseTransforms: { + complete: AzureAIInferenceCompleteResponseTransform(GITHUB), + chatComplete: AzureAIInferenceChatCompleteResponseTransform(GITHUB), + embed: AzureAIInferenceEmbedResponseTransform(GITHUB), + }, }; export { AzureAIInferenceAPIConfig, GithubModelAPiConfig }; diff --git a/src/providers/azure-ai-inference/messages.ts b/src/providers/azure-ai-inference/messages.ts index 08e11c1b3..ea30c99ad 100644 --- a/src/providers/azure-ai-inference/messages.ts +++ b/src/providers/azure-ai-inference/messages.ts @@ -1,26 +1,26 @@ -import { AZURE_AI_INFERENCE } from "../../globals"; -import { MessagesResponse } from "../../types/messagesResponse"; -import { getMessagesConfig } from "../anthropic-base/messages"; -import { AnthropicErrorResponse } from "../anthropic/types"; -import { AnthropicErrorResponseTransform } from "../anthropic/utils"; -import { ErrorResponse } from "../types"; -import { generateInvalidProviderResponseError } from "../utils"; +import { AZURE_AI_INFERENCE } from '../../globals'; +import { MessagesResponse } from '../../types/messagesResponse'; +import { getMessagesConfig } from '../anthropic-base/messages'; +import { AnthropicErrorResponse } from '../anthropic/types'; +import { AnthropicErrorResponseTransform } from '../anthropic/utils'; +import { ErrorResponse } from '../types'; +import { generateInvalidProviderResponseError } from '../utils'; export const AzureAIInferenceMessagesConfig = getMessagesConfig({}); export const AzureAIInferenceMessagesResponseTransform = ( - response: MessagesResponse | AnthropicErrorResponse, - responseStatus: number, + response: MessagesResponse | AnthropicErrorResponse, + responseStatus: number ): MessagesResponse | ErrorResponse => { - if (responseStatus !== 200) { - const errorResposne = AnthropicErrorResponseTransform( - response as AnthropicErrorResponse, - AZURE_AI_INFERENCE, - ); - if (errorResposne) return errorResposne; - } + if (responseStatus !== 200) { + const errorResposne = AnthropicErrorResponseTransform( + response as AnthropicErrorResponse, + AZURE_AI_INFERENCE + ); + if (errorResposne) return errorResposne; + } - if ("model" in response) return response; + if ('model' in response) return response; - return generateInvalidProviderResponseError(response, AZURE_AI_INFERENCE); + return generateInvalidProviderResponseError(response, AZURE_AI_INFERENCE); }; diff --git a/src/providers/google-vertex-ai/chatComplete.ts b/src/providers/google-vertex-ai/chatComplete.ts index 94dd87b0d..7d5739b1e 100644 --- a/src/providers/google-vertex-ai/chatComplete.ts +++ b/src/providers/google-vertex-ai/chatComplete.ts @@ -1,1295 +1,1245 @@ // Docs for REST API // https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/send-multimodal-prompts#gemini-send-multimodal-samples-drest -import { GOOGLE_VERTEX_AI } from "../../globals"; +import { GOOGLE_VERTEX_AI } from '../../globals'; import { - ContentType, - Message, - Params, - ToolCall, - SYSTEM_MESSAGE_ROLES, - MESSAGE_ROLES, - Options, -} from "../../types/requestBody"; + ContentType, + Message, + Params, + ToolCall, + SYSTEM_MESSAGE_ROLES, + MESSAGE_ROLES, + Options, +} from '../../types/requestBody'; import { - AnthropicChatCompleteConfig, - AnthropicChatCompleteResponse, - AnthropicChatCompleteStreamResponse, -} from "../anthropic/chatComplete"; + AnthropicChatCompleteConfig, + AnthropicChatCompleteResponse, + AnthropicChatCompleteStreamResponse, +} from '../anthropic/chatComplete'; import { - AnthropicStreamState, - AnthropicErrorResponse, -} from "../anthropic/types"; + AnthropicStreamState, + AnthropicErrorResponse, +} from '../anthropic/types'; import { - GoogleMessage, - GoogleMessagePart, - GoogleMessageRole, - GoogleToolConfig, - SYSTEM_INSTRUCTION_DISABLED_MODELS, - transformOpenAIRoleToGoogleRole, - transformToolChoiceForGemini, -} from "../google/chatComplete"; -import { GOOGLE_GENERATE_CONTENT_FINISH_REASON } from "../google/types"; + GoogleMessage, + GoogleMessagePart, + GoogleMessageRole, + GoogleToolConfig, + SYSTEM_INSTRUCTION_DISABLED_MODELS, + transformOpenAIRoleToGoogleRole, + transformToolChoiceForGemini, +} from '../google/chatComplete'; +import { GOOGLE_GENERATE_CONTENT_FINISH_REASON } from '../google/types'; import { - ChatCompletionResponse, - ErrorResponse, - Logprobs, - ProviderConfig, -} from "../types"; + ChatCompletionResponse, + ErrorResponse, + Logprobs, + ProviderConfig, +} from '../types'; import { - generateErrorResponse, - generateInvalidProviderResponseError, - transformFinishReason, -} from "../utils"; -import { transformGenerationConfig } from "./transformGenerationConfig"; -import { wasCacheCreated } from "./cachedContents"; + generateErrorResponse, + generateInvalidProviderResponseError, + transformFinishReason, +} from '../utils'; +import { transformGenerationConfig } from './transformGenerationConfig'; +import { wasCacheCreated } from './cachedContents'; import { - GoogleErrorResponse, - GoogleGenerateContentResponse, - VertexLlamaChatCompleteStreamChunk, - VertexLLamaChatCompleteResponse, - GoogleSearchRetrievalTool, - VERTEX_MODALITY, -} from "./types"; + GoogleErrorResponse, + GoogleGenerateContentResponse, + VertexLlamaChatCompleteStreamChunk, + VertexLLamaChatCompleteResponse, + GoogleSearchRetrievalTool, + VERTEX_MODALITY, +} from './types'; import { - getMimeType, - googleTools, - recursivelyDeleteUnsupportedParameters, - transformGoogleTools, - transformInputAudioPart, - transformVertexLogprobs, -} from "./utils"; + getMimeType, + googleTools, + recursivelyDeleteUnsupportedParameters, + transformGoogleTools, + transformInputAudioPart, + transformVertexLogprobs, +} from './utils'; export const VertexGoogleChatCompleteConfig: ProviderConfig = { - // https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versioning#gemini-model-versions - model: { - param: "model", - required: true, - default: "gemini-1.0-pro", - }, - messages: [ - { - param: "contents", - default: "", - transform: (params: Params) => { - let lastRole: GoogleMessageRole | undefined; - const messages: GoogleMessage[] = []; - - params.messages?.forEach((message: Message) => { - // From gemini-1.5 onwards, systemInstruction is supported - // Skipping system message and sending it in systemInstruction for gemini 1.5 models - if ( - SYSTEM_MESSAGE_ROLES.includes(message.role) && - !SYSTEM_INSTRUCTION_DISABLED_MODELS.includes( - params.model as string, - ) - ) - return; - - const role = transformOpenAIRoleToGoogleRole(message.role); - let parts: GoogleMessagePart[] = []; - - if (message.role === "assistant" && message.tool_calls) { - message.tool_calls.forEach((tool_call: ToolCall) => { - parts.push({ - functionCall: { - name: tool_call.function.name, - args: JSON.parse( - tool_call.function.arguments, - ), - }, - ...(tool_call.function.thought_signature && { - thoughtSignature: - tool_call.function.thought_signature, - }), - }); - }); - } else if (message.role === "tool") { - parts.push({ - functionResponse: { - name: - message.name ?? "gateway-tool-filler-name", - response: { - output: message.content ?? "", - }, - }, - }); - } else if ( - message.content && - typeof message.content === "object" - ) { - message.content.forEach((c: ContentType) => { - if (c.type === "text") { - parts.push({ - text: c.text ?? "", - }); - } else if (c.type === "input_audio") { - parts.push(transformInputAudioPart(c)); - } else if (c.type === "image_url") { - const { url, mime_type: passedMimeType } = - c.image_url || {}; - - if (!url) { - // Shouldn't throw error? - return; - } - - // Example: data:image/png;base64,abcdefg... - if (url.startsWith("data:")) { - const [mimeTypeWithPrefix, base64Image] = - url.split(";base64,"); - const mimeType = - mimeTypeWithPrefix.split(":")[1]; - - parts.push({ - inlineData: { - mimeType: mimeType, - data: base64Image, - }, - }); - - return; - } else if ( - url.startsWith("gs://") || - url.startsWith("https://") || - url.startsWith("http://") - ) { - parts.push({ - fileData: { - mimeType: - passedMimeType || - getMimeType(url), - fileUri: url, - }, - }); - } else { - // NOTE: This block is kept to maintain backward compatibility - // Earlier we were assuming that all images will be base64 with image/jpeg mimeType - parts.push({ - inlineData: { - mimeType: "image/jpeg", - data: c.image_url?.url ?? "", - }, - }); - } - } else if (c.type === "video_url") { - const { url, mime_type: passedMimeType } = - c.video_url || {}; - - if (!url) { - return; - } - - // Helper to detect YouTube URLs - const isYouTubeUrl = ( - videoUrl: string, - ): boolean => { - return ( - videoUrl.includes("youtube.com") || - videoUrl.includes("youtu.be") - ); - }; - - // Example: data:video/mp4;base64,abcdefg... - if (url.startsWith("data:")) { - const [mimeTypeWithPrefix, base64Video] = - url.split(";base64,"); - const mimeType = - mimeTypeWithPrefix.split(":")[1]; - - parts.push({ - inlineData: { - mimeType: mimeType, - data: base64Video, - }, - }); - } else { - // For URLs (gs://, https://, http://, YouTube) - const mimeType = - passedMimeType || - (isYouTubeUrl(url) - ? "video/mp4" - : getMimeType(url)); - parts.push({ - fileData: { - mimeType: mimeType, - fileUri: url, - }, - }); - } - } else if (c.type === "executable_code") { - parts.push({ - executableCode: c.executable_code, - }); - } else if (c.type === "code_execution_result") { - parts.push({ - codeExecutionResult: c.code_execution_result, - }); - } - }); - } else if (typeof message.content === "string") { - parts.push({ - text: message.content, - }); - } - - // @NOTE: This takes care of the "Please ensure that multiturn requests alternate between user and model." - // error that occurs when we have multiple user messages in a row. - const shouldCombineMessages = - lastRole === role && !params.model?.includes("vision"); - - if (shouldCombineMessages) { - messages[messages.length - 1].parts.push(...parts); - } else { - messages.push({ role, parts }); - } - - lastRole = role; - }); + // https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versioning#gemini-model-versions + model: { + param: 'model', + required: true, + default: 'gemini-1.0-pro', + }, + messages: [ + { + param: 'contents', + default: '', + transform: (params: Params) => { + let lastRole: GoogleMessageRole | undefined; + const messages: GoogleMessage[] = []; + + params.messages?.forEach((message: Message) => { + // From gemini-1.5 onwards, systemInstruction is supported + // Skipping system message and sending it in systemInstruction for gemini 1.5 models + if ( + SYSTEM_MESSAGE_ROLES.includes(message.role) && + !SYSTEM_INSTRUCTION_DISABLED_MODELS.includes(params.model as string) + ) + return; - return messages; - }, - }, - { - param: "systemInstruction", - default: "", - transform: (params: Params) => { - // systemInstruction is only supported from gemini 1.5 models - if ( - SYSTEM_INSTRUCTION_DISABLED_MODELS.includes( - params.model as string, - ) - ) - return; - // contents skips every system message, so all of them go here. - const parts = (params.messages ?? []) - .filter((message) => - SYSTEM_MESSAGE_ROLES.includes(message.role), - ) - .flatMap((message) => - typeof message.content === "string" - ? [{ text: message.content }] - : (message.content ?? []) - .filter((part) => part.text) - .map((part) => ({ text: part.text })), - ); - if (!parts.length) return; - return { parts, role: "system" }; - }, - }, - ], - temperature: { - param: "generationConfig", - transform: (params: Params) => transformGenerationConfig(params), - }, - top_p: { - param: "generationConfig", - transform: (params: Params) => transformGenerationConfig(params), - }, - top_k: { - param: "generationConfig", - transform: (params: Params) => transformGenerationConfig(params), - }, - max_tokens: { - param: "generationConfig", - transform: (params: Params) => transformGenerationConfig(params), - }, - max_completion_tokens: { - param: "generationConfig", - transform: (params: Params) => transformGenerationConfig(params), - }, - stop: { - param: "generationConfig", - transform: (params: Params) => transformGenerationConfig(params), - }, - response_format: { - param: "generationConfig", - transform: (params: Params) => transformGenerationConfig(params), - }, - logprobs: { - param: "generationConfig", - transform: (params: Params) => transformGenerationConfig(params), - }, - top_logprobs: { - param: "generationConfig", - transform: (params: Params) => transformGenerationConfig(params), - }, - // https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/configure-safety-attributes - // Example payload to be included in the request that sets the safety settings: - // "safety_settings": [ - // { - // "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - // "threshold": "BLOCK_NONE" - // }, - // { - // "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - // "threshold": "BLOCK_ONLY_HIGH" - // } - // ] - safety_settings: { - param: "safety_settings", - }, - tools: { - param: "tools", - default: "", - transform: (params: Params) => { - const functionDeclarations: any = []; - const tools: any = []; - params.tools?.forEach((tool) => { - if (tool.type === "function" && tool.function) { - // these are not supported by google - recursivelyDeleteUnsupportedParameters( - tool.function?.parameters, - ); - delete tool.function?.strict; - - if (googleTools.includes(tool.function.name)) { - tools.push(...transformGoogleTools(tool)); - } else { - functionDeclarations.push(tool.function); - } - } + const role = transformOpenAIRoleToGoogleRole(message.role); + let parts: GoogleMessagePart[] = []; + + if (message.role === 'assistant' && message.tool_calls) { + message.tool_calls.forEach((tool_call: ToolCall) => { + parts.push({ + functionCall: { + name: tool_call.function.name, + args: JSON.parse(tool_call.function.arguments), + }, + ...(tool_call.function.thought_signature && { + thoughtSignature: tool_call.function.thought_signature, + }), + }); }); - if (functionDeclarations.length) { - tools.push({ functionDeclarations }); - } - return tools; - }, - }, - tool_choice: { - param: "tool_config", - default: (params: Params) => { - const toolConfig = {} as GoogleToolConfig; - const googleMapsTool = params.tools?.find( - (tool) => - tool.function?.name === "googleMaps" || - tool.function?.name === "google_maps", - ); - if (googleMapsTool) { - toolConfig.retrievalConfig = - googleMapsTool.function?.parameters?.retrievalConfig; - return toolConfig; - } - return; - }, - required: true, - transform: (params: Params) => { - const toolConfig = {} as GoogleToolConfig; - if (params.tool_choice) { - const allowedFunctionNames: string[] = []; - if ( - typeof params.tool_choice === "object" && - params.tool_choice.type === "function" + } else if (message.role === 'tool') { + parts.push({ + functionResponse: { + name: message.name ?? 'gateway-tool-filler-name', + response: { + output: message.content ?? '', + }, + }, + }); + } else if (message.content && typeof message.content === 'object') { + message.content.forEach((c: ContentType) => { + if (c.type === 'text') { + parts.push({ + text: c.text ?? '', + }); + } else if (c.type === 'input_audio') { + parts.push(transformInputAudioPart(c)); + } else if (c.type === 'image_url') { + const { url, mime_type: passedMimeType } = c.image_url || {}; + + if (!url) { + // Shouldn't throw error? + return; + } + + // Example: data:image/png;base64,abcdefg... + if (url.startsWith('data:')) { + const [mimeTypeWithPrefix, base64Image] = + url.split(';base64,'); + const mimeType = mimeTypeWithPrefix.split(':')[1]; + + parts.push({ + inlineData: { + mimeType: mimeType, + data: base64Image, + }, + }); + + return; + } else if ( + url.startsWith('gs://') || + url.startsWith('https://') || + url.startsWith('http://') ) { - allowedFunctionNames.push(params.tool_choice.function.name); + parts.push({ + fileData: { + mimeType: passedMimeType || getMimeType(url), + fileUri: url, + }, + }); + } else { + // NOTE: This block is kept to maintain backward compatibility + // Earlier we were assuming that all images will be base64 with image/jpeg mimeType + parts.push({ + inlineData: { + mimeType: 'image/jpeg', + data: c.image_url?.url ?? '', + }, + }); } - toolConfig.function_calling_config = { - mode: transformToolChoiceForGemini(params.tool_choice), + } else if (c.type === 'video_url') { + const { url, mime_type: passedMimeType } = c.video_url || {}; + + if (!url) { + return; + } + + // Helper to detect YouTube URLs + const isYouTubeUrl = (videoUrl: string): boolean => { + return ( + videoUrl.includes('youtube.com') || + videoUrl.includes('youtu.be') + ); }; - if (allowedFunctionNames.length > 0) { - toolConfig.function_calling_config.allowed_function_names = - allowedFunctionNames; + + // Example: data:video/mp4;base64,abcdefg... + if (url.startsWith('data:')) { + const [mimeTypeWithPrefix, base64Video] = + url.split(';base64,'); + const mimeType = mimeTypeWithPrefix.split(':')[1]; + + parts.push({ + inlineData: { + mimeType: mimeType, + data: base64Video, + }, + }); + } else { + // For URLs (gs://, https://, http://, YouTube) + const mimeType = + passedMimeType || + (isYouTubeUrl(url) ? 'video/mp4' : getMimeType(url)); + parts.push({ + fileData: { + mimeType: mimeType, + fileUri: url, + }, + }); } - return toolConfig; - } - const googleMapsTool = params.tools?.find( - (tool) => - tool.function?.name === "googleMaps" || - tool.function?.name === "google_maps", - ); - if (googleMapsTool) { - toolConfig.retrievalConfig = - googleMapsTool.function?.parameters?.retrievalConfig; - } - return toolConfig; - }, - }, - labels: { - param: "labels", + } else if (c.type === 'executable_code') { + parts.push({ + executableCode: c.executable_code, + }); + } else if (c.type === 'code_execution_result') { + parts.push({ + codeExecutionResult: c.code_execution_result, + }); + } + }); + } else if (typeof message.content === 'string') { + parts.push({ + text: message.content, + }); + } + + // @NOTE: This takes care of the "Please ensure that multiturn requests alternate between user and model." + // error that occurs when we have multiple user messages in a row. + const shouldCombineMessages = + lastRole === role && !params.model?.includes('vision'); + + if (shouldCombineMessages) { + messages[messages.length - 1].parts.push(...parts); + } else { + messages.push({ role, parts }); + } + + lastRole = role; + }); + + return messages; + }, }, - thinking: { - param: "generationConfig", - transform: (params: Params) => transformGenerationConfig(params), + { + param: 'systemInstruction', + default: '', + transform: (params: Params) => { + // systemInstruction is only supported from gemini 1.5 models + if (SYSTEM_INSTRUCTION_DISABLED_MODELS.includes(params.model as string)) + return; + // contents skips every system message, so all of them go here. + const parts = (params.messages ?? []) + .filter((message) => SYSTEM_MESSAGE_ROLES.includes(message.role)) + .flatMap((message) => + typeof message.content === 'string' + ? [{ text: message.content }] + : (message.content ?? []) + .filter((part) => part.text) + .map((part) => ({ text: part.text })) + ); + if (!parts.length) return; + return { parts, role: 'system' }; + }, }, - modalities: { - param: "generationConfig", - transform: (params: Params) => transformGenerationConfig(params), + ], + temperature: { + param: 'generationConfig', + transform: (params: Params) => transformGenerationConfig(params), + }, + top_p: { + param: 'generationConfig', + transform: (params: Params) => transformGenerationConfig(params), + }, + top_k: { + param: 'generationConfig', + transform: (params: Params) => transformGenerationConfig(params), + }, + max_tokens: { + param: 'generationConfig', + transform: (params: Params) => transformGenerationConfig(params), + }, + max_completion_tokens: { + param: 'generationConfig', + transform: (params: Params) => transformGenerationConfig(params), + }, + stop: { + param: 'generationConfig', + transform: (params: Params) => transformGenerationConfig(params), + }, + response_format: { + param: 'generationConfig', + transform: (params: Params) => transformGenerationConfig(params), + }, + logprobs: { + param: 'generationConfig', + transform: (params: Params) => transformGenerationConfig(params), + }, + top_logprobs: { + param: 'generationConfig', + transform: (params: Params) => transformGenerationConfig(params), + }, + // https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/configure-safety-attributes + // Example payload to be included in the request that sets the safety settings: + // "safety_settings": [ + // { + // "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + // "threshold": "BLOCK_NONE" + // }, + // { + // "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", + // "threshold": "BLOCK_ONLY_HIGH" + // } + // ] + safety_settings: { + param: 'safety_settings', + }, + tools: { + param: 'tools', + default: '', + transform: (params: Params) => { + const functionDeclarations: any = []; + const tools: any = []; + params.tools?.forEach((tool) => { + if (tool.type === 'function' && tool.function) { + // these are not supported by google + recursivelyDeleteUnsupportedParameters(tool.function?.parameters); + delete tool.function?.strict; + + if (googleTools.includes(tool.function.name)) { + tools.push(...transformGoogleTools(tool)); + } else { + functionDeclarations.push(tool.function); + } + } + }); + if (functionDeclarations.length) { + tools.push({ functionDeclarations }); + } + return tools; }, - seed: { - param: "generationConfig", - transform: (params: Params) => transformGenerationConfig(params), + }, + tool_choice: { + param: 'tool_config', + default: (params: Params) => { + const toolConfig = {} as GoogleToolConfig; + const googleMapsTool = params.tools?.find( + (tool) => + tool.function?.name === 'googleMaps' || + tool.function?.name === 'google_maps' + ); + if (googleMapsTool) { + toolConfig.retrievalConfig = + googleMapsTool.function?.parameters?.retrievalConfig; + return toolConfig; + } + return; }, - reasoning_effort: { - param: "generationConfig", - transform: (params: Params) => transformGenerationConfig(params), + required: true, + transform: (params: Params) => { + const toolConfig = {} as GoogleToolConfig; + if (params.tool_choice) { + const allowedFunctionNames: string[] = []; + if ( + typeof params.tool_choice === 'object' && + params.tool_choice.type === 'function' + ) { + allowedFunctionNames.push(params.tool_choice.function.name); + } + toolConfig.function_calling_config = { + mode: transformToolChoiceForGemini(params.tool_choice), + }; + if (allowedFunctionNames.length > 0) { + toolConfig.function_calling_config.allowed_function_names = + allowedFunctionNames; + } + return toolConfig; + } + const googleMapsTool = params.tools?.find( + (tool) => + tool.function?.name === 'googleMaps' || + tool.function?.name === 'google_maps' + ); + if (googleMapsTool) { + toolConfig.retrievalConfig = + googleMapsTool.function?.parameters?.retrievalConfig; + } + return toolConfig; }, + }, + labels: { + param: 'labels', + }, + thinking: { + param: 'generationConfig', + transform: (params: Params) => transformGenerationConfig(params), + }, + modalities: { + param: 'generationConfig', + transform: (params: Params) => transformGenerationConfig(params), + }, + seed: { + param: 'generationConfig', + transform: (params: Params) => transformGenerationConfig(params), + }, + reasoning_effort: { + param: 'generationConfig', + transform: (params: Params) => transformGenerationConfig(params), + }, }; interface AnthorpicTextContentItem { - type: "text"; - text: string; + type: 'text'; + text: string; } interface AnthropicThinkingContentItem { - type: "thinking"; - thinking: string; - signature: string; + type: 'thinking'; + thinking: string; + signature: string; } interface AnthropicToolContentItem { - type: "tool_use"; - name: string; - id: string; - input: Record; + type: 'tool_use'; + name: string; + id: string; + input: Record; } type AnthropicContentItem = - | AnthorpicTextContentItem - | AnthropicThinkingContentItem - | AnthropicToolContentItem; + | AnthorpicTextContentItem + | AnthropicThinkingContentItem + | AnthropicToolContentItem; export const VertexAnthropicChatCompleteConfig: ProviderConfig = { - ...AnthropicChatCompleteConfig, - anthropic_version: { - param: "anthropic_version", - required: true, - default: "vertex-2023-10-16", - transform: (params: Params, providerOptions?: Options) => { - return ( - providerOptions?.anthropicVersion || - params.anthropic_version || - "vertex-2023-10-16" - ); - }, + ...AnthropicChatCompleteConfig, + anthropic_version: { + param: 'anthropic_version', + required: true, + default: 'vertex-2023-10-16', + transform: (params: Params, providerOptions?: Options) => { + return ( + providerOptions?.anthropicVersion || + params.anthropic_version || + 'vertex-2023-10-16' + ); }, - model: { - param: "model", - required: false, - transform: () => { - return undefined; - }, + }, + model: { + param: 'model', + required: false, + transform: () => { + return undefined; }, + }, }; export const GoogleChatCompleteResponseTransform: ( - response: - | GoogleGenerateContentResponse - | GoogleErrorResponse - | GoogleErrorResponse[], - responseStatus: number, - responseHeaders: Headers, - strictOpenAiCompliance: boolean, - gatewayRequestUrl?: string, - gatewayRequest?: Params, + response: + | GoogleGenerateContentResponse + | GoogleErrorResponse + | GoogleErrorResponse[], + responseStatus: number, + responseHeaders: Headers, + strictOpenAiCompliance: boolean, + gatewayRequestUrl?: string, + gatewayRequest?: Params ) => ChatCompletionResponse | ErrorResponse = ( - response, - responseStatus, - _responseHeaders, - strictOpenAiCompliance, - _gatewayRequestUrl, - gatewayRequest, + response, + responseStatus, + _responseHeaders, + strictOpenAiCompliance, + _gatewayRequestUrl, + gatewayRequest ) => { - // when error occurs on streaming request, the response is an array of errors. - if ( - responseStatus !== 200 && - Array.isArray(response) && - response.length > 0 && - "error" in response[0] - ) { - const { error } = response[0]; - - return generateErrorResponse( - { - message: error.message, - type: error.status, - param: null, - code: String(error.code), - }, - GOOGLE_VERTEX_AI, - ); - } - - if (responseStatus !== 200 && "error" in response) { - const { error } = response; - return generateErrorResponse( - { - message: error.message, - type: error.status, - param: null, - code: String(error.code), - }, - GOOGLE_VERTEX_AI, - ); - } - - // sometimes vertex gemini returns usageMetadata without candidates - const isValidResponse = - "candidates" in response || "usageMetadata" in response; - if (isValidResponse) { - const { - promptTokenCount = 0, - candidatesTokenCount = 0, - totalTokenCount = 0, - thoughtsTokenCount = 0, - cachedContentTokenCount = 0, - promptTokensDetails = [], - candidatesTokensDetails = [], - } = response.usageMetadata; - const inputAudioTokens = promptTokensDetails.reduce((acc, curr) => { - if (curr.modality === VERTEX_MODALITY.AUDIO) - return acc + curr.tokenCount; - return acc; - }, 0); - const outputAudioTokens = candidatesTokensDetails.reduce( - (acc, curr) => { - if (curr.modality === VERTEX_MODALITY.AUDIO) - return acc + curr.tokenCount; - return acc; - }, - 0, - ); - - return { - id: "portkey-" + crypto.randomUUID(), - object: "chat.completion", - created: Math.floor(Date.now() / 1000), - model: response.modelVersion, - provider: GOOGLE_VERTEX_AI, - choices: - response.candidates?.map((generation, index) => { - // transform tool calls and content by iterating over the content parts - const toolCalls: ToolCall[] = []; - let content: string | undefined; - const contentBlocks = []; - for (const part of generation.content?.parts ?? []) { - if (part.functionCall) { - toolCalls.push({ - id: "portkey-" + crypto.randomUUID(), - type: "function", - function: { - name: part.functionCall.name, - arguments: JSON.stringify( - part.functionCall.args, - ), - ...(!strictOpenAiCompliance && - part.thoughtSignature && { - thought_signature: - part.thoughtSignature, - }), - }, - }); - } else if (part.text) { - if (part.thought) { - contentBlocks.push({ - type: "thinking", - thinking: part.text, - }); - } else { - content = content - ? content + part.text - : part.text; - contentBlocks.push({ - type: "text", - text: part.text, - }); - } - } else if (part.inlineData) { - contentBlocks.push({ - type: "image_url", - image_url: { - url: `data:${part.inlineData.mimeType};base64,${part.inlineData.data}`, - }, - }); - } else if (part.executableCode) { - contentBlocks.push({ - type: "executable_code", - executable_code: part.executableCode, - }); - } else if (part.codeExecutionResult) { - contentBlocks.push({ - type: "code_execution_result", - code_execution_result: part.codeExecutionResult, - }); - } - } - - const message = { - role: MESSAGE_ROLES.ASSISTANT, - ...(toolCalls.length && { tool_calls: toolCalls }), - ...(content && { content }), - ...(!strictOpenAiCompliance && - contentBlocks.length && { - content_blocks: contentBlocks, - }), - }; - const logprobsContent: Logprobs[] | null = - transformVertexLogprobs(generation); - let logprobs; - if (logprobsContent) { - logprobs = { - content: logprobsContent, - }; - } - - return { - message: message, - index: index, - finish_reason: transformFinishReason( - generation.finishReason as GOOGLE_GENERATE_CONTENT_FINISH_REASON, - strictOpenAiCompliance, - ), - logprobs, - ...(!strictOpenAiCompliance && { - safetyRatings: generation.safetyRatings, - }), - ...(!strictOpenAiCompliance && - generation.groundingMetadata - ? { - groundingMetadata: - generation.groundingMetadata, - } - : {}), - }; - }) ?? [], - usage: { - prompt_tokens: promptTokenCount, - completion_tokens: candidatesTokenCount + thoughtsTokenCount, - total_tokens: totalTokenCount, - completion_tokens_details: { - reasoning_tokens: thoughtsTokenCount, - audio_tokens: outputAudioTokens, + // when error occurs on streaming request, the response is an array of errors. + if ( + responseStatus !== 200 && + Array.isArray(response) && + response.length > 0 && + 'error' in response[0] + ) { + const { error } = response[0]; + + return generateErrorResponse( + { + message: error.message, + type: error.status, + param: null, + code: String(error.code), + }, + GOOGLE_VERTEX_AI + ); + } + + if (responseStatus !== 200 && 'error' in response) { + const { error } = response; + return generateErrorResponse( + { + message: error.message, + type: error.status, + param: null, + code: String(error.code), + }, + GOOGLE_VERTEX_AI + ); + } + + // sometimes vertex gemini returns usageMetadata without candidates + const isValidResponse = + 'candidates' in response || 'usageMetadata' in response; + if (isValidResponse) { + const { + promptTokenCount = 0, + candidatesTokenCount = 0, + totalTokenCount = 0, + thoughtsTokenCount = 0, + cachedContentTokenCount = 0, + promptTokensDetails = [], + candidatesTokensDetails = [], + } = response.usageMetadata; + const inputAudioTokens = promptTokensDetails.reduce((acc, curr) => { + if (curr.modality === VERTEX_MODALITY.AUDIO) return acc + curr.tokenCount; + return acc; + }, 0); + const outputAudioTokens = candidatesTokensDetails.reduce((acc, curr) => { + if (curr.modality === VERTEX_MODALITY.AUDIO) return acc + curr.tokenCount; + return acc; + }, 0); + + return { + id: 'portkey-' + crypto.randomUUID(), + object: 'chat.completion', + created: Math.floor(Date.now() / 1000), + model: response.modelVersion, + provider: GOOGLE_VERTEX_AI, + choices: + response.candidates?.map((generation, index) => { + // transform tool calls and content by iterating over the content parts + const toolCalls: ToolCall[] = []; + let content: string | undefined; + const contentBlocks = []; + for (const part of generation.content?.parts ?? []) { + if (part.functionCall) { + toolCalls.push({ + id: 'portkey-' + crypto.randomUUID(), + type: 'function', + function: { + name: part.functionCall.name, + arguments: JSON.stringify(part.functionCall.args), + ...(!strictOpenAiCompliance && + part.thoughtSignature && { + thought_signature: part.thoughtSignature, + }), }, - prompt_tokens_details: { - // The request that created the cachedContents resource is - // billed the whole prefix at the standard input rate, so - // zero the cached discount for it (see cachedContents.ts). - cached_tokens: wasCacheCreated(gatewayRequest) - ? 0 - : cachedContentTokenCount, - audio_tokens: inputAudioTokens, + }); + } else if (part.text) { + if (part.thought) { + contentBlocks.push({ + type: 'thinking', + thinking: part.text, + }); + } else { + content = content ? content + part.text : part.text; + contentBlocks.push({ + type: 'text', + text: part.text, + }); + } + } else if (part.inlineData) { + contentBlocks.push({ + type: 'image_url', + image_url: { + url: `data:${part.inlineData.mimeType};base64,${part.inlineData.data}`, }, - // Report the written prefix on the creating request so - // downstream billing can price the create and its cache - // storage explicitly (Anthropic usage convention). - ...(wasCacheCreated(gatewayRequest) && - cachedContentTokenCount > 0 - ? { cache_creation_input_tokens: cachedContentTokenCount } - : {}), - }, - }; - } + }); + } else if (part.executableCode) { + contentBlocks.push({ + type: 'executable_code', + executable_code: part.executableCode, + }); + } else if (part.codeExecutionResult) { + contentBlocks.push({ + type: 'code_execution_result', + code_execution_result: part.codeExecutionResult, + }); + } + } + + const message = { + role: MESSAGE_ROLES.ASSISTANT, + ...(toolCalls.length && { tool_calls: toolCalls }), + ...(content && { content }), + ...(!strictOpenAiCompliance && + contentBlocks.length && { + content_blocks: contentBlocks, + }), + }; + const logprobsContent: Logprobs[] | null = + transformVertexLogprobs(generation); + let logprobs; + if (logprobsContent) { + logprobs = { + content: logprobsContent, + }; + } + + return { + message: message, + index: index, + finish_reason: transformFinishReason( + generation.finishReason as GOOGLE_GENERATE_CONTENT_FINISH_REASON, + strictOpenAiCompliance + ), + logprobs, + ...(!strictOpenAiCompliance && { + safetyRatings: generation.safetyRatings, + }), + ...(!strictOpenAiCompliance && generation.groundingMetadata + ? { + groundingMetadata: generation.groundingMetadata, + } + : {}), + }; + }) ?? [], + usage: { + prompt_tokens: promptTokenCount, + completion_tokens: candidatesTokenCount + thoughtsTokenCount, + total_tokens: totalTokenCount, + completion_tokens_details: { + reasoning_tokens: thoughtsTokenCount, + audio_tokens: outputAudioTokens, + }, + prompt_tokens_details: { + // The request that created the cachedContents resource is + // billed the whole prefix at the standard input rate, so + // zero the cached discount for it (see cachedContents.ts). + cached_tokens: wasCacheCreated(gatewayRequest) + ? 0 + : cachedContentTokenCount, + audio_tokens: inputAudioTokens, + }, + // Report the written prefix on the creating request so + // downstream billing can price the create and its cache + // storage explicitly (Anthropic usage convention). + ...(wasCacheCreated(gatewayRequest) && cachedContentTokenCount > 0 + ? { cache_creation_input_tokens: cachedContentTokenCount } + : {}), + }, + }; + } - return generateInvalidProviderResponseError(response, GOOGLE_VERTEX_AI); + return generateInvalidProviderResponseError(response, GOOGLE_VERTEX_AI); }; export const VertexLlamaChatCompleteConfig: ProviderConfig = { - model: { - param: "model", - required: true, - default: "meta/llama-3.1-405b-instruct-maas", - transform: (params: Params) => { - return ( - params.model?.replace("meta.", "meta/") || - "meta/llama-3.1-405b-instruct-maas" - ); - }, - }, - messages: { - param: "messages", - required: true, - default: [], - }, - max_tokens: { - param: "max_tokens", - default: 512, - min: 1, - max: 2048, - }, - max_completion_tokens: { - param: "max_tokens", - default: 512, - min: 1, - max: 2048, - }, - temperature: { - param: "temperature", - default: 0.5, - min: 0, - max: 1, - }, - top_p: { - param: "top_p", - default: 0.9, - min: 0, - max: 1, - }, - top_k: { - param: "top_k", - default: 0, - min: 0, - max: 2048, - }, - stream: { - param: "stream", - default: false, - }, - image_config: { - param: "generationConfig", - transform: (params: Params) => transformGenerationConfig(params), + model: { + param: 'model', + required: true, + default: 'meta/llama-3.1-405b-instruct-maas', + transform: (params: Params) => { + return ( + params.model?.replace('meta.', 'meta/') || + 'meta/llama-3.1-405b-instruct-maas' + ); }, + }, + messages: { + param: 'messages', + required: true, + default: [], + }, + max_tokens: { + param: 'max_tokens', + default: 512, + min: 1, + max: 2048, + }, + max_completion_tokens: { + param: 'max_tokens', + default: 512, + min: 1, + max: 2048, + }, + temperature: { + param: 'temperature', + default: 0.5, + min: 0, + max: 1, + }, + top_p: { + param: 'top_p', + default: 0.9, + min: 0, + max: 1, + }, + top_k: { + param: 'top_k', + default: 0, + min: 0, + max: 2048, + }, + stream: { + param: 'stream', + default: false, + }, + image_config: { + param: 'generationConfig', + transform: (params: Params) => transformGenerationConfig(params), + }, }; export const GoogleChatCompleteStreamChunkTransform: ( - response: string, - fallbackId: string, - streamState: any, - strictOpenAiCompliance: boolean, - gatewayRequest?: Params, + response: string, + fallbackId: string, + streamState: any, + strictOpenAiCompliance: boolean, + gatewayRequest?: Params ) => string = ( - responseChunk, - fallbackId, - streamState, - strictOpenAiCompliance, - gatewayRequest, + responseChunk, + fallbackId, + streamState, + strictOpenAiCompliance, + gatewayRequest ) => { - streamState.containsChainOfThoughtMessage = - streamState?.containsChainOfThoughtMessage ?? false; - const chunk = responseChunk - .trim() - .replace(/^data: /, "") - .trim(); - - if (chunk === "[DONE]") { - return `data: ${chunk}\n\n`; - } - - let parsedChunk: GoogleGenerateContentResponse = JSON.parse(chunk); - - let usageMetadata; - if (parsedChunk.usageMetadata) { - usageMetadata = { - prompt_tokens: parsedChunk.usageMetadata.promptTokenCount, - completion_tokens: parsedChunk.usageMetadata.candidatesTokenCount + (parsedChunk.usageMetadata.thoughtsTokenCount ?? 0), - total_tokens: parsedChunk.usageMetadata.totalTokenCount, - completion_tokens_details: { - reasoning_tokens: - parsedChunk.usageMetadata.thoughtsTokenCount ?? 0, - audio_tokens: - parsedChunk.usageMetadata?.candidatesTokensDetails?.reduce( - (acc, curr) => { - if (curr.modality === VERTEX_MODALITY.AUDIO) - return acc + curr.tokenCount; - return acc; - }, - 0, - ), - }, - prompt_tokens_details: { - // Zero the cached discount on the cache-creating request; it is - // billed the whole prefix at the standard input rate (see - // cachedContents.ts). - cached_tokens: wasCacheCreated(gatewayRequest) - ? 0 - : parsedChunk.usageMetadata.cachedContentTokenCount, - audio_tokens: - parsedChunk.usageMetadata?.promptTokensDetails?.reduce( - (acc, curr) => { - if (curr.modality === VERTEX_MODALITY.AUDIO) - return acc + curr.tokenCount; - return acc; - }, - 0, - ), + streamState.containsChainOfThoughtMessage = + streamState?.containsChainOfThoughtMessage ?? false; + const chunk = responseChunk + .trim() + .replace(/^data: /, '') + .trim(); + + if (chunk === '[DONE]') { + return `data: ${chunk}\n\n`; + } + + let parsedChunk: GoogleGenerateContentResponse = JSON.parse(chunk); + + let usageMetadata; + if (parsedChunk.usageMetadata) { + usageMetadata = { + prompt_tokens: parsedChunk.usageMetadata.promptTokenCount, + completion_tokens: + parsedChunk.usageMetadata.candidatesTokenCount + + (parsedChunk.usageMetadata.thoughtsTokenCount ?? 0), + total_tokens: parsedChunk.usageMetadata.totalTokenCount, + completion_tokens_details: { + reasoning_tokens: parsedChunk.usageMetadata.thoughtsTokenCount ?? 0, + audio_tokens: + parsedChunk.usageMetadata?.candidatesTokensDetails?.reduce( + (acc, curr) => { + if (curr.modality === VERTEX_MODALITY.AUDIO) + return acc + curr.tokenCount; + return acc; }, - // Report the written prefix on the creating request so downstream - // billing can price the create and its cache storage explicitly - // (Anthropic usage convention). - ...(wasCacheCreated(gatewayRequest) && - (parsedChunk.usageMetadata.cachedContentTokenCount ?? 0) > 0 - ? { - cache_creation_input_tokens: - parsedChunk.usageMetadata.cachedContentTokenCount, - } - : {}), - }; - } - - const dataChunk = { - id: fallbackId, - object: "chat.completion.chunk", - created: Math.floor(Date.now() / 1000), - model: parsedChunk.modelVersion, - provider: GOOGLE_VERTEX_AI, - choices: - parsedChunk.candidates?.map((generation, index) => { - const finishReason = generation.finishReason - ? transformFinishReason( - parsedChunk.candidates[0] - .finishReason as GOOGLE_GENERATE_CONTENT_FINISH_REASON, - strictOpenAiCompliance, - ) - : null; - let message: any = { role: "assistant", content: "" }; - if (generation.content?.parts[0]?.text) { - const contentBlocks = []; - let content = ""; - for (const part of generation.content.parts) { - if (part.thought) { - contentBlocks.push({ - index: 0, - delta: { thinking: part.text }, - }); - streamState.containsChainOfThoughtMessage = true; - } else { - content += part.text ?? ""; - contentBlocks.push({ - index: streamState.containsChainOfThoughtMessage - ? 1 - : 0, - delta: { text: part.text }, - }); - } - } - message = { - role: "assistant", - content, - ...(!strictOpenAiCompliance && - contentBlocks.length && { - content_blocks: contentBlocks, - }), - }; - } else if (generation.content?.parts[0]?.functionCall) { - message = { - role: "assistant", - tool_calls: generation.content.parts.map( - (part, idx) => { - if (part.functionCall) { - return { - index: idx, - id: "portkey-" + crypto.randomUUID(), - type: "function", - function: { - name: part.functionCall.name, - arguments: JSON.stringify( - part.functionCall.args, - ), - ...(!strictOpenAiCompliance && - part.thoughtSignature && { - thought_signature: - part.thoughtSignature, - }), - }, - }; - } - }, - ), - }; - } else if (generation.content?.parts[0]?.inlineData) { - const part = generation.content.parts[0]; - const contentBlocks = [ - { - index: streamState.containsChainOfThoughtMessage - ? 1 - : 0, - delta: { - type: "image_url", - image_url: { - url: `data:${part.inlineData?.mimeType};base64,${part.inlineData?.data}`, - }, - }, - }, - ]; - message = { - role: "assistant", - content_blocks: contentBlocks, - }; - } + 0 + ), + }, + prompt_tokens_details: { + // Zero the cached discount on the cache-creating request; it is + // billed the whole prefix at the standard input rate (see + // cachedContents.ts). + cached_tokens: wasCacheCreated(gatewayRequest) + ? 0 + : parsedChunk.usageMetadata.cachedContentTokenCount, + audio_tokens: parsedChunk.usageMetadata?.promptTokensDetails?.reduce( + (acc, curr) => { + if (curr.modality === VERTEX_MODALITY.AUDIO) + return acc + curr.tokenCount; + return acc; + }, + 0 + ), + }, + // Report the written prefix on the creating request so downstream + // billing can price the create and its cache storage explicitly + // (Anthropic usage convention). + ...(wasCacheCreated(gatewayRequest) && + (parsedChunk.usageMetadata.cachedContentTokenCount ?? 0) > 0 + ? { + cache_creation_input_tokens: + parsedChunk.usageMetadata.cachedContentTokenCount, + } + : {}), + }; + } + + const dataChunk = { + id: fallbackId, + object: 'chat.completion.chunk', + created: Math.floor(Date.now() / 1000), + model: parsedChunk.modelVersion, + provider: GOOGLE_VERTEX_AI, + choices: + parsedChunk.candidates?.map((generation, index) => { + const finishReason = generation.finishReason + ? transformFinishReason( + parsedChunk.candidates[0] + .finishReason as GOOGLE_GENERATE_CONTENT_FINISH_REASON, + strictOpenAiCompliance + ) + : null; + let message: any = { role: 'assistant', content: '' }; + if (generation.content?.parts[0]?.text) { + const contentBlocks = []; + let content = ''; + for (const part of generation.content.parts) { + if (part.thought) { + contentBlocks.push({ + index: 0, + delta: { thinking: part.text }, + }); + streamState.containsChainOfThoughtMessage = true; + } else { + content += part.text ?? ''; + contentBlocks.push({ + index: streamState.containsChainOfThoughtMessage ? 1 : 0, + delta: { text: part.text }, + }); + } + } + message = { + role: 'assistant', + content, + ...(!strictOpenAiCompliance && + contentBlocks.length && { + content_blocks: contentBlocks, + }), + }; + } else if (generation.content?.parts[0]?.functionCall) { + message = { + role: 'assistant', + tool_calls: generation.content.parts.map((part, idx) => { + if (part.functionCall) { return { - delta: message, - index: index, - finish_reason: finishReason, - ...(!strictOpenAiCompliance && { - safetyRatings: generation.safetyRatings, - }), - ...(!strictOpenAiCompliance && generation.groundingMetadata - ? { groundingMetadata: generation.groundingMetadata } - : {}), + index: idx, + id: 'portkey-' + crypto.randomUUID(), + type: 'function', + function: { + name: part.functionCall.name, + arguments: JSON.stringify(part.functionCall.args), + ...(!strictOpenAiCompliance && + part.thoughtSignature && { + thought_signature: part.thoughtSignature, + }), + }, }; - }) ?? [], - ...(parsedChunk.usageMetadata?.candidatesTokenCount && { - usage: usageMetadata, - }), - }; + } + }), + }; + } else if (generation.content?.parts[0]?.inlineData) { + const part = generation.content.parts[0]; + const contentBlocks = [ + { + index: streamState.containsChainOfThoughtMessage ? 1 : 0, + delta: { + type: 'image_url', + image_url: { + url: `data:${part.inlineData?.mimeType};base64,${part.inlineData?.data}`, + }, + }, + }, + ]; + message = { + role: 'assistant', + content_blocks: contentBlocks, + }; + } + return { + delta: message, + index: index, + finish_reason: finishReason, + ...(!strictOpenAiCompliance && { + safetyRatings: generation.safetyRatings, + }), + ...(!strictOpenAiCompliance && generation.groundingMetadata + ? { groundingMetadata: generation.groundingMetadata } + : {}), + }; + }) ?? [], + ...(parsedChunk.usageMetadata?.candidatesTokenCount && { + usage: usageMetadata, + }), + }; - return `data: ${JSON.stringify(dataChunk)}\n\n`; + return `data: ${JSON.stringify(dataChunk)}\n\n`; }; export const AnthropicErrorResponseTransform: ( - response: AnthropicErrorResponse, + response: AnthropicErrorResponse ) => ErrorResponse | undefined = (response) => { - if ("error" in response) { - return generateErrorResponse( - { - message: response.error?.message, - type: response.error?.type, - param: null, - code: null, - }, - GOOGLE_VERTEX_AI, - ); - } + if ('error' in response) { + return generateErrorResponse( + { + message: response.error?.message, + type: response.error?.type, + param: null, + code: null, + }, + GOOGLE_VERTEX_AI + ); + } - return undefined; + return undefined; }; export const VertexAnthropicChatCompleteResponseTransform: ( - response: AnthropicChatCompleteResponse | AnthropicErrorResponse, - responseStatus: number, - responseHeaders: Headers, - strictOpenAiCompliance: boolean, + response: AnthropicChatCompleteResponse | AnthropicErrorResponse, + responseStatus: number, + responseHeaders: Headers, + strictOpenAiCompliance: boolean ) => ChatCompletionResponse | ErrorResponse = ( - response, - responseStatus, - _responseHeaders, - strictOpenAiCompliance, + response, + responseStatus, + _responseHeaders, + strictOpenAiCompliance ) => { - if (responseStatus !== 200) { - const errorResposne = AnthropicErrorResponseTransform( - response as AnthropicErrorResponse, - ); - if (errorResposne) return errorResposne; - } - - if ("content" in response) { - const { - input_tokens = 0, - output_tokens = 0, - cache_creation_input_tokens = 0, - cache_read_input_tokens = 0, - } = response?.usage ?? {}; - - const totalTokens = - input_tokens + - output_tokens + - cache_creation_input_tokens + - cache_read_input_tokens; - - const shouldSendCacheUsage = - !strictOpenAiCompliance && - (cache_creation_input_tokens || cache_read_input_tokens); - - let content: AnthropicContentItem[] | string = strictOpenAiCompliance - ? "" - : []; - response.content.forEach((item) => { - if (!strictOpenAiCompliance && Array.isArray(content)) { - if (["text", "thinking"].includes(item.type)) { - content.push(item); - } - } else { - if (item.type === "text") { - content += item.text; - } - } - }); - - let toolCalls: any = []; - response.content.forEach((item) => { - if (item.type === "tool_use") { - toolCalls.push({ - id: item.id, - type: "function", - function: { - name: item.name, - arguments: JSON.stringify(item.input), - }, - }); - } + if (responseStatus !== 200) { + const errorResposne = AnthropicErrorResponseTransform( + response as AnthropicErrorResponse + ); + if (errorResposne) return errorResposne; + } + + if ('content' in response) { + const { + input_tokens = 0, + output_tokens = 0, + cache_creation_input_tokens = 0, + cache_read_input_tokens = 0, + } = response?.usage ?? {}; + + const totalTokens = + input_tokens + + output_tokens + + cache_creation_input_tokens + + cache_read_input_tokens; + + const shouldSendCacheUsage = + !strictOpenAiCompliance && + (cache_creation_input_tokens || cache_read_input_tokens); + + let content: AnthropicContentItem[] | string = strictOpenAiCompliance + ? '' + : []; + response.content.forEach((item) => { + if (!strictOpenAiCompliance && Array.isArray(content)) { + if (['text', 'thinking'].includes(item.type)) { + content.push(item); + } + } else { + if (item.type === 'text') { + content += item.text; + } + } + }); + + let toolCalls: any = []; + response.content.forEach((item) => { + if (item.type === 'tool_use') { + toolCalls.push({ + id: item.id, + type: 'function', + function: { + name: item.name, + arguments: JSON.stringify(item.input), + }, }); + } + }); + + return { + id: response.id, + object: 'chat.completion', + created: Math.floor(Date.now() / 1000), + model: response.model, + provider: GOOGLE_VERTEX_AI, + choices: [ + { + message: { + role: 'assistant', + content, + tool_calls: toolCalls.length ? toolCalls : undefined, + }, + index: 0, + logprobs: null, + finish_reason: transformFinishReason( + response.stop_reason, + strictOpenAiCompliance + ), + }, + ], + usage: { + prompt_tokens: input_tokens, + completion_tokens: output_tokens, + total_tokens: totalTokens, + prompt_tokens_details: { + cached_tokens: cache_read_input_tokens, + }, + ...(shouldSendCacheUsage && { + cache_read_input_tokens: cache_read_input_tokens, + cache_creation_input_tokens: cache_creation_input_tokens, + }), + }, + }; + } - return { - id: response.id, - object: "chat.completion", - created: Math.floor(Date.now() / 1000), - model: response.model, - provider: GOOGLE_VERTEX_AI, - choices: [ - { - message: { - role: "assistant", - content, - tool_calls: toolCalls.length ? toolCalls : undefined, - }, - index: 0, - logprobs: null, - finish_reason: transformFinishReason( - response.stop_reason, - strictOpenAiCompliance, - ), - }, - ], - usage: { - prompt_tokens: input_tokens, - completion_tokens: output_tokens, - total_tokens: totalTokens, - prompt_tokens_details: { - cached_tokens: cache_read_input_tokens, - }, - ...(shouldSendCacheUsage && { - cache_read_input_tokens: cache_read_input_tokens, - cache_creation_input_tokens: cache_creation_input_tokens, - }), - }, - }; - } - - return generateInvalidProviderResponseError(response, GOOGLE_VERTEX_AI); + return generateInvalidProviderResponseError(response, GOOGLE_VERTEX_AI); }; export const VertexAnthropicChatCompleteStreamChunkTransform: ( - response: string, - fallbackId: string, - streamState: AnthropicStreamState, - strictOpenAiCompliance: boolean, + response: string, + fallbackId: string, + streamState: AnthropicStreamState, + strictOpenAiCompliance: boolean ) => string | undefined = ( - responseChunk, - fallbackId, - streamState, - strictOpenAiCompliance, + responseChunk, + fallbackId, + streamState, + strictOpenAiCompliance ) => { - if (streamState.toolIndex == undefined) { - streamState.toolIndex = -1; - } - let chunk = responseChunk.trim(); - - if ( - chunk.startsWith("event: ping") || - chunk.startsWith("event: content_block_stop") || - chunk.startsWith("event: vertex_event") - ) { - return; - } - - if (chunk.startsWith("event: message_stop")) { - return "data: [DONE]\n\n"; - } - - chunk = chunk.replace(/^event: content_block_delta[\r\n]*/, ""); - chunk = chunk.replace(/^event: content_block_start[\r\n]*/, ""); - chunk = chunk.replace(/^event: message_delta[\r\n]*/, ""); - chunk = chunk.replace(/^event: message_start[\r\n]*/, ""); - chunk = chunk.replace(/^event: error[\r\n]*/, ""); - chunk = chunk.replace(/^data: /, ""); - chunk = chunk.trim(); - - const parsedChunk: AnthropicChatCompleteStreamResponse = JSON.parse(chunk); - - if (parsedChunk.type === "error" && parsedChunk.error) { - return ( - `data: ${JSON.stringify({ - id: fallbackId, - object: "chat.completion.chunk", - created: Math.floor(Date.now() / 1000), - model: "", - provider: GOOGLE_VERTEX_AI, - choices: [ - { - finish_reason: parsedChunk.error.type, - delta: { - content: "", - }, - }, - ], - })}` + - "\n\n" + - "data: [DONE]\n\n" - ); - } - - if (parsedChunk.type === "message_start" && parsedChunk.message?.usage) { - const shouldSendCacheUsage = - parsedChunk.message?.usage?.cache_read_input_tokens || - parsedChunk.message?.usage?.cache_creation_input_tokens; - - streamState.model = parsedChunk?.message?.model ?? ""; - - streamState.usage = { - prompt_tokens: parsedChunk.message.usage?.input_tokens, - ...(shouldSendCacheUsage && { - cache_read_input_tokens: - parsedChunk.message?.usage?.cache_read_input_tokens, - cache_creation_input_tokens: - parsedChunk.message?.usage?.cache_creation_input_tokens, - }), - }; - return ( - `data: ${JSON.stringify({ - id: fallbackId, - object: "chat.completion.chunk", - created: Math.floor(Date.now() / 1000), - model: streamState.model, - provider: GOOGLE_VERTEX_AI, - choices: [ - { - delta: { - content: "", - }, - index: 0, - logprobs: null, - finish_reason: null, - }, - ], - usage: { - prompt_tokens: streamState.usage.prompt_tokens, - }, - })}` + "\n\n" - ); - } - - if (parsedChunk.type === "message_delta" && parsedChunk.usage) { - const totalTokens = - (streamState?.usage?.prompt_tokens ?? 0) + - (streamState?.usage?.cache_creation_input_tokens ?? 0) + - (streamState?.usage?.cache_read_input_tokens ?? 0) + - (parsedChunk.usage.output_tokens ?? 0); - - return ( - `data: ${JSON.stringify({ - id: fallbackId, - object: "chat.completion.chunk", - created: Math.floor(Date.now() / 1000), - model: streamState.model, - provider: GOOGLE_VERTEX_AI, - choices: [ - { - index: 0, - delta: {}, - finish_reason: transformFinishReason( - parsedChunk.delta?.stop_reason, - strictOpenAiCompliance, - ), - }, - ], - usage: { - ...streamState.usage, - completion_tokens: parsedChunk.usage?.output_tokens, - total_tokens: totalTokens, - prompt_tokens_details: { - cached_tokens: - streamState.usage?.cache_read_input_tokens ?? 0, - }, - }, - })}` + "\n\n" - ); - } - - const toolCalls = []; - const isToolBlockStart: boolean = - parsedChunk.type === "content_block_start" && - parsedChunk.content_block?.type === "tool_use"; - if (isToolBlockStart) { - streamState.toolIndex = streamState.toolIndex + 1; - } - const isToolBlockDelta: boolean = - parsedChunk.type === "content_block_delta" && - parsedChunk.delta?.partial_json != undefined; - - if (isToolBlockStart && parsedChunk.content_block) { - toolCalls.push({ - index: streamState.toolIndex, - id: parsedChunk.content_block.id, - type: "function", - function: { - name: parsedChunk.content_block.name, - arguments: "", + if (streamState.toolIndex == undefined) { + streamState.toolIndex = -1; + } + let chunk = responseChunk.trim(); + + if ( + chunk.startsWith('event: ping') || + chunk.startsWith('event: content_block_stop') || + chunk.startsWith('event: vertex_event') + ) { + return; + } + + if (chunk.startsWith('event: message_stop')) { + return 'data: [DONE]\n\n'; + } + + chunk = chunk.replace(/^event: content_block_delta[\r\n]*/, ''); + chunk = chunk.replace(/^event: content_block_start[\r\n]*/, ''); + chunk = chunk.replace(/^event: message_delta[\r\n]*/, ''); + chunk = chunk.replace(/^event: message_start[\r\n]*/, ''); + chunk = chunk.replace(/^event: error[\r\n]*/, ''); + chunk = chunk.replace(/^data: /, ''); + chunk = chunk.trim(); + + const parsedChunk: AnthropicChatCompleteStreamResponse = JSON.parse(chunk); + + if (parsedChunk.type === 'error' && parsedChunk.error) { + return ( + `data: ${JSON.stringify({ + id: fallbackId, + object: 'chat.completion.chunk', + created: Math.floor(Date.now() / 1000), + model: '', + provider: GOOGLE_VERTEX_AI, + choices: [ + { + finish_reason: parsedChunk.error.type, + delta: { + content: '', }, - }); - } else if (isToolBlockDelta) { - toolCalls.push({ - index: streamState.toolIndex, - function: { - arguments: parsedChunk.delta.partial_json, + }, + ], + })}` + + '\n\n' + + 'data: [DONE]\n\n' + ); + } + + if (parsedChunk.type === 'message_start' && parsedChunk.message?.usage) { + const shouldSendCacheUsage = + parsedChunk.message?.usage?.cache_read_input_tokens || + parsedChunk.message?.usage?.cache_creation_input_tokens; + + streamState.model = parsedChunk?.message?.model ?? ''; + + streamState.usage = { + prompt_tokens: parsedChunk.message.usage?.input_tokens, + ...(shouldSendCacheUsage && { + cache_read_input_tokens: + parsedChunk.message?.usage?.cache_read_input_tokens, + cache_creation_input_tokens: + parsedChunk.message?.usage?.cache_creation_input_tokens, + }), + }; + return ( + `data: ${JSON.stringify({ + id: fallbackId, + object: 'chat.completion.chunk', + created: Math.floor(Date.now() / 1000), + model: streamState.model, + provider: GOOGLE_VERTEX_AI, + choices: [ + { + delta: { + content: '', }, - }); - } - - const content = parsedChunk.delta?.text; + index: 0, + logprobs: null, + finish_reason: null, + }, + ], + usage: { + prompt_tokens: streamState.usage.prompt_tokens, + }, + })}` + '\n\n' + ); + } - const contentBlockObject = { - index: parsedChunk.index, - delta: parsedChunk.delta ?? parsedChunk.content_block ?? {}, - }; - delete contentBlockObject.delta.type; + if (parsedChunk.type === 'message_delta' && parsedChunk.usage) { + const totalTokens = + (streamState?.usage?.prompt_tokens ?? 0) + + (streamState?.usage?.cache_creation_input_tokens ?? 0) + + (streamState?.usage?.cache_read_input_tokens ?? 0) + + (parsedChunk.usage.output_tokens ?? 0); return ( - `data: ${JSON.stringify({ - id: fallbackId, - object: "chat.completion.chunk", - created: Math.floor(Date.now() / 1000), - model: streamState.model, - provider: GOOGLE_VERTEX_AI, - choices: [ - { - delta: { - content, - tool_calls: toolCalls.length ? toolCalls : undefined, - ...(!strictOpenAiCompliance && - !toolCalls.length && { - content_blocks: [contentBlockObject], - }), - }, - index: 0, - logprobs: null, - finish_reason: parsedChunk.delta?.stop_reason ?? null, - }, - ], - })}` + "\n\n" + `data: ${JSON.stringify({ + id: fallbackId, + object: 'chat.completion.chunk', + created: Math.floor(Date.now() / 1000), + model: streamState.model, + provider: GOOGLE_VERTEX_AI, + choices: [ + { + index: 0, + delta: {}, + finish_reason: transformFinishReason( + parsedChunk.delta?.stop_reason, + strictOpenAiCompliance + ), + }, + ], + usage: { + ...streamState.usage, + completion_tokens: parsedChunk.usage?.output_tokens, + total_tokens: totalTokens, + prompt_tokens_details: { + cached_tokens: streamState.usage?.cache_read_input_tokens ?? 0, + }, + }, + })}` + '\n\n' ); + } + + const toolCalls = []; + const isToolBlockStart: boolean = + parsedChunk.type === 'content_block_start' && + parsedChunk.content_block?.type === 'tool_use'; + if (isToolBlockStart) { + streamState.toolIndex = streamState.toolIndex + 1; + } + const isToolBlockDelta: boolean = + parsedChunk.type === 'content_block_delta' && + parsedChunk.delta?.partial_json != undefined; + + if (isToolBlockStart && parsedChunk.content_block) { + toolCalls.push({ + index: streamState.toolIndex, + id: parsedChunk.content_block.id, + type: 'function', + function: { + name: parsedChunk.content_block.name, + arguments: '', + }, + }); + } else if (isToolBlockDelta) { + toolCalls.push({ + index: streamState.toolIndex, + function: { + arguments: parsedChunk.delta.partial_json, + }, + }); + } + + const content = parsedChunk.delta?.text; + + const contentBlockObject = { + index: parsedChunk.index, + delta: parsedChunk.delta ?? parsedChunk.content_block ?? {}, + }; + delete contentBlockObject.delta.type; + + return ( + `data: ${JSON.stringify({ + id: fallbackId, + object: 'chat.completion.chunk', + created: Math.floor(Date.now() / 1000), + model: streamState.model, + provider: GOOGLE_VERTEX_AI, + choices: [ + { + delta: { + content, + tool_calls: toolCalls.length ? toolCalls : undefined, + ...(!strictOpenAiCompliance && + !toolCalls.length && { + content_blocks: [contentBlockObject], + }), + }, + index: 0, + logprobs: null, + finish_reason: parsedChunk.delta?.stop_reason ?? null, + }, + ], + })}` + '\n\n' + ); }; export const VertexLlamaChatCompleteResponseTransform: ( - response: VertexLLamaChatCompleteResponse | GoogleErrorResponse, - responseStatus: number, + response: VertexLLamaChatCompleteResponse | GoogleErrorResponse, + responseStatus: number ) => ChatCompletionResponse | ErrorResponse = (response, responseStatus) => { - if ( - responseStatus !== 200 && - Array.isArray(response) && - response.length > 0 && - "error" in response[0] - ) { - const { error } = response[0]; - - return generateErrorResponse( - { - message: error.message, - type: error.status, - param: null, - code: String(error.code), - }, - GOOGLE_VERTEX_AI, - ); - } - if ("choices" in response) { - return { - id: crypto.randomUUID(), - created: Math.floor(Date.now() / 1000), - provider: GOOGLE_VERTEX_AI, - ...response, - }; - } - return generateInvalidProviderResponseError(response, GOOGLE_VERTEX_AI); + if ( + responseStatus !== 200 && + Array.isArray(response) && + response.length > 0 && + 'error' in response[0] + ) { + const { error } = response[0]; + + return generateErrorResponse( + { + message: error.message, + type: error.status, + param: null, + code: String(error.code), + }, + GOOGLE_VERTEX_AI + ); + } + if ('choices' in response) { + return { + id: crypto.randomUUID(), + created: Math.floor(Date.now() / 1000), + provider: GOOGLE_VERTEX_AI, + ...response, + }; + } + return generateInvalidProviderResponseError(response, GOOGLE_VERTEX_AI); }; export const VertexLlamaChatCompleteStreamChunkTransform: ( - response: string, - fallbackId: string, + response: string, + fallbackId: string ) => string = (responseChunk, fallbackId) => { - let chunk = responseChunk.trim(); - chunk = chunk.replace(/^data: /, ""); - chunk = chunk.trim(); - const parsedChunk: VertexLlamaChatCompleteStreamChunk = JSON.parse(chunk); - parsedChunk.id = fallbackId; - parsedChunk.created = Math.floor(Date.now() / 1000); - parsedChunk.provider = GOOGLE_VERTEX_AI; - return `data: ${JSON.stringify(parsedChunk)}` + "\n\n"; + let chunk = responseChunk.trim(); + chunk = chunk.replace(/^data: /, ''); + chunk = chunk.trim(); + const parsedChunk: VertexLlamaChatCompleteStreamChunk = JSON.parse(chunk); + parsedChunk.id = fallbackId; + parsedChunk.created = Math.floor(Date.now() / 1000); + parsedChunk.provider = GOOGLE_VERTEX_AI; + return `data: ${JSON.stringify(parsedChunk)}` + '\n\n'; }; diff --git a/src/providers/google-vertex-ai/transformGenerationConfig.ts b/src/providers/google-vertex-ai/transformGenerationConfig.ts index 8e35ce258..8e8b90708 100644 --- a/src/providers/google-vertex-ai/transformGenerationConfig.ts +++ b/src/providers/google-vertex-ai/transformGenerationConfig.ts @@ -56,7 +56,7 @@ export function transformGenerationConfig(params: PortkeyGeminiParams) { const { budget_tokens, type, reasoning_effort } = params.thinking; const thinkingConfig: Record = {}; thinkingConfig['include_thoughts'] = - (type === 'enabled' || budget_tokens || reasoning_effort) ? true : false; + type === 'enabled' || budget_tokens || reasoning_effort ? true : false; thinkingConfig['thinking_budget'] = budget_tokens; generationConfig['thinking_config'] = thinkingConfig; } diff --git a/src/providers/google-vertex-ai/utils.ts b/src/providers/google-vertex-ai/utils.ts index 3ada20ab2..3f5849786 100644 --- a/src/providers/google-vertex-ai/utils.ts +++ b/src/providers/google-vertex-ai/utils.ts @@ -1,21 +1,21 @@ import { - GoogleErrorResponse, - GoogleResponseCandidate, - GoogleBatchRecord, - GoogleFinetuneRecord, - GoogleSearchRetrievalTool, -} from "./types"; -import { generateErrorResponse } from "../utils"; + GoogleErrorResponse, + GoogleResponseCandidate, + GoogleBatchRecord, + GoogleFinetuneRecord, + GoogleSearchRetrievalTool, +} from './types'; +import { generateErrorResponse } from '../utils'; import { - BatchEndpoints, - GOOGLE_VERTEX_AI, - fileExtensionMimeTypeMap, -} from "../../globals"; -import { ErrorResponse, FinetuneRequest, Logprobs } from "../types"; -import { Context } from "hono"; -import { env } from "hono/adapter"; -import { ContentType, JsonSchema, Tool } from "../../types/requestBody"; -import { GoogleMessagePart } from "../google/chatComplete"; + BatchEndpoints, + GOOGLE_VERTEX_AI, + fileExtensionMimeTypeMap, +} from '../../globals'; +import { ErrorResponse, FinetuneRequest, Logprobs } from '../types'; +import { Context } from 'hono'; +import { env } from 'hono/adapter'; +import { ContentType, JsonSchema, Tool } from '../../types/requestBody'; +import { GoogleMessagePart } from '../google/chatComplete'; /** * Encodes an object as a Base64 URL-encoded string. @@ -23,39 +23,39 @@ import { GoogleMessagePart } from "../google/chatComplete"; * @returns The Base64 URL-encoded string. */ function base64UrlEncode(obj: Record): string { - return btoa(JSON.stringify(obj)) - .replace(/=/g, "") - .replace(/\+/g, "-") - .replace(/\//g, "_"); + return btoa(JSON.stringify(obj)) + .replace(/=/g, '') + .replace(/\+/g, '-') + .replace(/\//g, '_'); } const createJWT = async ( - header: Record, - payload: Record, - privateKey: CryptoKey, + header: Record, + payload: Record, + privateKey: CryptoKey ): Promise => { - const encodedHeader = base64UrlEncode(header); - const encodedPayload = base64UrlEncode(payload); - - const data = new TextEncoder().encode(`${encodedHeader}.${encodedPayload}`); - - const signature = await crypto.subtle.sign( - { - name: "RSASSA-PKCS1-v1_5", - hash: { name: "SHA-256" }, - }, - privateKey, - data, - ); - - const encodedSignature = btoa( - String.fromCharCode(...new Uint8Array(signature)), - ) - .replace(/=/g, "") - .replace(/\+/g, "-") - .replace(/\//g, "_"); - - return `${encodedHeader}.${encodedPayload}.${encodedSignature}`; + const encodedHeader = base64UrlEncode(header); + const encodedPayload = base64UrlEncode(payload); + + const data = new TextEncoder().encode(`${encodedHeader}.${encodedPayload}`); + + const signature = await crypto.subtle.sign( + { + name: 'RSASSA-PKCS1-v1_5', + hash: { name: 'SHA-256' }, + }, + privateKey, + data + ); + + const encodedSignature = btoa( + String.fromCharCode(...new Uint8Array(signature)) + ) + .replace(/=/g, '') + .replace(/\+/g, '-') + .replace(/\//g, '_'); + + return `${encodedHeader}.${encodedPayload}.${encodedSignature}`; }; /** @@ -64,775 +64,755 @@ const createJWT = async ( * @returns The imported private key. */ async function importPrivateKey(pem: string): Promise { - const pemContents = pem - .replace(/-----BEGIN PRIVATE KEY-----/g, "") - .replace(/-----END PRIVATE KEY-----/g, "") - .replace(/\s+/g, ""); - - const binaryDerString = atob(pemContents); - const binaryDer = new Uint8Array(binaryDerString.length); - - for (let i = 0; i < binaryDerString.length; i++) { - binaryDer[i] = binaryDerString.charCodeAt(i); - } - - return crypto.subtle.importKey( - "pkcs8", - binaryDer.buffer, - { - name: "RSASSA-PKCS1-v1_5", - hash: { name: "SHA-256" }, - }, - false, - ["sign"], - ); + const pemContents = pem + .replace(/-----BEGIN PRIVATE KEY-----/g, '') + .replace(/-----END PRIVATE KEY-----/g, '') + .replace(/\s+/g, ''); + + const binaryDerString = atob(pemContents); + const binaryDer = new Uint8Array(binaryDerString.length); + + for (let i = 0; i < binaryDerString.length; i++) { + binaryDer[i] = binaryDerString.charCodeAt(i); + } + + return crypto.subtle.importKey( + 'pkcs8', + binaryDer.buffer, + { + name: 'RSASSA-PKCS1-v1_5', + hash: { name: 'SHA-256' }, + }, + false, + ['sign'] + ); } export const getAccessToken = async ( - c: Context, - serviceAccountInfo: Record, + c: Context, + serviceAccountInfo: Record ): Promise => { + try { + let cacheKey = `${serviceAccountInfo.project_id}/${serviceAccountInfo.private_key_id}/${serviceAccountInfo.client_email}`; + // try to get from cache try { - let cacheKey = `${serviceAccountInfo.project_id}/${serviceAccountInfo.private_key_id}/${serviceAccountInfo.client_email}`; - // try to get from cache - try { - const getFromCacheByKey = c.get("getFromCacheByKey"); - const resp = getFromCacheByKey - ? await getFromCacheByKey(env(c), cacheKey) - : null; - if (resp) { - return resp; - } - } catch (err) {} - - const scope = "https://www.googleapis.com/auth/cloud-platform"; - const iat = Math.floor(Date.now() / 1000); - const exp = iat + 3600; // Token expiration time (1 hour) - - const payload = { - iss: serviceAccountInfo.client_email, - sub: serviceAccountInfo.client_email, - aud: "https://oauth2.googleapis.com/token", - iat: iat, - exp: exp, - scope: scope, - }; - - const header = { - alg: "RS256", - typ: "JWT", - kid: serviceAccountInfo.private_key_id, - }; - - const privateKey = await importPrivateKey( - serviceAccountInfo.private_key, - ); - - const jwtToken = await createJWT(header, payload, privateKey); - - const tokenUrl = "https://oauth2.googleapis.com/token"; - const tokenHeaders = { - "Content-Type": "application/x-www-form-urlencoded", - }; - const tokenData = new URLSearchParams({ - grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer", - assertion: jwtToken, - }); - - const tokenResponse = await fetch(tokenUrl, { - headers: tokenHeaders, - body: tokenData, - method: "POST", - }); - - const tokenJson: Record = await tokenResponse.json(); - const putInCacheWithValue = c.get("putInCacheWithValue"); - if (putInCacheWithValue && cacheKey) { - await putInCacheWithValue( - env(c), - cacheKey, - tokenJson.access_token, - 3000, - ); // 50 minutes - } + const getFromCacheByKey = c.get('getFromCacheByKey'); + const resp = getFromCacheByKey + ? await getFromCacheByKey(env(c), cacheKey) + : null; + if (resp) { + return resp; + } + } catch (err) {} + + const scope = 'https://www.googleapis.com/auth/cloud-platform'; + const iat = Math.floor(Date.now() / 1000); + const exp = iat + 3600; // Token expiration time (1 hour) + + const payload = { + iss: serviceAccountInfo.client_email, + sub: serviceAccountInfo.client_email, + aud: 'https://oauth2.googleapis.com/token', + iat: iat, + exp: exp, + scope: scope, + }; + + const header = { + alg: 'RS256', + typ: 'JWT', + kid: serviceAccountInfo.private_key_id, + }; + + const privateKey = await importPrivateKey(serviceAccountInfo.private_key); - return tokenJson.access_token; - } catch (err) { - return ""; + const jwtToken = await createJWT(header, payload, privateKey); + + const tokenUrl = 'https://oauth2.googleapis.com/token'; + const tokenHeaders = { + 'Content-Type': 'application/x-www-form-urlencoded', + }; + const tokenData = new URLSearchParams({ + grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer', + assertion: jwtToken, + }); + + const tokenResponse = await fetch(tokenUrl, { + headers: tokenHeaders, + body: tokenData, + method: 'POST', + }); + + const tokenJson: Record = await tokenResponse.json(); + const putInCacheWithValue = c.get('putInCacheWithValue'); + if (putInCacheWithValue && cacheKey) { + await putInCacheWithValue(env(c), cacheKey, tokenJson.access_token, 3000); // 50 minutes } + + return tokenJson.access_token; + } catch (err) { + return ''; + } }; export const getModelAndProvider = (modelString: string) => { - let provider = "google"; - let model = modelString; - const modelStringParts = modelString.split("."); - if ( - modelStringParts.length > 1 && - ["google", "anthropic", "meta", "endpoints", "mistralai"].includes( - modelStringParts[0], - ) - ) { - provider = modelStringParts[0]; - model = modelStringParts.slice(1).join("."); - } + let provider = 'google'; + let model = modelString; + const modelStringParts = modelString.split('.'); + if ( + modelStringParts.length > 1 && + ['google', 'anthropic', 'meta', 'endpoints', 'mistralai'].includes( + modelStringParts[0] + ) + ) { + provider = modelStringParts[0]; + model = modelStringParts.slice(1).join('.'); + } - return { provider, model }; + return { provider, model }; }; export const getMimeType = (url: string): string | undefined => { - const urlParts = url.split("."); - const extension = urlParts[ - urlParts.length - 1 - ] as keyof typeof fileExtensionMimeTypeMap; - return fileExtensionMimeTypeMap[extension]; + const urlParts = url.split('.'); + const extension = urlParts[ + urlParts.length - 1 + ] as keyof typeof fileExtensionMimeTypeMap; + return fileExtensionMimeTypeMap[extension]; }; export const GoogleErrorResponseTransform: ( - response: GoogleErrorResponse, - provider?: string, + response: GoogleErrorResponse, + provider?: string ) => ErrorResponse | undefined = (response, provider = GOOGLE_VERTEX_AI) => { - if ("error" in response) { - return generateErrorResponse( - { - message: response.error.message ?? "", - type: response.error.status ?? null, - param: null, - code: response.error.status ?? null, - }, - provider, - ); - } + if ('error' in response) { + return generateErrorResponse( + { + message: response.error.message ?? '', + type: response.error.status ?? null, + param: null, + code: response.error.status ?? null, + }, + provider + ); + } - return undefined; + return undefined; }; // Extract definition key from a JSON Schema $ref string const getDefFromRef = (ref: string): string | null => { - const match = ref.match(/^#\/\$defs\/(.+)$/); - return match ? match[1] : null; + const match = ref.match(/^#\/\$defs\/(.+)$/); + return match ? match[1] : null; }; const getDefObject = ( - defs: Record | undefined | null, - key: string | null, + defs: Record | undefined | null, + key: string | null ): any => (key && defs ? defs[key] : undefined); // Recursively expands $ref nodes in a JSON Schema object tree export const derefer = ( - schema: any, - defs: Record | null = null, - stack: Set = new Set(), + schema: any, + defs: Record | null = null, + stack: Set = new Set() ): any => { - if (schema === null || typeof schema !== "object") return schema; - if (Array.isArray(schema)) - return schema.map((item) => derefer(item, defs, stack)); - const node = { ...schema }; - const activeDefs = - defs ?? (node.$defs as Record | undefined) ?? null; - if ("$ref" in node && typeof node.$ref === "string") { - const defKey = getDefFromRef(node.$ref); - const target = getDefObject(activeDefs, defKey); - if (defKey && target) { - if (stack.has(defKey)) return node; - stack.add(defKey); - const resolved = derefer(target, activeDefs, stack); - stack.delete(defKey); - const keys = Object.keys(node); - if (keys.length === 1) return resolved; - const { $ref: _, ...siblings } = node; - for (const key of Object.keys(node)) delete (node as any)[key]; - Object.assign(node as any, resolved, siblings); - } + if (schema === null || typeof schema !== 'object') return schema; + if (Array.isArray(schema)) + return schema.map((item) => derefer(item, defs, stack)); + const node = { ...schema }; + const activeDefs = + defs ?? (node.$defs as Record | undefined) ?? null; + if ('$ref' in node && typeof node.$ref === 'string') { + const defKey = getDefFromRef(node.$ref); + const target = getDefObject(activeDefs, defKey); + if (defKey && target) { + if (stack.has(defKey)) return node; + stack.add(defKey); + const resolved = derefer(target, activeDefs, stack); + stack.delete(defKey); + const keys = Object.keys(node); + if (keys.length === 1) return resolved; + const { $ref: _, ...siblings } = node; + for (const key of Object.keys(node)) delete (node as any)[key]; + Object.assign(node as any, resolved, siblings); } - for (const [k, v] of Object.entries(node)) { - if (k === "$defs") continue; - node[k] = derefer(v, activeDefs, stack); - } - return node; + } + for (const [k, v] of Object.entries(node)) { + if (k === '$defs') continue; + node[k] = derefer(v, activeDefs, stack); + } + return node; }; export const transformGeminiToolParameters = ( - parameters: JsonSchema, + parameters: JsonSchema ): JsonSchema => { - if ( - !parameters || - typeof parameters !== "object" || - Array.isArray(parameters) - ) { - return parameters; - } - - let schema: JsonSchema = parameters; - if ("$defs" in schema && typeof schema.$defs === "object") { - schema = derefer(schema); - delete schema.$defs; + if ( + !parameters || + typeof parameters !== 'object' || + Array.isArray(parameters) + ) { + return parameters; + } + + let schema: JsonSchema = parameters; + if ('$defs' in schema && typeof schema.$defs === 'object') { + schema = derefer(schema); + delete schema.$defs; + } + + const isNullTypeNode = (node: any): boolean => + node && typeof node === 'object' && node.type === 'null'; + + const transformNode = (node: JsonSchema): JsonSchema => { + if (Array.isArray(node)) { + return node.map(transformNode); } + if (!node || typeof node !== 'object') return node; + + const transformed: JsonSchema = {}; + + for (const [key, value] of Object.entries(node)) { + if ((key === 'anyOf' || key === 'oneOf') && Array.isArray(value)) { + const nonNullItems = value.filter((item) => !isNullTypeNode(item)); + const hadNull = nonNullItems.length < value.length; + + if (nonNullItems.length === 1 && hadNull) { + // Flatten to single schema: get rid of anyOf/oneOf and set nullable: true + const single = transformNode(nonNullItems[0]); + if (single && typeof single === 'object') { + Object.assign(transformed, single); + transformed.nullable = true; + } + continue; + } - const isNullTypeNode = (node: any): boolean => - node && typeof node === "object" && node.type === "null"; + transformed[key] = transformNode(hadNull ? nonNullItems : value); + if (hadNull) transformed.nullable = true; + continue; + } - const transformNode = (node: JsonSchema): JsonSchema => { - if (Array.isArray(node)) { - return node.map(transformNode); - } - if (!node || typeof node !== "object") return node; - - const transformed: JsonSchema = {}; - - for (const [key, value] of Object.entries(node)) { - if ((key === "anyOf" || key === "oneOf") && Array.isArray(value)) { - const nonNullItems = value.filter( - (item) => !isNullTypeNode(item), - ); - const hadNull = nonNullItems.length < value.length; - - if (nonNullItems.length === 1 && hadNull) { - // Flatten to single schema: get rid of anyOf/oneOf and set nullable: true - const single = transformNode(nonNullItems[0]); - if (single && typeof single === "object") { - Object.assign(transformed, single); - transformed.nullable = true; - } - continue; - } - - transformed[key] = transformNode( - hadNull ? nonNullItems : value, - ); - if (hadNull) transformed.nullable = true; - continue; - } - - transformed[key] = transformNode(value); - } - return transformed; - }; + transformed[key] = transformNode(value); + } + return transformed; + }; - return transformNode(schema); + return transformNode(schema); }; // Vertex AI does not support additionalProperties in JSON Schema // https://cloud.google.com/vertex-ai/docs/reference/rest/v1/Schema export const recursivelyDeleteUnsupportedParameters = (obj: any) => { - if (typeof obj !== "object" || obj === null || Array.isArray(obj)) return; - delete obj.additional_properties; - delete obj.additionalProperties; - delete obj["$schema"]; - for (const key in obj) { - if (obj[key] !== null && typeof obj[key] === "object") { - recursivelyDeleteUnsupportedParameters(obj[key]); - } - if (key == "anyOf" && Array.isArray(obj[key])) { - obj[key].forEach((item: any) => { - recursivelyDeleteUnsupportedParameters(item); - }); - } + if (typeof obj !== 'object' || obj === null || Array.isArray(obj)) return; + delete obj.additional_properties; + delete obj.additionalProperties; + delete obj['$schema']; + for (const key in obj) { + if (obj[key] !== null && typeof obj[key] === 'object') { + recursivelyDeleteUnsupportedParameters(obj[key]); } + if (key == 'anyOf' && Array.isArray(obj[key])) { + obj[key].forEach((item: any) => { + recursivelyDeleteUnsupportedParameters(item); + }); + } + } }; // Generate Gateway specific response. export const GoogleResponseHandler = ( - response: Response | string | Record, - status: number, + response: Response | string | Record, + status: number ) => { - if (status !== 200) { - return new Response( - JSON.stringify({ - success: false, - error: response, - param: null, - provider: GOOGLE_VERTEX_AI, - }), - { - status: status || 500, - headers: { "Content-Type": "application/json" }, - }, - ); - } - - if (!(response instanceof Response)) { - const _response = - typeof response === "object" ? JSON.stringify(response) : response; - return new Response(_response as string, { - status: status || 200, - headers: { "Content-Type": "application/json" }, - }); - } - - return response as Response; + if (status !== 200) { + return new Response( + JSON.stringify({ + success: false, + error: response, + param: null, + provider: GOOGLE_VERTEX_AI, + }), + { + status: status || 500, + headers: { 'Content-Type': 'application/json' }, + } + ); + } + + if (!(response instanceof Response)) { + const _response = + typeof response === 'object' ? JSON.stringify(response) : response; + return new Response(_response as string, { + status: status || 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + + return response as Response; }; export const googleBatchStatusToOpenAI = ( - status: GoogleBatchRecord["state"], + status: GoogleBatchRecord['state'] ) => { - switch (status) { - case "JOB_STATE_CANCELLING": - return "cancelling"; - case "JOB_STATE_CANCELLED": - return "cancelled"; - case "JOB_STATE_EXPIRED": - return "expired"; - case "JOB_STATE_FAILED": - return "failed"; - case "JOB_STATE_PARTIALLY_SUCCEEDED": - case "JOB_STATE_SUCCEEDED": - return "completed"; - case "JOB_STATE_RUNNING": - case "JOB_STATE_UPDATING": - return "in_progress"; - case "JOB_STATE_PAUSED": - case "JOB_STATE_PENDING": - case "JOB_STATE_QUEUED": - case "JOB_STATE_UNSPECIFIED": - default: - return "validating"; - } + switch (status) { + case 'JOB_STATE_CANCELLING': + return 'cancelling'; + case 'JOB_STATE_CANCELLED': + return 'cancelled'; + case 'JOB_STATE_EXPIRED': + return 'expired'; + case 'JOB_STATE_FAILED': + return 'failed'; + case 'JOB_STATE_PARTIALLY_SUCCEEDED': + case 'JOB_STATE_SUCCEEDED': + return 'completed'; + case 'JOB_STATE_RUNNING': + case 'JOB_STATE_UPDATING': + return 'in_progress'; + case 'JOB_STATE_PAUSED': + case 'JOB_STATE_PENDING': + case 'JOB_STATE_QUEUED': + case 'JOB_STATE_UNSPECIFIED': + default: + return 'validating'; + } }; export const googleFinetuneStatusToOpenAI = ( - status: GoogleFinetuneRecord["state"], + status: GoogleFinetuneRecord['state'] ) => { - switch (status) { - case "JOB_STATE_CANCELLED": - case "JOB_STATE_CANCELLING": - case "JOB_STATE_EXPIRED": - return "cancelled"; - case "JOB_STATE_FAILED": - return "failed"; - case "JOB_STATE_PARTIALLY_SUCCEEDED": - case "JOB_STATE_SUCCEEDED": - return "succeeded"; - case "JOB_STATE_PAUSED": - case "JOB_STATE_PENDING": - case "JOB_STATE_QUEUED": - return "queued"; - case "JOB_STATE_RUNNING": - case "JOB_STATE_UPDATING": - return "running"; - case "JOB_STATE_UNSPECIFIED": - return "queued"; - default: - return "queued"; - } + switch (status) { + case 'JOB_STATE_CANCELLED': + case 'JOB_STATE_CANCELLING': + case 'JOB_STATE_EXPIRED': + return 'cancelled'; + case 'JOB_STATE_FAILED': + return 'failed'; + case 'JOB_STATE_PARTIALLY_SUCCEEDED': + case 'JOB_STATE_SUCCEEDED': + return 'succeeded'; + case 'JOB_STATE_PAUSED': + case 'JOB_STATE_PENDING': + case 'JOB_STATE_QUEUED': + return 'queued'; + case 'JOB_STATE_RUNNING': + case 'JOB_STATE_UPDATING': + return 'running'; + case 'JOB_STATE_UNSPECIFIED': + return 'queued'; + default: + return 'queued'; + } }; -const getTimeKey = (status: GoogleBatchRecord["state"], value: string) => { - if (status === "JOB_STATE_FAILED") { - return { failed_at: new Date(value).getTime() }; - } +const getTimeKey = (status: GoogleBatchRecord['state'], value: string) => { + if (status === 'JOB_STATE_FAILED') { + return { failed_at: new Date(value).getTime() }; + } - if (status === "JOB_STATE_SUCCEEDED") { - return { completed_at: new Date(value).getTime() }; - } + if (status === 'JOB_STATE_SUCCEEDED') { + return { completed_at: new Date(value).getTime() }; + } - if (status === "JOB_STATE_CANCELLED") { - return { cancelled_at: new Date(value).getTime() }; - } + if (status === 'JOB_STATE_CANCELLED') { + return { cancelled_at: new Date(value).getTime() }; + } - if (status === "JOB_STATE_EXPIRED") { - return { failed_at: new Date(value).getTime() }; - } - return {}; + if (status === 'JOB_STATE_EXPIRED') { + return { failed_at: new Date(value).getTime() }; + } + return {}; }; export const GoogleToOpenAIBatch = (response: GoogleBatchRecord) => { - const jobId = response.name.split("/").at(-1); - const total = Object.values(response.completionStats ?? {}).reduce( - (acc, current) => acc + Number.parseInt(current), - 0, - ); - - const endpoint = isEmbeddingModel(response.model) - ? BatchEndpoints.EMBEDDINGS - : BatchEndpoints.CHAT_COMPLETIONS; - - const fileSuffix = - endpoint === BatchEndpoints.EMBEDDINGS - ? "000000000000.jsonl" - : "predictions.jsonl"; - - const outputFileId = response.outputInfo - ? `${response.outputInfo?.gcsOutputDirectory}/${fileSuffix}` - : response.outputConfig.gcsDestination.outputUriPrefix; - - return { - id: jobId, - object: "batch", - endpoint: endpoint, - input_file_id: encodeURIComponent( - response.inputConfig.gcsSource?.uris?.at(0) ?? "", - ), - completion_window: null, - status: googleBatchStatusToOpenAI(response.state), - output_file_id: encodeURIComponent(outputFileId), - // Same as output_file_id - error_file_id: encodeURIComponent( - response.outputConfig.gcsDestination.outputUriPrefix ?? "", - ), - created_at: new Date(response.createTime).getTime(), - ...getTimeKey(response.state, response.endTime), - in_progress_at: new Date(response.startTime).getTime(), - ...getTimeKey(response.state, response.updateTime), - request_counts: { - total: total, - completed: response.completionStats?.successfulCount, - failed: response.completionStats?.failedCount, - }, - ...(response.error && { - errors: { - object: "list", - data: [response.error], - }, - }), - }; + const jobId = response.name.split('/').at(-1); + const total = Object.values(response.completionStats ?? {}).reduce( + (acc, current) => acc + Number.parseInt(current), + 0 + ); + + const endpoint = isEmbeddingModel(response.model) + ? BatchEndpoints.EMBEDDINGS + : BatchEndpoints.CHAT_COMPLETIONS; + + const fileSuffix = + endpoint === BatchEndpoints.EMBEDDINGS + ? '000000000000.jsonl' + : 'predictions.jsonl'; + + const outputFileId = response.outputInfo + ? `${response.outputInfo?.gcsOutputDirectory}/${fileSuffix}` + : response.outputConfig.gcsDestination.outputUriPrefix; + + return { + id: jobId, + object: 'batch', + endpoint: endpoint, + input_file_id: encodeURIComponent( + response.inputConfig.gcsSource?.uris?.at(0) ?? '' + ), + completion_window: null, + status: googleBatchStatusToOpenAI(response.state), + output_file_id: encodeURIComponent(outputFileId), + // Same as output_file_id + error_file_id: encodeURIComponent( + response.outputConfig.gcsDestination.outputUriPrefix ?? '' + ), + created_at: new Date(response.createTime).getTime(), + ...getTimeKey(response.state, response.endTime), + in_progress_at: new Date(response.startTime).getTime(), + ...getTimeKey(response.state, response.updateTime), + request_counts: { + total: total, + completed: response.completionStats?.successfulCount, + failed: response.completionStats?.failedCount, + }, + ...(response.error && { + errors: { + object: 'list', + data: [response.error], + }, + }), + }; }; export const transformVertexLogprobs = ( - generation: GoogleResponseCandidate, + generation: GoogleResponseCandidate ) => { - const logprobsContent: Logprobs[] = []; - if (!generation.logprobsResult) return null; - if (generation.logprobsResult?.chosenCandidates) { - generation.logprobsResult.chosenCandidates.forEach((candidate) => { - const bytes = []; - for (const char of candidate.token) { - bytes.push(char.charCodeAt(0)); - } - logprobsContent.push({ - token: candidate.token, - logprob: candidate.logProbability, - bytes: bytes, - }); - }); - } - if (generation.logprobsResult?.topCandidates) { - generation.logprobsResult.topCandidates.forEach( - (topCandidatesForIndex, index) => { - const topLogprobs = []; - for (const candidate of topCandidatesForIndex.candidates) { - const bytes = []; - for (const char of candidate.token) { - bytes.push(char.charCodeAt(0)); - } - topLogprobs.push({ - token: candidate.token, - logprob: candidate.logProbability, - bytes: bytes, - }); - } - logprobsContent[index].top_logprobs = topLogprobs; - }, - ); - } - return logprobsContent; + const logprobsContent: Logprobs[] = []; + if (!generation.logprobsResult) return null; + if (generation.logprobsResult?.chosenCandidates) { + generation.logprobsResult.chosenCandidates.forEach((candidate) => { + const bytes = []; + for (const char of candidate.token) { + bytes.push(char.charCodeAt(0)); + } + logprobsContent.push({ + token: candidate.token, + logprob: candidate.logProbability, + bytes: bytes, + }); + }); + } + if (generation.logprobsResult?.topCandidates) { + generation.logprobsResult.topCandidates.forEach( + (topCandidatesForIndex, index) => { + const topLogprobs = []; + for (const candidate of topCandidatesForIndex.candidates) { + const bytes = []; + for (const char of candidate.token) { + bytes.push(char.charCodeAt(0)); + } + topLogprobs.push({ + token: candidate.token, + logprob: candidate.logProbability, + bytes: bytes, + }); + } + logprobsContent[index].top_logprobs = topLogprobs; + } + ); + } + return logprobsContent; }; const populateHyperparameters = (value: FinetuneRequest) => { - let hyperParameters = value.hyperparameters ?? {}; - - if (value.method) { - const method = value.method.type; - hyperParameters = value.method?.[method]?.hyperparameters ?? {}; - } - - return { - epochCount: hyperParameters?.n_epochs, - learningRateMultiplier: hyperParameters?.learning_rate_multiplier, - adapterSize: hyperParameters?.batch_size, - }; + let hyperParameters = value.hyperparameters ?? {}; + + if (value.method) { + const method = value.method.type; + hyperParameters = value.method?.[method]?.hyperparameters ?? {}; + } + + return { + epochCount: hyperParameters?.n_epochs, + learningRateMultiplier: hyperParameters?.learning_rate_multiplier, + adapterSize: hyperParameters?.batch_size, + }; }; export const transformVertexFinetune = (params: FinetuneRequest) => { - const parameterSpec = { - training_dataset_uri: decodeURIComponent(params["training_file"] ?? ""), - ...(params["validation_file"] && { - validation_dataset_uri: decodeURIComponent( - params["validation_file"], - ), - }), - hyperParameters: populateHyperparameters(params), - }; - return parameterSpec; + const parameterSpec = { + training_dataset_uri: decodeURIComponent(params['training_file'] ?? ''), + ...(params['validation_file'] && { + validation_dataset_uri: decodeURIComponent(params['validation_file']), + }), + hyperParameters: populateHyperparameters(params), + }; + return parameterSpec; }; export const getBucketAndFile = (uri: string) => { - if (!uri) return { bucket: "", file: "" }; - let _url = decodeURIComponent(uri); - _url = _url.replaceAll("gs://", ""); - const parts = _url.split("/"); - const bucket = parts[0]; - const file = parts.slice(1).join("/"); - return { bucket, file }; + if (!uri) return { bucket: '', file: '' }; + let _url = decodeURIComponent(uri); + _url = _url.replaceAll('gs://', ''); + const parts = _url.split('/'); + const bucket = parts[0]; + const file = parts.slice(1).join('/'); + return { bucket, file }; }; export const GoogleToOpenAIFinetune = (response: GoogleFinetuneRecord) => { - return { - id: response.name.split("/").at(-1), - object: "finetune", - status: googleFinetuneStatusToOpenAI(response.state), - created_at: new Date(response.createTime).getTime(), - error: response.error, - fine_tuned_model: response.tunedModel?.model, - ...(response.endTime && { - finished_at: new Date(response.endTime).getTime(), - }), - hyperparameters: { - batch_size: - response.supervisedTuningSpec.hyperParameters?.adapterSize, - learning_rate_multiplier: - response.supervisedTuningSpec.hyperParameters - .learningRateMultiplier, - n_epochs: response.supervisedTuningSpec.hyperParameters.epochCount, - }, - model: response.baseModel ?? response.source_model?.baseModel, - trained_tokens: - response.tuningDataStats?.supervisedTuningDataStats - .totalBillableTokenCount, - training_file: encodeURIComponent( - response.supervisedTuningSpec.trainingDatasetUri, - ), - ...(response.supervisedTuningSpec.validationDatasetUri && { - validation_file: encodeURIComponent( - response.supervisedTuningSpec.validationDatasetUri, - ), - }), - }; + return { + id: response.name.split('/').at(-1), + object: 'finetune', + status: googleFinetuneStatusToOpenAI(response.state), + created_at: new Date(response.createTime).getTime(), + error: response.error, + fine_tuned_model: response.tunedModel?.model, + ...(response.endTime && { + finished_at: new Date(response.endTime).getTime(), + }), + hyperparameters: { + batch_size: response.supervisedTuningSpec.hyperParameters?.adapterSize, + learning_rate_multiplier: + response.supervisedTuningSpec.hyperParameters.learningRateMultiplier, + n_epochs: response.supervisedTuningSpec.hyperParameters.epochCount, + }, + model: response.baseModel ?? response.source_model?.baseModel, + trained_tokens: + response.tuningDataStats?.supervisedTuningDataStats + .totalBillableTokenCount, + training_file: encodeURIComponent( + response.supervisedTuningSpec.trainingDatasetUri + ), + ...(response.supervisedTuningSpec.validationDatasetUri && { + validation_file: encodeURIComponent( + response.supervisedTuningSpec.validationDatasetUri + ), + }), + }; }; export const vertexRequestLineHandler = ( - purpose: string, - vertexBatchEndpoint: BatchEndpoints, - transformedBody: any, - requestId: string, + purpose: string, + vertexBatchEndpoint: BatchEndpoints, + transformedBody: any, + requestId: string ) => { - switch (purpose) { - case "batch": - return vertexBatchEndpoint === BatchEndpoints.EMBEDDINGS - ? { ...transformedBody, requestId: requestId } - : { request: transformedBody, requestId: requestId }; - case "fine-tune": - return transformedBody; - } + switch (purpose) { + case 'batch': + return vertexBatchEndpoint === BatchEndpoints.EMBEDDINGS + ? { ...transformedBody, requestId: requestId } + : { request: transformedBody, requestId: requestId }; + case 'fine-tune': + return transformedBody; + } }; export const generateSignedURL = async ( - serviceAccountInfo: Record, - bucketName: string, - objectName: string, - expiration: number = 604800, - httpMethod: string = "GET", - queryParameters: Record = {}, - headers: Record = {}, + serviceAccountInfo: Record, + bucketName: string, + objectName: string, + expiration: number = 604800, + httpMethod: string = 'GET', + queryParameters: Record = {}, + headers: Record = {} ): Promise => { - if (expiration > 604800) { - throw new Error( - "Expiration Time can't be longer than 604800 seconds (7 days).", - ); - } - - const escapedObjectName = encodeURIComponent(objectName).replace( - /%2F/g, - "/", - ); - const canonicalUri = `/${escapedObjectName}`; - - const datetimeNow = new Date(); - const requestTimestamp = datetimeNow - .toISOString() - .replace(/[-:]/g, "") // Remove hyphens and colons - .replace(/\.\d{3}Z$/, "Z"); // Remove milliseconds and ensure Z at end - const datestamp = datetimeNow.toISOString().slice(0, 10).replace(/-/g, ""); - - const clientEmail = serviceAccountInfo.client_email; - const credentialScope = `${datestamp}/auto/storage/goog4_request`; - const credential = `${clientEmail}/${credentialScope}`; - - const host = `${bucketName}.storage.googleapis.com`; - headers["host"] = host; - - // Create canonical headers - let canonicalHeaders = ""; - const orderedHeaders = Object.keys(headers).sort(); - for (const key of orderedHeaders) { - const lowerKey = key.toLowerCase(); - const value = headers[key].toLowerCase(); - canonicalHeaders += `${lowerKey}:${value}\n`; - } - - // Create signed headers - const signedHeaders = orderedHeaders - .map((key) => key.toLowerCase()) - .join(";"); - - // Add required query parameters - const queryParams: Record = { - ...queryParameters, - "X-Goog-Algorithm": "GOOG4-RSA-SHA256", - "X-Goog-Credential": credential, - "X-Goog-Date": requestTimestamp, - "X-Goog-Expires": expiration.toString(), - "X-Goog-SignedHeaders": signedHeaders, - }; - - // Create canonical query string - const canonicalQueryString = Object.keys(queryParams) - .sort() - .map( - (key) => - `${encodeURIComponent(key)}=${encodeURIComponent(queryParams[key])}`, - ) - .join("&"); - - // Create canonical request - const canonicalRequest = [ - httpMethod, - canonicalUri, - canonicalQueryString, - canonicalHeaders, - signedHeaders, - "UNSIGNED-PAYLOAD", - ].join("\n"); - - // Hash the canonical request - const canonicalRequestHash = await crypto.subtle.digest( - "SHA-256", - new TextEncoder().encode(canonicalRequest), + if (expiration > 604800) { + throw new Error( + "Expiration Time can't be longer than 604800 seconds (7 days)." ); - - // Create string to sign - const stringToSign = [ - "GOOG4-RSA-SHA256", - requestTimestamp, - credentialScope, - Array.from(new Uint8Array(canonicalRequestHash)) - .map((b) => b.toString(16).padStart(2, "0")) - .join(""), - ].join("\n"); - - // Sign the string - const privateKey = await importPrivateKey(serviceAccountInfo.private_key); - const signature = await crypto.subtle.sign( - { - name: "RSASSA-PKCS1-v1_5", - hash: { name: "SHA-256" }, - }, - privateKey, - new TextEncoder().encode(stringToSign), - ); - - // Convert signature to hex - const signatureHex = Array.from(new Uint8Array(signature)) - .map((b) => b.toString(16).padStart(2, "0")) - .join(""); - - // Construct the final URL - const schemeAndHost = `https://${host}`; - return `${schemeAndHost}${canonicalUri}?${canonicalQueryString}&x-goog-signature=${signatureHex}`; + } + + const escapedObjectName = encodeURIComponent(objectName).replace(/%2F/g, '/'); + const canonicalUri = `/${escapedObjectName}`; + + const datetimeNow = new Date(); + const requestTimestamp = datetimeNow + .toISOString() + .replace(/[-:]/g, '') // Remove hyphens and colons + .replace(/\.\d{3}Z$/, 'Z'); // Remove milliseconds and ensure Z at end + const datestamp = datetimeNow.toISOString().slice(0, 10).replace(/-/g, ''); + + const clientEmail = serviceAccountInfo.client_email; + const credentialScope = `${datestamp}/auto/storage/goog4_request`; + const credential = `${clientEmail}/${credentialScope}`; + + const host = `${bucketName}.storage.googleapis.com`; + headers['host'] = host; + + // Create canonical headers + let canonicalHeaders = ''; + const orderedHeaders = Object.keys(headers).sort(); + for (const key of orderedHeaders) { + const lowerKey = key.toLowerCase(); + const value = headers[key].toLowerCase(); + canonicalHeaders += `${lowerKey}:${value}\n`; + } + + // Create signed headers + const signedHeaders = orderedHeaders + .map((key) => key.toLowerCase()) + .join(';'); + + // Add required query parameters + const queryParams: Record = { + ...queryParameters, + 'X-Goog-Algorithm': 'GOOG4-RSA-SHA256', + 'X-Goog-Credential': credential, + 'X-Goog-Date': requestTimestamp, + 'X-Goog-Expires': expiration.toString(), + 'X-Goog-SignedHeaders': signedHeaders, + }; + + // Create canonical query string + const canonicalQueryString = Object.keys(queryParams) + .sort() + .map( + (key) => + `${encodeURIComponent(key)}=${encodeURIComponent(queryParams[key])}` + ) + .join('&'); + + // Create canonical request + const canonicalRequest = [ + httpMethod, + canonicalUri, + canonicalQueryString, + canonicalHeaders, + signedHeaders, + 'UNSIGNED-PAYLOAD', + ].join('\n'); + + // Hash the canonical request + const canonicalRequestHash = await crypto.subtle.digest( + 'SHA-256', + new TextEncoder().encode(canonicalRequest) + ); + + // Create string to sign + const stringToSign = [ + 'GOOG4-RSA-SHA256', + requestTimestamp, + credentialScope, + Array.from(new Uint8Array(canonicalRequestHash)) + .map((b) => b.toString(16).padStart(2, '0')) + .join(''), + ].join('\n'); + + // Sign the string + const privateKey = await importPrivateKey(serviceAccountInfo.private_key); + const signature = await crypto.subtle.sign( + { + name: 'RSASSA-PKCS1-v1_5', + hash: { name: 'SHA-256' }, + }, + privateKey, + new TextEncoder().encode(stringToSign) + ); + + // Convert signature to hex + const signatureHex = Array.from(new Uint8Array(signature)) + .map((b) => b.toString(16).padStart(2, '0')) + .join(''); + + // Construct the final URL + const schemeAndHost = `https://${host}`; + return `${schemeAndHost}${canonicalUri}?${canonicalQueryString}&x-goog-signature=${signatureHex}`; }; export const isEmbeddingModel = (modelName: string) => { - return modelName.includes("embedding"); + return modelName.includes('embedding'); }; export const OPENAI_AUDIO_FORMAT_TO_VERTEX_MIME_TYPE_MAPPING = { - mp3: "audio/mp3", - wav: "audio/wav", - opus: "audio/ogg", - flac: "audio/flac", - pcm16: "audio/pcm", - "x-aac": "audio/aac", - "x-m4a": "audio/m4a", - mpeg: "audio/mpeg", - mpga: "audio/mpga", - mp4: "audio/mp4", - webm: "audio/webm", + mp3: 'audio/mp3', + wav: 'audio/wav', + opus: 'audio/ogg', + flac: 'audio/flac', + pcm16: 'audio/pcm', + 'x-aac': 'audio/aac', + 'x-m4a': 'audio/m4a', + mpeg: 'audio/mpeg', + mpga: 'audio/mpga', + mp4: 'audio/mp4', + webm: 'audio/webm', }; export const transformInputAudioPart = (c: ContentType): GoogleMessagePart => { - const data = c.input_audio?.data; - const mimeType = - OPENAI_AUDIO_FORMAT_TO_VERTEX_MIME_TYPE_MAPPING[ - c.input_audio - ?.format as keyof typeof OPENAI_AUDIO_FORMAT_TO_VERTEX_MIME_TYPE_MAPPING - ]; - return { - inlineData: { - data: data ?? "", - mimeType, - }, - }; + const data = c.input_audio?.data; + const mimeType = + OPENAI_AUDIO_FORMAT_TO_VERTEX_MIME_TYPE_MAPPING[ + c.input_audio + ?.format as keyof typeof OPENAI_AUDIO_FORMAT_TO_VERTEX_MIME_TYPE_MAPPING + ]; + return { + inlineData: { + data: data ?? '', + mimeType, + }, + }; }; export const googleTools = [ - "googleSearch", - "google_search", - "googleSearchRetrieval", - "google_search_retrieval", - "computerUse", - "computer_use", - "googleMaps", - "google_maps", - "codeExecution", - "code_execution", - "urlContext", - "url_context", + 'googleSearch', + 'google_search', + 'googleSearchRetrieval', + 'google_search_retrieval', + 'computerUse', + 'computer_use', + 'googleMaps', + 'google_maps', + 'codeExecution', + 'code_execution', + 'urlContext', + 'url_context', ]; export const transformGoogleTools = (tool: Tool) => { - const tools: any = []; - // This function is called only when tool.function exists - if (!tool.function) return tools; - - if (["googleSearch", "google_search"].includes(tool.function.name)) { - const timeRangeFilter = tool.function.parameters?.timeRangeFilter; - tools.push({ - googleSearch: { - // allow null - ...(timeRangeFilter !== undefined && { timeRangeFilter }), - }, - }); - } else if ( - ["googleSearchRetrieval", "google_search_retrieval"].includes( - tool.function.name, - ) - ) { - tools.push(buildGoogleSearchRetrievalTool(tool)); - } else if (["computerUse", "computer_use"].includes(tool.function.name)) { - tools.push({ - computerUse: { - environment: tool.function.parameters?.environment, - excludedPredefinedFunctions: - tool.function.parameters?.excluded_predefined_functions, - }, - }); - } else if (["googleMaps", "google_maps"].includes(tool.function.name)) { - tools.push({ - googleMaps: { - enableWidget: tool.function.parameters?.enableWidget, - }, - }); - } else if ( - ["codeExecution", "code_execution"].includes(tool.function.name) - ) { - tools.push({ - codeExecution: {}, - }); - } else if (["urlContext", "url_context"].includes(tool.function.name)) { - tools.push({ - urlContext: {}, - }); - } - return tools; + const tools: any = []; + // This function is called only when tool.function exists + if (!tool.function) return tools; + + if (['googleSearch', 'google_search'].includes(tool.function.name)) { + const timeRangeFilter = tool.function.parameters?.timeRangeFilter; + tools.push({ + googleSearch: { + // allow null + ...(timeRangeFilter !== undefined && { timeRangeFilter }), + }, + }); + } else if ( + ['googleSearchRetrieval', 'google_search_retrieval'].includes( + tool.function.name + ) + ) { + tools.push(buildGoogleSearchRetrievalTool(tool)); + } else if (['computerUse', 'computer_use'].includes(tool.function.name)) { + tools.push({ + computerUse: { + environment: tool.function.parameters?.environment, + excludedPredefinedFunctions: + tool.function.parameters?.excluded_predefined_functions, + }, + }); + } else if (['googleMaps', 'google_maps'].includes(tool.function.name)) { + tools.push({ + googleMaps: { + enableWidget: tool.function.parameters?.enableWidget, + }, + }); + } else if (['codeExecution', 'code_execution'].includes(tool.function.name)) { + tools.push({ + codeExecution: {}, + }); + } else if (['urlContext', 'url_context'].includes(tool.function.name)) { + tools.push({ + urlContext: {}, + }); + } + return tools; }; export const buildGoogleSearchRetrievalTool = (tool: Tool) => { - const googleSearchRetrievalTool: GoogleSearchRetrievalTool = { - googleSearchRetrieval: {}, - }; - // This function is called only when tool.function exists - if (tool.function?.parameters?.dynamicRetrievalConfig) { - googleSearchRetrievalTool.googleSearchRetrieval.dynamicRetrievalConfig = - tool.function.parameters.dynamicRetrievalConfig; - } - return googleSearchRetrievalTool; + const googleSearchRetrievalTool: GoogleSearchRetrievalTool = { + googleSearchRetrieval: {}, + }; + // This function is called only when tool.function exists + if (tool.function?.parameters?.dynamicRetrievalConfig) { + googleSearchRetrievalTool.googleSearchRetrieval.dynamicRetrievalConfig = + tool.function.parameters.dynamicRetrievalConfig; + } + return googleSearchRetrievalTool; }; diff --git a/src/types/requestBody.ts b/src/types/requestBody.ts index 7941f4438..e212c067e 100644 --- a/src/types/requestBody.ts +++ b/src/types/requestBody.ts @@ -1,41 +1,41 @@ -import { BatchEndpoints } from "../globals"; -import { HookObject } from "../middlewares/hooks/types"; +import { BatchEndpoints } from '../globals'; +import { HookObject } from '../middlewares/hooks/types'; /** * Settings for retrying requests. * @interface */ export interface RetrySettings { - /** The maximum number of retry attempts. */ - attempts: number; - /** The HTTP status codes on which to retry. */ - onStatusCodes: number[]; - /** Whether to use the provider's retry wait. */ - useRetryAfterHeader?: boolean; + /** The maximum number of retry attempts. */ + attempts: number; + /** The HTTP status codes on which to retry. */ + onStatusCodes: number[]; + /** Whether to use the provider's retry wait. */ + useRetryAfterHeader?: boolean; } export interface CacheSettings { - mode: string; - maxAge?: number; + mode: string; + maxAge?: number; } export enum StrategyModes { - LOADBALANCE = "loadbalance", - FALLBACK = "fallback", - SINGLE = "single", - CONDITIONAL = "conditional", + LOADBALANCE = 'loadbalance', + FALLBACK = 'fallback', + SINGLE = 'single', + CONDITIONAL = 'conditional', } interface Strategy { - mode: StrategyModes; - onStatusCodes?: Array; - conditions?: { - query: { - [key: string]: any; - }; - then: string; - }[]; - default?: string; + mode: StrategyModes; + onStatusCodes?: Array; + conditions?: { + query: { + [key: string]: any; + }; + then: string; + }[]; + default?: string; } /** @@ -43,140 +43,140 @@ interface Strategy { * @interface */ export interface Options { - /** The name of the provider. */ - provider: string; - /** The name of the API key for the provider. */ - virtualKey?: string; - /** The API key for the provider. */ - apiKey?: string; - /** The weight of the provider, used for load balancing. */ - weight?: number; - /** The retry settings for the provider. */ - retry?: RetrySettings; - /** The parameters to override in the request. */ - overrideParams?: Params; - /** The actual url used to make llm calls */ - urlToFetch?: string; - /** Azure specific */ - resourceName?: string; - deploymentId?: string; - apiVersion?: string; - adAuth?: string; - azureAuthMode?: string; - azureManagedClientId?: string; - azureWorkloadClientId?: string; - azureEntraClientId?: string; - azureEntraClientSecret?: string; - azureEntraTenantId?: string; - azureAdToken?: string; - azureModelName?: string; - /** Workers AI specific */ - workersAiAccountId?: string; - /** The parameter to set custom base url */ - customHost?: string; - /** The parameter to set list of headers to be forwarded as-is to the provider */ - forwardHeaders?: string[]; - /** provider option index picked based on weight in loadbalance mode */ - index?: number; - cache?: CacheSettings | string; - metadata?: Record; - requestTimeout?: number; - /** This is used to determine if the request should be transformed to formData Example: Stability V2 */ - transformToFormData?: boolean; - /** AWS specific (used for Bedrock and Sagemaker) */ - awsSecretAccessKey?: string; - awsAccessKeyId?: string; - awsSessionToken?: string; - awsRegion?: string; - awsAuthType?: string; - awsRoleArn?: string; - awsExternalId?: string; - awsS3Bucket?: string; - awsS3ObjectKey?: string; - awsBedrockModel?: string; - awsServerSideEncryption?: string; - awsServerSideEncryptionKMSKeyId?: string; - awsService?: string; - foundationModel?: string; - - /** Sagemaker specific */ - amznSagemakerCustomAttributes?: string; - amznSagemakerTargetModel?: string; - amznSagemakerTargetVariant?: string; - amznSagemakerTargetContainerHostname?: string; - amznSagemakerInferenceId?: string; - amznSagemakerEnableExplanations?: string; - amznSagemakerInferenceComponent?: string; - amznSagemakerSessionId?: string; - amznSagemakerModelName?: string; - - /** Stability AI specific */ - stabilityClientId?: string; - stabilityClientUserId?: string; - stabilityClientVersion?: string; - - /** Hugging Face specific */ - huggingfaceBaseUrl?: string; - - /** Google Vertex AI specific */ - vertexRegion?: string; - vertexProjectId?: string; - vertexServiceAccountJson?: Record; - vertexStorageBucketName?: string; - vertexModelName?: string; - vertexBatchEndpoint?: BatchEndpoints; - - // Required for file uploads with google. - filename?: string; - - afterRequestHooks?: HookObject[]; - beforeRequestHooks?: HookObject[]; - defaultInputGuardrails?: HookObject[]; - defaultOutputGuardrails?: HookObject[]; - /** OpenAI specific */ - openaiProject?: string; - openaiOrganization?: string; - openaiBeta?: string; - /** Azure Inference Specific */ - azureApiVersion?: string; - azureFoundryUrl?: string; - azureExtraParameters?: string; - azureDeploymentName?: string; - - /** The parameter to determine if extra non-openai compliant fields should be returned in response */ - strictOpenAiCompliance?: boolean; - - /** Parameter to determine if fim/completions endpoint is to be used */ - mistralFimCompletion?: string; - - /** Anthropic specific headers */ - anthropicBeta?: string; - anthropicVersion?: string; - anthropicApiKey?: string; - - /** Fireworks finetune required fields */ - fireworksAccountId?: string; - fireworksFileLength?: string; - - /** Cortex specific fields */ - snowflakeAccount?: string; - - /** Azure entra scope */ - azureEntraScope?: string; - - // Oracle specific fields - oracleApiVersion?: string; // example: 20160918 - oracleRegion?: string; // example: us-ashburn-1 - oracleCompartmentId?: string; // example: ocid1.compartment.oc1..aaaaaaaab7x77777777777777777 - oracleServingMode?: string; // supported values: ON_DEMAND, DEDICATED - oracleTenancy?: string; // example: ocid1.tenancy.oc1..aaaaaaaab7x77777777777777777 - oracleUser?: string; // example: ocid1.user.oc1..aaaaaaaab7x77777777777777777 - oracleFingerprint?: string; // example: 12:34:56:78:90:ab:cd:ef:12:34:56:78:90:ab:cd:ef - oraclePrivateKey?: string; // example: -----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA... - oracleKeyPassphrase?: string; // example: password - - /** Model pricing config */ - modelPricingConfig?: Record; + /** The name of the provider. */ + provider: string; + /** The name of the API key for the provider. */ + virtualKey?: string; + /** The API key for the provider. */ + apiKey?: string; + /** The weight of the provider, used for load balancing. */ + weight?: number; + /** The retry settings for the provider. */ + retry?: RetrySettings; + /** The parameters to override in the request. */ + overrideParams?: Params; + /** The actual url used to make llm calls */ + urlToFetch?: string; + /** Azure specific */ + resourceName?: string; + deploymentId?: string; + apiVersion?: string; + adAuth?: string; + azureAuthMode?: string; + azureManagedClientId?: string; + azureWorkloadClientId?: string; + azureEntraClientId?: string; + azureEntraClientSecret?: string; + azureEntraTenantId?: string; + azureAdToken?: string; + azureModelName?: string; + /** Workers AI specific */ + workersAiAccountId?: string; + /** The parameter to set custom base url */ + customHost?: string; + /** The parameter to set list of headers to be forwarded as-is to the provider */ + forwardHeaders?: string[]; + /** provider option index picked based on weight in loadbalance mode */ + index?: number; + cache?: CacheSettings | string; + metadata?: Record; + requestTimeout?: number; + /** This is used to determine if the request should be transformed to formData Example: Stability V2 */ + transformToFormData?: boolean; + /** AWS specific (used for Bedrock and Sagemaker) */ + awsSecretAccessKey?: string; + awsAccessKeyId?: string; + awsSessionToken?: string; + awsRegion?: string; + awsAuthType?: string; + awsRoleArn?: string; + awsExternalId?: string; + awsS3Bucket?: string; + awsS3ObjectKey?: string; + awsBedrockModel?: string; + awsServerSideEncryption?: string; + awsServerSideEncryptionKMSKeyId?: string; + awsService?: string; + foundationModel?: string; + + /** Sagemaker specific */ + amznSagemakerCustomAttributes?: string; + amznSagemakerTargetModel?: string; + amznSagemakerTargetVariant?: string; + amznSagemakerTargetContainerHostname?: string; + amznSagemakerInferenceId?: string; + amznSagemakerEnableExplanations?: string; + amznSagemakerInferenceComponent?: string; + amznSagemakerSessionId?: string; + amznSagemakerModelName?: string; + + /** Stability AI specific */ + stabilityClientId?: string; + stabilityClientUserId?: string; + stabilityClientVersion?: string; + + /** Hugging Face specific */ + huggingfaceBaseUrl?: string; + + /** Google Vertex AI specific */ + vertexRegion?: string; + vertexProjectId?: string; + vertexServiceAccountJson?: Record; + vertexStorageBucketName?: string; + vertexModelName?: string; + vertexBatchEndpoint?: BatchEndpoints; + + // Required for file uploads with google. + filename?: string; + + afterRequestHooks?: HookObject[]; + beforeRequestHooks?: HookObject[]; + defaultInputGuardrails?: HookObject[]; + defaultOutputGuardrails?: HookObject[]; + /** OpenAI specific */ + openaiProject?: string; + openaiOrganization?: string; + openaiBeta?: string; + /** Azure Inference Specific */ + azureApiVersion?: string; + azureFoundryUrl?: string; + azureExtraParameters?: string; + azureDeploymentName?: string; + + /** The parameter to determine if extra non-openai compliant fields should be returned in response */ + strictOpenAiCompliance?: boolean; + + /** Parameter to determine if fim/completions endpoint is to be used */ + mistralFimCompletion?: string; + + /** Anthropic specific headers */ + anthropicBeta?: string; + anthropicVersion?: string; + anthropicApiKey?: string; + + /** Fireworks finetune required fields */ + fireworksAccountId?: string; + fireworksFileLength?: string; + + /** Cortex specific fields */ + snowflakeAccount?: string; + + /** Azure entra scope */ + azureEntraScope?: string; + + // Oracle specific fields + oracleApiVersion?: string; // example: 20160918 + oracleRegion?: string; // example: us-ashburn-1 + oracleCompartmentId?: string; // example: ocid1.compartment.oc1..aaaaaaaab7x77777777777777777 + oracleServingMode?: string; // supported values: ON_DEMAND, DEDICATED + oracleTenancy?: string; // example: ocid1.tenancy.oc1..aaaaaaaab7x77777777777777777 + oracleUser?: string; // example: ocid1.user.oc1..aaaaaaaab7x77777777777777777 + oracleFingerprint?: string; // example: 12:34:56:78:90:ab:cd:ef:12:34:56:78:90:ab:cd:ef + oraclePrivateKey?: string; // example: -----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA... + oracleKeyPassphrase?: string; // example: password + + /** Model pricing config */ + modelPricingConfig?: Record; } /** @@ -184,44 +184,44 @@ export interface Options { * @interface */ export interface Targets { - name?: string; - strategy?: Strategy; - /** The name of the provider. */ - provider?: string | undefined; - /** The name of the API key for the provider. */ - virtualKey?: string; - /** The API key for the provider. */ - apiKey?: string; - /** The weight of the provider, used for load balancing. */ - weight?: number; - /** The retry settings for the provider. */ - retry?: RetrySettings; - /** The parameters to override in the request. */ - overrideParams?: Params; - /** The actual url used to make llm calls */ - urlToFetch?: string; - /** Azure specific */ - resourceName?: string; - deploymentId?: string; - apiVersion?: string; - adAuth?: string; - azureAuthMode?: string; - azureManagedClientId?: string; - azureEntraClientId?: string; - azureEntraClientSecret?: string; - azureEntraTenantId?: string; - azureModelName?: string; - /** provider option index picked based on weight in loadbalance mode */ - index?: number; - cache?: CacheSettings | string; - targets?: Targets[]; - - /** This is used to determine if the request should be transformed to formData Example: Stability V2 */ - transformToFormData?: boolean; - - defaultInputGuardrails?: HookObject[]; - defaultOutputGuardrails?: HookObject[]; - originalIndex?: number; + name?: string; + strategy?: Strategy; + /** The name of the provider. */ + provider?: string | undefined; + /** The name of the API key for the provider. */ + virtualKey?: string; + /** The API key for the provider. */ + apiKey?: string; + /** The weight of the provider, used for load balancing. */ + weight?: number; + /** The retry settings for the provider. */ + retry?: RetrySettings; + /** The parameters to override in the request. */ + overrideParams?: Params; + /** The actual url used to make llm calls */ + urlToFetch?: string; + /** Azure specific */ + resourceName?: string; + deploymentId?: string; + apiVersion?: string; + adAuth?: string; + azureAuthMode?: string; + azureManagedClientId?: string; + azureEntraClientId?: string; + azureEntraClientSecret?: string; + azureEntraTenantId?: string; + azureModelName?: string; + /** provider option index picked based on weight in loadbalance mode */ + index?: number; + cache?: CacheSettings | string; + targets?: Targets[]; + + /** This is used to determine if the request should be transformed to formData Example: Stability V2 */ + transformToFormData?: boolean; + + defaultInputGuardrails?: HookObject[]; + defaultOutputGuardrails?: HookObject[]; + originalIndex?: number; } /** @@ -229,15 +229,15 @@ export interface Targets { * @interface */ export interface Config { - /** The mode for handling the request. It can be "single", "fallback", "loadbalance", or "scientist". */ - mode: "single" | "fallback" | "loadbalance" | "scientist"; - /** The configuration for the provider(s). */ - options: Options[]; - targets?: Targets[]; - cache?: CacheSettings; - retry?: RetrySettings; - strategy?: Strategy; - customHost?: string; + /** The mode for handling the request. It can be "single", "fallback", "loadbalance", or "scientist". */ + mode: 'single' | 'fallback' | 'loadbalance' | 'scientist'; + /** The configuration for the provider(s). */ + options: Options[]; + targets?: Targets[]; + cache?: CacheSettings; + retry?: RetrySettings; + strategy?: Strategy; + customHost?: string; } /** @@ -246,66 +246,66 @@ export interface Config { * @interface */ export interface ContentType extends PromptCache { - type: string; - text?: string; - thinking?: string; - signature?: string; - image_url?: { - url: string; - detail?: string; - mime_type?: string; - }; - video_url?: { - url: string; - mime_type?: string; - }; - data?: string; - file?: { - file_data?: string; - file_id?: string; - file_name?: string; - file_url?: string; - mime_type?: string; - }; - input_audio?: { - data: string; - format: "mp3" | "wav" | string; //defaults to auto - }; + type: string; + text?: string; + thinking?: string; + signature?: string; + image_url?: { + url: string; + detail?: string; + mime_type?: string; + }; + video_url?: { + url: string; + mime_type?: string; + }; + data?: string; + file?: { + file_data?: string; + file_id?: string; + file_name?: string; + file_url?: string; + mime_type?: string; + }; + input_audio?: { + data: string; + format: 'mp3' | 'wav' | string; //defaults to auto + }; } export interface ToolCall { - id: string; - type: string; - function: { - name: string; - arguments: string; - description?: string; - thought_signature?: string; - }; + id: string; + type: string; + function: { + name: string; + arguments: string; + description?: string; + thought_signature?: string; + }; } export enum MESSAGE_ROLES { - SYSTEM = "system", - USER = "user", - ASSISTANT = "assistant", - FUNCTION = "function", - TOOL = "tool", - DEVELOPER = "developer", + SYSTEM = 'system', + USER = 'user', + ASSISTANT = 'assistant', + FUNCTION = 'function', + TOOL = 'tool', + DEVELOPER = 'developer', } -export const SYSTEM_MESSAGE_ROLES = ["system", "developer"]; +export const SYSTEM_MESSAGE_ROLES = ['system', 'developer']; export type OpenAIMessageRole = - | "system" - | "user" - | "assistant" - | "function" - | "tool" - | "developer"; - -export interface ContentBlockChunk extends Omit { - index: number; - type?: string; + | 'system' + | 'user' + | 'assistant' + | 'function' + | 'tool' + | 'developer'; + +export interface ContentBlockChunk extends Omit { + index: number; + type?: string; } /** @@ -313,36 +313,36 @@ export interface ContentBlockChunk extends Omit { * @interface */ export interface Message { - /** The role of the message sender. It can be 'system', 'user', 'assistant', or 'function'. */ - role: OpenAIMessageRole; - /** The content of the message. */ - content?: string | ContentType[]; - /** The content blocks of the message. */ - content_blocks?: ContentType[]; - /** The name of the function to call, if any. */ - name?: string; - /** The function call to make, if any. */ - function_call?: any; - tool_calls?: any; - tool_call_id?: string; - citationMetadata?: CitationMetadata; - /** Reasoning details for models that support extended thinking/reasoning. (Gemini) */ - reasoning_details?: any[]; + /** The role of the message sender. It can be 'system', 'user', 'assistant', or 'function'. */ + role: OpenAIMessageRole; + /** The content of the message. */ + content?: string | ContentType[]; + /** The content blocks of the message. */ + content_blocks?: ContentType[]; + /** The name of the function to call, if any. */ + name?: string; + /** The function call to make, if any. */ + function_call?: any; + tool_calls?: any; + tool_call_id?: string; + citationMetadata?: CitationMetadata; + /** Reasoning details for models that support extended thinking/reasoning. (Gemini) */ + reasoning_details?: any[]; } export interface PromptCache { - cache_control?: { type: "ephemeral" }; + cache_control?: { type: 'ephemeral' }; } export interface CitationMetadata { - citationSources?: CitationSource[]; + citationSources?: CitationSource[]; } export interface CitationSource { - startIndex?: number; - endIndex?: number; - uri?: string; - license?: string; + startIndex?: number; + endIndex?: number; + uri?: string; + license?: string; } /** @@ -350,8 +350,8 @@ export interface CitationSource { * @interface */ export interface JsonSchema { - /** The schema definition, indexed by key. */ - [key: string]: any; + /** The schema definition, indexed by key. */ + [key: string]: any; } /** @@ -359,54 +359,54 @@ export interface JsonSchema { * @interface */ export interface Function { - /** The name of the function. */ - name: string; - /** A description of the function. */ - description?: string; - /** The parameters for the function. */ - parameters?: JsonSchema; - /** Whether to enable strict schema adherence when generating the function call. If set to true, the model will follow the exact schema defined in the parameters field. Only a subset of JSON Schema is supported when strict is true */ - strict?: boolean; - /** - * When true, this tool is not loaded into context initially. - * Claude discovers it via Tool Search Tool on-demand. - * Part of Anthropic's advanced tool use beta features. - */ - defer_loading?: boolean; - /** - * List of tool types that can call this tool programmatically. - * E.g., ["code_execution_20250825"] enables Programmatic Tool Calling. - * Part of Anthropic's advanced tool use beta features. - */ - allowed_callers?: string[]; - /** - * Example inputs demonstrating how to use this tool. - * Helps Claude understand usage patterns beyond JSON schema. - * Part of Anthropic's advanced tool use beta features. - */ - input_examples?: Record[]; + /** The name of the function. */ + name: string; + /** A description of the function. */ + description?: string; + /** The parameters for the function. */ + parameters?: JsonSchema; + /** Whether to enable strict schema adherence when generating the function call. If set to true, the model will follow the exact schema defined in the parameters field. Only a subset of JSON Schema is supported when strict is true */ + strict?: boolean; + /** + * When true, this tool is not loaded into context initially. + * Claude discovers it via Tool Search Tool on-demand. + * Part of Anthropic's advanced tool use beta features. + */ + defer_loading?: boolean; + /** + * List of tool types that can call this tool programmatically. + * E.g., ["code_execution_20250825"] enables Programmatic Tool Calling. + * Part of Anthropic's advanced tool use beta features. + */ + allowed_callers?: string[]; + /** + * Example inputs demonstrating how to use this tool. + * Helps Claude understand usage patterns beyond JSON schema. + * Part of Anthropic's advanced tool use beta features. + */ + input_examples?: Record[]; } export interface ToolChoiceObject { - type: string; - function: { - name: string; - }; + type: string; + function: { + name: string; + }; } export interface CustomToolChoice { - type: "custom"; - custom: { - name?: string; - }; + type: 'custom'; + custom: { + name?: string; + }; } export type ToolChoice = - | ToolChoiceObject - | CustomToolChoice - | "none" - | "auto" - | "required"; + | ToolChoiceObject + | CustomToolChoice + | 'none' + | 'auto' + | 'required'; /** * A tool in the conversation. @@ -416,12 +416,12 @@ export type ToolChoice = * @interface */ export interface Tool extends PromptCache { - /** The name of the function. */ - type: string; - /** A description of the function. */ - function?: Function; - // this is used to support tools like computer, web_search, etc. - [key: string]: any; + /** The name of the function. */ + type: string; + /** A description of the function. */ + function?: Function; + // this is used to support tools like computer, web_search, etc. + [key: string]: any; } /** @@ -429,71 +429,71 @@ export interface Tool extends PromptCache { * @interface */ export interface Params { - model?: string; - prompt?: string | string[]; - messages?: Message[]; - functions?: Function[]; - function_call?: "none" | "auto" | { name: string }; - max_tokens?: number; - max_completion_tokens?: number; - temperature?: number; - top_p?: number; - n?: number; - stream?: boolean; - logprobs?: number; - top_logprobs?: boolean; - echo?: boolean; - stop?: string | string[]; - presence_penalty?: number; - frequency_penalty?: number; - best_of?: number; - logit_bias?: { [key: string]: number }; - user?: string; - context?: string; - examples?: Examples[]; - top_k?: number; - tools?: Tool[]; - tool_choice?: ToolChoice; - reasoning_effort?: "none" | "minimal" | "low" | "medium" | "high" | string; - response_format?: { - type: "json_object" | "text" | "json_schema"; - json_schema?: any; - }; - seed?: number; - store?: boolean; - metadata?: object; - modalities?: string[]; - audio?: { - voice: string; - format: string; - }; - service_tier?: string; - prediction?: { - type: string; - content: - | { - type: string; - text: string; - }[] - | string; - }; - // Google Vertex AI specific - safety_settings?: any; - // Anthropic specific - anthropic_beta?: string; - anthropic_version?: string; - thinking?: { - type?: string; - budget_tokens: number; - }; - // Embeddings specific - dimensions?: number; - parameters?: any; + model?: string; + prompt?: string | string[]; + messages?: Message[]; + functions?: Function[]; + function_call?: 'none' | 'auto' | { name: string }; + max_tokens?: number; + max_completion_tokens?: number; + temperature?: number; + top_p?: number; + n?: number; + stream?: boolean; + logprobs?: number; + top_logprobs?: boolean; + echo?: boolean; + stop?: string | string[]; + presence_penalty?: number; + frequency_penalty?: number; + best_of?: number; + logit_bias?: { [key: string]: number }; + user?: string; + context?: string; + examples?: Examples[]; + top_k?: number; + tools?: Tool[]; + tool_choice?: ToolChoice; + reasoning_effort?: 'none' | 'minimal' | 'low' | 'medium' | 'high' | string; + response_format?: { + type: 'json_object' | 'text' | 'json_schema'; + json_schema?: any; + }; + seed?: number; + store?: boolean; + metadata?: object; + modalities?: string[]; + audio?: { + voice: string; + format: string; + }; + service_tier?: string; + prediction?: { + type: string; + content: + | { + type: string; + text: string; + }[] + | string; + }; + // Google Vertex AI specific + safety_settings?: any; + // Anthropic specific + anthropic_beta?: string; + anthropic_version?: string; + thinking?: { + type?: string; + budget_tokens: number; + }; + // Embeddings specific + dimensions?: number; + parameters?: any; } interface Examples { - input?: Message; - output?: Message; + input?: Message; + output?: Message; } /** @@ -501,10 +501,10 @@ interface Examples { * @interface */ interface FullRequestBody { - /** The configuration for handling the request. */ - config: Config; - /** The parameters for the request. */ - params: Params; + /** The configuration for handling the request. */ + config: Config; + /** The parameters for the request. */ + params: Params; } /** @@ -512,28 +512,28 @@ interface FullRequestBody { * @interface */ export interface ShortConfig { - /** The name of the provider. */ - provider: string; - /** The name of the API key for the provider. */ - virtualKey?: string; - /** The API key for the provider. */ - apiKey?: string; - cache?: CacheSettings; - retry?: RetrySettings; - resourceName?: string; - deploymentId?: string; - workersAiAccountId?: string; - apiVersion?: string; - azureAuthMode?: string; - azureManagedClientId?: string; - azureEntraClientId?: string; - azureEntraClientSecret?: string; - azureEntraTenantId?: string; - azureModelName?: string; - customHost?: string; - // Google Vertex AI specific - vertexRegion?: string; - vertexProjectId?: string; + /** The name of the provider. */ + provider: string; + /** The name of the API key for the provider. */ + virtualKey?: string; + /** The API key for the provider. */ + apiKey?: string; + cache?: CacheSettings; + retry?: RetrySettings; + resourceName?: string; + deploymentId?: string; + workersAiAccountId?: string; + apiVersion?: string; + azureAuthMode?: string; + azureManagedClientId?: string; + azureEntraClientId?: string; + azureEntraClientSecret?: string; + azureEntraTenantId?: string; + azureModelName?: string; + customHost?: string; + // Google Vertex AI specific + vertexRegion?: string; + vertexProjectId?: string; } /** @@ -541,10 +541,10 @@ export interface ShortConfig { * @interface */ interface ShortRequestBody { - /** The simplified configuration for handling the request. */ - config: ShortConfig; - /** The parameters for the request. */ - params: Params; + /** The simplified configuration for handling the request. */ + config: ShortConfig; + /** The parameters for the request. */ + params: Params; } /**