From 74bfeb5fd602ee5c8ea5ef3630c3204021180e14 Mon Sep 17 00:00:00 2001 From: Abhishek Sharma Date: Mon, 14 Sep 2026 15:40:26 -0700 Subject: [PATCH 1/5] feat(google): pass Gemini agentic video through instead of flattening it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agentic video understanding could not be requested at all, because the request lost what it needed twice on the way in: 1. inputVideoBlockSchema did not declare `processing`, and z.object() strips undeclared keys, so the mode was gone before any adapter ran. 2. The Google adapter turned every non-data: video URL into a `[video: ]` text marker, so a YouTube or Files API URI never arrived as a video in the first place. `processing` now survives Chat ingress, the Responses schema, the IR and the adapter, and is emitted only when the caller sent it — no existing request gains an unknown upstream field. Fetchable URIs are an allowlist of the two forms Google documents, YouTube and the Files API, not "anything that is not a data: URL": file_data tells Gemini to dereference the URL, so a wildcard would make the proxy the reason a caller's private host got fetched by Google. Every other URL keeps the marker, which is what the existing does-not-mislabel-an-arbitrary-remote-URL test pins. Covers axis 2 of #3377; axis 1 (--text-only) already shipped. Closes #3271 --- src/adapters/google.ts | 57 ++++++++++++- src/chat/inbound.ts | 26 ++++-- src/responses/parser-content.ts | 5 +- src/responses/schema.ts | 4 + src/types/request.ts | 11 ++- tests/adapters/google/google-adapter.test.ts | 87 ++++++++++++++++++++ 6 files changed, 179 insertions(+), 11 deletions(-) diff --git a/src/adapters/google.ts b/src/adapters/google.ts index 449870c0b3e..d84e01d35e4 100644 --- a/src/adapters/google.ts +++ b/src/adapters/google.ts @@ -229,6 +229,39 @@ function toolResultImageParts(content: string | OcxContentPart[]): unknown[] { * surfaced on Claude-on-Antigravity; the guard lives here because this is where the parts are * built. Mirrors the Anthropic adapter's own empty-block guard (src/adapters/anthropic.ts). */ +/** + * A video URI Gemini fetches on its own behalf, as a `file_data` reference. + * + * Deliberately an allowlist of the two forms Google documents, not "anything + * that is not a data: URL". `file_data` tells Gemini to go and get the bytes; + * pointing it at an arbitrary host would either fail upstream or make the proxy + * the reason a caller's private URL got dereferenced by Google. Anything not + * matched here keeps the existing `[video: …]` text marker. + * + * YouTube is the case agentic video understanding is built around; the Files API + * uri is what `files.upload` hands back for a clip that was uploaded first. + */ +function geminiFetchableVideoUri(url: string): { uri: string; mimeType: string } | null { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return null; + } + if (parsed.protocol !== "https:") return null; + + const host = parsed.hostname.toLowerCase(); + const youtubeHosts = new Set(["youtube.com", "www.youtube.com", "m.youtube.com", "youtu.be"]); + if (youtubeHosts.has(host)) return { uri: url, mimeType: "video/*" }; + + // https://generativelanguage.googleapis.com/v1beta/files/ + if (host === "generativelanguage.googleapis.com" && /\/files\/[^/]+$/.test(parsed.pathname)) { + return { uri: url, mimeType: "video/*" }; + } + + return null; +} + const GEMINI_EMPTY_PLACEHOLDER = "(empty)"; const GEMINI_EMPTY_TOOL_OUTPUT_PLACEHOLDER = "(empty tool output)"; const GEMINI_MISSING_TOOL_RESULT = "[missing tool_result for this tool_use in history]"; @@ -339,10 +372,28 @@ function messagesToGeminiFormat( continue; } if (p.type === "video") { + // Gemini accepts inline video bytes in the same Part union as images. const data = parseDataUrl(p.videoUrl); - // Gemini accepts inline video bytes in the same Part union as images. Arbitrary - // remote URLs are not valid fileData references, so retain only a short marker. - parts.push(data ? { inline_data: { mime_type: data.mediaType, data: data.base64 } } : { text: `[video: ${p.videoUrl}]` }); + if (data) { + parts.push({ inline_data: { mime_type: data.mediaType, data: data.base64 } }); + continue; + } + // Two URI forms Gemini fetches itself: a YouTube watch URL and a Files API + // uri. Those ARE valid file_data references (#3271), and flattening them to + // a text marker was the whole reason agentic video could not be reached — + // the video never arrived as a video. Every other remote URL keeps the + // marker: we have no mime type for it and no evidence Gemini can fetch it. + const fileUri = geminiFetchableVideoUri(p.videoUrl); + if (fileUri) { + parts.push({ + file_data: { file_uri: fileUri.uri, mime_type: fileUri.mimeType }, + // Carried verbatim from the caller; emitted only when they asked for it, + // so no existing request gains an unknown field. + ...(p.processing ? { processing: p.processing } : {}), + }); + continue; + } + parts.push({ text: `[video: ${p.videoUrl}]` }); continue; } // Drop empty/malformed text instead of emitting `{ text: "" }` or a bare `{}` part. diff --git a/src/chat/inbound.ts b/src/chat/inbound.ts index b12761c8e4e..9d5522e280d 100644 --- a/src/chat/inbound.ts +++ b/src/chat/inbound.ts @@ -57,11 +57,21 @@ function contentToText(content: unknown): string { // route-eligibility predicate and this translator cannot drift apart again. const imageUrlFromPart = chatImageUrlFromPart; -function videoUrlFromPart(part: Rec): string | null { +/** + * A caller's video part, with the `processing` mode Gemini's agentic video + * understanding is requested by (#3271). The object form is the only one that + * can carry it — `video_url` as a bare string has nowhere to put it. + */ +function videoFromPart(part: Rec): { url: string; processing?: string } | null { if (part.type !== "video_url") return null; const videoUrl = part.video_url; - if (typeof videoUrl === "string" && videoUrl.length > 0) return videoUrl; - if (isRec(videoUrl) && typeof videoUrl.url === "string" && videoUrl.url.length > 0) return videoUrl.url; + if (typeof videoUrl === "string" && videoUrl.length > 0) return { url: videoUrl }; + if (isRec(videoUrl) && typeof videoUrl.url === "string" && videoUrl.url.length > 0) { + const processing = typeof videoUrl.processing === "string" && videoUrl.processing.length > 0 + ? videoUrl.processing + : undefined; + return { url: videoUrl.url, ...(processing ? { processing } : {}) }; + } return null; } @@ -91,8 +101,14 @@ function userContentToBlocks(content: unknown): Rec[] { }); continue; } - const videoUrl = videoUrlFromPart(raw); - if (videoUrl) blocks.push({ type: "input_video", video_url: videoUrl }); + const video = videoFromPart(raw); + if (video) { + blocks.push({ + type: "input_video", + video_url: video.url, + ...(video.processing ? { processing: video.processing } : {}), + }); + } } return blocks; } diff --git a/src/responses/parser-content.ts b/src/responses/parser-content.ts index 7675a42f7e8..ea76652242c 100644 --- a/src/responses/parser-content.ts +++ b/src/responses/parser-content.ts @@ -8,7 +8,7 @@ type InputBlock = | { type: "input_text"; text: string } | { type: "text"; text: string } | { type: "input_image"; image_url?: string; file_id?: string; detail?: string } - | { type: "input_video"; video_url?: string } + | { type: "input_video"; video_url?: string; processing?: string } // codex-rs protocol/src/models.rs sends audio as input_audio with an audio_url. | { type: "input_audio"; audio_url?: string; format?: string } | { type: "input_file"; file_id?: string; filename?: string; file_data?: string }; @@ -61,7 +61,8 @@ export function inputContentParts(blocks: unknown): string | OcxContentPart[] { // the request never carried, which is worse than dropping malformed input. } else if (block.type === "input_video") { const videoUrl = nonEmptyString(block.video_url); - if (videoUrl) parts.push({ type: "video", videoUrl }); + const processing = nonEmptyString((block as { processing?: string }).processing); + if (videoUrl) parts.push({ type: "video", videoUrl, ...(processing ? { processing } : {}) }); } else if (block.type === "input_audio") { // Upstream Codex sends input_audio with an audio_url (codex-rs // protocol/src/models.rs). The IR has no audio carrier and no adapter consumes diff --git a/src/responses/schema.ts b/src/responses/schema.ts index fced1a9e6e2..942cc2e9188 100644 --- a/src/responses/schema.ts +++ b/src/responses/schema.ts @@ -14,6 +14,10 @@ const inputImageBlockSchema = z.object({ const inputVideoBlockSchema = z.object({ type: z.literal("input_video"), video_url: z.string().min(1), + // Gemini agentic video understanding (#3271). z.object() strips unknown keys, + // so without declaring it here the mode is dropped before any adapter sees it + // and the request silently degrades to frame-by-frame decoding. + processing: z.string().min(1).optional(), }); const inputFileBlockSchema = z.object({ type: z.literal("input_file"), diff --git a/src/types/request.ts b/src/types/request.ts index a7c319ec403..4150072dea2 100644 --- a/src/types/request.ts +++ b/src/types/request.ts @@ -204,8 +204,17 @@ export interface OcxImageContent { export interface OcxVideoContent { type: "video"; - /** A base64 `data:` URL from an OpenAI-compatible `video_url` part. */ + /** + * A base64 `data:` URL from an OpenAI-compatible `video_url` part, or a URI + * the upstream can fetch itself (a YouTube watch URL, a Files API uri). + */ videoUrl: string; + /** + * Gemini's agentic video mode, carried verbatim from the caller's + * `video_url.processing` (#3271). Absent for every request that does not ask + * for it, so no existing traffic gains a field. + */ + processing?: string; } /** A user/developer message content part: text or native media. */ diff --git a/tests/adapters/google/google-adapter.test.ts b/tests/adapters/google/google-adapter.test.ts index 01d8e2602b8..b40e7aaeffd 100644 --- a/tests/adapters/google/google-adapter.test.ts +++ b/tests/adapters/google/google-adapter.test.ts @@ -139,6 +139,93 @@ describe("google adapter — Chat Completions video input", () => { }); expect(JSON.stringify(contents)).not.toContain("file_data"); }); + + test("a YouTube URL reaches Gemini as a video, carrying the agentic processing mode", async () => { + // #3271: the mode was dropped twice on the way in — z.object() strips an + // undeclared key, and the adapter then flattened the URL to a text marker, + // so agentic video understanding could not be requested at all. + const responsesBody = chatCompletionsToResponsesBody({ + model: "google-antigravity/gemini-3.7-flash", + messages: [{ + role: "user", + content: [ + { type: "text", text: "When do the arms pick up the gear?" }, + { + type: "video_url", + video_url: { url: "https://www.youtube.com/watch?v=example", processing: "agentic" }, + }, + ], + }], + }); + const parsed = parseRequest(responsesBody); + parsed.modelId = "gemini-3.7-flash"; + + const contents = await geminiContents(parsed); + + expect(contents).toContainEqual({ + role: "user", + parts: [ + { text: "When do the arms pick up the gear?" }, + { + file_data: { file_uri: "https://www.youtube.com/watch?v=example", mime_type: "video/*" }, + processing: "agentic", + }, + ], + }); + }); + + test("a Files API uri is fetchable too, and without a mode nothing is added", async () => { + const responsesBody = chatCompletionsToResponsesBody({ + model: "google-antigravity/gemini-3.7-flash", + messages: [{ + role: "user", + content: [{ + type: "video_url", + video_url: { url: "https://generativelanguage.googleapis.com/v1beta/files/abc123" }, + }], + }], + }); + const parsed = parseRequest(responsesBody); + parsed.modelId = "gemini-3.7-flash"; + + const contents = await geminiContents(parsed); + + expect(contents).toContainEqual({ + role: "user", + parts: [{ + file_data: { + file_uri: "https://generativelanguage.googleapis.com/v1beta/files/abc123", + mime_type: "video/*", + }, + }], + }); + // A caller who did not ask for a mode must not gain an unknown upstream field. + expect(JSON.stringify(contents)).not.toContain("processing"); + }); + + test("a look-alike host is not treated as fetchable", async () => { + // The allowlist matches the host, not a substring: `file_data` asks Gemini to + // dereference the URL, so a near-miss must stay a marker rather than send + // Google after an attacker-chosen host. + for (const url of [ + "https://youtube.com.evil.test/watch?v=x", + "https://notyoutube.com/watch?v=x", + "https://generativelanguage.googleapis.com.evil.test/v1beta/files/abc", + "http://www.youtube.com/watch?v=x", + ]) { + const responsesBody = chatCompletionsToResponsesBody({ + model: "google-antigravity/gemini-3.7-flash", + messages: [{ role: "user", content: [{ type: "video_url", video_url: { url, processing: "agentic" } }] }], + }); + const parsed = parseRequest(responsesBody); + parsed.modelId = "gemini-3.7-flash"; + + const contents = await geminiContents(parsed); + + expect(contents).toContainEqual({ role: "user", parts: [{ text: `[video: ${url}]` }] }); + expect(JSON.stringify(contents)).not.toContain("file_data"); + } + }); }); describe("google adapter — tool-call ids on the wire", () => { From d2e8c85c952f3415e630acaeaf8a24ac3a06f930 Mon Sep 17 00:00:00 2001 From: Abhishek Sharma Date: Mon, 14 Sep 2026 17:06:35 -0700 Subject: [PATCH 2/5] fix(google): emit media_processing, on every video part, without a guessed mime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three corrections from review, all confirmed against Google's video-understanding docs rather than taken on trust: 1. GenerateContent reads `media_processing` with an upper-case enum (STATIC | AGENTIC) on the part. `processing: "agentic"` is the Interactions API spelling and is ignored here, so forwarding the caller's field verbatim looked like a pass-through while agentic mode never engaged. Caught by CodeRabbit on #4663. 2. The field rides on the PART, so it applies to inline_data exactly as to file_data. Emitting it on only the fetched-uri branch dropped the mode for callers who inline their clip. 3. Dropped the invented `mime_type: "video/*"`. The documented REST example for a YouTube part carries file_uri alone, and the Files API knows the type of what it stored. Also adds music.youtube.com and youtube-nocookie.com to the allowlist — same service, and the omission was an oversight rather than a decision. Co-authored-by: Abhishek Sharma --- src/adapters/google.ts | 71 +++++++++++++++----- tests/adapters/google/google-adapter.test.ts | 63 +++++++++++++++-- 2 files changed, 109 insertions(+), 25 deletions(-) diff --git a/src/adapters/google.ts b/src/adapters/google.ts index d84e01d35e4..2df36114847 100644 --- a/src/adapters/google.ts +++ b/src/adapters/google.ts @@ -232,16 +232,19 @@ function toolResultImageParts(content: string | OcxContentPart[]): unknown[] { /** * A video URI Gemini fetches on its own behalf, as a `file_data` reference. * - * Deliberately an allowlist of the two forms Google documents, not "anything - * that is not a data: URL". `file_data` tells Gemini to go and get the bytes; - * pointing it at an arbitrary host would either fail upstream or make the proxy - * the reason a caller's private URL got dereferenced by Google. Anything not - * matched here keeps the existing `[video: …]` text marker. + * Deliberately an allowlist of the forms Google documents, not "anything that is + * not a data: URL". `file_data` tells Gemini to go and get the bytes; pointing it + * at an arbitrary host would either fail upstream or make the proxy the reason a + * caller's private URL got dereferenced by Google. Anything not matched here + * keeps the existing `[video: …]` text marker. * - * YouTube is the case agentic video understanding is built around; the Files API - * uri is what `files.upload` hands back for a clip that was uploaded first. + * Returns the uri alone: the documented REST example for a YouTube part carries + * `file_data.file_uri` and nothing else, and the Files API knows the type of what + * it stored. An invented `mime_type` would be a guess on both paths. + * + * https://ai.google.dev/gemini-api/docs/generate-content/video-understanding */ -function geminiFetchableVideoUri(url: string): { uri: string; mimeType: string } | null { +function geminiFetchableVideoUri(url: string): string | null { let parsed: URL; try { parsed = new URL(url); @@ -251,17 +254,43 @@ function geminiFetchableVideoUri(url: string): { uri: string; mimeType: string } if (parsed.protocol !== "https:") return null; const host = parsed.hostname.toLowerCase(); - const youtubeHosts = new Set(["youtube.com", "www.youtube.com", "m.youtube.com", "youtu.be"]); - if (youtubeHosts.has(host)) return { uri: url, mimeType: "video/*" }; + const youtubeHosts = new Set([ + "youtube.com", + "www.youtube.com", + "m.youtube.com", + "music.youtube.com", + "youtu.be", + "www.youtube-nocookie.com", + "youtube-nocookie.com", + ]); + if (youtubeHosts.has(host)) return url; // https://generativelanguage.googleapis.com/v1beta/files/ if (host === "generativelanguage.googleapis.com" && /\/files\/[^/]+$/.test(parsed.pathname)) { - return { uri: url, mimeType: "video/*" }; + return url; } return null; } +/** + * The caller's requested video mode as GenerateContent spells it. + * + * `media_processing` sits on the part beside `inline_data`/`file_data` and takes + * `STATIC` (the default) or `AGENTIC`. `processing: "agentic"` — the spelling in + * the original request and in Google's Interactions API — is a different API and + * is ignored here, so forwarding it verbatim would have looked like a + * pass-through while agentic mode never actually engaged. + * + * Upper-cased and forwarded rather than checked against our own copy of the enum: + * that list is Google's to extend, and a stale allowlist here would silently + * downgrade a caller using a newer mode. An unrecognized value fails upstream + * naming the field, which is a better failure than us dropping it. + */ +function geminiMediaProcessing(processing: string | undefined): string | undefined { + return processing ? processing.toUpperCase() : undefined; +} + const GEMINI_EMPTY_PLACEHOLDER = "(empty)"; const GEMINI_EMPTY_TOOL_OUTPUT_PLACEHOLDER = "(empty tool output)"; const GEMINI_MISSING_TOOL_RESULT = "[missing tool_result for this tool_use in history]"; @@ -372,10 +401,19 @@ function messagesToGeminiFormat( continue; } if (p.type === "video") { + // `media_processing` rides on the PART, so it applies to inline bytes + // exactly as it does to a fetched uri — emitting it on only one of the + // two would silently drop the mode for data: URLs. + const mediaProcessing = geminiMediaProcessing(p.processing); + const processingPart = mediaProcessing ? { media_processing: mediaProcessing } : {}; + // Gemini accepts inline video bytes in the same Part union as images. const data = parseDataUrl(p.videoUrl); if (data) { - parts.push({ inline_data: { mime_type: data.mediaType, data: data.base64 } }); + parts.push({ + inline_data: { mime_type: data.mediaType, data: data.base64 }, + ...processingPart, + }); continue; } // Two URI forms Gemini fetches itself: a YouTube watch URL and a Files API @@ -385,12 +423,9 @@ function messagesToGeminiFormat( // marker: we have no mime type for it and no evidence Gemini can fetch it. const fileUri = geminiFetchableVideoUri(p.videoUrl); if (fileUri) { - parts.push({ - file_data: { file_uri: fileUri.uri, mime_type: fileUri.mimeType }, - // Carried verbatim from the caller; emitted only when they asked for it, - // so no existing request gains an unknown field. - ...(p.processing ? { processing: p.processing } : {}), - }); + // Emitted only when the caller asked for a mode, so no existing + // request gains a field it did not have. + parts.push({ file_data: { file_uri: fileUri }, ...processingPart }); continue; } parts.push({ text: `[video: ${p.videoUrl}]` }); diff --git a/tests/adapters/google/google-adapter.test.ts b/tests/adapters/google/google-adapter.test.ts index b40e7aaeffd..c4b253463d7 100644 --- a/tests/adapters/google/google-adapter.test.ts +++ b/tests/adapters/google/google-adapter.test.ts @@ -167,8 +167,8 @@ describe("google adapter — Chat Completions video input", () => { parts: [ { text: "When do the arms pick up the gear?" }, { - file_data: { file_uri: "https://www.youtube.com/watch?v=example", mime_type: "video/*" }, - processing: "agentic", + file_data: { file_uri: "https://www.youtube.com/watch?v=example" }, + media_processing: "AGENTIC", }, ], }); @@ -193,14 +193,63 @@ describe("google adapter — Chat Completions video input", () => { expect(contents).toContainEqual({ role: "user", parts: [{ - file_data: { - file_uri: "https://generativelanguage.googleapis.com/v1beta/files/abc123", - mime_type: "video/*", - }, + file_data: { file_uri: "https://generativelanguage.googleapis.com/v1beta/files/abc123" }, }], }); // A caller who did not ask for a mode must not gain an unknown upstream field. - expect(JSON.stringify(contents)).not.toContain("processing"); + expect(JSON.stringify(contents)).not.toContain("media_processing"); + }); + + test("inline video bytes carry the mode too — it rides on the part, not the uri", async () => { + // `media_processing` sits beside `inline_data`/`file_data`, so a data: URL is + // just as eligible. Emitting it on only the fetched-uri branch silently + // dropped agentic mode for callers who inline their clip. + const responsesBody = chatCompletionsToResponsesBody({ + model: "google-antigravity/gemini-3.7-flash", + messages: [{ + role: "user", + content: [{ + type: "video_url", + video_url: { url: "data:video/mp4;base64,aGVsbG8=", processing: "agentic" }, + }], + }], + }); + const parsed = parseRequest(responsesBody); + parsed.modelId = "gemini-3.7-flash"; + + const contents = await geminiContents(parsed); + + expect(contents).toContainEqual({ + role: "user", + parts: [{ + inline_data: { mime_type: "video/mp4", data: "aGVsbG8=" }, + media_processing: "AGENTIC", + }], + }); + }); + + test("the mode is sent as GenerateContent spells it, not the caller's spelling", async () => { + // The caller sends `processing: "agentic"` (the Interactions API spelling, and + // what #3271 asked for). GenerateContent reads `media_processing` with an + // upper-case enum; forwarding the caller's spelling verbatim would have looked + // like a pass-through while agentic mode never engaged. + const responsesBody = chatCompletionsToResponsesBody({ + model: "google-antigravity/gemini-3.7-flash", + messages: [{ + role: "user", + content: [{ + type: "video_url", + video_url: { url: "https://youtu.be/example", processing: "agentic" }, + }], + }], + }); + const parsed = parseRequest(responsesBody); + parsed.modelId = "gemini-3.7-flash"; + + const wire = JSON.stringify(await geminiContents(parsed)); + + expect(wire).toContain('"media_processing":"AGENTIC"'); + expect(wire).not.toContain('"processing":"agentic"'); }); test("a look-alike host is not treated as fetchable", async () => { From 84f2994f56bf0bec70e39db1654e3dccfe138b50 Mon Sep 17 00:00:00 2001 From: Abhishek Sharma Date: Fri, 18 Sep 2026 14:12:35 -0700 Subject: [PATCH 3/5] docs(google): document the video part contract and the fetchable-URI boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the completion blocker on #4663: a new user-facing adapter input contract that neither docs-site/ nor structure/ described. docs-site gets the caller-facing half — the accepted `video_url` object (`url`, optional `processing`), the AGENTIC mapping onto the Part's `media_processing`, and a table of the three URL forms with the explicit statement that every other remote URL keeps the text marker. structure/ gets the internal half, including the two things that are decisions rather than description: `media_processing` rides on the Part so it must be emitted beside `inline_data` and `file_data` alike, and `geminiFetchableVideoUri` is the trust boundary that decides which URLs opencodex asks Gemini to fetch on its own behalf — matched on parsed host and pathname, never a substring. The translated adapter references were checked rather than assumed: the five that mention video do so in the `ollama-native` section about video being rejected there, which the google contract does not contradict. --- .../src/content/docs/reference/adapters.md | 23 ++++++++++++++++ structure/providers/google.md | 26 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/docs-site/src/content/docs/reference/adapters.md b/docs-site/src/content/docs/reference/adapters.md index 793a4522fe6..9637cd7b0dc 100644 --- a/docs-site/src/content/docs/reference/adapters.md +++ b/docs-site/src/content/docs/reference/adapters.md @@ -269,6 +269,29 @@ header and does not guarantee a provider cache hit. `functionResponse` per representable call. Interrupted histories receive an explicit missing-result marker; duplicate or standalone results are preserved as marked text (and image siblings) rather than emitted as invalid unpaired `functionResponse` parts. +- **Video input and agentic processing.** The OpenAI-compatible content part + `{"type": "video_url", "video_url": {"url": "…", "processing": "agentic"}}` is accepted on both + the Chat and Responses ingress routes. `url` is required; `processing` is optional and is + upper-cased onto the Gemini part as `media_processing` (`STATIC` is Gemini's default, `AGENTIC` + requests agentic video understanding). The field sits on the **part**, beside `inline_data` or + `file_data`, so it applies to inline bytes and fetched URIs alike — it is not the Interactions + API's `processing`. A request that omits `processing` gains no field, so existing callers are + unchanged. + + Three URL forms are handled, and only three: + + | `url` | sent as | + | --- | --- | + | `data:` URL | `inline_data` with the data URL's own media type | + | YouTube watch URL (`youtube.com`, `youtu.be`, `m.`/`music.`/`-nocookie` variants) | `file_data.file_uri` | + | `https://generativelanguage.googleapis.com/v1beta/files/` | `file_data.file_uri` | + + Any other remote URL is kept as the text marker `[video: ]`, because the adapter has no + media type for it and no evidence Gemini will fetch it. The allowlist is matched on the parsed + URL's host and path over HTTPS — not on a substring — so a look-alike host does not become a + `file_data` reference the proxy asks Gemini to fetch. `file_data` carries `file_uri` only; no + guessed `mime_type` is attached. + - **Inline image output:** when the model is one of the explicit image-capable chat IDs (`gemini-3.1-flash-image`, `gemini-2.0-flash-preview-image-generation`, or `gemini-3-pro-image-preview`), the adapter sends `responseModalities: ["TEXT", "IMAGE"]`. diff --git a/structure/providers/google.md b/structure/providers/google.md index 3da16fe08a3..1a64d0e4f1b 100644 --- a/structure/providers/google.md +++ b/structure/providers/google.md @@ -109,3 +109,29 @@ per-turn ordinal lists; the serialized ceiling, held at half `MAX_DEBUG_LINE_BYT turn detail from the tail until the summary fits. Without the second, a worst case inside the first serializes past the debug buffer's per-line cap, and the buffer truncates at a byte boundary: the consumer gets unparseable JSON whose retained prefix still reads `truncated: false`. + +## Video part boundary and agentic media processing + +The inbound contract is the OpenAI-compatible content part +`{ type: "video_url", video_url: { url, processing? } }`, normalized by both the Chat and +Responses ingress into an internal `{ type: "video", videoUrl, processing? }` part. `processing` +is caller-supplied and optional; nothing infers it. + +The outbound contract is a GenerateContent `contents[].parts[]` entry. `media_processing` is a +**Part** field, not a request field and not the Interactions API's `processing`, so it is emitted +beside `inline_data` and beside `file_data` alike — attaching it to only the fetched-URI branch +would silently drop the mode for `data:` URLs, which is the shape the first revision of #3271 had. +`geminiMediaProcessing` upper-cases the caller's value and returns `undefined` when absent, so a +request that did not ask for a mode gains no field. + +`geminiFetchableVideoUri` is the trust boundary: it decides which URLs opencodex will ask Gemini +to **fetch on its own behalf**. It parses the URL and requires HTTPS, then admits exactly two +families — the YouTube watch hosts (`youtube.com`, `www.`/`m.`/`music.` variants, `youtu.be`, and +the `-nocookie` forms) and `generativelanguage.googleapis.com` with a path matching +`/files/`. Matching is on the parsed host and pathname, never a substring of the URL, so a +look-alike host cannot become a `file_data` reference. Everything else keeps the +`[video: ]` text marker: without a media type there is nothing correct to send, and a +fetchable reference the proxy cannot vouch for is the SSRF-shaped half of this feature. + +`file_data` carries `file_uri` only. An earlier revision guessed a `mime_type` for it; the Files +API already knows the type of what it stores, and a wrong guess is worse than no guess. From 7174c294999dfda773f633d5c045ad872a812c70 Mon Sep 17 00:00:00 2001 From: Abhishek Sharma Date: Thu, 24 Sep 2026 21:23:00 -0700 Subject: [PATCH 4/5] fix(google): match the Files API resource path, not any /files/ suffix The allowlist tested parsed.pathname against an unanchored /\/files\/[^/]+$/, so it accepted more than the Files API resource form the comment described: /upload/v1beta/files/abc the resumable-upload endpoint /v1beta/tunedModels/x/files/abc a files path nested under another resource Neither is readable as a file resource, and file_data.file_uri asks Gemini to dereference the URL, so both were sent upstream instead of staying text markers. Anchored at the start instead. The version segment stays loose (v1[a-z0-9]*) rather than pinned to v1beta, because this service is reachable as v1, v1beta and v1alpha, and pinning would reject URLs that are valid today. --- src/adapters/google.ts | 8 ++++++-- tests/adapters/google/google-adapter.test.ts | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/adapters/google.ts b/src/adapters/google.ts index 9edc1115868..52e17210ea1 100644 --- a/src/adapters/google.ts +++ b/src/adapters/google.ts @@ -267,8 +267,12 @@ function geminiFetchableVideoUri(url: string): string | null { ]); if (youtubeHosts.has(host)) return url; - // https://generativelanguage.googleapis.com/v1beta/files/ - if (host === "generativelanguage.googleapis.com" && /\/files\/[^/]+$/.test(parsed.pathname)) { + // The Files API resource form, https://generativelanguage.googleapis.com/v1beta/files/. + // Anchored at the start so the resumable-upload endpoint (/upload/v1beta/files/) does + // not match: that URL is not a readable resource, and passing it as `file_data.file_uri` + // would have Gemini dereference something it cannot read. The version segment stays loose + // because this service is reachable as v1, v1beta and v1alpha. + if (host === "generativelanguage.googleapis.com" && /^\/v1[a-z0-9]*\/files\/[^/]+$/.test(parsed.pathname)) { return url; } diff --git a/tests/adapters/google/google-adapter.test.ts b/tests/adapters/google/google-adapter.test.ts index c4b253463d7..6adc1962b0a 100644 --- a/tests/adapters/google/google-adapter.test.ts +++ b/tests/adapters/google/google-adapter.test.ts @@ -260,6 +260,10 @@ describe("google adapter — Chat Completions video input", () => { "https://youtube.com.evil.test/watch?v=x", "https://notyoutube.com/watch?v=x", "https://generativelanguage.googleapis.com.evil.test/v1beta/files/abc", + // The resumable-upload endpoint, not the resource: Gemini cannot read it back. + "https://generativelanguage.googleapis.com/upload/v1beta/files/abc", + // A files path nested under something else is not the resource form either. + "https://generativelanguage.googleapis.com/v1beta/tunedModels/x/files/abc", "http://www.youtube.com/watch?v=x", ]) { const responsesBody = chatCompletionsToResponsesBody({ From 6c8b4a42dbd43362bfbba692e4654ee06bf9278d Mon Sep 17 00:00:00 2001 From: Abhishek Sharma Date: Thu, 24 Sep 2026 23:28:22 -0700 Subject: [PATCH 5/5] docs(adapters): match the widened Files API URL form The accepted resource form is v1, v1beta or v1alpha, not v1beta alone, and the resumable-upload path is explicitly not accepted. The table described the predicate as it was before 7174c29, which now understates what is accepted and says nothing about what was deliberately excluded. --- docs-site/src/content/docs/reference/adapters.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs-site/src/content/docs/reference/adapters.md b/docs-site/src/content/docs/reference/adapters.md index 7cd0afe1522..8ba84427117 100644 --- a/docs-site/src/content/docs/reference/adapters.md +++ b/docs-site/src/content/docs/reference/adapters.md @@ -297,12 +297,14 @@ freeform call echoed without its `` close counts as complete once | --- | --- | | `data:` URL | `inline_data` with the data URL's own media type | | YouTube watch URL (`youtube.com`, `youtu.be`, `m.`/`music.`/`-nocookie` variants) | `file_data.file_uri` | - | `https://generativelanguage.googleapis.com/v1beta/files/` | `file_data.file_uri` | + | `https://generativelanguage.googleapis.com//files/` — the Files API resource form, where `` is `v1`, `v1beta` or `v1alpha` | `file_data.file_uri` | Any other remote URL is kept as the text marker `[video: ]`, because the adapter has no media type for it and no evidence Gemini will fetch it. The allowlist is matched on the parsed URL's host and path over HTTPS — not on a substring — so a look-alike host does not become a - `file_data` reference the proxy asks Gemini to fetch. `file_data` carries `file_uri` only; no + `file_data` reference the proxy asks Gemini to fetch. The path is anchored at the start, so the + resumable-upload endpoint (`/upload//files/`) is *not* accepted: it is not a + readable resource, and `file_data.file_uri` asks Gemini to dereference what it is given. `file_data` carries `file_uri` only; no guessed `mime_type` is attached. - **Inline image output:** when the model is one of the explicit image-capable chat IDs