diff --git a/codex-rs/browser-computer-use/src/browser_playwright_evidence_test.mjs b/codex-rs/browser-computer-use/src/browser_playwright_evidence_test.mjs index 677edf873f76..49c71f63c72e 100644 --- a/codex-rs/browser-computer-use/src/browser_playwright_evidence_test.mjs +++ b/codex-rs/browser-computer-use/src/browser_playwright_evidence_test.mjs @@ -174,6 +174,68 @@ test("provider records selector and wheel settled trail in its manifest", async } }); +test("provider applies a per-call CSS viewport and scroll and returns measured screenshot evidence", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "codex-browser-provider-viewport-")); + const server = http.createServer((_request, response) => { + response.end("
viewport fixture
"); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + const request = { + tool: "browser_observe", + arguments: { + url: `http://127.0.0.1:${address.port}/`, + scope: "viewport_and_page", + view: { viewportWidth: 390, viewportHeight: 844, scrollY: 120 }, + }, + }; + const child = spawn(process.execPath, [fileURLToPath(new URL("./browser_playwright_provider.mjs", import.meta.url))], { + env: { + ...process.env, + CODEX_BROWSER_PLAYWRIGHT_STATE_DIR: stateDir, + CODEX_BROWSER_PLAYWRIGHT_ISOLATION: "shared", + CODEX_BROWSER_PLAYWRIGHT_HEADLESS: "1", + CODEX_BROWSER_PLAYWRIGHT_VIEWPORT_WIDTH: "1280", + CODEX_BROWSER_PLAYWRIGHT_VIEWPORT_HEIGHT: "720", + CODEX_BROWSER_PLAYWRIGHT_CAPTURE_MODE: "viewport", + CODEX_BROWSER_PLAYWRIGHT_ARTIFACT_POLICY: "off", + }, + stdio: ["pipe", "pipe", "inherit"], + }); + child.stdin.end(JSON.stringify(request)); + try { + const output = await new Promise((resolve, reject) => { + let text = ""; + child.stdout.on("data", (chunk) => { text += chunk; }); + child.on("error", reject); + child.on("close", (code, signal) => { + if (code !== 0) { + reject(new Error(`provider exited with code ${code} signal ${signal}: ${text}`)); + } else { + resolve(text); + } + }); + }); + const response = JSON.parse(output); + assert.equal(response.success, true, response.contentItems?.[0]?.text); + const text = response.contentItems[0].text; + assert.match(text, /viewport: 390x844 scroll=0,120 document=\d+x\d+/); + assert.match(text, /capture_metadata: \{"requestedViewport":\{"width":390,"height":844\},"effectiveViewport":\{"width":390,"height":844\},"clientViewport":\{"width":390,"height":844\}/); + + const images = response.contentItems.filter(({ type }) => type === "inputImage"); + assert.equal(images.length, 1); + assert.equal(images[0].detail, "high"); + assert.match(images[0].imageUrl, /^data:image\/png;base64,/); + const png = Buffer.from(images[0].imageUrl.slice("data:image/png;base64,".length), "base64"); + assert.deepEqual([...png.subarray(0, 8)], [137, 80, 78, 71, 13, 10, 26, 10]); + assert.equal(png.readUInt32BE(16), 390); + assert.equal(png.readUInt32BE(20), 844); + } finally { + await new Promise((resolve) => server.close(resolve)); + await fs.rm(stateDir, { recursive: true, force: true }); + } +}); + test("provider lets a first navigate action replace stale persisted state", async () => { const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "codex-browser-provider-navigate-")); const server = http.createServer((_request, response) => { diff --git a/codex-rs/browser-computer-use/src/lib.rs b/codex-rs/browser-computer-use/src/lib.rs index 805b8264898f..d64caa28965b 100644 --- a/codex-rs/browser-computer-use/src/lib.rs +++ b/codex-rs/browser-computer-use/src/lib.rs @@ -74,12 +74,12 @@ pub fn configured_browser_dynamic_tools_for_codex_home(codex_home: &Path) -> Vec vec![ browser_dynamic_tool( BROWSER_OBSERVE_TOOL_NAME, - "Capture the current browser viewport as a model-visible screenshot.", + "Capture the current browser viewport as a model-visible screenshot. Optional view.viewportWidth and view.viewportHeight set this call's CSS viewport size; view.scrollY sets its initial vertical scroll. The response reports measured geometry. This is not mobile-device emulation.", "non_mutating", ), browser_dynamic_tool( BROWSER_STEP_TOOL_NAME, - "Perform bounded browser actions, then return a fresh browser screenshot.", + "Perform bounded browser actions, then return a fresh browser screenshot. Optional view.viewportWidth and view.viewportHeight set this call's CSS viewport size; view.scrollY sets its initial vertical scroll. The response reports measured geometry. This is not mobile-device emulation.", "mutating", ), ] @@ -92,6 +92,29 @@ fn browser_dynamic_tool(name: &str, description: &str, mutation_class: &str) -> description: description.to_string(), input_schema: json!({ "type": "object", + "properties": { + "view": { + "type": "object", + "description": "Per-call CSS viewport and vertical scroll. The response reports measured effective viewport and scroll geometry; compare it with the requested values. This does not change the user agent, enable touch input, or change device scale factor.", + "properties": { + "viewportWidth": { + "type": "integer", + "minimum": 1, + "description": "CSS viewport width in pixels for this call." + }, + "viewportHeight": { + "type": "integer", + "minimum": 1, + "description": "CSS viewport height in pixels for this call." + }, + "scrollY": { + "type": "number", + "description": "Vertical page scroll position in CSS pixels to restore before capture or actions." + } + }, + "additionalProperties": true + } + }, "additionalProperties": true }), defer_loading: false, @@ -1524,6 +1547,21 @@ mod tests { .and_then(|capability| capability.family.as_deref()) == Some(COMPUTER_USE_ADAPTER_BROWSER) })); + for tool in tools { + let view = &tool.input_schema["properties"]["view"]; + assert_eq!(view["properties"]["viewportWidth"]["type"], "integer"); + assert_eq!(view["properties"]["viewportWidth"]["minimum"], 1); + assert_eq!(view["properties"]["viewportHeight"]["type"], "integer"); + assert_eq!(view["properties"]["viewportHeight"]["minimum"], 1); + assert_eq!(view["properties"]["scrollY"]["type"], "number"); + assert_eq!(tool.input_schema["additionalProperties"], true); + assert_eq!(view["additionalProperties"], true); + assert!( + view["description"].as_str().is_some_and( + |description| description.contains("does not change the user agent") + ) + ); + } } #[test]