From 60613f25f9f6ae23aadcbdab7e1994f42ef9ce36 Mon Sep 17 00:00:00 2001 From: Bob Lee Date: Sun, 18 Oct 2026 15:31:37 +0800 Subject: [PATCH 1/2] fix(browser-control): keep external CDP control in the background External browser control raised the user's browser on every attach: connect, switch_page and tab_new defaulted to activate=true and sent Page.bringToFront, new tabs used Target.createTarget's foreground default, and the macOS launch path used `open -na`, which activates the app. Browser control is background work now. Tabs are created with background: true, Page.bringToFront only runs for an explicit `activate` request, and the macOS launch path uses `open -g`. The legacy /json/new endpoint cannot create a background tab, so background creation goes through the browser-level WebSocket advertised by /json/version and falls back to the HTTP endpoint when that is unavailable. Launched automation instances also disable occluded-window backgrounding so a background page still loads, renders and captures. The two flows that genuinely need a visible window keep their activation: the browser-owned approval prompt before a real-profile connection, and a user-initiated restart. Windows and Linux have no supported non-activating launch, so only that first launch can take focus; every CDP operation afterwards stays in the background. Co-authored-by: OpenBitFun <318544290+bitfun-ai@users.noreply.github.com> --- .../desktop/src/api/browser_control_api.rs | 5 + .../agentic/tools/browser_control/AGENTS.md | 5 + .../tools/browser_control/browser_launcher.rs | 35 +++- .../tools/browser_control/cdp_client.rs | 149 ++++++++++++++++- .../tools/implementations/control_hub_tool.rs | 155 ++++++++++++++---- .../src/browser_control/launcher.rs | 135 +++++++++++++-- 6 files changed, 427 insertions(+), 57 deletions(-) diff --git a/src/apps/desktop/src/api/browser_control_api.rs b/src/apps/desktop/src/api/browser_control_api.rs index e23b71fbe1..d85e11ff85 100644 --- a/src/apps/desktop/src/api/browser_control_api.rs +++ b/src/apps/desktop/src/api/browser_control_api.rs @@ -354,6 +354,11 @@ async fn complete_launch( ) -> Result { match result { LaunchResult::UserProfileReady { endpoint } => { + // Attaching to a guarded real profile makes the browser show its own + // approval request, and that prompt is only answerable from a + // visible window. Browser control itself is background work, so + // this is one of the few flows that must surface the browser. + BrowserLauncher::activate_browser(kind); let connection = CdpClient::connect_user_profile_browser( logical_port, endpoint.port, diff --git a/src/crates/assembly/core/src/agentic/tools/browser_control/AGENTS.md b/src/crates/assembly/core/src/agentic/tools/browser_control/AGENTS.md index b76048f150..00a6c34384 100644 --- a/src/crates/assembly/core/src/agentic/tools/browser_control/AGENTS.md +++ b/src/crates/assembly/core/src/agentic/tools/browser_control/AGENTS.md @@ -16,6 +16,11 @@ observation/targeting helpers, used by both built-in and external browser client duplicate refs are errors, not successful empty snapshots. - Bounded text/value projections declare truncation. Inaccessible frames and offscreen elements are reported so absence is not mistaken for nonexistence. +- External browser control is background work. Creating or selecting a tab must + not raise the user's window or take focus: pass `background: true` to + `Target.createTarget` and keep `Page.bringToFront` behind an explicit + `activate` request. Launch paths follow the same rule (`open -g` on macOS), and + only a browser-owned consent prompt or a user-visible launch may activate. ## Focused verification diff --git a/src/crates/assembly/core/src/agentic/tools/browser_control/browser_launcher.rs b/src/crates/assembly/core/src/agentic/tools/browser_control/browser_launcher.rs index f4ca8df9da..03d682c0fb 100644 --- a/src/crates/assembly/core/src/agentic/tools/browser_control/browser_launcher.rs +++ b/src/crates/assembly/core/src/agentic/tools/browser_control/browser_launcher.rs @@ -76,7 +76,7 @@ impl BrowserLauncher { Ok(provider::BrowserLauncher::launch_with_cdp_options( kind, port, - Self::launch_options(None), + Self::launch_options(None, false), ) .await?) } @@ -89,13 +89,20 @@ impl BrowserLauncher { Ok(provider::BrowserLauncher::launch_with_cdp_options( kind, port, - Self::launch_options(user_data_dir), + Self::launch_options(user_data_dir, false), ) .await?) } + /// Restarting the user's browser is an explicit, user-initiated action, so + /// the relaunched browser is brought forward instead of staying hidden. pub async fn restart_with_cdp(kind: &BrowserKind, port: u16) -> OpenBitFunResult { - Self::launch_with_cdp(kind, port).await + Ok(provider::BrowserLauncher::launch_with_cdp_options( + kind, + port, + Self::launch_options(None, true), + ) + .await?) } /// Explicit Settings flow: keep the browser settings page open long enough @@ -105,16 +112,23 @@ impl BrowserLauncher { kind: &BrowserKind, port: u16, ) -> OpenBitFunResult { - let mut options = Self::launch_options(None); + let mut options = Self::launch_options(None, true); options.wait_for_user_profile_setup = true; Ok(provider::BrowserLauncher::launch_with_cdp_options(kind, port, options).await?) } - fn launch_options(user_data_dir: Option<&str>) -> BrowserLaunchOptions { + /// Surface an already-running browser so the user can answer a + /// browser-owned prompt. Returns whether the platform accepted the request. + pub fn activate_browser(kind: &BrowserKind) -> bool { + provider::BrowserLauncher::activate_browser(kind) + } + + fn launch_options(user_data_dir: Option<&str>, activate: bool) -> BrowserLaunchOptions { BrowserLaunchOptions { user_data_dir: user_data_dir.map(PathBuf::from), managed_profile_root: Some(get_path_manager_arc().user_data_dir()), wait_for_user_profile_setup: false, + activate, } } } @@ -127,19 +141,20 @@ mod tests { #[test] fn launch_options_injects_openbitfun_managed_profile_root() { - let options = BrowserLauncher::launch_options(None); + let options = BrowserLauncher::launch_options(None, false); assert_eq!(options.user_data_dir, None); assert_eq!( options.managed_profile_root, Some(get_path_manager_arc().user_data_dir()) ); + assert!(!options.activate); } #[test] fn launch_options_preserves_explicit_user_data_dir() { let explicit_profile = "custom-browser-profile"; - let options = BrowserLauncher::launch_options(Some(explicit_profile)); + let options = BrowserLauncher::launch_options(Some(explicit_profile), false); assert_eq!(options.user_data_dir, Some(PathBuf::from(explicit_profile))); assert_eq!( @@ -147,4 +162,10 @@ mod tests { Some(get_path_manager_arc().user_data_dir()) ); } + + #[test] + fn only_user_visible_launch_flows_activate_the_browser() { + assert!(!BrowserLauncher::launch_options(None, false).activate); + assert!(BrowserLauncher::launch_options(None, true).activate); + } } diff --git a/src/crates/assembly/core/src/agentic/tools/browser_control/cdp_client.rs b/src/crates/assembly/core/src/agentic/tools/browser_control/cdp_client.rs index 034620cd5e..6b46fc896c 100644 --- a/src/crates/assembly/core/src/agentic/tools/browser_control/cdp_client.rs +++ b/src/crates/assembly/core/src/agentic/tools/browser_control/cdp_client.rs @@ -102,6 +102,51 @@ impl CdpClient { .map_err(|error| OpenBitFunError::tool(error.to_string())) } + /// Create a new page/tab on a legacy fixed debug port without raising the + /// browser window. + /// + /// `PUT /json/new` always creates a foreground tab. Background creation + /// therefore goes through the browser-level WebSocket that `/json/version` + /// advertises and falls back to the HTTP endpoint when that WebSocket is + /// unavailable, so an old or restricted endpoint keeps working. + pub async fn create_page_in_background( + port: u16, + url: Option<&str>, + ) -> OpenBitFunResult { + let Some(ws_url) = Self::get_version(port) + .await + .ok() + .and_then(|version| version.web_socket_debugger_url) + else { + return Self::create_page(port, url).await; + }; + let Ok(browser) = Self::connect(&ws_url).await else { + return Self::create_page(port, url).await; + }; + let created = browser.create_browser_page(url, true).await; + browser.disconnect().await; + let page = created?; + + // A browser-level create response carries no page WebSocket URL, but + // legacy callers attach through it. Resolve the same target from `/json` + // so both creation paths return the identical shape. + for _ in 0..10 { + if let Ok(pages) = Self::list_pages(port).await { + if let Some(page) = pages.into_iter().find(|candidate| { + candidate.id == page.id && candidate.web_socket_debugger_url.is_some() + }) { + return Ok(page); + } + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + + Err(OpenBitFunError::tool(format!( + "Created tab {} but its page WebSocket never appeared on port {}. Connect to that page by id instead of creating another tab.", + page.id, port + ))) + } + /// Connect to a specific page by its legacy WebSocket debugger URL. pub async fn connect(ws_url: &str) -> OpenBitFunResult { info!("CDP connecting to page WebSocket"); @@ -426,12 +471,24 @@ impl CdpClient { } /// Create a target through the browser WebSocket and return its metadata. - pub async fn create_browser_page(&self, url: Option<&str>) -> OpenBitFunResult { + /// + /// `background` asks the browser not to activate the new tab. + /// `Target.createTarget` defaults to a foreground tab, and a foreground tab + /// raises the browser window and takes focus away from whatever the user is + /// doing. Browsers that ignore the parameter degrade to the old behavior + /// instead of failing the call. + pub async fn create_browser_page( + &self, + url: Option<&str>, + background: bool, + ) -> OpenBitFunResult { self.require_browser_connection()?; let target_url = url.unwrap_or("about:blank"); - let result = self - .send("Target.createTarget", Some(json!({ "url": target_url }))) - .await?; + let mut request = json!({ "url": target_url }); + if background { + request["background"] = json!(true); + } + let result = self.send("Target.createTarget", Some(request)).await?; let target_id = result .get("targetId") .and_then(Value::as_str) @@ -728,6 +785,90 @@ mod tests { assert_eq!(pages[0].web_socket_debugger_url, None); } + #[tokio::test] + async fn page_creation_only_asks_for_a_background_tab_when_requested() { + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("bind mock CDP server"); + let address = listener.local_addr().expect("mock CDP address"); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept CDP client"); + let mut socket = tokio_tungstenite::accept_async(stream) + .await + .expect("accept WebSocket"); + let mut requested_background = Vec::new(); + let mut created = 0; + let mut listed = 0; + + while let Some(message) = socket.next().await { + let Message::Text(text) = message.expect("read CDP command") else { + continue; + }; + let command: Value = serde_json::from_str(&text).expect("parse CDP command"); + let id = command + .get("id") + .and_then(Value::as_i64) + .expect("command id"); + + let result = match command + .get("method") + .and_then(Value::as_str) + .expect("command method") + { + "Target.createTarget" => { + created += 1; + requested_background.push(command["params"]["background"].clone()); + json!({ "targetId": format!("page-{created}") }) + } + "Target.getTargets" => { + listed += 1; + json!({ + "targetInfos": [{ + "targetId": format!("page-{created}"), + "type": "page", + "title": "", + "url": "about:blank" + }] + }) + } + other => panic!("unexpected CDP command: {other}"), + }; + + socket + .send(Message::Text( + json!({ "id": id, "result": result }).to_string().into(), + )) + .await + .expect("send CDP response"); + + if listed == 2 { + break; + } + } + + requested_background + }); + + let browser = CdpClient::connect(&format!("ws://{address}")) + .await + .expect("connect browser WebSocket"); + browser + .create_browser_page(None, true) + .await + .expect("create background page"); + browser + .create_browser_page(None, false) + .await + .expect("create foreground page"); + + let requested_background = server.await.expect("mock CDP server"); + assert_eq!( + requested_background, + vec![json!(true), Value::Null], + "background creation asks for a background tab; foreground creation keeps the browser default" + ); + } + #[tokio::test] async fn browser_websocket_flattens_commands_and_routes_page_events() { let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/control_hub_tool.rs b/src/crates/assembly/core/src/agentic/tools/implementations/control_hub_tool.rs index 077ab5cf69..8f59a05229 100644 --- a/src/crates/assembly/core/src/agentic/tools/implementations/control_hub_tool.rs +++ b/src/crates/assembly/core/src/agentic/tools/implementations/control_hub_tool.rs @@ -290,14 +290,38 @@ impl ControlHubTool { } } - async fn create_browser_page(port: u16, url: Option<&str>) -> OpenBitFunResult { + /// Create a new page/tab for an external browser session. + /// + /// Browser control is background work by default: the tab is created + /// without activating it so the user's window and focus stay where they + /// are. Only an explicit `activate` request creates a foreground tab. + async fn create_browser_page( + port: u16, + url: Option<&str>, + activate: bool, + ) -> OpenBitFunResult { if let Some(connection) = CdpClient::browser_connection(port).await { - connection.client.create_browser_page(url).await - } else { + connection.client.create_browser_page(url, !activate).await + } else if activate { CdpClient::create_page(port, url).await + } else { + CdpClient::create_page_in_background(port, url).await } } + /// Whether this call asked the browser to surface the page to the user. + /// + /// Background is the default for every browser action. `activate: true` is + /// the opt-in for the cases that genuinely need the visible window: the + /// user asked to watch or be shown the page, or a human step (sign-in, + /// consent, CAPTCHA) has to happen in it. + fn browser_activation_requested(params: &Value) -> bool { + params + .get("activate") + .and_then(Value::as_bool) + .unwrap_or(false) + } + async fn connect_page(port: u16, page: &CdpPageInfo) -> OpenBitFunResult { if let Some(connection) = CdpClient::browser_connection(port).await { connection.client.attach_to_page(&page.id).await @@ -470,6 +494,7 @@ impl ControlHubTool { ### domain: "browser" - `navigate { url, target?, session_id?, observe? }` opens a URL/domain and creates a page when needed. No preliminary connect/open/list call is needed. Default `target: "builtin"` uses Desktop's built-in browser; subsequent calls retain this conversation's page. `observe: "snapshot"` returns controls/@eN refs; `observe: "article"` returns readable text. Omit observe to only show the page; reuse returned observations. - For account/login-dependent work, prefer `target: "external"` only when a user-authorized connection is available. `connect { mode: "default", session_id? | target_url?, target_title?, activate? }` selects an existing tab without navigating; filters must match uniquely. Settings owns browser/profile selection and consent. Unavailable/disconnected access requires the user to connect there; never enable permissions or switch profiles yourself. +- External tabs are created and selected in the background: the browser window, its z-order, and the user's focus stay untouched. Pass `activate: true` (navigate/connect/tab_new/switch_page) only when the user asked to watch or be shown the page, or when the next step needs them to act inside it (sign-in, consent, payment, CAPTCHA, a native picker). Background control still reads and drives the page; never raise the window just to make an action "look" like it ran. - `connect { mode: "headless", port? }` only attaches to an already-running headless test browser (default 9223). It never launches or falls back to the user's browser. External calls inherit the selected page's port. - Both targets share the same action contract. Keep returned session_id across pages; stale ids never select another tab. To show the browser surface without a URL, use OpenBitFunControl `{ action: "open", capability_id: "feature.browser" }`. Do not invent `about:blank`. @@ -477,7 +502,7 @@ Actions (`selector` accepts observed @eN refs or CSS; `ref` is an alias): - Observe: `snapshot`, `read_article`, `get { selector, attribute? }` (text/html/value/href), `evaluate { expression, await_promise?, max_value_bytes? }` for structured extraction. Refresh only when the next decision needs new state. - Input: `click/hover/check/uncheck { selector }`, `fill { selector, value }` replaces text, `type { text }` appends at focus, `select { selector, option_text }`, `press_key { key }`, `scroll { direction?, amount? }`, `auto_scroll { direction?, max_scrolls?, delay_ms? }`. - Wait/capture: `wait { condition, timeout_ms? }` accepts a selector, load, domcontentloaded or networkidle; prefer the needed element. Built-in networkidle means native load completion. `screenshot { format?, quality?, full_page? }`: full-page requires CDP; text-only models use snapshot. -- Tabs: `list_pages`, `tab_query { url_contains?, title_contains?, limit? }`, `tab_new { url }`, `switch_page { page_id, activate? }`, `back/forward/reload/close`. +- Tabs: `list_pages`, `tab_query { url_contains?, title_contains?, limit? }`, `tab_new { url, activate? }`, `switch_page { page_id, activate? }`, `back/forward/reload/close`. - Page context: `fetch { url, method?, headers?, body? }` uses page credentials and CORS; `cookies { urls? }`, `set_cookies { cookies }`. WebFetch reads public HTTP without browser state; WebSearch discovers URLs. - CDP only: `set_file_input_files { selector, files }`, `cdp { method, params? }`, `network/console/errors { sub_command?, filter?, since?, limit? }`, `trace { sub_command }`, `dialog { response, prompt_text? }` arms the next dialog. @@ -1230,6 +1255,11 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o Err(error) => return Ok(err_response("browser", action, error)), }; let requested_id = params.get("session_id").and_then(Value::as_str); + // Browser control is background work: navigation reuses or creates a + // tab without touching the user's window unless the caller explicitly + // asked to show it. + let activate_on_create = + kind == BrowserTargetKind::External && Self::browser_activation_requested(params); let mut created = false; let mut opened_with_url = false; let target = if kind == BrowserTargetKind::BuiltIn { @@ -1341,7 +1371,8 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o { // A first URL visit must not overwrite an arbitrary user's // tab (or another conversation's selected page). - let page = Self::create_browser_page(connected.port, None).await?; + let page = + Self::create_browser_page(connected.port, None, activate_on_create).await?; let client = Self::connect_page(connected.port, &page).await?; let session = BrowserSession { session_id: page.id, @@ -1403,6 +1434,25 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o data["target"] = json!(kind.as_str()); data["session_id"] = json!(target.session_id()); data["created"] = json!(created); + if kind == BrowserTargetKind::External { + // A freshly created foreground tab is already visible; an existing + // page the caller asked to surface still needs the explicit raise. + let activated = if activate_on_create && !created { + target + .client() + .send("Page.bringToFront", None) + .await + .is_ok() + } else { + activate_on_create + }; + data["activated"] = json!(activated); + data["activation"] = json!(if activate_on_create { + "foreground" + } else { + "background" + }); + } if data["loaded"] == false && data.get("warning").is_none() { data["warning"] = json!("Navigation was submitted, but loading is not confirmed. Inspect this page or wait for the needed element; do not repeat navigation to repair an observation."); } @@ -1744,8 +1794,9 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o // (3 round-trips and one chance to pick the wrong id). With // `target_url` / `target_title` we collapse those into a // single `connect` call: pick the unique page whose URL or - // title contains the substring, register it as the default - // session, and bring it to the front. + // title contains the substring and register it as the default + // session. Selecting a tab stays a background operation; only + // `activate: true` surfaces it in the user's window. let target_url = params .get("target_url") .and_then(|v| v.as_str()) @@ -1754,10 +1805,7 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o .get("target_title") .and_then(|v| v.as_str()) .map(str::to_lowercase); - let activate = params - .get("activate") - .and_then(|v| v.as_bool()) - .unwrap_or(true); + let activate = Self::browser_activation_requested(params); let version = Self::browser_version(port).await?; if mode == "headless" { @@ -1771,7 +1819,7 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o && target_url.is_none() && target_title.is_none(); if created_page { - pages.push(Self::create_browser_page(port, None).await?); + pages.push(Self::create_browser_page(port, None, activate).await?); } let connected_browser = if mode == "headless" { "Headless test browser".to_string() @@ -1866,14 +1914,13 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o .enable_observers() .await; - // If the model targeted a specific tab AND wants it - // foregrounded (default), bring it to front the same - // way switch_page does. Failure here is non-fatal — - // we still return the connected session. + // Selecting a tab is background work. Only an explicit + // `activate: true` raises the browser window; a failed raise is + // still non-fatal because the session itself is connected. let mut activated = false; let mut activate_warning: Option = None; let targeted = matched_by_target.is_some(); - if targeted && activate { + if activate && !created_page { match session.client.send("Page.bringToFront", None).await { Ok(_) => activated = true, Err(e) => { @@ -1883,6 +1930,9 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o )); } } + } else if activate { + // The tab was just created in the foreground for this call. + activated = true; } let mut result = json!({ @@ -1899,6 +1949,7 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o "matched_by_target": targeted, "created": created_page, "activated": activated, + "activation": if activate { "foreground" } else { "background" }, "status": if mode == "headless" { "attached" } else if uses_user_profile { @@ -2170,11 +2221,8 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o )]); } let url = params.get("url").and_then(|v| v.as_str()); - let activate = params - .get("activate") - .and_then(|v| v.as_bool()) - .unwrap_or(true); - let page = Self::create_browser_page(port, url).await?; + let activate = Self::browser_activation_requested(params); + let page = Self::create_browser_page(port, url, activate).await?; let client = Self::connect_page(port, &page).await?; let session = BrowserSession { session_id: page.id.clone(), @@ -2189,9 +2237,13 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o let _ = BrowserActions::new(session.client.as_ref()) .enable_observers() .await; - if activate { - let _ = session.client.send("Page.bringToFront", None).await; - } + let activated = if activate { + // New tabs stay in the background unless the caller asked + // to hand this one to the user. + session.client.send("Page.bringToFront", None).await.is_ok() + } else { + false + }; Ok(vec![ToolResult::ok( json!({ "success": true, @@ -2199,7 +2251,8 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o "session_id": session.session_id, "page_url": page.url, "page_title": page.title, - "activated": activate, + "activated": activated, + "activation": if activate { "foreground" } else { "background" }, }), Some(format!( "New tab opened: {} (session {})", @@ -2242,16 +2295,11 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o )), )]); } - // Phase 2: by default ALSO surface the chosen tab in the - // user's actual browser window via `Page.bringToFront`. The - // legacy behavior only swapped the CDP session under the - // hood, leaving the user staring at the old tab while the - // model "drove" an invisible one. Models can opt out by - // passing `activate: false` for headless background tabs. - let activate = params - .get("activate") - .and_then(|v| v.as_bool()) - .unwrap_or(true); + // Switching the agent's target page is a background operation. + // `activate: true` additionally surfaces the tab in the user's + // window, which is only wanted when they asked to watch or when + // the next step needs them to interact with it. + let activate = Self::browser_activation_requested(params); let registry = browser_sessions(); let mut reused = false; @@ -2304,6 +2352,7 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o "session_id": session.session_id, "reused": reused, "activated": activated, + "activation": if activate { "foreground" } else { "background" }, }); if let Some(w) = &activate_warning { body["warning"] = json!(w); @@ -3507,6 +3556,7 @@ impl Tool for ControlHubTool { "session_id": { "type": "string", "description": "Exact browser page id returned by navigation, connection or inventory. Not the conversation id." }, "url": { "type": "string", "description": "Destination URL for navigate/tab_new, request URL for fetch." }, "observe": { "type": "string", "enum": ["none", "snapshot", "article"], "description": "Navigation's optional final observation; reuse it instead of a separate read." }, + "activate": { "type": "boolean", "description": "External browser only; false by default. When true, the created or selected tab is brought in front of the user's window. Use it only when the user must see the page or act in it." }, "selector": { "type": "string", "description": "Observed @eN ref or CSS selector for a page element." }, "value": { "type": "string", "description": "Replacement text for fill." }, "text": { "type": "string", "description": "Text to type into the focused control." }, @@ -4691,6 +4741,39 @@ mod control_hub_tests { ); } + #[test] + fn browser_activation_is_opt_in() { + // Browser control must not steal the user's window or focus unless the + // caller asked for it, so every missing or non-boolean value is + // background. + assert!(!ControlHubTool::browser_activation_requested(&json!({}))); + assert!(!ControlHubTool::browser_activation_requested(&json!({ + "activate": false + }))); + assert!(!ControlHubTool::browser_activation_requested(&json!({ + "activate": "true" + }))); + assert!(ControlHubTool::browser_activation_requested(&json!({ + "activate": true + }))); + } + + #[tokio::test] + async fn description_documents_background_browser_control() { + let desc = ControlHubTool::new().description().await.unwrap(); + assert!( + desc.contains("created and selected in the background") + && desc.contains("`activate: true`"), + "description must state that external tabs stay in the background and when activation is allowed" + ); + let schema = ControlHubTool::new().input_schema(); + assert_eq!( + schema.pointer("/properties/params/properties/activate/type"), + Some(&json!("boolean")), + "activate must be discoverable in the tool schema" + ); + } + #[tokio::test] async fn desktop_domain_returns_migration_error() { let tool = ControlHubTool::new(); diff --git a/src/crates/services/services-integrations/src/browser_control/launcher.rs b/src/crates/services/services-integrations/src/browser_control/launcher.rs index d613041729..49875c45cf 100644 --- a/src/crates/services/services-integrations/src/browser_control/launcher.rs +++ b/src/crates/services/services-integrations/src/browser_control/launcher.rs @@ -79,6 +79,11 @@ pub struct BrowserLaunchOptions { /// explicit setup action; ordinary agent connects should return guidance /// quickly instead of holding a tool call open. pub wait_for_user_profile_setup: bool, + /// Bring the launched browser to the foreground. Off by default: browser + /// control is background work, and activating the user's browser on every + /// attach steals focus from whatever they were doing. Opt in only for a + /// user-visible action such as restarting their browser from Settings. + pub activate: bool, } impl BrowserLauncher { @@ -648,17 +653,36 @@ impl BrowserLauncher { } } + /// `open` arguments for launching a macOS browser instance. + /// + /// Without `-g`, `open` activates the application, so every launch pulled + /// the user's browser in front of whatever they were doing. Background + /// control therefore launches with `-g` and lets an explicit user-visible + /// action opt back into activation. + #[cfg(target_os = "macos")] + fn macos_open_args(app_name: &str, browser_args: &[String], activate: bool) -> Vec { + let mut args = Vec::with_capacity(browser_args.len() + 5); + if !activate { + args.push("-g".to_string()); + } + args.push("-n".to_string()); + args.push("-a".to_string()); + args.push(app_name.to_string()); + args.push("--args".to_string()); + args.extend(browser_args.iter().cloned()); + args + } + #[cfg(target_os = "macos")] fn spawn_macos_browser( kind: &BrowserKind, exe: &str, args: &[String], + activate: bool, ) -> std::io::Result { if let Some(app_name) = Self::launch_app_name(kind) { - let mut command = silent_command("open"); - command.args(["-na", app_name, "--args"]); - command.args(args); - command.spawn() + let open_args = Self::macos_open_args(app_name, args, activate); + silent_command("open").args(open_args).spawn() } else { silent_command(exe).args(args).spawn() } @@ -669,6 +693,7 @@ impl BrowserLauncher { _kind: &BrowserKind, exe: &str, args: &[String], + _activate: bool, ) -> std::io::Result { silent_command(exe).args(args).spawn() } @@ -678,8 +703,47 @@ impl BrowserLauncher { kind: &BrowserKind, exe: &str, args: &[String], + activate: bool, ) -> std::io::Result { - Self::spawn_macos_browser(kind, exe, args) + Self::spawn_macos_browser(kind, exe, args, activate) + } + + /// Bring an already-running browser to the foreground. + /// + /// A browser-owned consent or approval prompt is only answerable from a + /// visible window, so the flows that request one must surface the browser + /// instead of leaving the user to hunt for a background window. Returns + /// whether the request was accepted; platforms without a supported + /// activation entry point report `false` and the caller keeps its normal + /// guidance. + pub fn activate_browser(kind: &BrowserKind) -> bool { + #[cfg(target_os = "macos")] + { + let Some(app_name) = Self::launch_app_name(kind) else { + return false; + }; + match silent_command("open").args(["-a", app_name]).output() { + Ok(output) if output.status.success() => true, + Ok(output) => { + debug!( + "Could not activate {}: {}", + kind, + String::from_utf8_lossy(&output.stderr).trim() + ); + false + } + Err(error) => { + debug!("Could not run open to activate {}: {}", kind, error); + false + } + } + } + + #[cfg(not(target_os = "macos"))] + { + let _ = kind; + false + } } /// Get the executable path or launch command for a browser kind. @@ -852,20 +916,32 @@ impl BrowserLauncher { }; let flag = format!("--remote-debugging-port={}", port); let profile_flag = format!("--user-data-dir={}", profile_dir.display()); - let extra: Vec = vec![ + let mut extra: Vec = vec![ flag.clone(), profile_flag, "--no-first-run".to_string(), "--no-default-browser-check".to_string(), ]; + // This instance exists for background automation: keep its renderer and + // timers running while the window is not focused or is occluded, so a + // background-driven page still loads, animates, and captures. + extra.extend( + [ + "--disable-backgrounding-occluded-windows", + "--disable-renderer-backgrounding", + "--disable-background-timer-throttling", + ] + .map(str::to_string), + ); info!( - "Launching {} with CDP on port {} (user_data_dir={})", + "Launching {} with CDP on port {} (user_data_dir={}, activate={})", kind, port, - profile_dir.display() + profile_dir.display(), + options.activate ); - let result = Self::spawn_browser(kind, &exe, &extra); + let result = Self::spawn_browser(kind, &exe, &extra, options.activate); match result { Ok(_child) => { @@ -888,8 +964,19 @@ impl BrowserLauncher { } } + /// Restart is a user-initiated, visible action: the browser was just quit, + /// so it comes back in the foreground instead of hiding behind the app the + /// user clicked Restart in. pub async fn restart_with_cdp(kind: &BrowserKind, port: u16) -> Result { - Self::launch_with_cdp_options(kind, port, BrowserLaunchOptions::default()).await + Self::launch_with_cdp_options( + kind, + port, + BrowserLaunchOptions { + activate: true, + ..BrowserLaunchOptions::default() + }, + ) + .await } #[allow(dead_code)] @@ -1168,4 +1255,32 @@ mod tests { )); assert!(!BrowserLauncher::default_cdp_preference_enabled("{}")); } + + #[test] + fn browser_launch_does_not_activate_by_default() { + assert!(!BrowserLaunchOptions::default().activate); + } + + #[cfg(target_os = "macos")] + #[test] + fn macos_background_launch_keeps_the_browser_out_of_the_foreground() { + let browser_args = vec!["--remote-debugging-port=9222".to_string()]; + + let background = BrowserLauncher::macos_open_args("Google Chrome", &browser_args, false); + assert_eq!( + background, + vec![ + "-g".to_string(), + "-n".to_string(), + "-a".to_string(), + "Google Chrome".to_string(), + "--args".to_string(), + "--remote-debugging-port=9222".to_string(), + ] + ); + + let foreground = BrowserLauncher::macos_open_args("Google Chrome", &browser_args, true); + assert!(!foreground.iter().any(|argument| argument == "-g")); + assert!(foreground.contains(&"--args".to_string())); + } } From 7cd33850ee347049a7042f488bbdd4faf47c9aaf Mon Sep 17 00:00:00 2001 From: Bob Lee Date: Sun, 18 Oct 2026 15:34:39 +0800 Subject: [PATCH 2/2] docs(architecture): state that external browser control is background work Record the browser activation contract next to the existing browser tool orchestration section so a future change does not reintroduce window raising: tabs are created in the background, page selection never surfaces the window, and only an explicit `activate` request, a browser-owned approval prompt, or a user-initiated restart may bring the browser forward. Co-authored-by: OpenBitFun <318544290+bitfun-ai@users.noreply.github.com> --- docs/architecture/computer-use-control.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/architecture/computer-use-control.md b/docs/architecture/computer-use-control.md index 8406be0452..6bc2d580e0 100644 --- a/docs/architecture/computer-use-control.md +++ b/docs/architecture/computer-use-control.md @@ -56,6 +56,16 @@ fall through to a different tab. Ending a conversation releases its selection, not the user's browser pages. `open_builtin` and URL-bearing `read_article` remain compatible aliases, including their legacy result keys. +External browser actions are background work. A conversation's tab is created in +the background and selecting a page never raises the user's window; only an +explicit `activate: true` request surfaces it, for steps that need the human +(show or watch, sign-in, consent, payment, CAPTCHA, a native picker). Launch +follows the same rule, so macOS starts the browser without activating it. The +exceptions are the browser-owned approval prompt for a guarded real-profile +connection and a user-initiated restart from Settings. A platform without a +non-activating launch can let the initial browser start take focus; every later +CDP action stays in the background. + Browser permissions are independent of `ai.computer_use_enabled`. External browser identity/availability is exposed only after a Settings connection has succeeded (or an opted-in startup connection receives browser-owned consent).