Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions docs/architecture/computer-use-control.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,16 @@ fall through to a different tab. Ending a conversation releases its selection,
not the user's browser pages. `open_builtin` and URL-bearing `read_article`
remain compatible aliases, including their legacy result keys.

External browser actions are background work. A conversation's tab is created in
the background and selecting a page never raises the user's window; only an
explicit `activate: true` request surfaces it, for steps that need the human
(show or watch, sign-in, consent, payment, CAPTCHA, a native picker). Launch
follows the same rule, so macOS starts the browser without activating it. The
exceptions are the browser-owned approval prompt for a guarded real-profile
connection and a user-initiated restart from Settings. A platform without a
non-activating launch can let the initial browser start take focus; every later
CDP action stays in the background.

Browser permissions are independent of `ai.computer_use_enabled`. External
browser identity/availability is exposed only after a Settings connection has
succeeded (or an opted-in startup connection receives browser-owned consent).
Expand Down
5 changes: 5 additions & 0 deletions src/apps/desktop/src/api/browser_control_api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,11 @@ async fn complete_launch(
) -> Result<BrowserControlLaunchResponse, String> {
match result {
LaunchResult::UserProfileReady { endpoint } => {
// Attaching to a guarded real profile makes the browser show its own
// approval request, and that prompt is only answerable from a
// visible window. Browser control itself is background work, so
// this is one of the few flows that must surface the browser.
BrowserLauncher::activate_browser(kind);
let connection = CdpClient::connect_user_profile_browser(
logical_port,
endpoint.port,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ observation/targeting helpers, used by both built-in and external browser client
duplicate refs are errors, not successful empty snapshots.
- Bounded text/value projections declare truncation. Inaccessible frames and
offscreen elements are reported so absence is not mistaken for nonexistence.
- External browser control is background work. Creating or selecting a tab must
not raise the user's window or take focus: pass `background: true` to
`Target.createTarget` and keep `Page.bringToFront` behind an explicit
`activate` request. Launch paths follow the same rule (`open -g` on macOS), and
only a browser-owned consent prompt or a user-visible launch may activate.

## Focused verification

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ impl BrowserLauncher {
Ok(provider::BrowserLauncher::launch_with_cdp_options(
kind,
port,
Self::launch_options(None),
Self::launch_options(None, false),
)
.await?)
}
Expand All @@ -89,13 +89,20 @@ impl BrowserLauncher {
Ok(provider::BrowserLauncher::launch_with_cdp_options(
kind,
port,
Self::launch_options(user_data_dir),
Self::launch_options(user_data_dir, false),
)
.await?)
}

/// Restarting the user's browser is an explicit, user-initiated action, so
/// the relaunched browser is brought forward instead of staying hidden.
pub async fn restart_with_cdp(kind: &BrowserKind, port: u16) -> OpenBitFunResult<LaunchResult> {
Self::launch_with_cdp(kind, port).await
Ok(provider::BrowserLauncher::launch_with_cdp_options(
kind,
port,
Self::launch_options(None, true),
)
.await?)
}

/// Explicit Settings flow: keep the browser settings page open long enough
Expand All @@ -105,16 +112,23 @@ impl BrowserLauncher {
kind: &BrowserKind,
port: u16,
) -> OpenBitFunResult<LaunchResult> {
let mut options = Self::launch_options(None);
let mut options = Self::launch_options(None, true);
options.wait_for_user_profile_setup = true;
Ok(provider::BrowserLauncher::launch_with_cdp_options(kind, port, options).await?)
}

fn launch_options(user_data_dir: Option<&str>) -> BrowserLaunchOptions {
/// Surface an already-running browser so the user can answer a
/// browser-owned prompt. Returns whether the platform accepted the request.
pub fn activate_browser(kind: &BrowserKind) -> bool {
provider::BrowserLauncher::activate_browser(kind)
}

fn launch_options(user_data_dir: Option<&str>, activate: bool) -> BrowserLaunchOptions {
BrowserLaunchOptions {
user_data_dir: user_data_dir.map(PathBuf::from),
managed_profile_root: Some(get_path_manager_arc().user_data_dir()),
wait_for_user_profile_setup: false,
activate,
}
}
}
Expand All @@ -127,24 +141,31 @@ mod tests {

#[test]
fn launch_options_injects_openbitfun_managed_profile_root() {
let options = BrowserLauncher::launch_options(None);
let options = BrowserLauncher::launch_options(None, false);

assert_eq!(options.user_data_dir, None);
assert_eq!(
options.managed_profile_root,
Some(get_path_manager_arc().user_data_dir())
);
assert!(!options.activate);
}

#[test]
fn launch_options_preserves_explicit_user_data_dir() {
let explicit_profile = "custom-browser-profile";
let options = BrowserLauncher::launch_options(Some(explicit_profile));
let options = BrowserLauncher::launch_options(Some(explicit_profile), false);

assert_eq!(options.user_data_dir, Some(PathBuf::from(explicit_profile)));
assert_eq!(
options.managed_profile_root,
Some(get_path_manager_arc().user_data_dir())
);
}

#[test]
fn only_user_visible_launch_flows_activate_the_browser() {
assert!(!BrowserLauncher::launch_options(None, false).activate);
assert!(BrowserLauncher::launch_options(None, true).activate);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,51 @@ impl CdpClient {
.map_err(|error| OpenBitFunError::tool(error.to_string()))
}

/// Create a new page/tab on a legacy fixed debug port without raising the
/// browser window.
///
/// `PUT /json/new` always creates a foreground tab. Background creation
/// therefore goes through the browser-level WebSocket that `/json/version`
/// advertises and falls back to the HTTP endpoint when that WebSocket is
/// unavailable, so an old or restricted endpoint keeps working.
pub async fn create_page_in_background(
port: u16,
url: Option<&str>,
) -> OpenBitFunResult<CdpPageInfo> {
let Some(ws_url) = Self::get_version(port)
.await
.ok()
.and_then(|version| version.web_socket_debugger_url)
else {
return Self::create_page(port, url).await;
};
let Ok(browser) = Self::connect(&ws_url).await else {
return Self::create_page(port, url).await;
};
let created = browser.create_browser_page(url, true).await;
browser.disconnect().await;
let page = created?;

// A browser-level create response carries no page WebSocket URL, but
// legacy callers attach through it. Resolve the same target from `/json`
// so both creation paths return the identical shape.
for _ in 0..10 {
if let Ok(pages) = Self::list_pages(port).await {
if let Some(page) = pages.into_iter().find(|candidate| {
candidate.id == page.id && candidate.web_socket_debugger_url.is_some()
}) {
return Ok(page);
}
}
tokio::time::sleep(Duration::from_millis(50)).await;
}

Err(OpenBitFunError::tool(format!(
"Created tab {} but its page WebSocket never appeared on port {}. Connect to that page by id instead of creating another tab.",
page.id, port
)))
}

/// Connect to a specific page by its legacy WebSocket debugger URL.
pub async fn connect(ws_url: &str) -> OpenBitFunResult<Self> {
info!("CDP connecting to page WebSocket");
Expand Down Expand Up @@ -426,12 +471,24 @@ impl CdpClient {
}

/// Create a target through the browser WebSocket and return its metadata.
pub async fn create_browser_page(&self, url: Option<&str>) -> OpenBitFunResult<CdpPageInfo> {
///
/// `background` asks the browser not to activate the new tab.
/// `Target.createTarget` defaults to a foreground tab, and a foreground tab
/// raises the browser window and takes focus away from whatever the user is
/// doing. Browsers that ignore the parameter degrade to the old behavior
/// instead of failing the call.
pub async fn create_browser_page(
&self,
url: Option<&str>,
background: bool,
) -> OpenBitFunResult<CdpPageInfo> {
self.require_browser_connection()?;
let target_url = url.unwrap_or("about:blank");
let result = self
.send("Target.createTarget", Some(json!({ "url": target_url })))
.await?;
let mut request = json!({ "url": target_url });
if background {
request["background"] = json!(true);
}
let result = self.send("Target.createTarget", Some(request)).await?;
let target_id = result
.get("targetId")
.and_then(Value::as_str)
Expand Down Expand Up @@ -728,6 +785,90 @@ mod tests {
assert_eq!(pages[0].web_socket_debugger_url, None);
}

#[tokio::test]
async fn page_creation_only_asks_for_a_background_tab_when_requested() {
let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0))
.await
.expect("bind mock CDP server");
let address = listener.local_addr().expect("mock CDP address");
let server = tokio::spawn(async move {
let (stream, _) = listener.accept().await.expect("accept CDP client");
let mut socket = tokio_tungstenite::accept_async(stream)
.await
.expect("accept WebSocket");
let mut requested_background = Vec::new();
let mut created = 0;
let mut listed = 0;

while let Some(message) = socket.next().await {
let Message::Text(text) = message.expect("read CDP command") else {
continue;
};
let command: Value = serde_json::from_str(&text).expect("parse CDP command");
let id = command
.get("id")
.and_then(Value::as_i64)
.expect("command id");

let result = match command
.get("method")
.and_then(Value::as_str)
.expect("command method")
{
"Target.createTarget" => {
created += 1;
requested_background.push(command["params"]["background"].clone());
json!({ "targetId": format!("page-{created}") })
}
"Target.getTargets" => {
listed += 1;
json!({
"targetInfos": [{
"targetId": format!("page-{created}"),
"type": "page",
"title": "",
"url": "about:blank"
}]
})
}
other => panic!("unexpected CDP command: {other}"),
};

socket
.send(Message::Text(
json!({ "id": id, "result": result }).to_string().into(),
))
.await
.expect("send CDP response");

if listed == 2 {
break;
}
}

requested_background
});

let browser = CdpClient::connect(&format!("ws://{address}"))
.await
.expect("connect browser WebSocket");
browser
.create_browser_page(None, true)
.await
.expect("create background page");
browser
.create_browser_page(None, false)
.await
.expect("create foreground page");

let requested_background = server.await.expect("mock CDP server");
assert_eq!(
requested_background,
vec![json!(true), Value::Null],
"background creation asks for a background tab; foreground creation keeps the browser default"
);
}

#[tokio::test]
async fn browser_websocket_flattens_commands_and_routes_page_events() {
let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0))
Expand Down
Loading
Loading