feat(browser): show sites a common Edge

crates/botloftd/src/browser/agent.rs

@@ -0,0 +1,110 @@
+//! What the browser tells sites it is (spec 21.3): the user agent of the
+//! Edge it runs without the word `Headless`, with the brands and platform a
+//! site reads from `navigator.userAgentData`. Overriding the user agent
+//! alone would leave those empty, which no common browser does.
+
+use std::time::Duration;
+
+use serde_json::{Value, json};
+use tokio::sync::mpsc;
+use tracing::debug;
+
+use super::BrowserError;
+use super::cdp::{Cdp, CdpEvent};
+
+/// A page of the browser's own that has `navigator.userAgentData`, which
+/// only secure pages have (`about:blank` does not).
+const SECURE_PAGE: &str = "chrome://version";
+/// What `navigator.userAgentData` says, once the page has it.
+const READ_METADATA: &str = "navigator.userAgentData ? navigator.userAgentData\
+    .getHighEntropyValues(['architecture', 'bitness', 'fullVersionList', 'model', \
+    'platformVersion', 'wow64']) : null";
+/// Tries while the page loads, 50 ms apart.
+const TRIES: usize = 40;
+
+/// The params of `Emulation.setUserAgentOverride` for every tab. Run before
+/// anything else reads `events`: the page it reads from opens and closes
+/// here, and its events are dropped.
+pub async fn read(
+    cdp: &Cdp,
+    events: &mut mpsc::UnboundedReceiver<CdpEvent>,
+) -> Result<Value, BrowserError> {
+    let version = cdp.call(None, "Browser.getVersion", json!({})).await?;
+    let user_agent = version["userAgent"]
+        .as_str()
+        .unwrap_or_default()
+        .replace("HeadlessChrome", "Chrome");
+    let mut params = json!({ "userAgent": user_agent });
+    match metadata(cdp).await {
+        Ok(Some(metadata)) => params["userAgentMetadata"] = metadata,
+        Ok(None) => debug!("browser: the user agent data did not come"),
+        Err(err) => debug!("browser: reading the user agent data: {err}"),
+    }
+    while events.try_recv().is_ok() {}
+    Ok(params)
+}
+
+/// The brands and platform a secure page reads, in the shape
+/// `Emulation.UserAgentMetadata` takes.
+async fn metadata(cdp: &Cdp) -> Result<Option<Value>, BrowserError> {
+    let created = cdp
+        .call(
+            None,
+            "Target.createTarget",
+            json!({ "url": SECURE_PAGE, "background": true }),
+        )
+        .await?;
+    let target = created["targetId"].clone();
+    let attached = cdp
+        .call(
+            None,
+            "Target.attachToTarget",
+            json!({ "targetId": target, "flatten": true }),
+        )
+        .await;
+    let read = match attached {
+        Ok(attached) => {
+            let session = attached["sessionId"].as_str().unwrap_or_default();
+            let read = evaluate(cdp, session).await;
+            let params = json!({ "sessionId": session });
+            let _ = cdp.call(None, "Target.detachFromTarget", params).await;
+            read
+        }
+        Err(err) => Err(err.into()),
+    };
+    let _ = cdp
+        .call(None, "Target.closeTarget", json!({ "targetId": target }))
+        .await;
+    // It closes a moment after the answer: still there, the tabs the daemon
+    // takes next would count it.
+    for _ in 0..TRIES {
+        let targets = cdp.call(None, "Target.getTargets", json!({})).await?;
+        let open = targets["targetInfos"]
+            .as_array()
+            .is_some_and(|all| all.iter().any(|info| info["targetId"] == target));
+        if !open {
+            break;
+        }
+        tokio::time::sleep(Duration::from_millis(50)).await;
+    }
+    read
+}
+
+async fn evaluate(cdp: &Cdp, session: &str) -> Result<Option<Value>, BrowserError> {
+    let params =
+        json!({ "expression": READ_METADATA, "awaitPromise": true, "returnByValue": true });
+    for _ in 0..TRIES {
+        // The page may still be blank, or its context gone as it loads.
+        if let Ok(mut read) = cdp
+            .call(Some(session), "Runtime.evaluate", params.clone())
+            .await
+        {
+            let value = read["result"]["value"].take();
+            if value.get("brands").is_some() {
+                return Ok(Some(value));
+            }
+        }
+        tokio::time::sleep(Duration::from_millis(50)).await;
+    }
+    Ok(None)
+}

crates/botloftd/src/browser/events.rs

@@ -57,10 +57,7 @@ impl Session {
                 json!({ "enabled": true }),
             ),
             ("Emulation.setDeviceMetricsOverride", viewport.metrics()),
-            (
-                "Emulation.setUserAgentOverride",
-                json!({ "userAgent": self.user_agent }),
-            ),
+            ("Emulation.setUserAgentOverride", self.user_agent.clone()),
         ];
         // A tab opened by a page waits for us, and some of these only
         // answer once it runs: send them all, the go-ahead last, then wait.

crates/botloftd/src/browser/launch.rs

@@ -65,6 +65,9 @@ pub async fn launch(
             // open their own tabs in the bot's browser.
             "--disable-extensions",
             "--disable-sync",
+            // `navigator.webdriver` stays `false`, as in a browser someone
+            // uses: some sites refuse to sign in to one a program drives.
+            "--disable-blink-features=AutomationControlled",
         ])
         .arg(format!("--user-data-dir={}", profile.display()))
         .arg("about:blank")

crates/botloftd/src/browser/mod.rs

@@ -3,6 +3,7 @@
 //! first browser tool, closes when nobody uses or watches it, and sends
 //! live frames to the app connection that watches it.
 
+mod agent;
 mod call;
 mod cdp;
 mod events;

crates/botloftd/src/browser/session.rs

@@ -13,11 +13,11 @@ use serde_json::json;
 use tokio::sync::{Notify, watch};
 use tracing::debug;
 
-use super::BrowserError;
 use super::cdp::Cdp;
 use super::events::pump;
 use super::launch::{self, BrowserProcess};
 use super::viewport::{MAX_HEIGHT, Viewport, WIDTH};
+use super::{BrowserError, agent};
 
 /// Most tabs a browser keeps; past this, the one active longest ago closes.
 pub(super) const MAX_TABS: usize = 6;
@@ -125,7 +125,8 @@ pub struct Session {
     /// The next element ref, shared by every page of this browser.
     pub next_ref: AtomicU64,
     process: Mutex<Option<BrowserProcess>>,
-    pub(super) user_agent: String,
+    /// The params of `Emulation.setUserAgentOverride` (`agent.rs`).
+    pub(super) user_agent: serde_json::Value,
     /// One change of what the app wants at a time (`watch.rs`).
     pub(super) syncing: tokio::sync::Mutex<()>,
 }
@@ -141,12 +142,8 @@ impl Session {
         hooks: Hooks,
     ) -> Result<Arc<Self>, BrowserError> {
         let (process, url) = launch::launch(program, profile).await?;
-        let (cdp, events) = Cdp::connect(&url).await?;
-        let version = cdp.call(None, "Browser.getVersion", json!({})).await?;
-        let user_agent = version["userAgent"]
-            .as_str()
-            .unwrap_or_default()
-            .replace("HeadlessChrome", "Chrome");
+        let (cdp, mut events) = Cdp::connect(&url).await?;
+        let user_agent = agent::read(&cdp, &mut events).await?;
         let session = Arc::new(Self {
             cdp,
             tabs: Mutex::new(Tabs {

crates/botloftd/tests/browser.rs

@@ -102,6 +102,27 @@ async fn words_made_of_one_element_per_letter_keep_their_spaces() {
     assert!(page.contains("Two words"), "{page}");
 }
 
+#[tokio::test(flavor = "multi_thread")]
+async fn sites_see_a_common_browser() {
+    let Some(mut b) = setup().await else { return };
+    let opening = call(
+        &b.mcp,
+        "browser_open",
+        json!({ "url": format!("{}/signals", b.site) }),
+    );
+    answer_site(&mut b.app, true, None).await;
+    let page = opening.await.expect("task").expect("page");
+    assert!(page.contains("Webdriver: false"), "{page}");
+    // The order of the brands changes from one version to the next.
+    let brands = page
+        .split("Brands: ")
+        .nth(1)
+        .and_then(|rest| rest.split(';').next())
+        .unwrap_or_default();
+    assert!(brands.contains("Chromium"), "{page}");
+    assert!(!page.contains("Headless"), "{page}");
+}
+
 #[tokio::test(flavor = "multi_thread")]
 async fn a_site_the_owner_denies_stays_closed() {
     let Some(mut b) = setup().await else { return };

crates/botloftd/tests/common/site.rs

@@ -2,8 +2,8 @@
 //! it leads to, a link that opens a new tab, a button that shows a dialog,
 //! a sign-in page with its fields at fixed points, for the owner's hands,
 //! a page that says how big its window is, one that counts its loads, one
-//! that counts the times it went out of sight and one with each letter in
-//! its own element.
+//! that counts the times it went out of sight, one with each letter in its
+//! own element and one that shows what tells a site a program drives it.
 
 use std::collections::HashMap;
 use std::net::SocketAddr;
@@ -50,6 +50,12 @@ const SIGHT: &str = "<!doctype html><title>Sight</title><p id=\"hidden\">Hidden
     if (document.hidden) { hidden += 1; document.getElementById('hidden').textContent = \
     'Hidden ' + hidden + ' times'; } });</script>";
 
+/// What sites look at to tell a browser a program drives it.
+const SIGNALS: &str = "<!doctype html><title>Signals</title><p id=\"signals\"></p><script>\
+    document.getElementById('signals').textContent = 'Webdriver: ' + navigator.webdriver + \
+    '; Brands: ' + navigator.userAgentData.brands.map(b => b.brand).join(', ') + \
+    '; Agent: ' + navigator.userAgent + '; Outer: ' + outerWidth + ' x ' + outerHeight;</script>";
+
 async fn account(Form(form): Form<HashMap<String, String>>) -> Html<String> {
     let user = form.get("user").cloned().unwrap_or_default();
     let length = form
@@ -93,6 +99,7 @@ pub async fn serve() -> SocketAddr {
         .route("/size", get(|| async { Html(SIZE) }))
         .route("/sight", get(|| async { Html(SIGHT) }))
         .route("/letters", get(|| async { Html(LETTERS) }))
+        .route("/signals", get(|| async { Html(SIGNALS) }))
         .route("/account", post(account));
     let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
     let addr = listener.local_addr().expect("addr");