fix(browser): clicks land on a scaled screen, and the bot's browser stops waiting for nothing

app/src/lib/protocol.gen.ts

@@ -1745,8 +1745,9 @@ export type BrowserWatchParams = { botId: BotId, };
 export type BrowserResizeParams = { botId: BotId, width: number, height: number, 
 /**
  * The screen's pixels per pixel of the app, in percent (its
- * `devicePixelRatio` times 100): the page is drawn that sharp for the
- * owner, up to twice (spec 21.3). Absent, 100.
+ * `devicePixelRatio` times 100). The daemon ignores it: the page is
+ * always drawn in its own pixels, because a sharper page made clicks
+ * land at the wrong point (spec 21.3, 19). Absent, 100.
  */
 scale?: number, };
 

app/src/motion.css

@@ -174,8 +174,8 @@
   pointer-events: none;
   translate: -4px -3px;
   transition:
-    left 520ms var(--ease-out),
-    top 520ms var(--ease-out);
+    left 260ms var(--ease-out),
+    top 260ms var(--ease-out);
 }
 
 .bot-cursor > span {
@@ -190,7 +190,7 @@
   border: 3px solid;
   border-radius: 9999px;
   pointer-events: none;
-  animation: ripple 700ms var(--ease-out) 520ms both;
+  animation: ripple 700ms var(--ease-out) 260ms both;
 }
 
 .bot-typing {

crates/botloft-core/src/protocol/browser.rs

@@ -167,8 +167,9 @@ pub struct BrowserResizeParams {
     pub width: u32,
     pub height: u32,
     /// The screen's pixels per pixel of the app, in percent (its
-    /// `devicePixelRatio` times 100): the page is drawn that sharp for the
-    /// owner, up to twice (spec 21.3). Absent, 100.
+    /// `devicePixelRatio` times 100). The daemon ignores it: the page is
+    /// always drawn in its own pixels, because a sharper page made clicks
+    /// land at the wrong point (spec 21.3, 19). Absent, 100.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     #[cfg_attr(test, ts(optional))]
     pub scale: Option<u32>,

crates/botloftd/src/browser/actions.js

@@ -19,30 +19,73 @@
     return { left, top, width: rect.width, height: rect.height };
   };
 
-  // Where to click an element, after scrolling it into the middle of the
-  // screen, and what else is on top of that point, if anything.
+  // Whether something else is on top of a point of an element: true when
+  // the point shows the element, a part of it or what it sits in.
+  const shows = (el, x, y) => {
+    const hit = el.ownerDocument.elementFromPoint(x, y);
+    return !!hit && (hit === el || el.contains(hit) || hit.contains(el));
+  };
+
+  // A point inside the element that is not covered, the middle first, then
+  // the corners and edges of its middle part, or null. Returns the fraction
+  // of the box it is at.
+  const freePoint = (el, local) => {
+    const view = el.ownerDocument.defaultView;
+    for (const fy of [0.5, 0.2, 0.8]) {
+      for (const fx of [0.5, 0.2, 0.8]) {
+        const x = local.left + local.width * fx;
+        const y = local.top + local.height * fy;
+        if (x < 0 || y < 0 || x >= view.innerWidth || y >= view.innerHeight) continue;
+        if (shows(el, x, y)) return { fx, fy };
+      }
+    }
+    return null;
+  };
+
+  // Where to click an element. It scrolls into the middle of the screen;
+  // when a fixed header, a banner or a chat widget covers it there, it tries
+  // the other alignments and the other points of the element, and when
+  // nothing is free it says so (`script`), so the click can go through the
+  // page instead of the mouse. Also what is still on top, if anything.
   const point = (ref) => {
     const el = get(ref);
     if (!el) return { missing: true, next: next() };
-    el.scrollIntoView({ block: "center", inline: "center", behavior: "instant" });
+    let pick = null;
+    for (const block of ["center", "end", "start", "nearest"]) {
+      el.scrollIntoView({ block, inline: "center", behavior: "instant" });
+      const local = el.getBoundingClientRect();
+      if (local.width === 0 || local.height === 0) break;
+      pick = freePoint(el, local);
+      if (pick) break;
+    }
+    if (!pick) el.scrollIntoView({ block: "center", inline: "center", behavior: "instant" });
     const box = topRect(el);
     const role = roleOf(el, styleOf(el), "auto") ?? el.localName;
     const label = clean(nameOf(el), 60);
     if (box.width === 0 || box.height === 0) return { hidden: true, role, label, next: next() };
-    const x = Math.min(Math.max(box.left + box.width / 2, 0), innerWidth - 1);
-    const y = Math.min(Math.max(box.top + box.height / 2, 0), innerHeight - 1);
+    const [fx, fy] = pick ? [pick.fx, pick.fy] : [0.5, 0.5];
+    const x = Math.min(Math.max(box.left + box.width * fx, 0), innerWidth - 1);
+    const y = Math.min(Math.max(box.top + box.height * fy, 0), innerHeight - 1);
     const local = el.getBoundingClientRect();
     const hit = el.ownerDocument.elementFromPoint(
-      local.left + local.width / 2,
-      local.top + local.height / 2,
+      local.left + local.width * fx,
+      local.top + local.height * fy,
     );
     let cover = null;
-    if (hit && hit !== el && !el.contains(hit) && !hit.contains(el)) {
+    if (!pick && hit && hit !== el && !el.contains(hit) && !hit.contains(el)) {
       let shown = hit;
       while (shown && !roleOf(shown, styleOf(shown), "auto")) shown = shown.parentElement;
       cover = shown ? token(shown, roleOf(shown, styleOf(shown), "auto")) : `<${hit.localName}>`;
     }
-    return { x, y, role, label, text: isTextField(el), cover, next: next() };
+    return { x, y, role, label, text: isTextField(el), cover, script: !pick && cover !== null, next: next() };
+  };
+
+  // The page's own click on an element, for when the mouse cannot reach it.
+  const clickScript = (ref) => {
+    const el = get(ref);
+    if (!el) return { missing: true };
+    el.click();
+    return { ok: true };
   };
 
   // Focuses a text field and selects what it holds, so typing replaces it.
@@ -112,5 +155,5 @@
     return el && el !== document.body && isTextField(el) ? meaning(el) : { none: true };
   };
 
-  return { point, prepareType, select, scrollEnd, at, focused };
+  return { point, clickScript, prepareType, select, scrollEnd, at, focused };
 }

crates/botloftd/src/browser/events.rs

@@ -211,14 +211,7 @@ impl Session {
             size("deviceWidth", viewport.width),
             size("deviceHeight", viewport.height),
         );
-        // The screencast draws in the page's own pixels, however sharp the
-        // page is (spec 19): on a sharper screen its frame only says the page
-        // changed, and the frame sent is a screenshot, which is that sharp.
-        let sharp = active && viewport.scale > 100;
-        if active
-            && !sharp
-            && let Some(data) = params["data"].as_str()
-        {
+        if active && let Some(data) = params["data"].as_str() {
             hooks.frames.send_replace(Some(Arc::new(BrowserFrame {
                 bot_id: hooks.bot.clone(),
                 data: data.to_owned(),
@@ -229,26 +222,7 @@ impl Session {
         let cdp = self.cdp.clone();
         let session = session.to_owned();
         let ack = json!({ "sessionId": params["sessionId"] });
-        let (frames, bot) = (hooks.frames.clone(), hooks.bot.clone());
         tokio::spawn(async move {
-            if sharp {
-                let shot = json!({ "format": "jpeg", "quality": 60, "optimizeForSpeed": true });
-                let taken = cdp
-                    .call(Some(&session), "Page.captureScreenshot", shot)
-                    .await;
-                if let Some(data) = taken
-                    .ok()
-                    .and_then(|shot| shot["data"].as_str().map(str::to_owned))
-                {
-                    frames.send_replace(Some(Arc::new(BrowserFrame {
-                        bot_id: bot,
-                        data,
-                        width,
-                        height,
-                    })));
-                }
-            }
-            // Confirmed only after, so screenshots never pile up.
             tokio::time::sleep(gap).await;
             cdp.send(Some(&session), "Page.screencastFrameAck", ack);
         });

crates/botloftd/src/browser/page.rs

@@ -19,6 +19,9 @@ pub struct Done {
     pub label: Option<String>,
     /// Another element was on top of the one clicked.
     pub cover: Option<String>,
+    /// Nothing of the element was free of what covered it, so the click went
+    /// through the page's own script and not the mouse.
+    pub scripted: bool,
     /// What `browser_select` chose.
     pub chosen: Option<String>,
 }
@@ -28,6 +31,8 @@ pub struct Done {
 pub struct Aim {
     reference: String,
     point: Value,
+    /// Covered whatever the point: only the page's own script can click it.
+    script: bool,
     pub done: Done,
 }
 
@@ -81,10 +86,12 @@ impl Session {
             point: point["x"].as_f64().zip(point["y"].as_f64()),
             label,
             cover: point["cover"].as_str().map(str::to_owned),
+            scripted: false,
             chosen: None,
         };
         Ok(Aim {
             reference: reference.to_owned(),
+            script: point["script"] == true,
             point,
             done,
         })
@@ -114,8 +121,27 @@ impl Session {
         Ok(())
     }
 
-    pub async fn click(&self, Aim { done, .. }: Aim) -> Result<Done, BrowserError> {
+    pub async fn click(
+        &self,
+        Aim {
+            reference,
+            script,
+            mut done,
+            ..
+        }: Aim,
+    ) -> Result<Done, BrowserError> {
         let marks = self.marks();
+        if script {
+            let result = self
+                .reader(&format!("clickScript({})", json!(reference)))
+                .await?;
+            if result["missing"] == true {
+                return Err(BrowserError::Stale(reference));
+            }
+            done.scripted = true;
+            self.settle(marks).await;
+            return Ok(done);
+        }
         self.mouse_click(done.point.unwrap_or_default()).await?;
         self.settle(marks).await;
         Ok(done)
@@ -129,10 +155,16 @@ impl Session {
         submit: bool,
     ) -> Result<Done, BrowserError> {
         let Aim {
-            reference, done, ..
+            reference,
+            script,
+            done,
+            ..
         } = aim;
         let marks = self.marks();
-        self.mouse_click(done.point.unwrap_or_default()).await?;
+        // A field nothing frees is still focused by `prepareType` below.
+        if !script {
+            self.mouse_click(done.point.unwrap_or_default()).await?;
+        }
         let ready = self
             .reader(&format!("prepareType({})", json!(reference)))
             .await?;

crates/botloftd/src/browser/session.rs

@@ -27,8 +27,6 @@ pub(super) const FRAME_GAP: Duration = Duration::from_millis(66);
 /// ...and ~30 while the browser is in the owner's hands, so the page keeps
 /// up with them as they scroll and point (spec 21.8).
 pub(super) const QUICK_GAP: Duration = Duration::from_millis(33);
-/// The largest side of a frame that only says the page changed.
-const SIGNAL: u32 = 64;
 /// How long the first tab may take to show up.
 const FIRST_TAB_TIMEOUT: Duration = Duration::from_secs(5);
 
@@ -259,16 +257,9 @@ impl Session {
 
     pub(super) async fn cast(&self, session: &str, on: bool) {
         let result = if on {
-            // Frames come the size of the page, however tall it is. On a
-            // sharper screen they only say the page changed (`events.rs`),
-            // so they come small.
-            let (max_width, max_height) = if self.viewport().scale > 100 {
-                (SIGNAL, SIGNAL)
-            } else {
-                (MAX_WIDTH, MAX_HEIGHT)
-            };
+            // Frames come the size of the page, however tall it is.
             let params = json!({
-                "format": "jpeg", "quality": 60, "maxWidth": max_width, "maxHeight": max_height,
+                "format": "jpeg", "quality": 60, "maxWidth": MAX_WIDTH, "maxHeight": MAX_HEIGHT,
             });
             self.cdp
                 .call(Some(session), "Page.startScreencast", params)

crates/botloftd/src/browser/settle.rs

@@ -8,11 +8,18 @@ use super::session::Session;
 const LOAD_WAIT: Duration = Duration::from_secs(15);
 const QUIET: Duration = Duration::from_millis(500);
 const QUIET_WAIT: Duration = Duration::from_secs(3);
+/// The same wait on a page that was never quiet before the action (an
+/// analytics script, a chat widget, a live feed): its network will not go
+/// quiet because of the click, so waiting for it only costs the bot 3 s per
+/// action (measured: 3.2 s a click, against 0.17 s on a quiet page).
+const NOISY_WAIT: Duration = Duration::from_millis(600);
 
 /// The active tab before an action, to tell whether it navigated.
 pub(super) struct Marks {
     target: String,
     navigations: u64,
+    /// The network was busy just before the action, whatever the action does.
+    noisy: bool,
 }
 
 impl Session {
@@ -22,6 +29,8 @@ impl Session {
         Marks {
             target: tab.map(|tab| tab.target.clone()).unwrap_or_default(),
             navigations: tab.map_or(0, |tab| tab.navigations),
+            noisy: tab
+                .is_some_and(|tab| !tab.inflight.is_empty() || tab.network_at.elapsed() < QUIET),
         }
     }
 
@@ -46,7 +55,7 @@ impl Session {
             }
             let _ = tokio::time::timeout(left, notified).await;
         }
-        let deadline = Instant::now() + QUIET_WAIT;
+        let deadline = Instant::now() + if before.noisy { NOISY_WAIT } else { QUIET_WAIT };
         loop {
             let notified = self.changed.notified();
             let quiet_for = {