Merge pull request #167 from httpsphl/feat/browser-sharp

feat(browser): the owner sees the page as sharp as their screen, and quicker in their hands

Author: httpsphl

app/src/features/browser/browser.test.tsx

@@ -82,7 +82,9 @@ describe("browser panel", () => {
     fireEvent.click(toggle());
     const resized = () =>
       fake.calls.filter((call) => call.method === "browser.resize").map((call) => call.params);
-    await waitFor(() => expect(resized()).toEqual([{ botId: scout.id, width: 560, height: 804 }]));
+    await waitFor(() =>
+      expect(resized()).toEqual([{ botId: scout.id, width: 560, height: 804, scale: 100 }]),
+    );
     // A narrow room keeps the desktop layout's width, in the room's shape.
     expect(fake.browser.size(scout.id)).toEqual({ width: 800, height: 1148 });
 

app/src/features/browser/useRoom.ts

@@ -49,21 +49,24 @@ export function useRoom(): [Size | null, (node: HTMLElement | null) => void] {
 
 /**
  * Tells the daemon the room the page has, once it stops changing, while
- * this connection watches the bot's browser.
+ * this connection watches the bot's browser, and how sharp the screen is,
+ * so the page is drawn with as many pixels as show it (spec 21.3).
  */
 export function useFitPage(bot: Pick<Bot, "id">, room: Size | null, watched: boolean): void {
   const api = useApi();
   const known = room !== null;
   const width = Math.max(1, room?.width ?? 0);
   const height = Math.max(1, room?.height ?? 0);
+  // A change of the app's zoom changes the room too, which reads it again.
+  const scale = Math.round((typeof devicePixelRatio === "number" ? devicePixelRatio : 1) * 100);
   useEffect(() => {
     if (!watched || !known) {
       return;
     }
     const timer = setTimeout(() => {
       // The next change of the room asks again.
-      api.call("browser.resize", { botId: bot.id, width, height }).catch(() => {});
+      api.call("browser.resize", { botId: bot.id, width, height, scale }).catch(() => {});
     }, SETTLED_MS);
     return () => clearTimeout(timer);
-  }, [api, bot.id, watched, known, width, height]);
+  }, [api, bot.id, watched, known, width, height, scale]);
 }

app/src/lib/protocol.gen.ts

@@ -1340,7 +1340,13 @@ export type BrowserWatchParams = { botId: BotId, };
  * `browser.resize`: the room the app's panel has for the page, in the
  * app's pixels. The page takes its shape (spec 21.3).
  */
-export type BrowserResizeParams = { botId: BotId, width: number, height: number, };
+export type BrowserResizeParams = { botId: BotId, width: number, height: number, 
+/**
+ * The screen's pixels per pixel of the app, in percent (its
+ * `devicePixelRatio` times 100): the page is drawn that sharp for the
+ * owner, up to twice (spec 21.3). Absent, 100.
+ */
+scale?: number, };
 
 export type MouseAction = "move" | "down" | "up";
 

crates/botloft-core/src/protocol/browser.rs

@@ -166,4 +166,10 @@ pub struct BrowserResizeParams {
     pub bot_id: BotId,
     pub width: u32,
     pub height: u32,
+    /// The screen's pixels per pixel of the app, in percent (its
+    /// `devicePixelRatio` times 100): the page is drawn that sharp for the
+    /// owner, up to twice (spec 21.3). Absent, 100.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    #[cfg_attr(test, ts(optional))]
+    pub scale: Option<u32>,
 }

crates/botloftd/src/browser/events.rs

@@ -11,7 +11,7 @@ use tokio::sync::mpsc;
 use tracing::debug;
 
 use super::cdp::CdpEvent;
-use super::session::{FRAME_GAP, Hooks, MAX_TABS, PageInfo, Session, Tab};
+use super::session::{FRAME_GAP, Hooks, MAX_TABS, PageInfo, QUICK_GAP, Session, Tab};
 use super::titles::retitles;
 
 impl Session {
@@ -196,29 +196,60 @@ impl Session {
     }
 
     fn frame(&self, session: &str, params: &Value, hooks: &Hooks) {
-        let (active, viewport) = {
+        let (active, viewport, gap) = {
             let tabs = self.lock();
             let active = tabs.active().is_some_and(|tab| tab.session == session);
-            (active, tabs.viewport)
+            let gap = if tabs.quick { QUICK_GAP } else { FRAME_GAP };
+            (active, tabs.viewport, gap)
         };
-        if active && let Some(data) = params["data"].as_str() {
-            let size = |key: &str, fallback: u32| {
-                params["metadata"][key]
-                    .as_f64()
-                    .map_or(fallback, |value| value.round() as u32)
-            };
+        let size = |key: &str, fallback: u32| {
+            params["metadata"][key]
+                .as_f64()
+                .map_or(fallback, |value| value.round() as u32)
+        };
+        let (width, height) = (
+            size("deviceWidth", viewport.width),
+            size("deviceHeight", viewport.height),
+        );
+        // The screencast draws in the page's own pixels, however sharp the
+        // page is (spec 19): on a sharper screen its frame only says the page
+        // changed, and the frame sent is a screenshot, which is that sharp.
+        let sharp = active && viewport.scale > 100;
+        if active
+            && !sharp
+            && let Some(data) = params["data"].as_str()
+        {
             hooks.frames.send_replace(Some(Arc::new(BrowserFrame {
                 bot_id: hooks.bot.clone(),
                 data: data.to_owned(),
-                width: size("deviceWidth", viewport.width),
-                height: size("deviceHeight", viewport.height),
+                width,
+                height,
             })));
         }
         let cdp = self.cdp.clone();
         let session = session.to_owned();
         let ack = json!({ "sessionId": params["sessionId"] });
+        let (frames, bot) = (hooks.frames.clone(), hooks.bot.clone());
         tokio::spawn(async move {
-            tokio::time::sleep(FRAME_GAP).await;
+            if sharp {
+                let shot = json!({ "format": "jpeg", "quality": 60, "optimizeForSpeed": true });
+                let taken = cdp
+                    .call(Some(&session), "Page.captureScreenshot", shot)
+                    .await;
+                if let Some(data) = taken
+                    .ok()
+                    .and_then(|shot| shot["data"].as_str().map(str::to_owned))
+                {
+                    frames.send_replace(Some(Arc::new(BrowserFrame {
+                        bot_id: bot,
+                        data,
+                        width,
+                        height,
+                    })));
+                }
+            }
+            // Confirmed only after, so screenshots never pile up.
+            tokio::time::sleep(gap).await;
             cdp.send(Some(&session), "Page.screencastFrameAck", ack);
         });
     }

crates/botloftd/src/browser/hands.rs

@@ -281,6 +281,7 @@ impl Drop for Hands {
             mine
         });
         if released {
+            self.session.owner_gave_back();
             update(
                 &self.slots,
                 &self.events,

crates/botloftd/src/browser/moves.rs

@@ -136,14 +136,21 @@ impl Session {
         Ok(())
     }
 
-    /// The owner took the browser: remember the tab the bot was on.
+    /// The owner took the browser: remember the tab the bot was on, and
+    /// send them frames quicker.
     pub(super) fn owner_took(&self) {
         let mut tabs = self.lock();
+        tabs.quick = true;
         if tabs.owner_from.is_none() {
             tabs.owner_from = tabs.active().map(|tab| tab.target.clone());
         }
     }
 
+    /// The owner gave the browser back: frames as for watching the bot.
+    pub(super) fn owner_gave_back(&self) {
+        self.lock().quick = false;
+    }
+
     /// Whether the owner had the browser and left the bot on another tab
     /// than the one it was on; asked once, by the bot's next tool (spec
     /// 21.10).

crates/botloftd/src/browser/page.rs

@@ -294,21 +294,4 @@ impl Session {
         self.settle(marks).await;
         Ok(true)
     }
-
-    /// The screen as a JPEG, base64.
-    pub async fn screenshot(&self) -> Result<String, BrowserError> {
-        let (session, _) = self.page()?;
-        let shot = self
-            .cdp
-            .call(
-                Some(&session),
-                "Page.captureScreenshot",
-                json!({ "format": "jpeg", "quality": 70 }),
-            )
-            .await?;
-        shot["data"]
-            .as_str()
-            .map(str::to_owned)
-            .ok_or_else(|| BrowserError::Page("the browser returned no picture".to_owned()))
-    }
 }

crates/botloftd/src/browser/rest.rs

@@ -123,7 +123,7 @@ impl Browsers {
             if held || session.is_resting() {
                 return;
             }
-            if let Ok(data) = session.screenshot().await {
+            if let Ok(data) = session.picture().await {
                 let page = session.viewport();
                 frames.send_replace(Some(Arc::new(BrowserFrame {
                     bot_id: bot.clone(),

crates/botloftd/src/browser/session.rs

@@ -21,8 +21,14 @@ use super::{BrowserError, agent};
 
 /// Most tabs a browser keeps; past this, the one active longest ago closes.
 pub(super) const MAX_TABS: usize = 6;
-/// Frames are confirmed after this, so the browser sends ~15 a second.
+/// Frames are confirmed after this, so the browser sends ~15 a second
+/// while the owner watches the bot work...
 pub(super) const FRAME_GAP: Duration = Duration::from_millis(66);
+/// ...and ~30 while the browser is in the owner's hands, so the page keeps
+/// up with them as they scroll and point (spec 21.8).
+pub(super) const QUICK_GAP: Duration = Duration::from_millis(33);
+/// The largest side of a frame that only says the page changed.
+const SIGNAL: u32 = 64;
 /// How long the first tab may take to show up.
 const FIRST_TAB_TIMEOUT: Duration = Duration::from_secs(5);
 
@@ -76,6 +82,8 @@ pub struct Tabs {
     pub(super) resting: bool,
     pub closed: bool,
     pub viewport: Viewport,
+    /// The owner has it in their hands: frames come quicker.
+    pub(super) quick: bool,
     /// The tab the bot was on when the owner took the browser, until the
     /// bot's next tool (spec 21.10).
     pub(super) owner_from: Option<String>,
@@ -251,9 +259,16 @@ impl Session {
 
     pub(super) async fn cast(&self, session: &str, on: bool) {
         let result = if on {
-            // Frames come the size of the page, however tall it is.
+            // Frames come the size of the page, however tall it is. On a
+            // sharper screen they only say the page changed (`events.rs`),
+            // so they come small.
+            let (max_width, max_height) = if self.viewport().scale > 100 {
+                (SIGNAL, SIGNAL)
+            } else {
+                (MAX_WIDTH, MAX_HEIGHT)
+            };
             let params = json!({
-                "format": "jpeg", "quality": 60, "maxWidth": MAX_WIDTH, "maxHeight": MAX_HEIGHT,
+                "format": "jpeg", "quality": 60, "maxWidth": max_width, "maxHeight": max_height,
             });
             self.cdp
                 .call(Some(session), "Page.startScreencast", params)