Merge pull request #93 from httpsphl/feat/cursor-before-click

feat(browser): the owner's cursor gets there before the click

Author: httpsphl

app/src/motion.css

@@ -214,7 +214,7 @@
   border: 3px solid;
   border-radius: 9999px;
   pointer-events: none;
-  animation: ripple 700ms var(--ease-out) 420ms both;
+  animation: ripple 700ms var(--ease-out) 520ms both;
 }
 
 .bot-typing {

crates/botloftd/src/browser/mod.rs

@@ -39,7 +39,7 @@ pub use self::call::Call;
 pub use self::hands::{Asking, Hands, InputError, OWNER_WAIT, TakeError};
 pub use self::keys::{Key, find as find_key, names as key_names};
 pub use self::launch::find as find_program;
-pub use self::page::{Done, Scroll};
+pub use self::page::{Aim, Done, Scroll};
 pub use self::read::{READ_MAX, Reading};
 pub use self::session::Session;
 pub use self::sweep::{Want, run};

crates/botloftd/src/browser/page.rs

@@ -1,6 +1,8 @@
 //! What a bot does in its active tab (spec 21.4): open, read, click, type,
 //! choose, press, scroll, go back and look. Each action waits for the page
-//! to settle (spec 21.3).
+//! to settle (spec 21.3). Acting on an element takes two steps: `aim` finds
+//! it and scrolls it into view, so the owner's cursor can get there first
+//! (spec 21.7), and then the action itself.
 
 use serde_json::{Value, json};
 
@@ -21,6 +23,14 @@ pub struct Done {
     pub chosen: Option<String>,
 }
 
+/// An element found and in view, before anything is done to it.
+#[derive(Debug, Clone)]
+pub struct Aim {
+    reference: String,
+    point: Value,
+    pub done: Done,
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum Scroll {
     Down,
@@ -52,7 +62,7 @@ impl Session {
     }
 
     /// Scrolls the element into view and finds where to click it.
-    async fn locate(&self, reference: &str) -> Result<(Value, Done), BrowserError> {
+    pub async fn aim(&self, reference: &str) -> Result<Aim, BrowserError> {
         let point = self.reader(&format!("point({})", json!(reference))).await?;
         if point["missing"] == true {
             return Err(BrowserError::Stale(reference.to_owned()));
@@ -73,7 +83,24 @@ impl Session {
             cover: point["cover"].as_str().map(str::to_owned),
             chosen: None,
         };
-        Ok((point, done))
+        Ok(Aim {
+            reference: reference.to_owned(),
+            point,
+            done,
+        })
+    }
+
+    /// `aim` for a text field: anything else fails before the cursor goes
+    /// there.
+    pub async fn aim_field(&self, reference: &str) -> Result<Aim, BrowserError> {
+        let aim = self.aim(reference).await?;
+        if aim.point["text"] != true {
+            let role = aim.point["role"].as_str().unwrap_or("element");
+            return Err(BrowserError::Page(format!(
+                "{reference} is a {role}, not a text field; browser_type only types into text fields"
+            )));
+        }
+        Ok(aim)
     }
 
     async fn mouse_click(&self, (x, y): (f64, f64)) -> Result<(), BrowserError> {
@@ -87,27 +114,23 @@ impl Session {
         Ok(())
     }
 
-    pub async fn click(&self, reference: &str) -> Result<Done, BrowserError> {
-        let (_, done) = self.locate(reference).await?;
+    pub async fn click(&self, Aim { done, .. }: Aim) -> Result<Done, BrowserError> {
         let marks = self.marks();
         self.mouse_click(done.point.unwrap_or_default()).await?;
         self.settle(marks).await;
         Ok(done)
     }
 
+    /// Types into a field from `aim_field`.
     pub async fn type_text(
         &self,
-        reference: &str,
+        aim: Aim,
         text: &str,
         submit: bool,
     ) -> Result<Done, BrowserError> {
-        let (point, done) = self.locate(reference).await?;
-        if point["text"] != true {
-            let role = point["role"].as_str().unwrap_or("element");
-            return Err(BrowserError::Page(format!(
-                "{reference} is a {role}, not a text field; browser_type only types into text fields"
-            )));
-        }
+        let Aim {
+            reference, done, ..
+        } = aim;
         let marks = self.marks();
         self.mouse_click(done.point.unwrap_or_default()).await?;
         let ready = self
@@ -135,8 +158,12 @@ impl Session {
         Ok(done)
     }
 
-    pub async fn choose(&self, reference: &str, option: &str) -> Result<Done, BrowserError> {
-        let (_, mut done) = self.locate(reference).await?;
+    pub async fn choose(&self, aim: Aim, option: &str) -> Result<Done, BrowserError> {
+        let Aim {
+            reference,
+            mut done,
+            ..
+        } = aim;
         let marks = self.marks();
         let result = self
             .reader(&format!("select({}, {})", json!(reference), json!(option)))
@@ -237,7 +264,7 @@ impl Session {
     /// Scrolls until the element is in the middle of the screen.
     pub async fn scroll_to(&self, reference: &str) -> Result<Done, BrowserError> {
         let marks = self.marks();
-        let (_, done) = self.locate(reference).await?;
+        let Aim { done, .. } = self.aim(reference).await?;
         self.settle(marks).await;
         Ok(done)
     }

crates/botloftd/src/browser/watch.rs

@@ -34,6 +34,13 @@ impl Browsers {
         }
     }
 
+    /// Someone has the bot's browser open in the app.
+    pub fn watched(&self, bot: &BotId) -> bool {
+        lock(&self.slots)
+            .get(bot)
+            .is_some_and(|slot| slot.watchers > 0)
+    }
+
     /// The room a watching app has for the page, `width` by `height`: the
     /// page takes its shape, now or when the browser opens (spec 21.3).
     pub fn resize(&self, bot: &BotId, width: u32, height: u32) {

crates/botloftd/src/tools/browser.rs

@@ -12,7 +12,7 @@ use tracing::debug;
 
 use super::browser_args::Tool;
 use super::browser_help::ask_owner;
-use super::browser_reply::{answer, describe, report, unavailable};
+use super::browser_reply::{answer, describe, point, report, unavailable};
 use super::browser_sites::{allowed, page_allowed};
 use super::calls::explain;
 use crate::browser::{Done, OWNER_WAIT, Scroll, sites};
@@ -168,19 +168,38 @@ async fn run(
             }
             return Ok(Reply::Picture(data, line));
         }
-        Tool::Click { reference } => (BrowserActionKind::Click, session.click(&reference).await),
+        // The cursor goes first; these report before they act.
+        Tool::Click { reference } => {
+            let kind = BrowserActionKind::Click;
+            let aim = session
+                .aim(&reference)
+                .await
+                .map_err(|err| describe(&err))?;
+            point(daemon, id, kind, &aim).await;
+            (kind, session.click(aim).await)
+        }
         Tool::Type {
             reference,
             text,
             submit,
-        } => (
-            BrowserActionKind::Type,
-            session.type_text(&reference, &text, submit).await,
-        ),
-        Tool::Select { reference, option } => (
-            BrowserActionKind::Select,
-            session.choose(&reference, &option).await,
-        ),
+        } => {
+            let kind = BrowserActionKind::Type;
+            let aim = session
+                .aim_field(&reference)
+                .await
+                .map_err(|err| describe(&err))?;
+            point(daemon, id, kind, &aim).await;
+            (kind, session.type_text(aim, &text, submit).await)
+        }
+        Tool::Select { reference, option } => {
+            let kind = BrowserActionKind::Select;
+            let aim = session
+                .aim(&reference)
+                .await
+                .map_err(|err| describe(&err))?;
+            point(daemon, id, kind, &aim).await;
+            (kind, session.choose(aim, &option).await)
+        }
         Tool::Press { key } => {
             let label = Some(key.key.trim().to_owned()).filter(|key| !key.is_empty());
             let result = session.press(key).await.map(|()| Done {
@@ -206,7 +225,13 @@ async fn run(
         Tool::Open { .. } | Tool::Close | Tool::AskOwner { .. } => unreachable!("handled above"),
     };
     let done = done.map_err(|err| describe(&err))?;
-    report(daemon, id, kind, &done);
+    let pointed = matches!(
+        kind,
+        BrowserActionKind::Click | BrowserActionKind::Type | BrowserActionKind::Select
+    );
+    if !pointed {
+        report(daemon, id, kind, &done);
+    }
     let mut notes = Vec::new();
     if let Some(cover) = &done.cover {
         notes.push(format!(

crates/botloftd/src/tools/browser_reply.rs

@@ -2,12 +2,14 @@
 //! the action for the owner's cursor, and errors in words the bot can act
 //! on.
 
+use std::time::Duration;
+
 use botloft_core::ids::BotId;
 use botloft_core::protocol::{BrowserAction, BrowserActionKind};
 
 use super::browser::{Bot, Reply};
 use super::browser_sites::outside_file;
-use crate::browser::{BrowserError, Done, Session, sites};
+use crate::browser::{Aim, BrowserError, Done, Session, sites};
 use crate::state::Daemon;
 
 /// The page as the bot reads it, with what happened on the way.
@@ -54,6 +56,20 @@ pub(super) async fn answer(
     Ok(Reply::Text(out))
 }
 
+/// How long the owner's cursor takes to glide to a point (`.bot-cursor` in
+/// the app's `motion.css`, 520 ms), with a little to spare.
+const GLIDE: Duration = Duration::from_millis(560);
+
+/// Shows the owner where the bot is about to act and, while someone
+/// watches, waits for the cursor to get there, so the page changes after
+/// the cursor arrives and not before (spec 21.7).
+pub(super) async fn point(daemon: &Daemon, bot: &BotId, kind: BrowserActionKind, aim: &Aim) {
+    report(daemon, bot, kind, &aim.done);
+    if daemon.browsers.watched(bot) {
+        tokio::time::sleep(GLIDE).await;
+    }
+}
+
 pub(super) fn report(daemon: &Daemon, bot: &BotId, kind: BrowserActionKind, done: &Done) {
     daemon.browsers.action(BrowserAction {
         bot_id: bot.clone(),

crates/botloftd/tests/browser.rs

@@ -4,6 +4,8 @@
 
 mod common;
 
+use std::time::{Duration, Instant};
+
 use common::browsing::{answer_site, call, ref_of, setup};
 use serde_json::{Value, json};
 
@@ -197,3 +199,36 @@ async fn new_tabs_and_dialogs_reach_the_bot_and_the_owner_watches() {
         .await
         .expect("unwatch");
 }
+
+#[tokio::test(flavor = "multi_thread")]
+async fn the_owner_sees_the_cursor_get_there_before_the_click() {
+    let Some(mut b) = setup().await else { return };
+    b.app
+        .call("browser.watch", json!({ "botId": b.bot["id"] }))
+        .await
+        .expect("watch");
+    let opening = call(
+        &b.mcp,
+        "browser_open",
+        json!({ "url": format!("{}/links", b.site) }),
+    );
+    answer_site(&mut b.app, true, None).await;
+    let page = opening.await.expect("task").expect("page");
+
+    let warn = ref_of(&page, "button \"Warn me\"");
+    let clicking = call(&b.mcp, "browser_click", json!({ "ref": warn }));
+    let action = loop {
+        let action = b.app.notification("browser.action").await;
+        if action["kind"] == "click" {
+            break action;
+        }
+    };
+    let pointed = Instant::now();
+    assert_eq!(action["label"], "Warn me");
+    assert!(action["x"].as_f64().expect("x") > 0.0);
+    // The click waits for the cursor's glide in the panel.
+    assert!(!clicking.is_finished());
+    let warned = clicking.await.expect("task").expect("clicked");
+    assert!(warned.contains("\"Hi there\""), "{warned}");
+    assert!(pointed.elapsed() >= Duration::from_millis(500));
+}

docs/spec.md

@@ -1264,7 +1264,7 @@ Esqueceu a senha? [e5 link "Recuperar acesso"]
 
 - O que muda o navegador responde assim que o pedido é aceito; o resultado chega em `browser.changed` e nos quadros.
 - `BrowserState`: `botId`, `status` (`closed`, `starting`, `open`, `failed`), `url`, `title` e `loading` (da aba ativa), `tabs` (as abas abertas, na ordem em que abriram: `BrowserTab {id, title, url, active}`, com `id` opaco), `error` (por que não abriu, em `failed`), `control` (`bot` ou `owner`, 21.10), `resting` (o navegador descansa, 21.2), `ask` (o que o bot pediu ao dono com `browser_ask_owner`, enquanto o pedido está aberto) e `updatedAt`.
-- Notificações para todos: `browser.changed` (`BrowserState`) e `browser.action {botId, kind, x, y, label, at}`, com `kind` `open`, `click`, `type`, `select`, `press`, `scroll` ou `back`; `x` e `y` em pixels da página (o tamanho que vem em `browser.frame`) quando a ação tem um ponto; `label` é o nome do elemento, a tecla ou o site, nunca o texto digitado.
+- Notificações para todos: `browser.changed` (`BrowserState`) e `browser.action {botId, kind, x, y, label, at}`, com `kind` `open`, `click`, `type`, `select`, `press`, `scroll` ou `back`; `x` e `y` em pixels da página (o tamanho que vem em `browser.frame`) quando a ação tem um ponto; `label` é o nome do elemento, a tecla ou o site, nunca o texto digitado. Clicar, digitar e escolher avisam antes de agir: o daemon acha o elemento e o traz para a tela, manda `browser.action` com o ponto e, se alguém assiste, espera o cursor deslizar até lá no painel (560 ms, o tempo da animação mais uma folga) antes do clique de verdade; sem ninguém assistindo, não espera. As outras ações avisam depois de feitas. Assim a página muda depois que o cursor chega, e não antes.
 - Só para a conexão que assiste: `browser.frame {botId, data, width, height}`, um JPEG em base64 e o tamanho da página que ele mostra. Cada conexão assiste um bot por vez; `browser.watch` de outro troca, e fechar a conexão para.
 - Os quadros vêm do `Page.startScreencast` (JPEG, qualidade 60, do tamanho da página, 21.3), que roda só enquanto alguém assiste, e só quando a tela muda. O daemon confirma cada quadro no máximo ~15 vezes por segundo. Um app lento não acumula quadros: a conexão manda sempre o mais novo quando consegue, e quadros não contam no limite de 1024 notificações (11.1).