diff --git a/app/src/motion.css b/app/src/motion.css index 4a37a20b..2abbbefe 100644 --- a/app/src/motion.css +++ b/app/src/motion.css @@ -214,7 +214,7 @@ border: 3px solid; border-radius: 9999px; pointer-events: none; - animation: ripple 700ms var(--ease-out) 420ms both; + animation: ripple 700ms var(--ease-out) 520ms both; } .bot-typing { diff --git a/crates/botloftd/src/browser/mod.rs b/crates/botloftd/src/browser/mod.rs index 04f0902a..f6eb3d1d 100644 --- a/crates/botloftd/src/browser/mod.rs +++ b/crates/botloftd/src/browser/mod.rs @@ -39,7 +39,7 @@ pub use self::call::Call; pub use self::hands::{Asking, Hands, InputError, OWNER_WAIT, TakeError}; pub use self::keys::{Key, find as find_key, names as key_names}; pub use self::launch::find as find_program; -pub use self::page::{Done, Scroll}; +pub use self::page::{Aim, Done, Scroll}; pub use self::read::{READ_MAX, Reading}; pub use self::session::Session; pub use self::sweep::{Want, run}; diff --git a/crates/botloftd/src/browser/page.rs b/crates/botloftd/src/browser/page.rs index 2f5baaa2..4b629e87 100644 --- a/crates/botloftd/src/browser/page.rs +++ b/crates/botloftd/src/browser/page.rs @@ -1,6 +1,8 @@ //! What a bot does in its active tab (spec 21.4): open, read, click, type, //! choose, press, scroll, go back and look. Each action waits for the page -//! to settle (spec 21.3). +//! to settle (spec 21.3). Acting on an element takes two steps: `aim` finds +//! it and scrolls it into view, so the owner's cursor can get there first +//! (spec 21.7), and then the action itself. use serde_json::{Value, json}; @@ -21,6 +23,14 @@ pub struct Done { pub chosen: Option, } +/// An element found and in view, before anything is done to it. +#[derive(Debug, Clone)] +pub struct Aim { + reference: String, + point: Value, + pub done: Done, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Scroll { Down, @@ -52,7 +62,7 @@ impl Session { } /// Scrolls the element into view and finds where to click it. - async fn locate(&self, reference: &str) -> Result<(Value, Done), BrowserError> { + pub async fn aim(&self, reference: &str) -> Result { let point = self.reader(&format!("point({})", json!(reference))).await?; if point["missing"] == true { return Err(BrowserError::Stale(reference.to_owned())); @@ -73,7 +83,24 @@ impl Session { cover: point["cover"].as_str().map(str::to_owned), chosen: None, }; - Ok((point, done)) + Ok(Aim { + reference: reference.to_owned(), + point, + done, + }) + } + + /// `aim` for a text field: anything else fails before the cursor goes + /// there. + pub async fn aim_field(&self, reference: &str) -> Result { + let aim = self.aim(reference).await?; + if aim.point["text"] != true { + let role = aim.point["role"].as_str().unwrap_or("element"); + return Err(BrowserError::Page(format!( + "{reference} is a {role}, not a text field; browser_type only types into text fields" + ))); + } + Ok(aim) } async fn mouse_click(&self, (x, y): (f64, f64)) -> Result<(), BrowserError> { @@ -87,27 +114,23 @@ impl Session { Ok(()) } - pub async fn click(&self, reference: &str) -> Result { - let (_, done) = self.locate(reference).await?; + pub async fn click(&self, Aim { done, .. }: Aim) -> Result { let marks = self.marks(); self.mouse_click(done.point.unwrap_or_default()).await?; self.settle(marks).await; Ok(done) } + /// Types into a field from `aim_field`. pub async fn type_text( &self, - reference: &str, + aim: Aim, text: &str, submit: bool, ) -> Result { - let (point, done) = self.locate(reference).await?; - if point["text"] != true { - let role = point["role"].as_str().unwrap_or("element"); - return Err(BrowserError::Page(format!( - "{reference} is a {role}, not a text field; browser_type only types into text fields" - ))); - } + let Aim { + reference, done, .. + } = aim; let marks = self.marks(); self.mouse_click(done.point.unwrap_or_default()).await?; let ready = self @@ -135,8 +158,12 @@ impl Session { Ok(done) } - pub async fn choose(&self, reference: &str, option: &str) -> Result { - let (_, mut done) = self.locate(reference).await?; + pub async fn choose(&self, aim: Aim, option: &str) -> Result { + let Aim { + reference, + mut done, + .. + } = aim; let marks = self.marks(); let result = self .reader(&format!("select({}, {})", json!(reference), json!(option))) @@ -237,7 +264,7 @@ impl Session { /// Scrolls until the element is in the middle of the screen. pub async fn scroll_to(&self, reference: &str) -> Result { let marks = self.marks(); - let (_, done) = self.locate(reference).await?; + let Aim { done, .. } = self.aim(reference).await?; self.settle(marks).await; Ok(done) } diff --git a/crates/botloftd/src/browser/watch.rs b/crates/botloftd/src/browser/watch.rs index d4cccf98..9d73cc06 100644 --- a/crates/botloftd/src/browser/watch.rs +++ b/crates/botloftd/src/browser/watch.rs @@ -34,6 +34,13 @@ impl Browsers { } } + /// Someone has the bot's browser open in the app. + pub fn watched(&self, bot: &BotId) -> bool { + lock(&self.slots) + .get(bot) + .is_some_and(|slot| slot.watchers > 0) + } + /// The room a watching app has for the page, `width` by `height`: the /// page takes its shape, now or when the browser opens (spec 21.3). pub fn resize(&self, bot: &BotId, width: u32, height: u32) { diff --git a/crates/botloftd/src/tools/browser.rs b/crates/botloftd/src/tools/browser.rs index 78b28d47..db56d8a7 100644 --- a/crates/botloftd/src/tools/browser.rs +++ b/crates/botloftd/src/tools/browser.rs @@ -12,7 +12,7 @@ use tracing::debug; use super::browser_args::Tool; use super::browser_help::ask_owner; -use super::browser_reply::{answer, describe, report, unavailable}; +use super::browser_reply::{answer, describe, point, report, unavailable}; use super::browser_sites::{allowed, page_allowed}; use super::calls::explain; use crate::browser::{Done, OWNER_WAIT, Scroll, sites}; @@ -168,19 +168,38 @@ async fn run( } return Ok(Reply::Picture(data, line)); } - Tool::Click { reference } => (BrowserActionKind::Click, session.click(&reference).await), + // The cursor goes first; these report before they act. + Tool::Click { reference } => { + let kind = BrowserActionKind::Click; + let aim = session + .aim(&reference) + .await + .map_err(|err| describe(&err))?; + point(daemon, id, kind, &aim).await; + (kind, session.click(aim).await) + } Tool::Type { reference, text, submit, - } => ( - BrowserActionKind::Type, - session.type_text(&reference, &text, submit).await, - ), - Tool::Select { reference, option } => ( - BrowserActionKind::Select, - session.choose(&reference, &option).await, - ), + } => { + let kind = BrowserActionKind::Type; + let aim = session + .aim_field(&reference) + .await + .map_err(|err| describe(&err))?; + point(daemon, id, kind, &aim).await; + (kind, session.type_text(aim, &text, submit).await) + } + Tool::Select { reference, option } => { + let kind = BrowserActionKind::Select; + let aim = session + .aim(&reference) + .await + .map_err(|err| describe(&err))?; + point(daemon, id, kind, &aim).await; + (kind, session.choose(aim, &option).await) + } Tool::Press { key } => { let label = Some(key.key.trim().to_owned()).filter(|key| !key.is_empty()); let result = session.press(key).await.map(|()| Done { @@ -206,7 +225,13 @@ async fn run( Tool::Open { .. } | Tool::Close | Tool::AskOwner { .. } => unreachable!("handled above"), }; let done = done.map_err(|err| describe(&err))?; - report(daemon, id, kind, &done); + let pointed = matches!( + kind, + BrowserActionKind::Click | BrowserActionKind::Type | BrowserActionKind::Select + ); + if !pointed { + report(daemon, id, kind, &done); + } let mut notes = Vec::new(); if let Some(cover) = &done.cover { notes.push(format!( diff --git a/crates/botloftd/src/tools/browser_reply.rs b/crates/botloftd/src/tools/browser_reply.rs index 7245a691..e39ee9c0 100644 --- a/crates/botloftd/src/tools/browser_reply.rs +++ b/crates/botloftd/src/tools/browser_reply.rs @@ -2,12 +2,14 @@ //! the action for the owner's cursor, and errors in words the bot can act //! on. +use std::time::Duration; + use botloft_core::ids::BotId; use botloft_core::protocol::{BrowserAction, BrowserActionKind}; use super::browser::{Bot, Reply}; use super::browser_sites::outside_file; -use crate::browser::{BrowserError, Done, Session, sites}; +use crate::browser::{Aim, BrowserError, Done, Session, sites}; use crate::state::Daemon; /// The page as the bot reads it, with what happened on the way. @@ -54,6 +56,20 @@ pub(super) async fn answer( Ok(Reply::Text(out)) } +/// How long the owner's cursor takes to glide to a point (`.bot-cursor` in +/// the app's `motion.css`, 520 ms), with a little to spare. +const GLIDE: Duration = Duration::from_millis(560); + +/// Shows the owner where the bot is about to act and, while someone +/// watches, waits for the cursor to get there, so the page changes after +/// the cursor arrives and not before (spec 21.7). +pub(super) async fn point(daemon: &Daemon, bot: &BotId, kind: BrowserActionKind, aim: &Aim) { + report(daemon, bot, kind, &aim.done); + if daemon.browsers.watched(bot) { + tokio::time::sleep(GLIDE).await; + } +} + pub(super) fn report(daemon: &Daemon, bot: &BotId, kind: BrowserActionKind, done: &Done) { daemon.browsers.action(BrowserAction { bot_id: bot.clone(), diff --git a/crates/botloftd/tests/browser.rs b/crates/botloftd/tests/browser.rs index d039a1a3..6ebd3a7e 100644 --- a/crates/botloftd/tests/browser.rs +++ b/crates/botloftd/tests/browser.rs @@ -4,6 +4,8 @@ mod common; +use std::time::{Duration, Instant}; + use common::browsing::{answer_site, call, ref_of, setup}; use serde_json::{Value, json}; @@ -197,3 +199,36 @@ async fn new_tabs_and_dialogs_reach_the_bot_and_the_owner_watches() { .await .expect("unwatch"); } + +#[tokio::test(flavor = "multi_thread")] +async fn the_owner_sees_the_cursor_get_there_before_the_click() { + let Some(mut b) = setup().await else { return }; + b.app + .call("browser.watch", json!({ "botId": b.bot["id"] })) + .await + .expect("watch"); + let opening = call( + &b.mcp, + "browser_open", + json!({ "url": format!("{}/links", b.site) }), + ); + answer_site(&mut b.app, true, None).await; + let page = opening.await.expect("task").expect("page"); + + let warn = ref_of(&page, "button \"Warn me\""); + let clicking = call(&b.mcp, "browser_click", json!({ "ref": warn })); + let action = loop { + let action = b.app.notification("browser.action").await; + if action["kind"] == "click" { + break action; + } + }; + let pointed = Instant::now(); + assert_eq!(action["label"], "Warn me"); + assert!(action["x"].as_f64().expect("x") > 0.0); + // The click waits for the cursor's glide in the panel. + assert!(!clicking.is_finished()); + let warned = clicking.await.expect("task").expect("clicked"); + assert!(warned.contains("\"Hi there\""), "{warned}"); + assert!(pointed.elapsed() >= Duration::from_millis(500)); +} diff --git a/docs/spec.md b/docs/spec.md index 9c50d4e0..7cdf4c7c 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1264,7 +1264,7 @@ Esqueceu a senha? [e5 link "Recuperar acesso"] - O que muda o navegador responde assim que o pedido é aceito; o resultado chega em `browser.changed` e nos quadros. - `BrowserState`: `botId`, `status` (`closed`, `starting`, `open`, `failed`), `url`, `title` e `loading` (da aba ativa), `tabs` (as abas abertas, na ordem em que abriram: `BrowserTab {id, title, url, active}`, com `id` opaco), `error` (por que não abriu, em `failed`), `control` (`bot` ou `owner`, 21.10), `resting` (o navegador descansa, 21.2), `ask` (o que o bot pediu ao dono com `browser_ask_owner`, enquanto o pedido está aberto) e `updatedAt`. -- Notificações para todos: `browser.changed` (`BrowserState`) e `browser.action {botId, kind, x, y, label, at}`, com `kind` `open`, `click`, `type`, `select`, `press`, `scroll` ou `back`; `x` e `y` em pixels da página (o tamanho que vem em `browser.frame`) quando a ação tem um ponto; `label` é o nome do elemento, a tecla ou o site, nunca o texto digitado. +- Notificações para todos: `browser.changed` (`BrowserState`) e `browser.action {botId, kind, x, y, label, at}`, com `kind` `open`, `click`, `type`, `select`, `press`, `scroll` ou `back`; `x` e `y` em pixels da página (o tamanho que vem em `browser.frame`) quando a ação tem um ponto; `label` é o nome do elemento, a tecla ou o site, nunca o texto digitado. Clicar, digitar e escolher avisam antes de agir: o daemon acha o elemento e o traz para a tela, manda `browser.action` com o ponto e, se alguém assiste, espera o cursor deslizar até lá no painel (560 ms, o tempo da animação mais uma folga) antes do clique de verdade; sem ninguém assistindo, não espera. As outras ações avisam depois de feitas. Assim a página muda depois que o cursor chega, e não antes. - Só para a conexão que assiste: `browser.frame {botId, data, width, height}`, um JPEG em base64 e o tamanho da página que ele mostra. Cada conexão assiste um bot por vez; `browser.watch` de outro troca, e fechar a conexão para. - Os quadros vêm do `Page.startScreencast` (JPEG, qualidade 60, do tamanho da página, 21.3), que roda só enquanto alguém assiste, e só quando a tela muda. O daemon confirma cada quadro no máximo ~15 vezes por segundo. Um app lento não acumula quadros: a conexão manda sempre o mais novo quando consegue, e quadros não contam no limite de 1024 notificações (11.1).