From b7d5587928348e84258a57bb2898d1d88919336e Mon Sep 17 00:00:00 2001 From: Phelipe Lorran <112786693+httpsphl@users.noreply.github.com> Date: Wed, 30 Sep 2026 10:37:54 -0300 Subject: [PATCH 1/3] feat(browser): the page takes the shape of the panel watching it The page was always 1280 x 800, so a tall panel showed it small with an empty area below. While an app connection watches, `browser.resize` tells the daemon the room its panel has for the page: the page keeps its 1280 of width, so sites keep their desktop layout, and takes the height that gives it the room's shape, between 600 and 2000. Frames, the owner's points and the bot's scroll follow the size the page has. When the last one watching stops, the page is 1280 x 800 again. Spec 21.3, 21.7, 21.8 and 21.10 say so, and section 19 records what the real Edge does when the size changes with a page open. --- app/src/lib/fakeBrowser.ts | 68 +++++++++- app/src/lib/protocol.gen.ts | 7 + crates/botloft-core/src/protocol/browser.rs | 11 ++ .../src/protocol/export_browser.rs | 6 + crates/botloft-core/src/protocol/mod.rs | 1 + crates/botloftd/src/browser/call.rs | 18 +-- crates/botloftd/src/browser/events.rs | 24 ++-- crates/botloftd/src/browser/hands.rs | 29 +++-- crates/botloftd/src/browser/mod.rs | 5 + crates/botloftd/src/browser/page.rs | 11 +- crates/botloftd/src/browser/session.rs | 20 ++- crates/botloftd/src/browser/viewport.rs | 121 ++++++++++++++++++ crates/botloftd/src/browser/watch.rs | 62 ++++++--- crates/botloftd/src/rpc/mod.rs | 9 +- crates/botloftd/src/rpc/watching.rs | 45 +++++-- crates/botloftd/tests/browser_panel.rs | 113 ++++++++++++++++ crates/botloftd/tests/common/site.rs | 10 +- docs/spec.md | 16 ++- 18 files changed, 487 insertions(+), 89 deletions(-) create mode 100644 crates/botloftd/src/browser/viewport.rs create mode 100644 crates/botloftd/tests/browser_panel.rs diff --git a/app/src/lib/fakeBrowser.ts b/app/src/lib/fakeBrowser.ts index 90ffedfe..af71fcd9 100644 --- a/app/src/lib/fakeBrowser.ts +++ b/app/src/lib/fakeBrowser.ts @@ -1,6 +1,7 @@ // The fake daemon's browsers (spec 21.7): each bot's state, the live frames // of the one being watched and what the bot does, for the browser panel, -// and the owner's hands in it (spec 21.10). +// the size the panel gives the page (spec 21.3), and the owner's hands in +// it (spec 21.10). import type { FakeBotloft, Handlers } from "./fake"; import { conflict } from "./fakeRules"; @@ -14,6 +15,20 @@ import type { BrowserState, } from "./protocol.gen"; +/** The page's size: as wide as ever, as tall as the panel's room asks. */ +export interface PageSize { + width: number; + height: number; +} + +const PAGE: PageSize = { width: 1280, height: 800 }; + +/** The daemon's rule for the page in a room of the panel (spec 21.3). */ +function fitting(width: number, height: number): PageSize { + const tall = Math.floor((PAGE.width * height) / width); + return { width: PAGE.width, height: Math.min(2000, Math.max(600, tall)) }; +} + export class FakeBrowser { private readonly states = new Map(); private readonly frames = new Map(); @@ -23,6 +38,10 @@ export class FakeBrowser { readonly watches: (BotId | null)[] = []; /** What the owner sent to the page, in order. */ readonly inputs: BrowserInput[] = []; + /** Each bot's page size, while the panel asks for one. */ + private readonly sizes = new Map(); + /** What draws each bot's page, for the preview. */ + private readonly painters = new Map string>(); /** Each bot's open request for help. */ private readonly asks = new Map(); /** Pages that answer the owner's hands, for the preview. */ @@ -96,9 +115,28 @@ export class FakeBrowser { this.set(botId, { ask: null, control: "bot" }); } + /** The size of the bot's page now. */ + size(botId: BotId): PageSize { + return this.sizes.get(botId) ?? PAGE; + } + + /** Draws the bot's page with `draw`, now and whenever its size changes. */ + paint(botId: BotId, draw: (size: PageSize) => string): void { + this.painters.set(botId, draw); + this.repaint(botId); + } + + /** The page changed: a new picture of it, if something draws it. */ + repaint(botId: BotId): void { + const draw = this.painters.get(botId); + if (draw) { + this.frame(botId, draw(this.size(botId))); + } + } + /** A new picture of the page; sent only while the app watches that bot. */ - frame(botId: BotId, data: string, width = 1280, height = 800): void { - const frame: BrowserFrame = { botId, data, width, height }; + frame(botId: BotId, data: string, size: PageSize = this.size(botId)): void { + const frame: BrowserFrame = { botId, data, ...size }; this.frames.set(botId, frame); if (this.watching === botId) { this.fake.emit({ name: "browser.frame", params: frame }); @@ -132,11 +170,22 @@ export class FakeBrowser { return this.set(botId, { control: "bot" }); } + /** Nobody watches: the page goes back to the size bots work in alone. */ + private unwatch(): void { + const botId = this.watching; + this.release(); + this.watching = null; + if (botId !== null && this.sizes.delete(botId)) { + this.repaint(botId); + } + } + handlers(): Pick< Handlers, | "browser.list" | "browser.watch" | "browser.unwatch" + | "browser.resize" | "browser.take" | "browser.release" | "browser.input" @@ -145,17 +194,24 @@ export class FakeBrowser { "browser.list": () => [...this.states.values()].filter((state) => state.status !== "closed"), "browser.watch": ({ botId }) => { this.fake.bot(botId, false); - this.release(); + this.unwatch(); this.watching = botId; this.watches.push(botId); return { state: this.state(botId), frame: this.frames.get(botId) ?? null }; }, "browser.unwatch": () => { - this.release(); - this.watching = null; + this.unwatch(); this.watches.push(null); return null; }, + "browser.resize": ({ botId, width, height }) => { + if (this.watching !== botId) { + throw conflict("watch this browser before sizing it"); + } + this.sizes.set(botId, fitting(width, height)); + this.repaint(botId); + return null; + }, "browser.take": ({ botId }) => { if (this.watching !== botId) { throw conflict("watch this browser before taking it"); diff --git a/app/src/lib/protocol.gen.ts b/app/src/lib/protocol.gen.ts index 1aa00419..0d3f277e 100644 --- a/app/src/lib/protocol.gen.ts +++ b/app/src/lib/protocol.gen.ts @@ -892,6 +892,12 @@ export type BrowserView = { state: BrowserState, frame: BrowserFrame | null, }; export type BrowserWatchParams = { botId: BotId, }; +/** + * `browser.resize`: the room the app's panel has for the page, in the + * app's pixels. The page takes its shape (spec 21.3). + */ +export type BrowserResizeParams = { botId: BotId, width: number, height: number, }; + export type MouseAction = "move" | "down" | "up"; export type MouseButton = "none" | "left" | "middle" | "right"; @@ -1020,6 +1026,7 @@ export interface RpcMethods { "browser.list": { params: undefined; result: Array }; "browser.watch": { params: BrowserWatchParams; result: BrowserView }; "browser.unwatch": { params: undefined; result: null }; + "browser.resize": { params: BrowserResizeParams; result: null }; "browser.take": { params: BrowserControlParams; result: BrowserState }; "browser.release": { params: BrowserControlParams; result: BrowserState }; "browser.input": { params: BrowserInputParams; result: null }; diff --git a/crates/botloft-core/src/protocol/browser.rs b/crates/botloft-core/src/protocol/browser.rs index 194fb53d..6b839f20 100644 --- a/crates/botloft-core/src/protocol/browser.rs +++ b/crates/botloft-core/src/protocol/browser.rs @@ -128,3 +128,14 @@ pub struct BrowserView { pub struct BrowserWatchParams { pub bot_id: BotId, } + +/// `browser.resize`: the room the app's panel has for the page, in the +/// app's pixels. The page takes its shape (spec 21.3). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct BrowserResizeParams { + pub bot_id: BotId, + pub width: u32, + pub height: u32, +} diff --git a/crates/botloft-core/src/protocol/export_browser.rs b/crates/botloft-core/src/protocol/export_browser.rs index dfc9c191..c99f4908 100644 --- a/crates/botloft-core/src/protocol/export_browser.rs +++ b/crates/botloft-core/src/protocol/export_browser.rs @@ -14,6 +14,7 @@ pub(super) fn decls(out: &mut Out) { out.decl::(); out.decl::(); out.decl::(); + out.decl::(); out.decl::(); out.decl::(); out.decl::(); @@ -37,6 +38,11 @@ pub(super) fn methods(out: &mut Out) { &out.name::(), ); out.method(method::BROWSER_UNWATCH, "undefined", "null"); + out.method( + method::BROWSER_RESIZE, + &out.name::(), + "null", + ); let control = out.name::(); let state = out.name::(); out.method(method::BROWSER_TAKE, &control, &state); diff --git a/crates/botloft-core/src/protocol/mod.rs b/crates/botloft-core/src/protocol/mod.rs index 43054c83..df1e5651 100644 --- a/crates/botloft-core/src/protocol/mod.rs +++ b/crates/botloft-core/src/protocol/mod.rs @@ -117,6 +117,7 @@ pub mod method { pub const BROWSER_LIST: &str = "browser.list"; pub const BROWSER_WATCH: &str = "browser.watch"; pub const BROWSER_UNWATCH: &str = "browser.unwatch"; + pub const BROWSER_RESIZE: &str = "browser.resize"; pub const BROWSER_TAKE: &str = "browser.take"; pub const BROWSER_RELEASE: &str = "browser.release"; pub const BROWSER_INPUT: &str = "browser.input"; diff --git a/crates/botloftd/src/browser/call.rs b/crates/botloftd/src/browser/call.rs index 84b683fe..61092e45 100644 --- a/crates/botloftd/src/browser/call.rs +++ b/crates/botloftd/src/browser/call.rs @@ -11,7 +11,7 @@ use tokio::sync::OwnedMutexGuard; use tracing::debug; use super::session::{Hooks, PageInfo, Session}; -use super::{BrowserError, Browsers, launch, lock, update}; +use super::{BrowserError, Browsers, launch, lock, update, watch}; pub struct Call<'a> { pub(super) browsers: &'a Browsers, @@ -43,7 +43,11 @@ impl Call<'_> { None => Err(BrowserError::NotFound), Some(program) => { let profile = browsers.profiles.join(bot.as_str()); - Session::start(&program, &profile, downloads, self.hooks(start)).await + let viewport = { + let mut slots = lock(&browsers.slots); + browsers.slot(&mut slots, bot).viewport + }; + Session::start(&program, &profile, downloads, viewport, self.hooks(start)).await } }; let session = match started { @@ -57,16 +61,14 @@ impl Call<'_> { return Err(err); } }; - let watched = { + { let mut slots = lock(&browsers.slots); let slot = browsers.slot(&mut slots, bot); slot.session = Some((start, Arc::clone(&session))); - slot.watchers > 0 - }; - browsers.set_state(bot, |state| state.status = BrowserStatus::Open); - if watched { - session.set_watching(true).await; } + browsers.set_state(bot, |state| state.status = BrowserStatus::Open); + // Whoever watches already gets frames, the size they asked for. + watch::sync(Arc::clone(&browsers.slots), bot.clone()).await; debug!(bot = %bot, "browser: open"); Ok(session) } diff --git a/crates/botloftd/src/browser/events.rs b/crates/botloftd/src/browser/events.rs index 675f720c..13c6d234 100644 --- a/crates/botloftd/src/browser/events.rs +++ b/crates/botloftd/src/browser/events.rs @@ -11,7 +11,7 @@ use tokio::sync::mpsc; use tracing::debug; use super::cdp::CdpEvent; -use super::session::{FRAME_GAP, HEIGHT, Hooks, MAX_TABS, PageInfo, Session, Tab, WIDTH}; +use super::session::{FRAME_GAP, Hooks, MAX_TABS, PageInfo, Session, Tab}; impl Session { /// Sets up a new tab before it runs, and makes it the active one. @@ -25,7 +25,7 @@ impl Session { else { return; }; - let (previous, watching, extra) = { + let (previous, watching, extra, viewport) = { let mut tabs = self.lock(); let previous = tabs.active().map(|tab| tab.session.clone()); tabs.list.push(Tab { @@ -39,7 +39,7 @@ impl Session { network_at: Instant::now(), }); let extra = (tabs.list.len() > MAX_TABS).then(|| tabs.list[0].target.clone()); - (previous, tabs.watching, extra) + (previous, tabs.watching, extra, tabs.viewport) }; let setup = [ ("Page.enable", json!({})), @@ -48,10 +48,7 @@ impl Session { "Page.setInterceptFileChooserDialog", json!({ "enabled": true }), ), - ( - "Emulation.setDeviceMetricsOverride", - json!({ "width": WIDTH, "height": HEIGHT, "deviceScaleFactor": 1, "mobile": false }), - ), + ("Emulation.setDeviceMetricsOverride", viewport.metrics()), ( "Emulation.setUserAgentOverride", json!({ "userAgent": self.user_agent }), @@ -188,10 +185,11 @@ impl Session { } fn frame(&self, session: &str, params: &Value, hooks: &Hooks) { - let active = self - .lock() - .active() - .is_some_and(|tab| tab.session == session); + let (active, viewport) = { + let tabs = self.lock(); + let active = tabs.active().is_some_and(|tab| tab.session == session); + (active, tabs.viewport) + }; if active && let Some(data) = params["data"].as_str() { let size = |key: &str, fallback: u32| { params["metadata"][key] @@ -201,8 +199,8 @@ impl Session { hooks.frames.send_replace(Some(Arc::new(BrowserFrame { bot_id: hooks.bot.clone(), data: data.to_owned(), - width: size("deviceWidth", WIDTH), - height: size("deviceHeight", HEIGHT), + width: size("deviceWidth", viewport.width), + height: size("deviceHeight", viewport.height), }))); } let cdp = self.cdp.clone(); diff --git a/crates/botloftd/src/browser/hands.rs b/crates/botloftd/src/browser/hands.rs index 34a8f932..7d3d3a88 100644 --- a/crates/botloftd/src/browser/hands.rs +++ b/crates/botloftd/src/browser/hands.rs @@ -11,8 +11,8 @@ use botloft_core::protocol::{BrowserControl, BrowserInput, MouseAction}; use tokio::sync::{broadcast, mpsc}; use tracing::debug; -use super::session::{HEIGHT, Session, WIDTH}; -use super::{Browsers, Slots, lock, update}; +use super::session::Session; +use super::{Browsers, Slots, Viewport, lock, update}; use crate::clock::Clock; use crate::state::Event; @@ -145,14 +145,15 @@ impl Drop for Asking<'_> { impl Hands { /// Sends one of the owner's events to the page, after the ones before. pub fn send(&self, input: BrowserInput) -> Result<(), InputError> { - if !fits(&input) { - return Err(InputError::Invalid); - } - let holds = lock(&self.slots) + let page = lock(&self.slots) .get(&self.bot) - .is_some_and(|slot| *slot.held.borrow() == Some(self.hold)); - if !holds { + .filter(|slot| *slot.held.borrow() == Some(self.hold)) + .map(|slot| slot.viewport); + let Some(page) = page else { return Err(InputError::NotHeld); + }; + if !fits(&input, page) { + return Err(InputError::Invalid); } self.input.send(input).map_err(|_| InputError::Closed) } @@ -180,11 +181,9 @@ impl Drop for Hands { } } -/// Whether an event makes sense on the page. -fn fits(input: &BrowserInput) -> bool { - let on_page = |x: f64, y: f64| { - (0.0..=f64::from(WIDTH)).contains(&x) && (0.0..=f64::from(HEIGHT)).contains(&y) - }; +/// Whether an event makes sense on a page of that size. +fn fits(input: &BrowserInput, page: Viewport) -> bool { + let on_page = |x: f64, y: f64| page.contains(x, y); let modifiers_ok = |modifiers: u32| modifiers <= 15; match input { BrowserInput::Mouse { @@ -276,11 +275,15 @@ mod tests { #[test] fn events_off_the_page_or_too_big_are_refused() { + let fits = |input: &BrowserInput| super::fits(input, Viewport::default()); assert!(fits(&mouse(640.0, 400.0))); assert!(fits(&mouse(1280.0, 800.0))); assert!(!fits(&mouse(-1.0, 10.0))); assert!(!fits(&mouse(f64::NAN, 10.0))); assert!(!fits(&mouse(10.0, 900.0))); + // A taller page takes points further down. + let tall = Viewport::fitting(640, 700); + assert!(super::fits(&mouse(10.0, 900.0), tall)); let long = BrowserInput::Text { text: "x".repeat(TEXT_MAX + 1), }; diff --git a/crates/botloftd/src/browser/mod.rs b/crates/botloftd/src/browser/mod.rs index 662a3f4b..15684bce 100644 --- a/crates/botloftd/src/browser/mod.rs +++ b/crates/botloftd/src/browser/mod.rs @@ -16,6 +16,7 @@ mod session; mod settle; pub mod sites; mod sweep; +mod viewport; mod watch; use std::collections::HashMap; @@ -39,6 +40,7 @@ pub use self::page::{Done, Scroll}; pub use self::read::{READ_MAX, Reading}; pub use self::session::Session; pub use self::sweep::run; +pub use self::viewport::Viewport; pub use self::watch::Watching; use crate::clock::Clock; use crate::config::Config; @@ -111,6 +113,8 @@ struct Slot { state: BrowserState, frames: Frames, watchers: usize, + /// The page size the app watching asked for (spec 21.3). + viewport: Viewport, used: Instant, held: Held, /// The bot's open request for the owner's help. @@ -161,6 +165,7 @@ impl Browsers { state: BrowserState::closed(bot.clone(), self.clock.now_ms()), frames: tokio::sync::watch::channel(None).0, watchers: 0, + viewport: Viewport::default(), used: Instant::now(), held: tokio::sync::watch::channel(None).0, asking: None, diff --git a/crates/botloftd/src/browser/page.rs b/crates/botloftd/src/browser/page.rs index e0c628aa..2f5baaa2 100644 --- a/crates/botloftd/src/browser/page.rs +++ b/crates/botloftd/src/browser/page.rs @@ -6,7 +6,7 @@ use serde_json::{Value, json}; use super::BrowserError; use super::keys::{self, Key}; -use super::session::{HEIGHT, Session, WIDTH}; +use super::session::Session; /// Where an action happened, for the owner's view, and what the bot should /// know about it. @@ -204,10 +204,15 @@ impl Session { pub async fn scroll(&self, to: Scroll) -> Result { let (session, _) = self.page()?; let marks = self.marks(); - let center = (f64::from(WIDTH) / 2.0, f64::from(HEIGHT) / 2.0); + let screen = self.viewport(); + let center = ( + f64::from(screen.width) / 2.0, + f64::from(screen.height) / 2.0, + ); let done = match to { Scroll::Down | Scroll::Up => { - let delta = f64::from(HEIGHT) * 0.8 * if to == Scroll::Down { 1.0 } else { -1.0 }; + let delta = + f64::from(screen.height) * 0.8 * if to == Scroll::Down { 1.0 } else { -1.0 }; let params = json!({ "type": "mouseWheel", "x": center.0, "y": center.1, "deltaX": 0, "deltaY": delta, }); diff --git a/crates/botloftd/src/browser/session.rs b/crates/botloftd/src/browser/session.rs index 345b8f79..e51e0e39 100644 --- a/crates/botloftd/src/browser/session.rs +++ b/crates/botloftd/src/browser/session.rs @@ -17,10 +17,8 @@ use super::BrowserError; use super::cdp::Cdp; use super::events::pump; use super::launch::{self, BrowserProcess}; +use super::viewport::{MAX_HEIGHT, Viewport, WIDTH}; -/// The page size bots and the owner see, in CSS pixels (spec 21.3). -pub const WIDTH: u32 = 1280; -pub const HEIGHT: u32 = 800; /// Most tabs a browser keeps; the oldest inactive one closes past this. pub(super) const MAX_TABS: usize = 6; /// Frames are confirmed after this, so the browser sends ~15 a second. @@ -68,6 +66,7 @@ pub struct Tabs { pub(super) downloads: HashMap, pub watching: bool, pub closed: bool, + pub viewport: Viewport, } impl Tabs { @@ -101,14 +100,18 @@ pub struct Session { pub next_ref: AtomicU64, process: Mutex>, pub(super) user_agent: String, + /// One change of what the app wants at a time (`watch.rs`). + pub(super) syncing: tokio::sync::Mutex<()>, } impl Session { - /// Starts the browser and waits for its first tab. + /// Starts the browser, its pages `viewport` in size, and waits for its + /// first tab. pub async fn start( program: &Path, profile: &Path, downloads: &Path, + viewport: Viewport, hooks: Hooks, ) -> Result, BrowserError> { let (process, url) = launch::launch(program, profile).await?; @@ -120,11 +123,15 @@ impl Session { .replace("HeadlessChrome", "Chrome"); let session = Arc::new(Self { cdp, - tabs: Mutex::new(Tabs::default()), + tabs: Mutex::new(Tabs { + viewport, + ..Tabs::default() + }), changed: Notify::new(), next_ref: AtomicU64::new(1), process: Mutex::new(Some(process)), user_agent, + syncing: tokio::sync::Mutex::new(()), }); tokio::spawn(pump(Arc::clone(&session), events, hooks)); @@ -218,8 +225,9 @@ impl Session { pub(super) async fn cast(&self, session: &str, on: bool) { let result = if on { + // Frames come the size of the page, however tall it is. let params = json!({ - "format": "jpeg", "quality": 60, "maxWidth": WIDTH, "maxHeight": HEIGHT, + "format": "jpeg", "quality": 60, "maxWidth": WIDTH, "maxHeight": MAX_HEIGHT, }); self.cdp .call(Some(session), "Page.startScreencast", params) diff --git a/crates/botloftd/src/browser/viewport.rs b/crates/botloftd/src/browser/viewport.rs new file mode 100644 index 00000000..927b04e6 --- /dev/null +++ b/crates/botloftd/src/browser/viewport.rs @@ -0,0 +1,121 @@ +//! The size of the page (spec 21.3): as wide as a desktop screen always, +//! and as tall as the owner's panel asks while they watch. + +use serde_json::{Value, json}; +use tracing::debug; + +use super::session::Session; + +/// The page size bots and the owner see, in CSS pixels. The height is the +/// one nobody asked to change. +pub const WIDTH: u32 = 1280; +pub const HEIGHT: u32 = 800; +/// How short and how tall the owner's panel may make the page. +pub const MIN_HEIGHT: u32 = 600; +pub const MAX_HEIGHT: u32 = 2000; + +/// The size of the page, the same in every tab. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Viewport { + pub width: u32, + pub height: u32, +} + +impl Default for Viewport { + fn default() -> Self { + Self { + width: WIDTH, + height: HEIGHT, + } + } +} + +impl Viewport { + /// The page for a panel with `width` by `height` of room: as wide as + /// ever, so sites keep their desktop layout, and as tall as the room's + /// shape asks, within limits. + pub fn fitting(width: u32, height: u32) -> Self { + let tall = u64::from(WIDTH) * u64::from(height) / u64::from(width.max(1)); + Self { + width: WIDTH, + height: u32::try_from(tall) + .unwrap_or(MAX_HEIGHT) + .clamp(MIN_HEIGHT, MAX_HEIGHT), + } + } + + pub(super) fn metrics(self) -> Value { + json!({ + "width": self.width, "height": self.height, "deviceScaleFactor": 1, "mobile": false, + }) + } + + /// Whether a point is on the page. + pub fn contains(self, x: f64, y: f64) -> bool { + (0.0..=f64::from(self.width)).contains(&x) && (0.0..=f64::from(self.height)).contains(&y) + } +} + +impl Session { + pub fn viewport(&self) -> Viewport { + self.lock().viewport + } + + /// Makes the page `viewport` in size, in every tab. + pub async fn set_viewport(&self, viewport: Viewport) { + let pages: Vec = { + let mut tabs = self.lock(); + if tabs.viewport == viewport { + return; + } + tabs.viewport = viewport; + tabs.list.iter().map(|tab| tab.session.clone()).collect() + }; + for page in pages { + let sized = self + .cdp + .call( + Some(&page), + "Emulation.setDeviceMetricsOverride", + viewport.metrics(), + ) + .await; + if let Err(err) = sized { + debug!("browser: sizing a tab: {err}"); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_page_takes_the_shape_of_the_room_within_limits() { + let page = |width, height| Viewport::fitting(width, height); + // A panel 536 wide with 700 of height: the same shape, 1280 wide. + assert_eq!( + page(536, 700), + Viewport { + width: 1280, + height: 1671 + } + ); + assert_eq!(page(1280, 800), Viewport::default()); + // Too flat or too tall a room stops at the limits. + assert_eq!(page(1600, 300).height, MIN_HEIGHT); + assert_eq!(page(256, 1400).height, MAX_HEIGHT); + assert_eq!(page(0, 0).height, MIN_HEIGHT); + assert_eq!(page(1, u32::MAX).height, MAX_HEIGHT); + } + + #[test] + fn points_are_on_the_page_up_to_its_edges() { + let page = Viewport::fitting(640, 700); + assert!(page.contains(1280.0, 1400.0)); + assert!(!page.contains(10.0, 1400.5)); + assert!(!page.contains(-1.0, 10.0)); + assert!(!page.contains(f64::NAN, 10.0)); + } +} diff --git a/crates/botloftd/src/browser/watch.rs b/crates/botloftd/src/browser/watch.rs index c3bee5ae..c02a4126 100644 --- a/crates/botloftd/src/browser/watch.rs +++ b/crates/botloftd/src/browser/watch.rs @@ -1,5 +1,6 @@ //! An app connection watching a bot's browser (spec 21.7). While anyone -//! watches, the active tab sends frames; when the last one stops, it stops. +//! watches, the active tab sends frames and the page is the size their +//! panel asks for (spec 21.3); when the last one stops, both stop. use std::sync::Arc; @@ -7,7 +8,7 @@ use botloft_core::ids::BotId; use botloft_core::protocol::BrowserFrame; use tokio::sync::watch; -use super::{Browsers, Slots, lock}; +use super::{Browsers, Slots, Viewport, lock}; /// Held by the connection; dropping it stops watching. pub struct Watching { @@ -25,38 +26,63 @@ impl Browsers { slot.watchers += 1; slot.frames.subscribe() }; - sync(Arc::clone(&self.slots), bot.clone()); + sync_soon(&self.slots, bot); Watching { slots: Arc::clone(&self.slots), bot: bot.clone(), frames, } } + + /// The room a watching app has for the page, `width` by `height`: the + /// page takes its shape, now or when the browser opens (spec 21.3). + pub fn resize(&self, bot: &BotId, width: u32, height: u32) { + { + let mut slots = lock(&self.slots); + self.slot(&mut slots, bot).viewport = Viewport::fitting(width, height); + } + sync_soon(&self.slots, bot); + } } impl Drop for Watching { fn drop(&mut self) { if let Some(slot) = lock(&self.slots).get_mut(&self.bot) { slot.watchers = slot.watchers.saturating_sub(1); + if slot.watchers == 0 { + // No panel to fit anymore. + slot.viewport = Viewport::default(); + } } - sync(Arc::clone(&self.slots), self.bot.clone()); + sync_soon(&self.slots, &self.bot); } } -/// Turns the frames on or off to match whether anyone watches. It reads -/// the count when it runs, so calls that finish out of order still end on -/// the right answer. -fn sync(slots: Slots, bot: BotId) { - let Ok(runtime) = tokio::runtime::Handle::try_current() else { +/// Makes the running browser match what the apps want: frames while anyone +/// watches, and the page the size they asked for. One at a time, each +/// reading what is wanted when its turn comes, so changes that finish out +/// of order still end on the right answer. +pub(super) async fn sync(slots: Slots, bot: BotId) { + let running = lock(&slots).get(&bot).and_then(|slot| { + slot.session + .as_ref() + .map(|(_, session)| Arc::clone(session)) + }); + let Some(session) = running else { return; }; - runtime.spawn(async move { - let found = lock(&slots).get(&bot).and_then(|slot| { - let session = slot.session.as_ref()?.1.clone(); - Some((session, slot.watchers > 0)) - }); - if let Some((session, on)) = found { - session.set_watching(on).await; - } - }); + let _turn = session.syncing.lock().await; + let wanted = lock(&slots) + .get(&bot) + .map(|slot| (slot.watchers > 0, slot.viewport)); + if let Some((on, viewport)) = wanted { + session.set_viewport(viewport).await; + session.set_watching(on).await; + } +} + +fn sync_soon(slots: &Slots, bot: &BotId) { + if let Ok(runtime) = tokio::runtime::Handle::try_current() { + runtime.spawn(sync(Arc::clone(slots), bot.clone())); + } } diff --git a/crates/botloftd/src/rpc/mod.rs b/crates/botloftd/src/rpc/mod.rs index b5105ef1..41d45c19 100644 --- a/crates/botloftd/src/rpc/mod.rs +++ b/crates/botloftd/src/rpc/mod.rs @@ -184,10 +184,11 @@ fn handle( frames.send_replace(None); watch.request(daemon, &request.method, request.params) } - // So is the owner's hold on it (spec 21.10). - method::BROWSER_TAKE | method::BROWSER_RELEASE | method::BROWSER_INPUT => { - watch.request(daemon, &request.method, request.params) - } + // So are the size of its page and the owner's hold on it (spec 21.10). + method::BROWSER_RESIZE + | method::BROWSER_TAKE + | method::BROWSER_RELEASE + | method::BROWSER_INPUT => watch.request(daemon, &request.method, request.params), name => dispatch::dispatch(daemon, name, request.params), }; Some(match result { diff --git a/crates/botloftd/src/rpc/watching.rs b/crates/botloftd/src/rpc/watching.rs index 978db07b..630261a9 100644 --- a/crates/botloftd/src/rpc/watching.rs +++ b/crates/botloftd/src/rpc/watching.rs @@ -1,10 +1,12 @@ //! A connection watching one bot's browser (spec 21.7): `browser.watch` -//! and `browser.unwatch`, and the frames that follow. The same connection -//! takes the browser into the owner's hands and gives it back (spec 21.10). +//! and `browser.unwatch`, the frames that follow, and the size its panel +//! gives the page. The same connection takes the browser into the owner's +//! hands and gives it back (spec 21.10). +use botloft_core::ids::BotId; use botloft_core::protocol::{ - ApprovalsAnswerParams, BrowserControlParams, BrowserInputParams, BrowserWatchParams, - error_code, method, notification, + ApprovalsAnswerParams, BrowserControlParams, BrowserInputParams, BrowserResizeParams, + BrowserWatchParams, error_code, method, notification, }; use serde::Serialize; use serde_json::Value; @@ -16,6 +18,9 @@ use crate::browser::{Hands, InputError, Watching}; use crate::service::{ApiError, bots}; use crate::state::Daemon; +/// The most room a panel may say it has, in pixels each way. +const ROOM_MAX: u32 = 10_000; + #[derive(Default)] pub(super) struct Watch { current: Option, @@ -31,6 +36,7 @@ impl Watch { params: Option, ) -> Result { match name { + method::BROWSER_RESIZE => return self.resize(daemon, &parse(params)?), method::BROWSER_TAKE => return self.take(daemon, parse(params)?), method::BROWSER_RELEASE => return self.release(daemon, &parse(params)?), method::BROWSER_INPUT => return self.input(parse(params)?), @@ -49,13 +55,34 @@ impl Watch { to_value(&daemon.browsers.view(¶ms.bot_id)) } + fn watches(&self, bot: &BotId) -> bool { + self.current + .as_ref() + .is_some_and(|watching| watching.bot == *bot) + } + + /// The page takes the shape of the room this connection's panel has. + fn resize(&self, daemon: &Daemon, params: &BrowserResizeParams) -> Result { + if !self.watches(¶ms.bot_id) { + return Err( + ApiError::Conflict("watch this browser before sizing it".to_owned()).into(), + ); + } + let fits = |side: u32| (1..=ROOM_MAX).contains(&side); + if !fits(params.width) || !fits(params.height) { + return Err( + ApiError::validation(format!("width and height go from 1 to {ROOM_MAX}")).into(), + ); + } + daemon + .browsers + .resize(¶ms.bot_id, params.width, params.height); + Ok(Value::Null) + } + fn take(&mut self, daemon: &Daemon, params: BrowserControlParams) -> Result { let bot = ¶ms.bot_id; - if self - .current - .as_ref() - .is_none_or(|watching| watching.bot != *bot) - { + if !self.watches(bot) { return Err( ApiError::Conflict("watch this browser before taking it".to_owned()).into(), ); diff --git a/crates/botloftd/tests/browser_panel.rs b/crates/botloftd/tests/browser_panel.rs new file mode 100644 index 00000000..4e1fe3d5 --- /dev/null +++ b/crates/botloftd/tests/browser_panel.rs @@ -0,0 +1,113 @@ +//! The owner's panel on a bot's browser with the real Microsoft Edge (spec +//! 21.3, 21.7): the page taking the shape of the room the panel has. +//! Skipped where Edge is not installed. + +mod common; + +use std::time::Duration; + +use common::Client; +use common::browsing::{Browsing, answer_site, call, setup}; +use serde_json::{Value, json}; + +/// Reads the page until it says its window is `size`. +async fn until_window(b: &mut Browsing, size: &str) -> String { + let mut page = String::new(); + for _ in 0..50 { + page = b + .mcp + .tool_text("browser_look", json!({})) + .await + .expect("look"); + if page.contains(&format!("Window: {size}")) { + return page; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!("the window never became {size}:\n{page}"); +} + +/// The next frame that is `height` tall. +async fn frame_of(app: &mut Client, height: u64) -> Value { + for _ in 0..50 { + let frame = app.notification("browser.frame").await; + if frame["height"] == height { + return frame; + } + } + panic!("no frame came {height} tall"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn the_page_takes_the_shape_of_the_panel_while_the_owner_watches() { + let Some(mut b) = setup().await else { return }; + let id = b.bot["id"].clone(); + let room = |width: u32, height: u32| json!({ "botId": id, "width": width, "height": height }); + + // Only a connection watching the browser sizes it, and with sane numbers. + assert!(b.app.call("browser.resize", room(536, 700)).await.is_err()); + b.app + .call("browser.watch", json!({ "botId": id })) + .await + .expect("watch"); + assert!(b.app.call("browser.resize", room(0, 700)).await.is_err()); + assert!( + b.app + .call("browser.resize", room(536, 20_000)) + .await + .is_err() + ); + + // Asked before the browser opens: it opens that size. + b.app + .call("browser.resize", room(536, 700)) + .await + .expect("resize"); + let opening = call( + &b.mcp, + "browser_open", + json!({ "url": format!("{}/size", b.site) }), + ); + answer_site(&mut b.app, true, None).await; + let page = opening.await.expect("task").expect("page"); + assert!(page.contains("Window: 1280 x 1671"), "{page}"); + let frame = frame_of(&mut b.app, 1671).await; + assert_eq!(frame["width"], 1280); + + // The panel changes: the page follows, and so do the frames. + b.app + .call("browser.resize", room(640, 400)) + .await + .expect("resize"); + until_window(&mut b, "1280 x 800").await; + frame_of(&mut b.app, 800).await; + + // The owner's points go as far as the page does now. + b.app + .call("browser.resize", room(640, 700)) + .await + .expect("resize"); + until_window(&mut b, "1280 x 1400").await; + b.app + .call("browser.take", json!({ "botId": id })) + .await + .expect("take"); + let click = |y: f64| { + json!({ "botId": id, "input": { + "kind": "mouse", "action": "move", "x": 10.0, "y": y, "button": "none", + "buttons": 0, "clicks": 0, "modifiers": 0, + }}) + }; + b.app + .call("browser.input", click(1390.0)) + .await + .expect("a point on the taller page"); + assert!(b.app.call("browser.input", click(1410.0)).await.is_err()); + + // Nobody watches anymore: back to the size bots work in alone. + b.app + .call("browser.unwatch", Value::Null) + .await + .expect("unwatch"); + until_window(&mut b, "1280 x 800").await; +} diff --git a/crates/botloftd/tests/common/site.rs b/crates/botloftd/tests/common/site.rs index 24d7c222..f1891b47 100644 --- a/crates/botloftd/tests/common/site.rs +++ b/crates/botloftd/tests/common/site.rs @@ -1,7 +1,7 @@ //! A small web site on 127.0.0.1 for the browser tests: a form, the page //! it leads to, a link that opens a new tab, a button that shows a dialog, -//! and a sign-in page with its fields at fixed points, for the owner's -//! hands. +//! a sign-in page with its fields at fixed points, for the owner's hands, +//! and a page that says how big its window is. use std::collections::HashMap; use std::net::SocketAddr; @@ -30,6 +30,11 @@ const LOGIN: &str = "Sign in