{status === "failed" ? (
{t.failedBody}
@@ -94,41 +112,44 @@ export function BrowserPanel({
<>
{live && ask && !hands.held &&
{frame && (
![]()
{
+ cleanup();
+ vi.restoreAllMocks();
+});
/** A crew "Ops" with @scout, idle. */
function crew() {
@@ -68,6 +71,37 @@ describe("browser panel", () => {
expect(within(panel()).queryByText("Opened elsewhere.com")).toBeNull();
});
+ test("gives the page the shape of the room the panel has, and shows it as large as fits", async () => {
+ const { fake, scout } = crew();
+ fake.browser.open(scout.id, "https://example.com/", "Example");
+ // The panel under its address row, as the app measures it: 536 by 700
+ // for the page, once what goes around and under it is set aside.
+ vi.spyOn(Element.prototype, "clientWidth", "get").mockReturnValue(560);
+ vi.spyOn(Element.prototype, "clientHeight", "get").mockReturnValue(804);
+ await openScout(fake);
+ fireEvent.click(toggle());
+ const resized = () =>
+ fake.calls.filter((call) => call.method === "browser.resize").map((call) => call.params);
+ await waitFor(() => expect(resized()).toEqual([{ botId: scout.id, width: 536, height: 700 }]));
+ // As wide as ever, and as tall as the room's shape asks.
+ expect(fake.browser.size(scout.id)).toEqual({ width: 1280, height: 1671 });
+
+ act(() => {
+ fake.browser.frame(scout.id, "AAAA");
+ });
+ // The bot's cursor lands where it acted on the taller page.
+ const screenshot = within(panel()).getByRole("figure", { name: "What Scout sees" });
+ act(() => {
+ fake.browser.act(scout.id, "click", { x: 640, y: 1671, label: "More" });
+ });
+ const cursor = screenshot.querySelector(".bot-cursor") as HTMLElement;
+ expect(cursor.style.top).toBe("100%");
+
+ // Nobody watches anymore: the page is back to its own size.
+ fireEvent.click(within(panel()).getByRole("button", { name: "Close" }));
+ await waitFor(() => expect(fake.browser.size(scout.id)).toEqual({ width: 1280, height: 800 }));
+ });
+
test("opens by itself when the bot starts browsing, and the button marks it once closed", async () => {
const { fake, scout } = crew();
await openScout(fake);
diff --git a/app/src/features/browser/useRoom.ts b/app/src/features/browser/useRoom.ts
new file mode 100644
index 00000000..f0b56842
--- /dev/null
+++ b/app/src/features/browser/useRoom.ts
@@ -0,0 +1,69 @@
+// The room the panel has for the bot's page (spec 21.8): measured as the
+// panel changes and told to the daemon, so the page takes its shape (spec
+// 21.3).
+
+import { useEffect, useLayoutEffect, useState } from "react";
+import type { Bot } from "../../lib/protocol.gen";
+import { useApi } from "../../store/context";
+
+export interface Size {
+ width: number;
+ height: number;
+}
+
+/** How long the room stays the same before the daemon hears about it. */
+const SETTLED_MS = 300;
+
+/**
+ * The room inside an element, following it as it changes: `null` until it
+ * has any. The second value goes in the element's `ref`.
+ */
+export function useRoom(): [Size | null, (node: HTMLElement | null) => void] {
+ const [node, setNode] = useState(null);
+ const [room, setRoom] = useState(null);
+ useLayoutEffect(() => {
+ if (!node) {
+ setRoom(null);
+ return;
+ }
+ const measure = () => {
+ const width = node.clientWidth;
+ const height = node.clientHeight;
+ setRoom((was) => {
+ if (width <= 0 || height <= 0) {
+ return null;
+ }
+ return was?.width === width && was.height === height ? was : { width, height };
+ });
+ };
+ measure();
+ if (typeof ResizeObserver === "undefined") {
+ return;
+ }
+ const observer = new ResizeObserver(measure);
+ observer.observe(node);
+ return () => observer.disconnect();
+ }, [node]);
+ return [room, setNode];
+}
+
+/**
+ * Tells the daemon the room the page has, once it stops changing, while
+ * this connection watches the bot's browser.
+ */
+export function useFitPage(bot: Pick, room: Size | null, watched: boolean): void {
+ const api = useApi();
+ const known = room !== null;
+ const width = Math.max(1, room?.width ?? 0);
+ const height = Math.max(1, room?.height ?? 0);
+ useEffect(() => {
+ if (!watched || !known) {
+ return;
+ }
+ const timer = setTimeout(() => {
+ // The next change of the room asks again.
+ api.call("browser.resize", { botId: bot.id, width, height }).catch(() => {});
+ }, SETTLED_MS);
+ return () => clearTimeout(timer);
+ }, [api, bot.id, watched, known, width, height]);
+}
diff --git a/app/src/lib/fakeBrowser.ts b/app/src/lib/fakeBrowser.ts
index 90ffedfe..af71fcd9 100644
--- a/app/src/lib/fakeBrowser.ts
+++ b/app/src/lib/fakeBrowser.ts
@@ -1,6 +1,7 @@
// The fake daemon's browsers (spec 21.7): each bot's state, the live frames
// of the one being watched and what the bot does, for the browser panel,
-// and the owner's hands in it (spec 21.10).
+// the size the panel gives the page (spec 21.3), and the owner's hands in
+// it (spec 21.10).
import type { FakeBotloft, Handlers } from "./fake";
import { conflict } from "./fakeRules";
@@ -14,6 +15,20 @@ import type {
BrowserState,
} from "./protocol.gen";
+/** The page's size: as wide as ever, as tall as the panel's room asks. */
+export interface PageSize {
+ width: number;
+ height: number;
+}
+
+const PAGE: PageSize = { width: 1280, height: 800 };
+
+/** The daemon's rule for the page in a room of the panel (spec 21.3). */
+function fitting(width: number, height: number): PageSize {
+ const tall = Math.floor((PAGE.width * height) / width);
+ return { width: PAGE.width, height: Math.min(2000, Math.max(600, tall)) };
+}
+
export class FakeBrowser {
private readonly states = new Map();
private readonly frames = new Map();
@@ -23,6 +38,10 @@ export class FakeBrowser {
readonly watches: (BotId | null)[] = [];
/** What the owner sent to the page, in order. */
readonly inputs: BrowserInput[] = [];
+ /** Each bot's page size, while the panel asks for one. */
+ private readonly sizes = new Map();
+ /** What draws each bot's page, for the preview. */
+ private readonly painters = new Map string>();
/** Each bot's open request for help. */
private readonly asks = new Map();
/** Pages that answer the owner's hands, for the preview. */
@@ -96,9 +115,28 @@ export class FakeBrowser {
this.set(botId, { ask: null, control: "bot" });
}
+ /** The size of the bot's page now. */
+ size(botId: BotId): PageSize {
+ return this.sizes.get(botId) ?? PAGE;
+ }
+
+ /** Draws the bot's page with `draw`, now and whenever its size changes. */
+ paint(botId: BotId, draw: (size: PageSize) => string): void {
+ this.painters.set(botId, draw);
+ this.repaint(botId);
+ }
+
+ /** The page changed: a new picture of it, if something draws it. */
+ repaint(botId: BotId): void {
+ const draw = this.painters.get(botId);
+ if (draw) {
+ this.frame(botId, draw(this.size(botId)));
+ }
+ }
+
/** A new picture of the page; sent only while the app watches that bot. */
- frame(botId: BotId, data: string, width = 1280, height = 800): void {
- const frame: BrowserFrame = { botId, data, width, height };
+ frame(botId: BotId, data: string, size: PageSize = this.size(botId)): void {
+ const frame: BrowserFrame = { botId, data, ...size };
this.frames.set(botId, frame);
if (this.watching === botId) {
this.fake.emit({ name: "browser.frame", params: frame });
@@ -132,11 +170,22 @@ export class FakeBrowser {
return this.set(botId, { control: "bot" });
}
+ /** Nobody watches: the page goes back to the size bots work in alone. */
+ private unwatch(): void {
+ const botId = this.watching;
+ this.release();
+ this.watching = null;
+ if (botId !== null && this.sizes.delete(botId)) {
+ this.repaint(botId);
+ }
+ }
+
handlers(): Pick<
Handlers,
| "browser.list"
| "browser.watch"
| "browser.unwatch"
+ | "browser.resize"
| "browser.take"
| "browser.release"
| "browser.input"
@@ -145,17 +194,24 @@ export class FakeBrowser {
"browser.list": () => [...this.states.values()].filter((state) => state.status !== "closed"),
"browser.watch": ({ botId }) => {
this.fake.bot(botId, false);
- this.release();
+ this.unwatch();
this.watching = botId;
this.watches.push(botId);
return { state: this.state(botId), frame: this.frames.get(botId) ?? null };
},
"browser.unwatch": () => {
- this.release();
- this.watching = null;
+ this.unwatch();
this.watches.push(null);
return null;
},
+ "browser.resize": ({ botId, width, height }) => {
+ if (this.watching !== botId) {
+ throw conflict("watch this browser before sizing it");
+ }
+ this.sizes.set(botId, fitting(width, height));
+ this.repaint(botId);
+ return null;
+ },
"browser.take": ({ botId }) => {
if (this.watching !== botId) {
throw conflict("watch this browser before taking it");
diff --git a/app/src/lib/protocol.gen.ts b/app/src/lib/protocol.gen.ts
index 1aa00419..0d3f277e 100644
--- a/app/src/lib/protocol.gen.ts
+++ b/app/src/lib/protocol.gen.ts
@@ -892,6 +892,12 @@ export type BrowserView = { state: BrowserState, frame: BrowserFrame | null, };
export type BrowserWatchParams = { botId: BotId, };
+/**
+ * `browser.resize`: the room the app's panel has for the page, in the
+ * app's pixels. The page takes its shape (spec 21.3).
+ */
+export type BrowserResizeParams = { botId: BotId, width: number, height: number, };
+
export type MouseAction = "move" | "down" | "up";
export type MouseButton = "none" | "left" | "middle" | "right";
@@ -1020,6 +1026,7 @@ export interface RpcMethods {
"browser.list": { params: undefined; result: Array };
"browser.watch": { params: BrowserWatchParams; result: BrowserView };
"browser.unwatch": { params: undefined; result: null };
+ "browser.resize": { params: BrowserResizeParams; result: null };
"browser.take": { params: BrowserControlParams; result: BrowserState };
"browser.release": { params: BrowserControlParams; result: BrowserState };
"browser.input": { params: BrowserInputParams; result: null };
diff --git a/crates/botloft-core/src/protocol/browser.rs b/crates/botloft-core/src/protocol/browser.rs
index 194fb53d..6b839f20 100644
--- a/crates/botloft-core/src/protocol/browser.rs
+++ b/crates/botloft-core/src/protocol/browser.rs
@@ -128,3 +128,14 @@ pub struct BrowserView {
pub struct BrowserWatchParams {
pub bot_id: BotId,
}
+
+/// `browser.resize`: the room the app's panel has for the page, in the
+/// app's pixels. The page takes its shape (spec 21.3).
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+#[cfg_attr(test, derive(ts_rs::TS))]
+pub struct BrowserResizeParams {
+ pub bot_id: BotId,
+ pub width: u32,
+ pub height: u32,
+}
diff --git a/crates/botloft-core/src/protocol/export_browser.rs b/crates/botloft-core/src/protocol/export_browser.rs
index dfc9c191..c99f4908 100644
--- a/crates/botloft-core/src/protocol/export_browser.rs
+++ b/crates/botloft-core/src/protocol/export_browser.rs
@@ -14,6 +14,7 @@ pub(super) fn decls(out: &mut Out) {
out.decl::();
out.decl::();
out.decl::();
+ out.decl::();
out.decl::();
out.decl::();
out.decl::();
@@ -37,6 +38,11 @@ pub(super) fn methods(out: &mut Out) {
&out.name::(),
);
out.method(method::BROWSER_UNWATCH, "undefined", "null");
+ out.method(
+ method::BROWSER_RESIZE,
+ &out.name::(),
+ "null",
+ );
let control = out.name::();
let state = out.name::();
out.method(method::BROWSER_TAKE, &control, &state);
diff --git a/crates/botloft-core/src/protocol/mod.rs b/crates/botloft-core/src/protocol/mod.rs
index 43054c83..df1e5651 100644
--- a/crates/botloft-core/src/protocol/mod.rs
+++ b/crates/botloft-core/src/protocol/mod.rs
@@ -117,6 +117,7 @@ pub mod method {
pub const BROWSER_LIST: &str = "browser.list";
pub const BROWSER_WATCH: &str = "browser.watch";
pub const BROWSER_UNWATCH: &str = "browser.unwatch";
+ pub const BROWSER_RESIZE: &str = "browser.resize";
pub const BROWSER_TAKE: &str = "browser.take";
pub const BROWSER_RELEASE: &str = "browser.release";
pub const BROWSER_INPUT: &str = "browser.input";
diff --git a/crates/botloftd/src/browser/call.rs b/crates/botloftd/src/browser/call.rs
index 84b683fe..61092e45 100644
--- a/crates/botloftd/src/browser/call.rs
+++ b/crates/botloftd/src/browser/call.rs
@@ -11,7 +11,7 @@ use tokio::sync::OwnedMutexGuard;
use tracing::debug;
use super::session::{Hooks, PageInfo, Session};
-use super::{BrowserError, Browsers, launch, lock, update};
+use super::{BrowserError, Browsers, launch, lock, update, watch};
pub struct Call<'a> {
pub(super) browsers: &'a Browsers,
@@ -43,7 +43,11 @@ impl Call<'_> {
None => Err(BrowserError::NotFound),
Some(program) => {
let profile = browsers.profiles.join(bot.as_str());
- Session::start(&program, &profile, downloads, self.hooks(start)).await
+ let viewport = {
+ let mut slots = lock(&browsers.slots);
+ browsers.slot(&mut slots, bot).viewport
+ };
+ Session::start(&program, &profile, downloads, viewport, self.hooks(start)).await
}
};
let session = match started {
@@ -57,16 +61,14 @@ impl Call<'_> {
return Err(err);
}
};
- let watched = {
+ {
let mut slots = lock(&browsers.slots);
let slot = browsers.slot(&mut slots, bot);
slot.session = Some((start, Arc::clone(&session)));
- slot.watchers > 0
- };
- browsers.set_state(bot, |state| state.status = BrowserStatus::Open);
- if watched {
- session.set_watching(true).await;
}
+ browsers.set_state(bot, |state| state.status = BrowserStatus::Open);
+ // Whoever watches already gets frames, the size they asked for.
+ watch::sync(Arc::clone(&browsers.slots), bot.clone()).await;
debug!(bot = %bot, "browser: open");
Ok(session)
}
diff --git a/crates/botloftd/src/browser/events.rs b/crates/botloftd/src/browser/events.rs
index 675f720c..13c6d234 100644
--- a/crates/botloftd/src/browser/events.rs
+++ b/crates/botloftd/src/browser/events.rs
@@ -11,7 +11,7 @@ use tokio::sync::mpsc;
use tracing::debug;
use super::cdp::CdpEvent;
-use super::session::{FRAME_GAP, HEIGHT, Hooks, MAX_TABS, PageInfo, Session, Tab, WIDTH};
+use super::session::{FRAME_GAP, Hooks, MAX_TABS, PageInfo, Session, Tab};
impl Session {
/// Sets up a new tab before it runs, and makes it the active one.
@@ -25,7 +25,7 @@ impl Session {
else {
return;
};
- let (previous, watching, extra) = {
+ let (previous, watching, extra, viewport) = {
let mut tabs = self.lock();
let previous = tabs.active().map(|tab| tab.session.clone());
tabs.list.push(Tab {
@@ -39,7 +39,7 @@ impl Session {
network_at: Instant::now(),
});
let extra = (tabs.list.len() > MAX_TABS).then(|| tabs.list[0].target.clone());
- (previous, tabs.watching, extra)
+ (previous, tabs.watching, extra, tabs.viewport)
};
let setup = [
("Page.enable", json!({})),
@@ -48,10 +48,7 @@ impl Session {
"Page.setInterceptFileChooserDialog",
json!({ "enabled": true }),
),
- (
- "Emulation.setDeviceMetricsOverride",
- json!({ "width": WIDTH, "height": HEIGHT, "deviceScaleFactor": 1, "mobile": false }),
- ),
+ ("Emulation.setDeviceMetricsOverride", viewport.metrics()),
(
"Emulation.setUserAgentOverride",
json!({ "userAgent": self.user_agent }),
@@ -188,10 +185,11 @@ impl Session {
}
fn frame(&self, session: &str, params: &Value, hooks: &Hooks) {
- let active = self
- .lock()
- .active()
- .is_some_and(|tab| tab.session == session);
+ let (active, viewport) = {
+ let tabs = self.lock();
+ let active = tabs.active().is_some_and(|tab| tab.session == session);
+ (active, tabs.viewport)
+ };
if active && let Some(data) = params["data"].as_str() {
let size = |key: &str, fallback: u32| {
params["metadata"][key]
@@ -201,8 +199,8 @@ impl Session {
hooks.frames.send_replace(Some(Arc::new(BrowserFrame {
bot_id: hooks.bot.clone(),
data: data.to_owned(),
- width: size("deviceWidth", WIDTH),
- height: size("deviceHeight", HEIGHT),
+ width: size("deviceWidth", viewport.width),
+ height: size("deviceHeight", viewport.height),
})));
}
let cdp = self.cdp.clone();
diff --git a/crates/botloftd/src/browser/hands.rs b/crates/botloftd/src/browser/hands.rs
index 34a8f932..7d3d3a88 100644
--- a/crates/botloftd/src/browser/hands.rs
+++ b/crates/botloftd/src/browser/hands.rs
@@ -11,8 +11,8 @@ use botloft_core::protocol::{BrowserControl, BrowserInput, MouseAction};
use tokio::sync::{broadcast, mpsc};
use tracing::debug;
-use super::session::{HEIGHT, Session, WIDTH};
-use super::{Browsers, Slots, lock, update};
+use super::session::Session;
+use super::{Browsers, Slots, Viewport, lock, update};
use crate::clock::Clock;
use crate::state::Event;
@@ -145,14 +145,15 @@ impl Drop for Asking<'_> {
impl Hands {
/// Sends one of the owner's events to the page, after the ones before.
pub fn send(&self, input: BrowserInput) -> Result<(), InputError> {
- if !fits(&input) {
- return Err(InputError::Invalid);
- }
- let holds = lock(&self.slots)
+ let page = lock(&self.slots)
.get(&self.bot)
- .is_some_and(|slot| *slot.held.borrow() == Some(self.hold));
- if !holds {
+ .filter(|slot| *slot.held.borrow() == Some(self.hold))
+ .map(|slot| slot.viewport);
+ let Some(page) = page else {
return Err(InputError::NotHeld);
+ };
+ if !fits(&input, page) {
+ return Err(InputError::Invalid);
}
self.input.send(input).map_err(|_| InputError::Closed)
}
@@ -180,11 +181,9 @@ impl Drop for Hands {
}
}
-/// Whether an event makes sense on the page.
-fn fits(input: &BrowserInput) -> bool {
- let on_page = |x: f64, y: f64| {
- (0.0..=f64::from(WIDTH)).contains(&x) && (0.0..=f64::from(HEIGHT)).contains(&y)
- };
+/// Whether an event makes sense on a page of that size.
+fn fits(input: &BrowserInput, page: Viewport) -> bool {
+ let on_page = |x: f64, y: f64| page.contains(x, y);
let modifiers_ok = |modifiers: u32| modifiers <= 15;
match input {
BrowserInput::Mouse {
@@ -276,11 +275,15 @@ mod tests {
#[test]
fn events_off_the_page_or_too_big_are_refused() {
+ let fits = |input: &BrowserInput| super::fits(input, Viewport::default());
assert!(fits(&mouse(640.0, 400.0)));
assert!(fits(&mouse(1280.0, 800.0)));
assert!(!fits(&mouse(-1.0, 10.0)));
assert!(!fits(&mouse(f64::NAN, 10.0)));
assert!(!fits(&mouse(10.0, 900.0)));
+ // A taller page takes points further down.
+ let tall = Viewport::fitting(640, 700);
+ assert!(super::fits(&mouse(10.0, 900.0), tall));
let long = BrowserInput::Text {
text: "x".repeat(TEXT_MAX + 1),
};
diff --git a/crates/botloftd/src/browser/mod.rs b/crates/botloftd/src/browser/mod.rs
index 662a3f4b..15684bce 100644
--- a/crates/botloftd/src/browser/mod.rs
+++ b/crates/botloftd/src/browser/mod.rs
@@ -16,6 +16,7 @@ mod session;
mod settle;
pub mod sites;
mod sweep;
+mod viewport;
mod watch;
use std::collections::HashMap;
@@ -39,6 +40,7 @@ pub use self::page::{Done, Scroll};
pub use self::read::{READ_MAX, Reading};
pub use self::session::Session;
pub use self::sweep::run;
+pub use self::viewport::Viewport;
pub use self::watch::Watching;
use crate::clock::Clock;
use crate::config::Config;
@@ -111,6 +113,8 @@ struct Slot {
state: BrowserState,
frames: Frames,
watchers: usize,
+ /// The page size the app watching asked for (spec 21.3).
+ viewport: Viewport,
used: Instant,
held: Held,
/// The bot's open request for the owner's help.
@@ -161,6 +165,7 @@ impl Browsers {
state: BrowserState::closed(bot.clone(), self.clock.now_ms()),
frames: tokio::sync::watch::channel(None).0,
watchers: 0,
+ viewport: Viewport::default(),
used: Instant::now(),
held: tokio::sync::watch::channel(None).0,
asking: None,
diff --git a/crates/botloftd/src/browser/page.rs b/crates/botloftd/src/browser/page.rs
index e0c628aa..2f5baaa2 100644
--- a/crates/botloftd/src/browser/page.rs
+++ b/crates/botloftd/src/browser/page.rs
@@ -6,7 +6,7 @@ use serde_json::{Value, json};
use super::BrowserError;
use super::keys::{self, Key};
-use super::session::{HEIGHT, Session, WIDTH};
+use super::session::Session;
/// Where an action happened, for the owner's view, and what the bot should
/// know about it.
@@ -204,10 +204,15 @@ impl Session {
pub async fn scroll(&self, to: Scroll) -> Result {
let (session, _) = self.page()?;
let marks = self.marks();
- let center = (f64::from(WIDTH) / 2.0, f64::from(HEIGHT) / 2.0);
+ let screen = self.viewport();
+ let center = (
+ f64::from(screen.width) / 2.0,
+ f64::from(screen.height) / 2.0,
+ );
let done = match to {
Scroll::Down | Scroll::Up => {
- let delta = f64::from(HEIGHT) * 0.8 * if to == Scroll::Down { 1.0 } else { -1.0 };
+ let delta =
+ f64::from(screen.height) * 0.8 * if to == Scroll::Down { 1.0 } else { -1.0 };
let params = json!({
"type": "mouseWheel", "x": center.0, "y": center.1, "deltaX": 0, "deltaY": delta,
});
diff --git a/crates/botloftd/src/browser/session.rs b/crates/botloftd/src/browser/session.rs
index 345b8f79..e51e0e39 100644
--- a/crates/botloftd/src/browser/session.rs
+++ b/crates/botloftd/src/browser/session.rs
@@ -17,10 +17,8 @@ use super::BrowserError;
use super::cdp::Cdp;
use super::events::pump;
use super::launch::{self, BrowserProcess};
+use super::viewport::{MAX_HEIGHT, Viewport, WIDTH};
-/// The page size bots and the owner see, in CSS pixels (spec 21.3).
-pub const WIDTH: u32 = 1280;
-pub const HEIGHT: u32 = 800;
/// Most tabs a browser keeps; the oldest inactive one closes past this.
pub(super) const MAX_TABS: usize = 6;
/// Frames are confirmed after this, so the browser sends ~15 a second.
@@ -68,6 +66,7 @@ pub struct Tabs {
pub(super) downloads: HashMap,
pub watching: bool,
pub closed: bool,
+ pub viewport: Viewport,
}
impl Tabs {
@@ -101,14 +100,18 @@ pub struct Session {
pub next_ref: AtomicU64,
process: Mutex>,
pub(super) user_agent: String,
+ /// One change of what the app wants at a time (`watch.rs`).
+ pub(super) syncing: tokio::sync::Mutex<()>,
}
impl Session {
- /// Starts the browser and waits for its first tab.
+ /// Starts the browser, its pages `viewport` in size, and waits for its
+ /// first tab.
pub async fn start(
program: &Path,
profile: &Path,
downloads: &Path,
+ viewport: Viewport,
hooks: Hooks,
) -> Result, BrowserError> {
let (process, url) = launch::launch(program, profile).await?;
@@ -120,11 +123,15 @@ impl Session {
.replace("HeadlessChrome", "Chrome");
let session = Arc::new(Self {
cdp,
- tabs: Mutex::new(Tabs::default()),
+ tabs: Mutex::new(Tabs {
+ viewport,
+ ..Tabs::default()
+ }),
changed: Notify::new(),
next_ref: AtomicU64::new(1),
process: Mutex::new(Some(process)),
user_agent,
+ syncing: tokio::sync::Mutex::new(()),
});
tokio::spawn(pump(Arc::clone(&session), events, hooks));
@@ -218,8 +225,9 @@ impl Session {
pub(super) async fn cast(&self, session: &str, on: bool) {
let result = if on {
+ // Frames come the size of the page, however tall it is.
let params = json!({
- "format": "jpeg", "quality": 60, "maxWidth": WIDTH, "maxHeight": HEIGHT,
+ "format": "jpeg", "quality": 60, "maxWidth": WIDTH, "maxHeight": MAX_HEIGHT,
});
self.cdp
.call(Some(session), "Page.startScreencast", params)
diff --git a/crates/botloftd/src/browser/viewport.rs b/crates/botloftd/src/browser/viewport.rs
new file mode 100644
index 00000000..411d9bd8
--- /dev/null
+++ b/crates/botloftd/src/browser/viewport.rs
@@ -0,0 +1,136 @@
+//! The size of the page (spec 21.3): as wide as a desktop screen always,
+//! and as tall as the owner's panel asks while they watch.
+
+use serde_json::{Value, json};
+use tracing::debug;
+
+use super::session::{FRAME_GAP, Session};
+
+/// The page size bots and the owner see, in CSS pixels. The height is the
+/// one nobody asked to change.
+pub const WIDTH: u32 = 1280;
+pub const HEIGHT: u32 = 800;
+/// How short and how tall the owner's panel may make the page.
+pub const MIN_HEIGHT: u32 = 600;
+pub const MAX_HEIGHT: u32 = 2000;
+
+/// The size of the page, the same in every tab.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct Viewport {
+ pub width: u32,
+ pub height: u32,
+}
+
+impl Default for Viewport {
+ fn default() -> Self {
+ Self {
+ width: WIDTH,
+ height: HEIGHT,
+ }
+ }
+}
+
+impl Viewport {
+ /// The page for a panel with `width` by `height` of room: as wide as
+ /// ever, so sites keep their desktop layout, and as tall as the room's
+ /// shape asks, within limits.
+ pub fn fitting(width: u32, height: u32) -> Self {
+ let tall = u64::from(WIDTH) * u64::from(height) / u64::from(width.max(1));
+ Self {
+ width: WIDTH,
+ height: u32::try_from(tall)
+ .unwrap_or(MAX_HEIGHT)
+ .clamp(MIN_HEIGHT, MAX_HEIGHT),
+ }
+ }
+
+ pub(super) fn metrics(self) -> Value {
+ json!({
+ "width": self.width, "height": self.height, "deviceScaleFactor": 1, "mobile": false,
+ })
+ }
+
+ /// Whether a point is on the page.
+ pub fn contains(self, x: f64, y: f64) -> bool {
+ (0.0..=f64::from(self.width)).contains(&x) && (0.0..=f64::from(self.height)).contains(&y)
+ }
+}
+
+impl Session {
+ pub fn viewport(&self) -> Viewport {
+ self.lock().viewport
+ }
+
+ /// Makes the page `viewport` in size, in every tab, and sends whoever
+ /// watches a frame of that size.
+ pub async fn set_viewport(&self, viewport: Viewport) {
+ let pages: Vec = {
+ let mut tabs = self.lock();
+ if tabs.viewport == viewport {
+ return;
+ }
+ tabs.viewport = viewport;
+ tabs.list.iter().map(|tab| tab.session.clone()).collect()
+ };
+ for page in pages {
+ let sized = self
+ .cdp
+ .call(
+ Some(&page),
+ "Emulation.setDeviceMetricsOverride",
+ viewport.metrics(),
+ )
+ .await;
+ if let Err(err) = sized {
+ debug!("browser: sizing a tab: {err}");
+ }
+ }
+ // The browser drops the frame of the new size while older ones wait
+ // to be confirmed, and a page that stands still sends no other
+ // (spec 19): once those are confirmed, the frames start over.
+ tokio::time::sleep(FRAME_GAP * 2).await;
+ let watched = {
+ let tabs = self.lock();
+ tabs.active()
+ .filter(|_| tabs.watching)
+ .map(|tab| tab.session.clone())
+ };
+ if let Some(page) = watched {
+ self.cast(&page, false).await;
+ self.cast(&page, true).await;
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn the_page_takes_the_shape_of_the_room_within_limits() {
+ let page = |width, height| Viewport::fitting(width, height);
+ // A panel 536 wide with 700 of height: the same shape, 1280 wide.
+ assert_eq!(
+ page(536, 700),
+ Viewport {
+ width: 1280,
+ height: 1671
+ }
+ );
+ assert_eq!(page(1280, 800), Viewport::default());
+ // Too flat or too tall a room stops at the limits.
+ assert_eq!(page(1600, 300).height, MIN_HEIGHT);
+ assert_eq!(page(256, 1400).height, MAX_HEIGHT);
+ assert_eq!(page(0, 0).height, MIN_HEIGHT);
+ assert_eq!(page(1, u32::MAX).height, MAX_HEIGHT);
+ }
+
+ #[test]
+ fn points_are_on_the_page_up_to_its_edges() {
+ let page = Viewport::fitting(640, 700);
+ assert!(page.contains(1280.0, 1400.0));
+ assert!(!page.contains(10.0, 1400.5));
+ assert!(!page.contains(-1.0, 10.0));
+ assert!(!page.contains(f64::NAN, 10.0));
+ }
+}
diff --git a/crates/botloftd/src/browser/watch.rs b/crates/botloftd/src/browser/watch.rs
index c3bee5ae..d4cccf98 100644
--- a/crates/botloftd/src/browser/watch.rs
+++ b/crates/botloftd/src/browser/watch.rs
@@ -1,5 +1,6 @@
//! An app connection watching a bot's browser (spec 21.7). While anyone
-//! watches, the active tab sends frames; when the last one stops, it stops.
+//! watches, the active tab sends frames and the page is the size their
+//! panel asks for (spec 21.3); when the last one stops, both stop.
use std::sync::Arc;
@@ -7,7 +8,7 @@ use botloft_core::ids::BotId;
use botloft_core::protocol::BrowserFrame;
use tokio::sync::watch;
-use super::{Browsers, Slots, lock};
+use super::{Browsers, Slots, Viewport, lock};
/// Held by the connection; dropping it stops watching.
pub struct Watching {
@@ -25,38 +26,63 @@ impl Browsers {
slot.watchers += 1;
slot.frames.subscribe()
};
- sync(Arc::clone(&self.slots), bot.clone());
+ sync_soon(&self.slots, bot);
Watching {
slots: Arc::clone(&self.slots),
bot: bot.clone(),
frames,
}
}
+
+ /// The room a watching app has for the page, `width` by `height`: the
+ /// page takes its shape, now or when the browser opens (spec 21.3).
+ pub fn resize(&self, bot: &BotId, width: u32, height: u32) {
+ {
+ let mut slots = lock(&self.slots);
+ self.slot(&mut slots, bot).viewport = Viewport::fitting(width, height);
+ }
+ sync_soon(&self.slots, bot);
+ }
}
impl Drop for Watching {
fn drop(&mut self) {
if let Some(slot) = lock(&self.slots).get_mut(&self.bot) {
slot.watchers = slot.watchers.saturating_sub(1);
+ if slot.watchers == 0 {
+ // No panel to fit anymore.
+ slot.viewport = Viewport::default();
+ }
}
- sync(Arc::clone(&self.slots), self.bot.clone());
+ sync_soon(&self.slots, &self.bot);
}
}
-/// Turns the frames on or off to match whether anyone watches. It reads
-/// the count when it runs, so calls that finish out of order still end on
-/// the right answer.
-fn sync(slots: Slots, bot: BotId) {
- let Ok(runtime) = tokio::runtime::Handle::try_current() else {
+/// Makes the running browser match what the apps want: frames while anyone
+/// watches, and the page the size they asked for. One at a time, each
+/// reading what is wanted when its turn comes, so changes that finish out
+/// of order still end on the right answer.
+pub(super) async fn sync(slots: Slots, bot: BotId) {
+ let running = lock(&slots).get(&bot).and_then(|slot| {
+ slot.session
+ .as_ref()
+ .map(|(_, session)| Arc::clone(session))
+ });
+ let Some(session) = running else {
return;
};
- runtime.spawn(async move {
- let found = lock(&slots).get(&bot).and_then(|slot| {
- let session = slot.session.as_ref()?.1.clone();
- Some((session, slot.watchers > 0))
- });
- if let Some((session, on)) = found {
- session.set_watching(on).await;
- }
- });
+ let _turn = session.syncing.lock().await;
+ let wanted = lock(&slots)
+ .get(&bot)
+ .map(|slot| (slot.watchers > 0, slot.viewport));
+ if let Some((on, viewport)) = wanted {
+ session.set_watching(on).await;
+ session.set_viewport(viewport).await;
+ }
+}
+
+fn sync_soon(slots: &Slots, bot: &BotId) {
+ if let Ok(runtime) = tokio::runtime::Handle::try_current() {
+ runtime.spawn(sync(Arc::clone(slots), bot.clone()));
+ }
}
diff --git a/crates/botloftd/src/rpc/mod.rs b/crates/botloftd/src/rpc/mod.rs
index b5105ef1..41d45c19 100644
--- a/crates/botloftd/src/rpc/mod.rs
+++ b/crates/botloftd/src/rpc/mod.rs
@@ -184,10 +184,11 @@ fn handle(
frames.send_replace(None);
watch.request(daemon, &request.method, request.params)
}
- // So is the owner's hold on it (spec 21.10).
- method::BROWSER_TAKE | method::BROWSER_RELEASE | method::BROWSER_INPUT => {
- watch.request(daemon, &request.method, request.params)
- }
+ // So are the size of its page and the owner's hold on it (spec 21.10).
+ method::BROWSER_RESIZE
+ | method::BROWSER_TAKE
+ | method::BROWSER_RELEASE
+ | method::BROWSER_INPUT => watch.request(daemon, &request.method, request.params),
name => dispatch::dispatch(daemon, name, request.params),
};
Some(match result {
diff --git a/crates/botloftd/src/rpc/watching.rs b/crates/botloftd/src/rpc/watching.rs
index 978db07b..630261a9 100644
--- a/crates/botloftd/src/rpc/watching.rs
+++ b/crates/botloftd/src/rpc/watching.rs
@@ -1,10 +1,12 @@
//! A connection watching one bot's browser (spec 21.7): `browser.watch`
-//! and `browser.unwatch`, and the frames that follow. The same connection
-//! takes the browser into the owner's hands and gives it back (spec 21.10).
+//! and `browser.unwatch`, the frames that follow, and the size its panel
+//! gives the page. The same connection takes the browser into the owner's
+//! hands and gives it back (spec 21.10).
+use botloft_core::ids::BotId;
use botloft_core::protocol::{
- ApprovalsAnswerParams, BrowserControlParams, BrowserInputParams, BrowserWatchParams,
- error_code, method, notification,
+ ApprovalsAnswerParams, BrowserControlParams, BrowserInputParams, BrowserResizeParams,
+ BrowserWatchParams, error_code, method, notification,
};
use serde::Serialize;
use serde_json::Value;
@@ -16,6 +18,9 @@ use crate::browser::{Hands, InputError, Watching};
use crate::service::{ApiError, bots};
use crate::state::Daemon;
+/// The most room a panel may say it has, in pixels each way.
+const ROOM_MAX: u32 = 10_000;
+
#[derive(Default)]
pub(super) struct Watch {
current: Option,
@@ -31,6 +36,7 @@ impl Watch {
params: Option,
) -> Result {
match name {
+ method::BROWSER_RESIZE => return self.resize(daemon, &parse(params)?),
method::BROWSER_TAKE => return self.take(daemon, parse(params)?),
method::BROWSER_RELEASE => return self.release(daemon, &parse(params)?),
method::BROWSER_INPUT => return self.input(parse(params)?),
@@ -49,13 +55,34 @@ impl Watch {
to_value(&daemon.browsers.view(¶ms.bot_id))
}
+ fn watches(&self, bot: &BotId) -> bool {
+ self.current
+ .as_ref()
+ .is_some_and(|watching| watching.bot == *bot)
+ }
+
+ /// The page takes the shape of the room this connection's panel has.
+ fn resize(&self, daemon: &Daemon, params: &BrowserResizeParams) -> Result {
+ if !self.watches(¶ms.bot_id) {
+ return Err(
+ ApiError::Conflict("watch this browser before sizing it".to_owned()).into(),
+ );
+ }
+ let fits = |side: u32| (1..=ROOM_MAX).contains(&side);
+ if !fits(params.width) || !fits(params.height) {
+ return Err(
+ ApiError::validation(format!("width and height go from 1 to {ROOM_MAX}")).into(),
+ );
+ }
+ daemon
+ .browsers
+ .resize(¶ms.bot_id, params.width, params.height);
+ Ok(Value::Null)
+ }
+
fn take(&mut self, daemon: &Daemon, params: BrowserControlParams) -> Result {
let bot = ¶ms.bot_id;
- if self
- .current
- .as_ref()
- .is_none_or(|watching| watching.bot != *bot)
- {
+ if !self.watches(bot) {
return Err(
ApiError::Conflict("watch this browser before taking it".to_owned()).into(),
);
diff --git a/crates/botloftd/tests/browser_panel.rs b/crates/botloftd/tests/browser_panel.rs
new file mode 100644
index 00000000..7fd9943f
--- /dev/null
+++ b/crates/botloftd/tests/browser_panel.rs
@@ -0,0 +1,114 @@
+//! The owner's panel on a bot's browser with the real Microsoft Edge (spec
+//! 21.3, 21.7): the page taking the shape of the room the panel has.
+//! Skipped where Edge is not installed.
+
+mod common;
+
+use std::time::Duration;
+
+use common::Client;
+use common::browsing::{Browsing, answer_site, call, setup};
+use serde_json::{Value, json};
+
+/// Reads the page until it says its window is `size`.
+async fn until_window(b: &mut Browsing, size: &str) -> String {
+ let mut page = String::new();
+ for _ in 0..50 {
+ page = b
+ .mcp
+ .tool_text("browser_look", json!({}))
+ .await
+ .expect("look");
+ if page.contains(&format!("Window: {size}")) {
+ return page;
+ }
+ tokio::time::sleep(Duration::from_millis(100)).await;
+ }
+ panic!("the window never became {size}:\n{page}");
+}
+
+/// The next frame that is `height` tall.
+async fn frame_of(app: &mut Client, height: u64) -> Value {
+ for _ in 0..50 {
+ let frame = app.notification("browser.frame").await;
+ if frame["height"] == height {
+ return frame;
+ }
+ }
+ panic!("no frame came {height} tall");
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn the_page_takes_the_shape_of_the_panel_while_the_owner_watches() {
+ let Some(mut b) = setup().await else { return };
+ let id = b.bot["id"].clone();
+ let room = |width: u32, height: u32| json!({ "botId": id, "width": width, "height": height });
+
+ // Only a connection watching the browser sizes it, and with sane numbers.
+ assert!(b.app.call("browser.resize", room(536, 700)).await.is_err());
+ b.app
+ .call("browser.watch", json!({ "botId": id }))
+ .await
+ .expect("watch");
+ assert!(b.app.call("browser.resize", room(0, 700)).await.is_err());
+ assert!(
+ b.app
+ .call("browser.resize", room(536, 20_000))
+ .await
+ .is_err()
+ );
+
+ // Asked before the browser opens: it opens that size.
+ b.app
+ .call("browser.resize", room(536, 700))
+ .await
+ .expect("resize");
+ let opening = call(
+ &b.mcp,
+ "browser_open",
+ json!({ "url": format!("{}/size", b.site) }),
+ );
+ answer_site(&mut b.app, true, None).await;
+ let page = opening.await.expect("task").expect("page");
+ assert!(page.contains("Window: 1280 x 1671"), "{page}");
+ let frame = frame_of(&mut b.app, 1671).await;
+ assert_eq!(frame["width"], 1280);
+
+ // The panel changes: the page follows, and so do the frames.
+ b.app
+ .call("browser.resize", room(640, 400))
+ .await
+ .expect("resize");
+ until_window(&mut b, "1280 x 800").await;
+ frame_of(&mut b.app, 800).await;
+
+ // The owner's points go as far as the page does now.
+ b.app
+ .call("browser.resize", room(640, 700))
+ .await
+ .expect("resize");
+ until_window(&mut b, "1280 x 1400").await;
+ frame_of(&mut b.app, 1400).await;
+ b.app
+ .call("browser.take", json!({ "botId": id }))
+ .await
+ .expect("take");
+ let click = |y: f64| {
+ json!({ "botId": id, "input": {
+ "kind": "mouse", "action": "move", "x": 10.0, "y": y, "button": "none",
+ "buttons": 0, "clicks": 0, "modifiers": 0,
+ }})
+ };
+ b.app
+ .call("browser.input", click(1390.0))
+ .await
+ .expect("a point on the taller page");
+ assert!(b.app.call("browser.input", click(1410.0)).await.is_err());
+
+ // Nobody watches anymore: back to the size bots work in alone.
+ b.app
+ .call("browser.unwatch", Value::Null)
+ .await
+ .expect("unwatch");
+ until_window(&mut b, "1280 x 800").await;
+}
diff --git a/crates/botloftd/tests/common/site.rs b/crates/botloftd/tests/common/site.rs
index 24d7c222..f1891b47 100644
--- a/crates/botloftd/tests/common/site.rs
+++ b/crates/botloftd/tests/common/site.rs
@@ -1,7 +1,7 @@
//! A small web site on 127.0.0.1 for the browser tests: a form, the page
//! it leads to, a link that opens a new tab, a button that shows a dialog,
-//! and a sign-in page with its fields at fixed points, for the owner's
-//! hands.
+//! a sign-in page with its fields at fixed points, for the owner's hands,
+//! and a page that says how big its window is.
use std::collections::HashMap;
use std::net::SocketAddr;
@@ -30,6 +30,11 @@ const LOGIN: &str = "Sign in
-
- {hands.held && frame && (
-
+
- {status === "open" && caption && !hands.held && (
-
+
+ {hands.held && frame && (
+
+ {(status !== "open" || !frame) && (
+
-
+ {caption && (
+
+