diff --git a/app/src/dev/seedBrowser.ts b/app/src/dev/seedBrowser.ts index e968486d..5b31ad52 100644 --- a/app/src/dev/seedBrowser.ts +++ b/app/src/dev/seedBrowser.ts @@ -1,12 +1,16 @@ // Dev only: Scout searching in its browser in the fake preview, so the -// browser panel has something live to show. The "page" is drawn on a -// canvas and sent as JPEG frames, like the daemon's screencast. +// browser panel has something live to show: three tabs, the search in the +// active one. The "page" is drawn on a canvas, the size the panel asks for, +// and sent as JPEG frames, like the daemon's screencast. import type { FakeBotloft } from "../lib/fake"; +import type { PageSize } from "../lib/fakePages"; import type { BotId } from "../lib/protocol.gen"; +import { activeTab, drawPage } from "./seedPage"; -const W = 1280; -const H = 800; +const SITE = "https://recipes.example/"; + +const RECIPES = ["Classic sourdough loaf", "Starter in 5 days", "No-knead rye", "Focaccia"]; interface Scene { url: string; @@ -15,16 +19,16 @@ interface Scene { results: boolean; } -function draw(scene: Scene): string { +function draw(scene: Scene, size: PageSize): string { const canvas = document.createElement("canvas"); - canvas.width = W; - canvas.height = H; + canvas.width = size.width; + canvas.height = size.height; const g = canvas.getContext("2d"); if (!g) { return ""; } g.fillStyle = "#ffffff"; - g.fillRect(0, 0, W, H); + g.fillRect(0, 0, size.width, size.height); g.fillStyle = "#1f2937"; g.font = "600 34px system-ui, sans-serif"; g.fillText(scene.results ? "Results" : "Find a recipe", 120, 150); @@ -43,15 +47,12 @@ function draw(scene: Scene): string { g.fillStyle = "#ffffff"; g.fillText("Search", 938, 226); if (scene.results) { - for (let row = 0; row < 4; row++) { + // A taller page shows more of the list. + for (let row = 0; 300 + row * 110 < size.height - 60; row++) { const y = 300 + row * 110; g.fillStyle = "#1d4ed8"; g.font = "600 24px system-ui, sans-serif"; - g.fillText( - ["Classic sourdough loaf", "Starter in 5 days", "No-knead rye", "Focaccia"][row] ?? "", - 120, - y, - ); + g.fillText(RECIPES[row % RECIPES.length] ?? "", 120, y); g.fillStyle = "#e5e7eb"; g.fillRect(120, y + 18, 820, 14); g.fillRect(120, y + 42, 560, 14); @@ -60,10 +61,10 @@ function draw(scene: Scene): string { return canvas.toDataURL("image/jpeg", 0.7).split(",")[1] ?? ""; } -/** Scout searches for a recipe, over and over. */ +/** Scout searches for a recipe, over and over, while the browser is its. */ export function seedBrowser(fake: FakeBotloft, scout: BotId): void { const home: Scene = { - url: "https://recipes.example/", + url: SITE, title: "Recipes", query: "", results: false, @@ -116,15 +117,25 @@ export function seedBrowser(fake: FakeBotloft, scout: BotId): void { ); let scene = home; let step = 0; - fake.browser.open(scout, home.url, home.title); - fake.browser.frame(scout, draw(scene)); + // Two pages it opened on the way, then the one it works in. + fake.browser.open(scout, "https://bakers.example/forum/hydration", "Hydration tips · Bakers"); + fake.browser.openTab(scout, "https://mill.example/flour/rye", "Rye flour · The Mill"); + fake.browser.openTab(scout, home.url, home.title); + fake.browser.paint(scout, (size) => { + const tab = activeTab(fake, scout); + return tab?.url.startsWith(SITE) ? draw(scene, size) : drawPage(tab, size); + }); setInterval(() => { + // In the owner's hands, the bot waits. + if (fake.browser.state(scout).control === "owner") { + return; + } const next = steps[step % steps.length]; step += 1; if (next) { scene = next(scene); fake.browser.open(scout, scene.url, scene.title); - fake.browser.frame(scout, draw(scene)); + fake.browser.repaint(scout); } }, 2200); } diff --git a/app/src/dev/seedHands.ts b/app/src/dev/seedHands.ts index e55c910a..90970b44 100644 --- a/app/src/dev/seedHands.ts +++ b/app/src/dev/seedHands.ts @@ -3,10 +3,12 @@ // the owner's clicks and keys, so taking the browser can be tried end to end. import type { FakeBotloft } from "../lib/fake"; +import type { PageSize } from "../lib/fakePages"; import type { BrowserInput, CrewId } from "../lib/protocol.gen"; +import { activeTab, drawPage } from "./seedPage"; + +const SITE = "https://flour.example/"; -const W = 1280; -const H = 800; const FIELDS = { user: 300, password: 390 } as const; type Field = keyof typeof FIELDS; @@ -17,16 +19,16 @@ interface Form { signedIn: boolean; } -function draw(form: Form): string { +function draw(form: Form, size: PageSize): string { const canvas = document.createElement("canvas"); - canvas.width = W; - canvas.height = H; + canvas.width = size.width; + canvas.height = size.height; const g = canvas.getContext("2d"); if (!g) { return ""; } g.fillStyle = "#f7f4ee"; - g.fillRect(0, 0, W, H); + g.fillRect(0, 0, size.width, size.height); g.fillStyle = "#3b2a1a"; g.font = "600 36px Georgia, serif"; if (form.signedIn) { @@ -90,21 +92,24 @@ export function seedHands(fake: FakeBotloft, crewId: CrewId): void { const clerk = fake.addBot(crewId, "Clerk", "Places the bakery's orders"); fake.setBotState(clerk.id, "needs_approval", 2); const form: Form = { user: "", password: "", field: null, signedIn: false }; - const login = "https://flour.example/login"; + const login = `${SITE}login`; + const onSite = () => activeTab(fake, clerk.id)?.url.startsWith(SITE) ?? false; fake.browser.open(clerk.id, login, "Sign in · Flour Co."); - fake.browser.frame(clerk.id, draw(form)); + fake.browser.paint(clerk.id, (size) => + onSite() ? draw(form, size) : drawPage(activeTab(fake, clerk.id), size), + ); fake.chat.tool(clerk.id, "mcp__botloft__browser_open", { summary: login, status: "done" }); const task = "Sign in to Flour Co. with the bakery's account"; fake.chat.tool(clerk.id, "mcp__botloft__browser_ask_owner", { summary: task, status: "running" }); fake.browser.ask(clerk.id, task); fake.browser.onInput(clerk.id, (input) => { - if (form.signedIn) { + if (form.signedIn || !onSite()) { return; } if (handle(form, input)) { form.signedIn = true; - fake.browser.open(clerk.id, "https://flour.example/orders", "Orders · Flour Co."); + fake.browser.open(clerk.id, `${SITE}orders`, "Orders · Flour Co."); } - fake.browser.frame(clerk.id, draw(form)); + fake.browser.repaint(clerk.id); }); } diff --git a/app/src/dev/seedPage.ts b/app/src/dev/seedPage.ts new file mode 100644 index 00000000..655178ac --- /dev/null +++ b/app/src/dev/seedPage.ts @@ -0,0 +1,37 @@ +// Dev only: any page of the fake preview's browsers that no seed draws +// itself, such as a tab in the background or an address the owner typed. +// A heading with the tab's title, its site, and lines of grey "text". + +import type { FakeBotloft } from "../lib/fake"; +import type { PageSize } from "../lib/fakePages"; +import type { BotId, BrowserTab } from "../lib/protocol.gen"; + +/** The bot's active tab, if its browser has any. */ +export function activeTab(fake: FakeBotloft, botId: BotId): BrowserTab | null { + return fake.browser.state(botId).tabs.find((tab) => tab.active) ?? null; +} + +export function drawPage(tab: BrowserTab | null, size: PageSize): string { + const canvas = document.createElement("canvas"); + canvas.width = size.width; + canvas.height = size.height; + const g = canvas.getContext("2d"); + if (!g) { + return ""; + } + g.fillStyle = "#ffffff"; + g.fillRect(0, 0, size.width, size.height); + if (tab && tab.url !== "about:blank") { + g.fillStyle = "#111827"; + g.font = "600 40px Georgia, serif"; + g.fillText(tab.title || tab.url, 120, 160); + g.fillStyle = "#6b7280"; + g.font = "20px system-ui, sans-serif"; + g.fillText(tab.url, 120, 204); + g.fillStyle = "#e5e7eb"; + for (let y = 270; y < size.height - 80; y += 34) { + g.fillRect(120, y, y % 5 === 0 ? 640 : 960, 14); + } + } + return canvas.toDataURL("image/jpeg", 0.7).split(",")[1] ?? ""; +} diff --git a/app/src/features/browser/AddressBar.tsx b/app/src/features/browser/AddressBar.tsx new file mode 100644 index 00000000..25a4c1e9 --- /dev/null +++ b/app/src/features/browser/AddressBar.tsx @@ -0,0 +1,107 @@ +// The address row of a bot's browser (spec 21.8): reloading the page, where +// it is, and opening it in the owner's own browser. With the browser in +// the owner's hands the address is theirs to type (spec 21.10). + +import { ExternalLink, Globe, LoaderCircle, Lock, RotateCw } from "lucide-react"; +import { type RefObject, useEffect, useState } from "react"; +import { useT } from "../../i18n"; +import type { Bot, BrowserState } from "../../lib/protocol.gen"; +import { useApi, useHost } from "../../store/context"; +import { Button } from "../../ui/Button"; +import { attempt } from "../../ui/toast"; +import type { Hands } from "./useHands"; + +export function AddressBar({ + bot, + state, + hands, + field, +}: { + bot: Pick; + state: BrowserState; + hands: Hands; + /** The address field, for whoever sends the owner to it. */ + field: RefObject; +}) { + const t = useT().browser; + const api = useApi(); + const host = useHost(); + const url = state.url && state.url !== "about:blank" ? state.url : null; + const web = url !== null && /^https?:\/\//i.test(url); + const Icon = state.loading ? LoaderCircle : url?.startsWith("https://") ? Lock : Globe; + const held = hands.held; + const tab = state.tabs.find((tab) => tab.active)?.id; + // What the owner is typing; `null` shows where the page is. + const [draft, setDraft] = useState(null); + // biome-ignore lint/correctness/useExhaustiveDependencies: another tab, or the bot's turn again, drops what was typed + useEffect(() => setDraft(null), [tab, held]); + + const go = () => { + const typed = draft?.trim(); + setDraft(null); + field.current?.blur(); + if (typed) { + void hands.open(typed); + } + }; + + return ( +
+
+ ); +} diff --git a/app/src/features/browser/BrowserPanel.tsx b/app/src/features/browser/BrowserPanel.tsx index 4c5e60ea..a23ce0ae 100644 --- a/app/src/features/browser/BrowserPanel.tsx +++ b/app/src/features/browser/BrowserPanel.tsx @@ -1,23 +1,30 @@ -// The bot's own browser beside its chat (spec 21.8): the page it is on, -// live, with its cursor and a line about what it just did. Watching starts -// when the panel opens and stops when it closes. The owner can take it into -// their own hands and give it back (spec 21.10). +// The bot's own browser beside its chat (spec 21.8): its tabs, the address +// and the page it is on, live, with its cursor and a line about what it +// just did. Watching starts when the panel opens and stops when it closes. +// The owner can take it into their own hands and give it back (spec 21.10). -import { ExternalLink, Globe, LoaderCircle, Lock, Maximize2, Minimize2, X } from "lucide-react"; +import { Globe, LoaderCircle, Maximize2, Minimize2, X } from "lucide-react"; import { useEffect, useRef, useState } from "react"; import { useT } from "../../i18n"; -import type { Bot, BrowserState } from "../../lib/protocol.gen"; -import { useApp, useHost } from "../../store/context"; +import type { Bot } from "../../lib/protocol.gen"; +import { useApp } from "../../store/context"; import { Button } from "../../ui/Button"; import { Callout } from "../../ui/Callout"; import { SidePanel } from "../../ui/SidePanel"; -import { attempt } from "../../ui/toast"; import { BotAvatar } from "../bots/BotAvatar"; +import { AddressBar } from "./AddressBar"; import { AskCallout, HeldBar, TakeBar } from "./HandsBars"; import { HandsLayer } from "./HandsLayer"; import { LiveView, useCaption } from "./LiveView"; +import { TabStrip } from "./TabStrip"; import { useBrowserView } from "./useBrowserView"; import { useHands } from "./useHands"; +import { useFitPage, useRoom } from "./useRoom"; + +/** Around the page in the panel, in px. */ +const PAD = 12; +/** Kept under the page, for what the bot just did and the button to take it. */ +const UNDER = 80; export function BrowserPanel({ bot, @@ -39,6 +46,21 @@ export function BrowserPanel({ const hands = useHands(bot, state); const [focused, setFocused] = useState(false); const ask = state?.ask ?? null; + // The page takes the shape of the room the panel has for it (spec 21.3). + // Notices that come and go are not counted: they never resize the page. + const [inside, body] = useRoom(); + const room = inside && { + width: inside.width - 2 * PAD, + height: inside.height - 2 * PAD - UNDER, + }; + useFitPage(bot, room, watched); + // A new tab is blank: the owner says where it goes. + const address = useRef(null); + const addTab = async () => { + if (await hands.newTab()) { + address.current?.focus(); + } + }; // Taking needs the watch first: the daemon ties the hands to it. const taken = useRef(0); useEffect(() => { @@ -74,8 +96,17 @@ export function BrowserPanel({ + ); +} diff --git a/app/src/features/browser/browser.test.tsx b/app/src/features/browser/browser.test.tsx index 53511f32..41d96e48 100644 --- a/app/src/features/browser/browser.test.tsx +++ b/app/src/features/browser/browser.test.tsx @@ -1,9 +1,12 @@ import { act, cleanup, fireEvent, screen, waitFor, within } from "@testing-library/react"; -import { afterEach, describe, expect, test } from "vitest"; +import { afterEach, describe, expect, test, vi } from "vitest"; import { FakeBotloft } from "../../lib/fake"; import { crewOpened, openBot, renderApp } from "../../test/app"; -afterEach(cleanup); +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); +}); /** A crew "Ops" with @scout, idle. */ function crew() { @@ -68,6 +71,37 @@ describe("browser panel", () => { expect(within(panel()).queryByText("Opened elsewhere.com")).toBeNull(); }); + test("gives the page the shape of the room the panel has, and shows it as large as fits", async () => { + const { fake, scout } = crew(); + fake.browser.open(scout.id, "https://example.com/", "Example"); + // The panel under its address row, as the app measures it: 536 by 700 + // for the page, once what goes around and under it is set aside. + vi.spyOn(Element.prototype, "clientWidth", "get").mockReturnValue(560); + vi.spyOn(Element.prototype, "clientHeight", "get").mockReturnValue(804); + await openScout(fake); + fireEvent.click(toggle()); + const resized = () => + fake.calls.filter((call) => call.method === "browser.resize").map((call) => call.params); + await waitFor(() => expect(resized()).toEqual([{ botId: scout.id, width: 536, height: 700 }])); + // As wide as ever, and as tall as the room's shape asks. + expect(fake.browser.size(scout.id)).toEqual({ width: 1280, height: 1671 }); + + act(() => { + fake.browser.frame(scout.id, "AAAA"); + }); + // The bot's cursor lands where it acted on the taller page. + const screenshot = within(panel()).getByRole("figure", { name: "What Scout sees" }); + act(() => { + fake.browser.act(scout.id, "click", { x: 640, y: 1671, label: "More" }); + }); + const cursor = screenshot.querySelector(".bot-cursor") as HTMLElement; + expect(cursor.style.top).toBe("100%"); + + // Nobody watches anymore: the page is back to its own size. + fireEvent.click(within(panel()).getByRole("button", { name: "Close" })); + await waitFor(() => expect(fake.browser.size(scout.id)).toEqual({ width: 1280, height: 800 })); + }); + test("opens by itself when the bot starts browsing, and the button marks it once closed", async () => { const { fake, scout } = crew(); await openScout(fake); diff --git a/app/src/features/browser/tabs.test.tsx b/app/src/features/browser/tabs.test.tsx new file mode 100644 index 00000000..e39be004 --- /dev/null +++ b/app/src/features/browser/tabs.test.tsx @@ -0,0 +1,128 @@ +import { act, cleanup, fireEvent, screen, waitFor, within } from "@testing-library/react"; +import { afterEach, describe, expect, test } from "vitest"; +import { FakeBotloft } from "../../lib/fake"; +import { crewOpened, openBot, renderApp } from "../../test/app"; + +afterEach(cleanup); + +const SEARCH = "https://www.example.com/search?q=rye"; +const MILL = "https://mill.example/flour/rye"; + +/** A crew "Ops" with @scout, its browser on a page with another tab behind it. */ +function crew() { + const fake = new FakeBotloft(); + const ops = fake.addCrew("Ops"); + const scout = fake.addBot(ops.id, "Scout", "Finds sources"); + fake.setBotState(scout.id, "busy"); + fake.browser.open(scout.id, SEARCH, "Rye · Example"); + fake.browser.openTab(scout.id, MILL, "Rye flour · The Mill"); + return { fake, scout }; +} + +async function openPanel(fake: FakeBotloft, botId: string) { + renderApp(fake); + await crewOpened("Ops"); + openBot("Scout"); + await screen.findByRole("list", { name: "Messages" }); + fireEvent.click(screen.getByRole("button", { name: /^Show browser/ })); + await waitFor(() => expect(fake.browser.watching).toBe(botId)); + act(() => { + fake.browser.frame(botId, "AAAA"); + }); +} + +const panel = () => screen.getByRole("complementary", { name: "Scout's browser" }); +const tabs = () => within(within(panel()).getByRole("tablist", { name: "Tabs" })); +const tab = (name: string) => tabs().getByRole("tab", { name }); +const selected = () => + tabs() + .getAllByRole("tab") + .map((one) => `${one.getAttribute("aria-label")}${one.ariaSelected === "true" ? " *" : ""}`); +const address = () => within(panel()).getByRole("textbox", { name: "Address" }) as HTMLInputElement; +const called = (fake: FakeBotloft, method: string) => + fake.calls.filter((call) => call.method === method).map((call) => call.params); + +describe("the tabs of the bot's browser", () => { + test("show what the bot has open, by title and site, and only show until the owner takes it", async () => { + const { fake, scout } = crew(); + await openPanel(fake, scout.id); + expect(selected()).toEqual([ + "Rye · Example, example.com", + "Rye flour · The Mill, mill.example *", + ]); + expect(address().value).toBe(MILL); + expect(address().readOnly).toBe(true); + + // A page opens another tab: it shows up, and is the active one. + act(() => { + fake.browser.openTab(scout.id, "https://papers.example/rye.pdf", ""); + }); + expect(selected()).toEqual([ + "Rye · Example, example.com", + "Rye flour · The Mill, mill.example", + "papers.example *", + ]); + + // The browser is the bot's: clicking moves nothing. + fireEvent.click(tab("Rye · Example, example.com")); + const add = within(panel()).getByRole("button", { name: "New tab" }); + expect(add.getAttribute("aria-disabled")).toBe("true"); + expect(add.title).toBe("Take control to switch tabs or open a new one"); + fireEvent.click(add); + expect(called(fake, "browser.switchTab")).toEqual([]); + expect(called(fake, "browser.newTab")).toEqual([]); + }); + + test("the page reloads at any time, without taking the browser", async () => { + const { fake, scout } = crew(); + await openPanel(fake, scout.id); + fireEvent.click(within(panel()).getByRole("button", { name: "Reload" })); + await waitFor(() => expect(fake.browser.reloads).toEqual([scout.id])); + expect(called(fake, "browser.take")).toEqual([]); + expect(fake.browser.state(scout.id).control).toBe("bot"); + }); + + test("in the owner's hands they switch, a new one opens, and the address is theirs to type", async () => { + const { fake, scout } = crew(); + await openPanel(fake, scout.id); + fireEvent.click(within(panel()).getByRole("button", { name: "Take control" })); + expect(await within(panel()).findByText("You are in control")).toBeDefined(); + expect(within(panel()).getByText("Scout goes on in the tab you leave open.")).toBeDefined(); + + fireEvent.click(tab("Rye · Example, example.com")); + await waitFor(() => + expect(selected()).toEqual([ + "Rye · Example, example.com *", + "Rye flour · The Mill, mill.example", + ]), + ); + expect(address().value).toBe(SEARCH); + + // A new tab is blank, and the owner says where it goes. + fireEvent.click(within(panel()).getByRole("button", { name: "New tab" })); + await waitFor(() => expect(selected()[2]).toBe("New tab *")); + await waitFor(() => expect(document.activeElement).toBe(address())); + expect(address().value).toBe(""); + fireEvent.change(address(), { target: { value: "bakery.example/orders" } }); + fireEvent.keyDown(address(), { key: "Enter" }); + await waitFor(() => expect(address().value).toBe("https://bakery.example/orders")); + expect(called(fake, "browser.open")).toEqual([ + { botId: scout.id, url: "bakery.example/orders" }, + ]); + expect(selected()[2]).toBe("bakery.example *"); + + // Escape drops what was typed; what is no address says so. + fireEvent.change(address(), { target: { value: "somewhere" } }); + fireEvent.keyDown(address(), { key: "Escape" }); + expect(address().value).toBe("https://bakery.example/orders"); + fireEvent.change(address(), { target: { value: "two words" } }); + fireEvent.keyDown(address(), { key: "Enter" }); + expect(await screen.findByText(/^Could not open that address/)).toBeDefined(); + + // Back with the bot, the tabs only show again. + fireEvent.click(within(panel()).getByRole("button", { name: "Done, give it back to Scout" })); + await waitFor(() => expect(address().readOnly).toBe(true)); + fireEvent.click(tab("Rye · Example, example.com")); + expect(called(fake, "browser.switchTab")).toHaveLength(1); + }); +}); diff --git a/app/src/features/browser/useHands.ts b/app/src/features/browser/useHands.ts index 405a728c..28150f12 100644 --- a/app/src/features/browser/useHands.ts +++ b/app/src/features/browser/useHands.ts @@ -1,6 +1,7 @@ // The owner's hands in a bot's browser (spec 21.10): taking it, giving it -// back, and sending what they do on the page. Events go out in order and -// without waiting for each other: the daemon queues them for the page. +// back, sending what they do on the page, and moving between its tabs. +// Events go out in order and without waiting for each other: the daemon +// queues them for the page. import { useCallback, useState } from "react"; import { useT } from "../../i18n"; @@ -15,30 +16,35 @@ export interface Hands { take(): Promise; release(): Promise; send(input: BrowserInput): void; + /** Opens a blank tab; `true` once the daemon took the request. */ + newTab(): Promise; + switchTab(tabId: string): Promise; + /** Takes the active tab to the address the owner typed. */ + open(url: string): Promise; } export function useHands(bot: Pick, state: BrowserState | null): Hands { const api = useApi(); - const t = useT().browser.hands; + const t = useT().browser; const putBrowser = useApp((app) => app.putBrowser); const [busy, setBusy] = useState(false); const botId = bot.id; const take = useCallback(async () => { setBusy(true); - await attempt(t.takeFailed, async () => { + await attempt(t.hands.takeFailed, async () => { putBrowser(await api.call("browser.take", { botId })); }); setBusy(false); - }, [api, botId, putBrowser, t.takeFailed]); + }, [api, botId, putBrowser, t.hands.takeFailed]); const release = useCallback(async () => { setBusy(true); - await attempt(t.giveBackFailed, async () => { + await attempt(t.hands.giveBackFailed, async () => { putBrowser(await api.call("browser.release", { botId })); }); setBusy(false); - }, [api, botId, putBrowser, t.giveBackFailed]); + }, [api, botId, putBrowser, t.hands.giveBackFailed]); const send = useCallback( (input: BrowserInput) => { @@ -48,11 +54,28 @@ export function useHands(bot: Pick, state: BrowserState | null): Hand [api, botId], ); + const newTab = useCallback( + () => attempt(t.tabs.addFailed, () => api.call("browser.newTab", { botId })), + [api, botId, t.tabs.addFailed], + ); + const switchTab = useCallback( + (tabId: string) => + attempt(t.tabs.switchFailed, () => api.call("browser.switchTab", { botId, tabId })), + [api, botId, t.tabs.switchFailed], + ); + const open = useCallback( + (url: string) => attempt(t.goFailed, () => api.call("browser.open", { botId, url })), + [api, botId, t.goFailed], + ); + return { held: state?.status === "open" && state.control === "owner", busy, take, release, send, + newTab, + switchTab, + open, }; } diff --git a/app/src/features/browser/useRoom.ts b/app/src/features/browser/useRoom.ts new file mode 100644 index 00000000..f0b56842 --- /dev/null +++ b/app/src/features/browser/useRoom.ts @@ -0,0 +1,69 @@ +// The room the panel has for the bot's page (spec 21.8): measured as the +// panel changes and told to the daemon, so the page takes its shape (spec +// 21.3). + +import { useEffect, useLayoutEffect, useState } from "react"; +import type { Bot } from "../../lib/protocol.gen"; +import { useApi } from "../../store/context"; + +export interface Size { + width: number; + height: number; +} + +/** How long the room stays the same before the daemon hears about it. */ +const SETTLED_MS = 300; + +/** + * The room inside an element, following it as it changes: `null` until it + * has any. The second value goes in the element's `ref`. + */ +export function useRoom(): [Size | null, (node: HTMLElement | null) => void] { + const [node, setNode] = useState(null); + const [room, setRoom] = useState(null); + useLayoutEffect(() => { + if (!node) { + setRoom(null); + return; + } + const measure = () => { + const width = node.clientWidth; + const height = node.clientHeight; + setRoom((was) => { + if (width <= 0 || height <= 0) { + return null; + } + return was?.width === width && was.height === height ? was : { width, height }; + }); + }; + measure(); + if (typeof ResizeObserver === "undefined") { + return; + } + const observer = new ResizeObserver(measure); + observer.observe(node); + return () => observer.disconnect(); + }, [node]); + return [room, setNode]; +} + +/** + * Tells the daemon the room the page has, once it stops changing, while + * this connection watches the bot's browser. + */ +export function useFitPage(bot: Pick, room: Size | null, watched: boolean): void { + const api = useApi(); + const known = room !== null; + const width = Math.max(1, room?.width ?? 0); + const height = Math.max(1, room?.height ?? 0); + useEffect(() => { + if (!watched || !known) { + return; + } + const timer = setTimeout(() => { + // The next change of the room asks again. + api.call("browser.resize", { botId: bot.id, width, height }).catch(() => {}); + }, SETTLED_MS); + return () => clearTimeout(timer); + }, [api, bot.id, watched, known, width, height]); +} diff --git a/app/src/i18n/en/browser.ts b/app/src/i18n/en/browser.ts index 42c16796..6aaabaad 100644 --- a/app/src/i18n/en/browser.ts +++ b/app/src/i18n/en/browser.ts @@ -1,6 +1,7 @@ // Each bot's own browser (spec 21.8): the panel beside its chat where the -// owner watches it live, the card where it asks to use a new site, and the -// owner taking the browser into their own hands (spec 21.10). +// owner watches it live, its tabs and address, the card where it asks to +// use a new site, and the owner taking the browser into their own hands +// (spec 21.10). export const browser = { heading: "Browser", @@ -17,6 +18,10 @@ export const browser = { live: "Live", loading: "Loading…", address: "Address", + addressHint: "Type an address and press Enter", + goFailed: "Could not open that address", + reload: "Reload", + reloadFailed: "Could not reload the page", screen: (bot: string) => `What ${bot} sees`, emptyTitle: (bot: string) => `${bot} hasn't opened the browser yet`, emptyBody: @@ -28,7 +33,14 @@ export const browser = { failedBody: "Botloft uses Microsoft Edge, which comes with Windows. Check that it's installed, then ask the bot to try again.", details: "Details", - tabs: (count: number) => (count === 1 ? "1 tab" : `${count} tabs`), + tabs: { + label: "Tabs", + blank: "New tab", + add: "New tab", + addFailed: "Could not open a new tab", + switchFailed: "Could not switch tabs", + takeFirst: "Take control to switch tabs or open a new one", + }, did: { open: (site: string) => `Opened ${site}`, click: (what: string) => `Clicked ${what}`, @@ -55,6 +67,7 @@ export const browser = { takeFailed: "Could not take the browser", holding: "You are in control", waits: (bot: string) => `${bot}'s actions in the browser wait until you give it back.`, + leaves: (bot: string) => `${bot} goes on in the tab you leave open.`, giveBack: (bot: string) => `Done, give it back to ${bot}`, giveBackFailed: "Could not give the browser back", clickToType: "Click the screen to type", diff --git a/app/src/i18n/es/browser.ts b/app/src/i18n/es/browser.ts index 9cd167eb..2fcf8b84 100644 --- a/app/src/i18n/es/browser.ts +++ b/app/src/i18n/es/browser.ts @@ -1,5 +1,6 @@ // El navegador de cada bot (spec 21.8): el panel junto al chat donde el -// dueño lo mira en vivo, y la tarjeta en la que pide usar un sitio nuevo. +// dueño lo mira en vivo, sus pestañas y la dirección, y la tarjeta en la +// que pide usar un sitio nuevo. import type { Messages } from "../en"; @@ -18,6 +19,10 @@ export const browser: Messages["browser"] = { live: "En vivo", loading: "Cargando…", address: "Dirección", + addressHint: "Escribe una dirección y pulsa Enter", + goFailed: "No se pudo abrir esa dirección", + reload: "Recargar", + reloadFailed: "No se pudo recargar la página", screen: (bot) => `Lo que ve ${bot}`, emptyTitle: (bot) => `${bot} todavía no abrió el navegador`, emptyBody: @@ -29,7 +34,14 @@ export const browser: Messages["browser"] = { failedBody: "Botloft usa Microsoft Edge, que viene con Windows. Comprueba que esté instalado y pide al bot que lo intente de nuevo.", details: "Detalles", - tabs: (count) => (count === 1 ? "1 pestaña" : `${count} pestañas`), + tabs: { + label: "Pestañas", + blank: "Nueva pestaña", + add: "Nueva pestaña", + addFailed: "No se pudo abrir una pestaña nueva", + switchFailed: "No se pudo cambiar de pestaña", + takeFirst: "Toma el control para cambiar de pestaña o abrir una nueva", + }, did: { open: (site) => `Abrió ${site}`, click: (what) => `Hizo clic en ${what}`, @@ -55,6 +67,7 @@ export const browser: Messages["browser"] = { takeFailed: "No se pudo tomar el navegador", holding: "Tienes el control", waits: (bot) => `Las acciones de ${bot} en el navegador esperan a que lo devuelvas.`, + leaves: (bot) => `${bot} sigue en la pestaña que dejes abierta.`, giveBack: (bot) => `Listo, devolver a ${bot}`, giveBackFailed: "No se pudo devolver el navegador", clickToType: "Haz clic en la pantalla para escribir", diff --git a/app/src/i18n/pt-BR/browser.ts b/app/src/i18n/pt-BR/browser.ts index 279e8fd9..e26e9b54 100644 --- a/app/src/i18n/pt-BR/browser.ts +++ b/app/src/i18n/pt-BR/browser.ts @@ -1,6 +1,7 @@ // O navegador de cada bot (spec 21.8): o painel ao lado do chat onde o dono -// assiste ao vivo, o cartão em que o bot pede para usar um site novo, e o -// dono assumindo o navegador com as próprias mãos (spec 21.10). +// assiste ao vivo, as abas e o endereço, o cartão em que o bot pede para +// usar um site novo, e o dono assumindo o navegador com as próprias mãos +// (spec 21.10). import type { Messages } from "../en"; @@ -19,6 +20,10 @@ export const browser: Messages["browser"] = { live: "Ao vivo", loading: "Carregando…", address: "Endereço", + addressHint: "Digite um endereço e aperte Enter", + goFailed: "Não foi possível abrir esse endereço", + reload: "Recarregar", + reloadFailed: "Não foi possível recarregar a página", screen: (bot) => `O que ${bot} vê`, emptyTitle: (bot) => `${bot} ainda não abriu o navegador`, emptyBody: @@ -30,7 +35,14 @@ export const browser: Messages["browser"] = { failedBody: "O Botloft usa o Microsoft Edge, que vem com o Windows. Confira se ele está instalado e peça ao bot para tentar de novo.", details: "Detalhes", - tabs: (count) => (count === 1 ? "1 aba" : `${count} abas`), + tabs: { + label: "Abas", + blank: "Nova aba", + add: "Nova aba", + addFailed: "Não foi possível abrir uma nova aba", + switchFailed: "Não foi possível trocar de aba", + takeFirst: "Assuma o controle para trocar de aba ou abrir uma nova", + }, did: { open: (site) => `Abriu ${site}`, click: (what) => `Clicou em ${what}`, @@ -57,6 +69,7 @@ export const browser: Messages["browser"] = { takeFailed: "Não foi possível assumir o navegador", holding: "Você está no controle", waits: (bot) => `As ações de ${bot} no navegador esperam você devolver.`, + leaves: (bot) => `${bot} continua na aba que você deixar aberta.`, giveBack: (bot) => `Pronto, devolver para ${bot}`, giveBackFailed: "Não foi possível devolver o navegador", clickToType: "Clique na tela para digitar", diff --git a/app/src/lib/fakeBrowser.ts b/app/src/lib/fakeBrowser.ts index 90ffedfe..256c8ad1 100644 --- a/app/src/lib/fakeBrowser.ts +++ b/app/src/lib/fakeBrowser.ts @@ -1,9 +1,11 @@ -// The fake daemon's browsers (spec 21.7): each bot's state, the live frames -// of the one being watched and what the bot does, for the browser panel, -// and the owner's hands in it (spec 21.10). +// The fake daemon's browsers (spec 21.7): each bot's state and tabs, the +// live frames of the one being watched and what the bot does, for the +// browser panel, the size the panel gives the page (spec 21.3), and the +// owner's hands in it (spec 21.10). import type { FakeBotloft, Handlers } from "./fake"; -import { conflict } from "./fakeRules"; +import { fitting, PAGE, type PageSize, typed } from "./fakePages"; +import { conflict, invalid, notFound } from "./fakeRules"; import type { ApprovalId, BotId, @@ -12,6 +14,7 @@ import type { BrowserFrame, BrowserInput, BrowserState, + BrowserTab, } from "./protocol.gen"; export class FakeBrowser { @@ -23,6 +26,12 @@ export class FakeBrowser { readonly watches: (BotId | null)[] = []; /** What the owner sent to the page, in order. */ readonly inputs: BrowserInput[] = []; + /** Each time the owner reloaded a bot's page. */ + readonly reloads: BotId[] = []; + /** Each bot's page size, while the panel asks for one. */ + private readonly sizes = new Map(); + /** What draws each bot's page, for the preview. */ + private readonly painters = new Map string>(); /** Each bot's open request for help. */ private readonly asks = new Map(); /** Pages that answer the owner's hands, for the preview. */ @@ -38,7 +47,7 @@ export class FakeBrowser { url: null, title: null, loading: false, - tabs: 0, + tabs: [], error: null, control: "bot", ask: null, @@ -56,22 +65,34 @@ export class FakeBrowser { return next; } - /** The bot opened a page. */ + /** Puts `tabs` in the bot's browser, the active one's page on top. */ + private setTabs(botId: BotId, tabs: BrowserTab[], more: Partial = {}) { + const active = tabs.find((tab) => tab.active); + const page = { url: active?.url ?? null, title: active?.title || null }; + return this.set(botId, { tabs, ...page, ...more }); + } + + /** A page opened in the active tab; the first tab, if there is none. */ open(botId: BotId, url: string, title: string): BrowserState { - return this.set(botId, { status: "open", url, title, loading: false, tabs: 1, error: null }); + const tabs = this.state(botId).tabs; + const next = + tabs.length === 0 + ? [{ id: this.fake.id("tab"), url, title, active: true }] + : tabs.map((tab) => (tab.active ? { ...tab, url, title } : tab)); + return this.setTabs(botId, next, { status: "open", loading: false, error: null }); + } + + /** Another tab opened, and became the active one. */ + openTab(botId: BotId, url: string, title: string): BrowserTab { + const tab = { id: this.fake.id("tab"), url, title, active: true }; + const others = this.state(botId).tabs.map((other) => ({ ...other, active: false })); + this.setTabs(botId, [...others, tab]); + return tab; } close(botId: BotId): BrowserState { this.frames.delete(botId); - return this.set(botId, { - status: "closed", - url: null, - title: null, - loading: false, - tabs: 0, - control: "bot", - ask: null, - }); + return this.setTabs(botId, [], { status: "closed", loading: false, control: "bot", ask: null }); } /** The bot asks the owner for a hand: a request in the chat and in the panel. */ @@ -96,9 +117,28 @@ export class FakeBrowser { this.set(botId, { ask: null, control: "bot" }); } + /** The size of the bot's page now. */ + size(botId: BotId): PageSize { + return this.sizes.get(botId) ?? PAGE; + } + + /** Draws the bot's page with `draw`, now and whenever its size changes. */ + paint(botId: BotId, draw: (size: PageSize) => string): void { + this.painters.set(botId, draw); + this.repaint(botId); + } + + /** The page changed: a new picture of it, if something draws it. */ + repaint(botId: BotId): void { + const draw = this.painters.get(botId); + if (draw) { + this.frame(botId, draw(this.size(botId))); + } + } + /** A new picture of the page; sent only while the app watches that bot. */ - frame(botId: BotId, data: string, width = 1280, height = 800): void { - const frame: BrowserFrame = { botId, data, width, height }; + frame(botId: BotId, data: string, size: PageSize = this.size(botId)): void { + const frame: BrowserFrame = { botId, data, ...size }; this.frames.set(botId, frame); if (this.watching === botId) { this.fake.emit({ name: "browser.frame", params: frame }); @@ -132,30 +172,59 @@ export class FakeBrowser { return this.set(botId, { control: "bot" }); } + /** The owner has the watched browser, or the call is refused. */ + private held(botId: BotId): void { + if (this.watching !== botId || this.state(botId).control !== "owner") { + throw conflict("take the browser before using it"); + } + } + + /** Nobody watches: the page goes back to the size bots work in alone. */ + private unwatch(): void { + const botId = this.watching; + this.release(); + this.watching = null; + if (botId !== null && this.sizes.delete(botId)) { + this.repaint(botId); + } + } + handlers(): Pick< Handlers, | "browser.list" | "browser.watch" | "browser.unwatch" + | "browser.resize" | "browser.take" | "browser.release" | "browser.input" + | "browser.reload" + | "browser.newTab" + | "browser.switchTab" + | "browser.open" > { return { "browser.list": () => [...this.states.values()].filter((state) => state.status !== "closed"), "browser.watch": ({ botId }) => { this.fake.bot(botId, false); - this.release(); + this.unwatch(); this.watching = botId; this.watches.push(botId); return { state: this.state(botId), frame: this.frames.get(botId) ?? null }; }, "browser.unwatch": () => { - this.release(); - this.watching = null; + this.unwatch(); this.watches.push(null); return null; }, + "browser.resize": ({ botId, width, height }) => { + if (this.watching !== botId) { + throw conflict("watch this browser before sizing it"); + } + this.sizes.set(botId, fitting(width, height)); + this.repaint(botId); + return null; + }, "browser.take": ({ botId }) => { if (this.watching !== botId) { throw conflict("watch this browser before taking it"); @@ -173,13 +242,51 @@ export class FakeBrowser { return this.release() ?? this.state(botId); }, "browser.input": ({ botId, input }) => { - if (this.watching !== botId || this.state(botId).control !== "owner") { - throw conflict("take the browser before using it"); - } + this.held(botId); this.inputs.push(input); this.pages.get(botId)?.(input); return null; }, + "browser.reload": ({ botId }) => { + if (this.watching !== botId) { + throw conflict("watch this browser before reloading it"); + } + if (this.state(botId).status !== "open") { + throw conflict("the browser is not open"); + } + this.reloads.push(botId); + this.repaint(botId); + return null; + }, + "browser.newTab": ({ botId }) => { + this.held(botId); + this.openTab(botId, "about:blank", ""); + this.repaint(botId); + return null; + }, + "browser.switchTab": ({ botId, tabId }) => { + this.held(botId); + const tabs = this.state(botId).tabs; + if (!tabs.some((tab) => tab.id === tabId)) { + throw notFound("that tab"); + } + this.setTabs( + botId, + tabs.map((tab) => ({ ...tab, active: tab.id === tabId })), + ); + this.repaint(botId); + return null; + }, + "browser.open": ({ botId, url }) => { + this.held(botId); + const address = typed(url); + if (!address) { + throw invalid("that is not a web address"); + } + this.open(botId, address, new URL(address).hostname); + this.repaint(botId); + return null; + }, }; } } diff --git a/app/src/lib/fakePages.ts b/app/src/lib/fakePages.ts new file mode 100644 index 00000000..ebb98d63 --- /dev/null +++ b/app/src/lib/fakePages.ts @@ -0,0 +1,39 @@ +// The fake daemon's rules for a browser's pages, close enough to the +// daemon's for the UI: the size of a page in a panel (spec 21.3) and the +// addresses the owner may type (spec 21.10). + +/** The page's size: as wide as ever, as tall as the panel's room asks. */ +export interface PageSize { + width: number; + height: number; +} + +export const PAGE: PageSize = { width: 1280, height: 800 }; + +/** The page for a panel with `width` by `height` of room. */ +export function fitting(width: number, height: number): PageSize { + const tall = Math.floor((PAGE.width * height) / width); + return { width: PAGE.width, height: Math.min(2000, Math.max(600, tall)) }; +} + +/** + * What the owner typed, as the web address to open: `https://` goes in + * front of one without a scheme. `null` for anything else. + */ +export function typed(input: string): string | null { + const text = input.trim(); + if (!text || /\s/.test(text)) { + return null; + } + const web = /^https?:\/\//i.test(text); + // `mailto:x` has a scheme; `localhost:3000` has a port. + if (!web && /^[a-z][a-z0-9+.-]+:(?!\d+([/?#]|$))/i.test(text)) { + return null; + } + const url = web ? text : `https://${text}`; + try { + return new URL(url).hostname ? url : null; + } catch { + return null; + } +} diff --git a/app/src/lib/protocol.gen.ts b/app/src/lib/protocol.gen.ts index 1aa00419..fd866774 100644 --- a/app/src/lib/protocol.gen.ts +++ b/app/src/lib/protocol.gen.ts @@ -822,6 +822,23 @@ export type BrowserStatus = "closed" | "starting" | "open" | "failed"; */ export type BrowserControl = "bot" | "owner"; +/** + * One of the browser's open tabs. + */ +export type BrowserTab = { +/** + * Names the tab in `browser.switchTab`; means nothing else. + */ +id: string, +/** + * Empty until the page has one. + */ +title: string, url: string, +/** + * The tab the bot's tools and the owner's hands act on. + */ +active: boolean, }; + export type BrowserState = { botId: BotId, status: BrowserStatus, /** * The active tab's address. @@ -832,9 +849,9 @@ url: string | null, title: string | null, */ loading: boolean, /** - * Open tabs. + * The open tabs, in the order they opened. */ -tabs: number, +tabs: Array, /** * Why it could not start, when `failed`. */ @@ -892,6 +909,12 @@ export type BrowserView = { state: BrowserState, frame: BrowserFrame | null, }; export type BrowserWatchParams = { botId: BotId, }; +/** + * `browser.resize`: the room the app's panel has for the page, in the + * app's pixels. The page takes its shape (spec 21.3). + */ +export type BrowserResizeParams = { botId: BotId, width: number, height: number, }; + export type MouseAction = "move" | "down" | "up"; export type MouseButton = "none" | "left" | "middle" | "right"; @@ -911,12 +934,23 @@ buttons: number, clicks: number, modifiers: number, } | { "kind": "wheel", x: number, y: number, dx: number, dy: number, modifiers: number, } | { "kind": "key", key: string, code: string, modifiers: number, } | { "kind": "text", text: string, }; /** - * `browser.take` and `browser.release`. + * `browser.take`, `browser.release`, `browser.reload` and + * `browser.newTab`. */ export type BrowserControlParams = { botId: BotId, }; export type BrowserInputParams = { botId: BotId, input: BrowserInput, }; +/** + * `browser.switchTab`: the tab that becomes the active one. + */ +export type BrowserTabParams = { botId: BotId, tabId: string, }; + +/** + * `browser.open`: the address the owner typed for the active tab. + */ +export type BrowserOpenParams = { botId: BotId, url: string, }; + /** * The device a screen is drawn for. */ @@ -1020,9 +1054,14 @@ export interface RpcMethods { "browser.list": { params: undefined; result: Array }; "browser.watch": { params: BrowserWatchParams; result: BrowserView }; "browser.unwatch": { params: undefined; result: null }; + "browser.resize": { params: BrowserResizeParams; result: null }; "browser.take": { params: BrowserControlParams; result: BrowserState }; "browser.release": { params: BrowserControlParams; result: BrowserState }; "browser.input": { params: BrowserInputParams; result: null }; + "browser.reload": { params: BrowserControlParams; result: null }; + "browser.newTab": { params: BrowserControlParams; result: null }; + "browser.switchTab": { params: BrowserTabParams; result: null }; + "browser.open": { params: BrowserOpenParams; result: null }; "screens.list": { params: ScreensListParams; result: Array }; } diff --git a/crates/botloft-core/src/protocol/browser.rs b/crates/botloft-core/src/protocol/browser.rs index 194fb53d..9b2ab766 100644 --- a/crates/botloft-core/src/protocol/browser.rs +++ b/crates/botloft-core/src/protocol/browser.rs @@ -19,6 +19,20 @@ pub enum BrowserStatus { Failed, } +/// One of the browser's open tabs. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct BrowserTab { + /// Names the tab in `browser.switchTab`; means nothing else. + pub id: String, + /// Empty until the page has one. + pub title: String, + pub url: String, + /// The tab the bot's tools and the owner's hands act on. + pub active: bool, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] #[cfg_attr(test, derive(ts_rs::TS))] @@ -30,8 +44,8 @@ pub struct BrowserState { pub title: Option, /// The active tab is loading a page. pub loading: bool, - /// Open tabs. - pub tabs: u32, + /// The open tabs, in the order they opened. + pub tabs: Vec, /// Why it could not start, when `failed`. pub error: Option, /// Who uses it now (spec 21.10). @@ -50,7 +64,7 @@ impl BrowserState { url: None, title: None, loading: false, - tabs: 0, + tabs: Vec::new(), error: None, control: BrowserControl::Bot, ask: None, @@ -128,3 +142,14 @@ pub struct BrowserView { pub struct BrowserWatchParams { pub bot_id: BotId, } + +/// `browser.resize`: the room the app's panel has for the page, in the +/// app's pixels. The page takes its shape (spec 21.3). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct BrowserResizeParams { + pub bot_id: BotId, + pub width: u32, + pub height: u32, +} diff --git a/crates/botloft-core/src/protocol/browser_input.rs b/crates/botloft-core/src/protocol/browser_input.rs index a082e809..0677aaf2 100644 --- a/crates/botloft-core/src/protocol/browser_input.rs +++ b/crates/botloft-core/src/protocol/browser_input.rs @@ -91,7 +91,8 @@ impl std::fmt::Debug for BrowserInput { } } -/// `browser.take` and `browser.release`. +/// `browser.take`, `browser.release`, `browser.reload` and +/// `browser.newTab`. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] #[cfg_attr(test, derive(ts_rs::TS))] @@ -99,6 +100,24 @@ pub struct BrowserControlParams { pub bot_id: BotId, } +/// `browser.switchTab`: the tab that becomes the active one. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct BrowserTabParams { + pub bot_id: BotId, + pub tab_id: String, +} + +/// `browser.open`: the address the owner typed for the active tab. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct BrowserOpenParams { + pub bot_id: BotId, + pub url: String, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] #[cfg_attr(test, derive(ts_rs::TS))] diff --git a/crates/botloft-core/src/protocol/export_browser.rs b/crates/botloft-core/src/protocol/export_browser.rs index dfc9c191..e0dfcb00 100644 --- a/crates/botloft-core/src/protocol/export_browser.rs +++ b/crates/botloft-core/src/protocol/export_browser.rs @@ -8,17 +8,21 @@ use super::Out; pub(super) fn decls(out: &mut Out) { out.decl::(); out.decl::(); + out.decl::(); out.decl::(); out.decl::(); out.decl::(); out.decl::(); out.decl::(); out.decl::(); + out.decl::(); out.decl::(); out.decl::(); out.decl::(); out.decl::(); out.decl::(); + out.decl::(); + out.decl::(); out.decl::(); out.decl::(); out.decl::(); @@ -37,6 +41,11 @@ pub(super) fn methods(out: &mut Out) { &out.name::(), ); out.method(method::BROWSER_UNWATCH, "undefined", "null"); + out.method( + method::BROWSER_RESIZE, + &out.name::(), + "null", + ); let control = out.name::(); let state = out.name::(); out.method(method::BROWSER_TAKE, &control, &state); @@ -46,6 +55,18 @@ pub(super) fn methods(out: &mut Out) { &out.name::(), "null", ); + out.method(method::BROWSER_RELOAD, &control, "null"); + out.method(method::BROWSER_NEW_TAB, &control, "null"); + out.method( + method::BROWSER_SWITCH_TAB, + &out.name::(), + "null", + ); + out.method( + method::BROWSER_OPEN, + &out.name::(), + "null", + ); out.method( method::SCREENS_LIST, &out.name::(), diff --git a/crates/botloft-core/src/protocol/mod.rs b/crates/botloft-core/src/protocol/mod.rs index 43054c83..e61eedea 100644 --- a/crates/botloft-core/src/protocol/mod.rs +++ b/crates/botloft-core/src/protocol/mod.rs @@ -117,9 +117,14 @@ pub mod method { pub const BROWSER_LIST: &str = "browser.list"; pub const BROWSER_WATCH: &str = "browser.watch"; pub const BROWSER_UNWATCH: &str = "browser.unwatch"; + pub const BROWSER_RESIZE: &str = "browser.resize"; pub const BROWSER_TAKE: &str = "browser.take"; pub const BROWSER_RELEASE: &str = "browser.release"; pub const BROWSER_INPUT: &str = "browser.input"; + pub const BROWSER_RELOAD: &str = "browser.reload"; + pub const BROWSER_NEW_TAB: &str = "browser.newTab"; + pub const BROWSER_SWITCH_TAB: &str = "browser.switchTab"; + pub const BROWSER_OPEN: &str = "browser.open"; pub const SCREENS_LIST: &str = "screens.list"; pub const SETTINGS_GET: &str = "settings.get"; pub const SETTINGS_UPDATE: &str = "settings.update"; diff --git a/crates/botloftd/src/browser/call.rs b/crates/botloftd/src/browser/call.rs index 84b683fe..61092e45 100644 --- a/crates/botloftd/src/browser/call.rs +++ b/crates/botloftd/src/browser/call.rs @@ -11,7 +11,7 @@ use tokio::sync::OwnedMutexGuard; use tracing::debug; use super::session::{Hooks, PageInfo, Session}; -use super::{BrowserError, Browsers, launch, lock, update}; +use super::{BrowserError, Browsers, launch, lock, update, watch}; pub struct Call<'a> { pub(super) browsers: &'a Browsers, @@ -43,7 +43,11 @@ impl Call<'_> { None => Err(BrowserError::NotFound), Some(program) => { let profile = browsers.profiles.join(bot.as_str()); - Session::start(&program, &profile, downloads, self.hooks(start)).await + let viewport = { + let mut slots = lock(&browsers.slots); + browsers.slot(&mut slots, bot).viewport + }; + Session::start(&program, &profile, downloads, viewport, self.hooks(start)).await } }; let session = match started { @@ -57,16 +61,14 @@ impl Call<'_> { return Err(err); } }; - let watched = { + { let mut slots = lock(&browsers.slots); let slot = browsers.slot(&mut slots, bot); slot.session = Some((start, Arc::clone(&session))); - slot.watchers > 0 - }; - browsers.set_state(bot, |state| state.status = BrowserStatus::Open); - if watched { - session.set_watching(true).await; } + browsers.set_state(bot, |state| state.status = BrowserStatus::Open); + // Whoever watches already gets frames, the size they asked for. + watch::sync(Arc::clone(&browsers.slots), bot.clone()).await; debug!(bot = %bot, "browser: open"); Ok(session) } diff --git a/crates/botloftd/src/browser/events.rs b/crates/botloftd/src/browser/events.rs index 675f720c..3577092a 100644 --- a/crates/botloftd/src/browser/events.rs +++ b/crates/botloftd/src/browser/events.rs @@ -11,7 +11,8 @@ use tokio::sync::mpsc; use tracing::debug; use super::cdp::CdpEvent; -use super::session::{FRAME_GAP, HEIGHT, Hooks, MAX_TABS, PageInfo, Session, Tab, WIDTH}; +use super::session::{FRAME_GAP, Hooks, MAX_TABS, PageInfo, Session, Tab}; +use super::titles::retitles; impl Session { /// Sets up a new tab before it runs, and makes it the active one. @@ -25,21 +26,25 @@ impl Session { else { return; }; - let (previous, watching, extra) = { + let (previous, watching, extra, viewport) = { let mut tabs = self.lock(); let previous = tabs.active().map(|tab| tab.session.clone()); + let opened = tabs.opened; + tabs.opened += 1; tabs.list.push(Tab { target: target.to_owned(), session: session.to_owned(), url: info["url"].as_str().unwrap_or_default().to_owned(), - title: info["title"].as_str().unwrap_or_default().to_owned(), + title: String::new(), loading: false, navigations: 0, inflight: HashSet::new(), network_at: Instant::now(), + opened, + ready: false, }); let extra = (tabs.list.len() > MAX_TABS).then(|| tabs.list[0].target.clone()); - (previous, tabs.watching, extra) + (previous, tabs.watching, extra, tabs.viewport) }; let setup = [ ("Page.enable", json!({})), @@ -48,10 +53,7 @@ impl Session { "Page.setInterceptFileChooserDialog", json!({ "enabled": true }), ), - ( - "Emulation.setDeviceMetricsOverride", - json!({ "width": WIDTH, "height": HEIGHT, "deviceScaleFactor": 1, "mobile": false }), - ), + ("Emulation.setDeviceMetricsOverride", viewport.metrics()), ( "Emulation.setUserAgentOverride", json!({ "userAgent": self.user_agent }), @@ -74,6 +76,9 @@ impl Session { debug!("browser: setting up a new tab: {err}"); } } + if let Some(tab) = self.lock().by_session(session) { + tab.ready = true; + } if let Some(target) = extra { self.cdp .send(None, "Target.closeTarget", json!({ "targetId": target })); @@ -95,18 +100,22 @@ impl Session { let mut tabs = self.lock(); let was_active = tabs.active().is_some_and(|tab| tab.session == session); tabs.list.retain(|tab| tab.session != session); - let now_active = tabs.active().map(|tab| tab.session.clone()); + let now_active = tabs + .active() + .map(|tab| (tab.session.clone(), tab.target.clone())); (was_active, now_active, tabs.watching, tabs.list.is_empty()) }; if empty && !self.is_closed() { self.cdp .send(None, "Target.createTarget", json!({ "url": "about:blank" })); } - if was_active - && watching - && let Some(active) = now_active - { - self.cast(&active, true).await; + if was_active && let Some((active, target)) = now_active { + // The browser may have picked another tab to show. + self.cdp + .send(None, "Target.activateTarget", json!({ "targetId": target })); + if watching { + self.cast(&active, true).await; + } } } @@ -162,12 +171,11 @@ impl Session { }; let mut tabs = self.lock(); if let Some(tab) = tabs.list.iter_mut().find(|tab| tab.target == target) { + // The title here is the address again, never the page's + // (`titles.rs`). if let Some(url) = info["url"].as_str() { url.clone_into(&mut tab.url); } - if let Some(title) = info["title"].as_str() { - title.clone_into(&mut tab.title); - } } } @@ -188,10 +196,11 @@ impl Session { } fn frame(&self, session: &str, params: &Value, hooks: &Hooks) { - let active = self - .lock() - .active() - .is_some_and(|tab| tab.session == session); + let (active, viewport) = { + let tabs = self.lock(); + let active = tabs.active().is_some_and(|tab| tab.session == session); + (active, tabs.viewport) + }; if active && let Some(data) = params["data"].as_str() { let size = |key: &str, fallback: u32| { params["metadata"][key] @@ -201,8 +210,8 @@ impl Session { hooks.frames.send_replace(Some(Arc::new(BrowserFrame { bot_id: hooks.bot.clone(), data: data.to_owned(), - width: size("deviceWidth", WIDTH), - height: size("deviceHeight", HEIGHT), + width: size("deviceWidth", viewport.width), + height: size("deviceHeight", viewport.height), }))); } let cdp = self.cdp.clone(); @@ -231,6 +240,8 @@ impl Session { let fragment = params["frame"]["urlFragment"].as_str().unwrap_or_default(); tab.url = format!("{url}{fragment}"); } + // The new page's title is read once it loads. + tab.title.clear(); tab.inflight.clear(); } "Page.navigatedWithinDocument" if main => { @@ -265,9 +276,20 @@ pub(super) async fn pump( hooks: Hooks, ) { let mut last = PageInfo::default(); - while let Some(event) = events.recv().await { - session.handle(&event, &hooks).await; - session.changed.notify_waiters(); + loop { + tokio::select! { + event = events.recv() => { + let Some(event) = event else { break }; + session.handle(&event, &hooks).await; + session.changed.notify_waiters(); + if let Some(page) = event.session.filter(|_| retitles(&event.method)) { + let session = Arc::clone(&session); + tokio::spawn(async move { session.retitle(&page).await }); + } + } + // The owner moved between tabs: no event says so. + () = session.moved.notified() => {} + } let info = session.lock().info(); if info != last { (hooks.changed)(info.clone()); diff --git a/crates/botloftd/src/browser/hands.rs b/crates/botloftd/src/browser/hands.rs index 34a8f932..e861347a 100644 --- a/crates/botloftd/src/browser/hands.rs +++ b/crates/botloftd/src/browser/hands.rs @@ -1,29 +1,25 @@ //! The owner's hands in a bot's browser (spec 21.10): taking it and giving -//! it back, the owner's events reaching the page in order, the bot's tools -//! waiting meanwhile, and the bot asking the owner for help. +//! it back, what the owner does reaching the browser in order, the bot's +//! tools waiting meanwhile, and the bot asking the owner for help. use std::sync::Arc; use std::sync::atomic::Ordering; use std::time::Duration; use botloft_core::ids::{ApprovalId, BotId}; -use botloft_core::protocol::{BrowserControl, BrowserInput, MouseAction}; +use botloft_core::protocol::{BrowserControl, BrowserInput}; use tokio::sync::{broadcast, mpsc}; use tracing::debug; -use super::session::{HEIGHT, Session, WIDTH}; -use super::{Browsers, Slots, lock, update}; +use super::input::fits; +use super::moves::{Move, feed}; +use super::session::Session; +use super::{Browsers, Slots, Viewport, lock, sites, update}; use crate::clock::Clock; use crate::state::Event; /// Longest a bot's tool waits for the owner to give the browser back. pub const OWNER_WAIT: Duration = Duration::from_secs(10 * 60); -/// Longest text the owner sends at once, in characters. -const TEXT_MAX: usize = 10_000; -/// Longest key or code name. -const KEY_MAX: usize = 32; -/// Farthest one turn of the wheel scrolls, in pixels. -const WHEEL_MAX: f64 = 10_000.0; #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] pub enum TakeError { @@ -33,11 +29,15 @@ pub enum TakeError { Taken, } -/// Why an event of the owner's did not go to the page. +/// Why something the owner did was not done in the browser. #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] pub enum InputError { #[error("that event does not fit the page")] Invalid, + #[error("that is not a web address")] + Address, + #[error("that tab is not open anymore")] + NoTab, #[error("the browser is back with the bot; take it again")] NotHeld, #[error("the browser closed")] @@ -52,7 +52,8 @@ pub struct Hands { clock: Arc, pub bot: BotId, hold: u64, - input: mpsc::UnboundedSender, + session: Arc, + moves: mpsc::UnboundedSender, } impl Browsers { @@ -67,31 +68,43 @@ impl Browsers { if slot.held.borrow().is_some() { return Err(TakeError::Taken); } + session.owner_took(); slot.held.send_replace(Some(hold)); } self.set_state(bot, |state| state.control = BrowserControl::Owner); debug!(bot = %bot, "browser: the owner took it"); - let (input, queue) = mpsc::unbounded_channel(); - tokio::spawn(feed(session, queue)); + let (moves, queue) = mpsc::unbounded_channel(); + tokio::spawn(feed(Arc::clone(&session), queue)); Ok(Hands { slots: Arc::clone(&self.slots), events: self.events.clone(), clock: Arc::clone(&self.clock), bot: bot.clone(), hold, - input, + session, + moves, }) } - /// Waits while the owner holds the bot's browser. `false` if they still - /// hold it after `limit`. + /// Waits while the owner holds the bot's browser, and for what they did + /// last to reach it. `false` if they still hold it after `limit`. pub async fn wait_for_owner(&self, bot: &BotId, limit: Duration) -> bool { let mut held = { let mut slots = lock(&self.slots); self.slot(&mut slots, bot).held.subscribe() }; - let waited = tokio::time::timeout(limit, held.wait_for(Option::is_none)).await; - matches!(waited, Ok(Ok(_))) + let given_back = async { + if held.wait_for(Option::is_none).await.is_err() { + return false; + } + if let Some(session) = self.running(bot) { + session.owner_settled().await; + } + true + }; + tokio::time::timeout(limit, given_back) + .await + .unwrap_or(false) } /// The bot asks the owner for help: the app shows `task` until the @@ -143,18 +156,54 @@ impl Drop for Asking<'_> { } impl Hands { - /// Sends one of the owner's events to the page, after the ones before. + /// The page's size, while the owner still has the browser. + fn page(&self) -> Result { + lock(&self.slots) + .get(&self.bot) + .filter(|slot| *slot.held.borrow() == Some(self.hold)) + .map(|slot| slot.viewport) + .ok_or(InputError::NotHeld) + } + + /// Queues one thing the owner did, after the ones before. + fn make(&self, step: Move) -> Result<(), InputError> { + self.session.owner_moves.send_modify(|left| *left += 1); + self.moves.send(step).map_err(|_| { + self.session + .owner_moves + .send_modify(|left| *left = left.saturating_sub(1)); + InputError::Closed + }) + } + + /// Sends one of the owner's events to the page. pub fn send(&self, input: BrowserInput) -> Result<(), InputError> { - if !fits(&input) { + if !fits(&input, self.page()?) { return Err(InputError::Invalid); } - let holds = lock(&self.slots) - .get(&self.bot) - .is_some_and(|slot| *slot.held.borrow() == Some(self.hold)); - if !holds { - return Err(InputError::NotHeld); + self.make(Move::Input(input)) + } + + /// Opens a blank tab, which becomes the active one. + pub fn new_tab(&self) -> Result<(), InputError> { + self.page()?; + self.make(Move::NewTab) + } + + /// Makes the tab `id` the active one. + pub fn switch_tab(&self, id: &str) -> Result<(), InputError> { + self.page()?; + if !self.session.has_tab(id) { + return Err(InputError::NoTab); } - self.input.send(input).map_err(|_| InputError::Closed) + self.make(Move::SwitchTab(id.to_owned())) + } + + /// Takes the active tab to the web address the owner typed. + pub fn open(&self, address: &str) -> Result<(), InputError> { + self.page()?; + let url = sites::typed(address).ok_or(InputError::Address)?; + self.make(Move::Open(url)) } } @@ -179,121 +228,3 @@ impl Drop for Hands { } } } - -/// Whether an event makes sense on the page. -fn fits(input: &BrowserInput) -> bool { - let on_page = |x: f64, y: f64| { - (0.0..=f64::from(WIDTH)).contains(&x) && (0.0..=f64::from(HEIGHT)).contains(&y) - }; - let modifiers_ok = |modifiers: u32| modifiers <= 15; - match input { - BrowserInput::Mouse { - x, - y, - buttons, - clicks, - modifiers, - .. - } => on_page(*x, *y) && *buttons <= 7 && *clicks <= 3 && modifiers_ok(*modifiers), - BrowserInput::Wheel { - x, - y, - dx, - dy, - modifiers, - } => { - on_page(*x, *y) - && dx.abs() <= WHEEL_MAX - && dy.abs() <= WHEEL_MAX - && modifiers_ok(*modifiers) - } - BrowserInput::Key { - key, - code, - modifiers, - } => { - !key.is_empty() - && key.chars().count() <= KEY_MAX - && code.len() <= KEY_MAX - && modifiers_ok(*modifiers) - } - BrowserInput::Text { text } => !text.is_empty() && text.chars().count() <= TEXT_MAX, - } -} - -/// Sends the owner's events to the page one by one, in order. Of several -/// mouse moves in a row only the newest matters. -async fn feed(session: Arc, mut queue: mpsc::UnboundedReceiver) { - let mut next = queue.recv().await; - while let Some(mut input) = next.take() { - while let Ok(later) = queue.try_recv() { - if moves(&input) && moves(&later) { - input = later; - } else { - next = Some(later); - break; - } - } - if session.is_closed() { - return; - } - if let Err(err) = session.owner_input(&input).await { - debug!("browser: an event of the owner's did not reach the page: {err}"); - } - if next.is_none() { - next = queue.recv().await; - } - } -} - -fn moves(input: &BrowserInput) -> bool { - matches!( - input, - BrowserInput::Mouse { - action: MouseAction::Move, - .. - } - ) -} - -#[cfg(test)] -mod tests { - use botloft_core::protocol::MouseButton; - - use super::*; - - fn mouse(x: f64, y: f64) -> BrowserInput { - BrowserInput::Mouse { - action: MouseAction::Down, - x, - y, - button: MouseButton::Left, - buttons: 1, - clicks: 1, - modifiers: 0, - } - } - - #[test] - fn events_off_the_page_or_too_big_are_refused() { - assert!(fits(&mouse(640.0, 400.0))); - assert!(fits(&mouse(1280.0, 800.0))); - assert!(!fits(&mouse(-1.0, 10.0))); - assert!(!fits(&mouse(f64::NAN, 10.0))); - assert!(!fits(&mouse(10.0, 900.0))); - let long = BrowserInput::Text { - text: "x".repeat(TEXT_MAX + 1), - }; - assert!(!fits(&long)); - let empty = BrowserInput::Text { - text: String::new(), - }; - assert!(!fits(&empty)); - let key = BrowserInput::Key { - key: "a".to_owned(), - code: "KeyA".to_owned(), - modifiers: 16, - }; - assert!(!fits(&key)); - } -} diff --git a/crates/botloftd/src/browser/input.rs b/crates/botloftd/src/browser/input.rs index b1d74600..b892c9c2 100644 --- a/crates/botloftd/src/browser/input.rs +++ b/crates/botloftd/src/browser/input.rs @@ -1,13 +1,59 @@ -//! The owner's events on the page (spec 21.10), as the DevTools protocol -//! takes them: mouse and wheel, keys with or without text, and text pasted -//! or composed. +//! The owner's events on the page (spec 21.10): which ones make sense, and +//! each as the DevTools protocol takes it: mouse and wheel, keys with or +//! without text, and text pasted or composed. use botloft_core::protocol::{BrowserInput, MouseAction, MouseButton, modifier}; use serde_json::{Value, json}; -use super::BrowserError; use super::keys; use super::session::Session; +use super::{BrowserError, Viewport}; + +/// Longest text the owner sends at once, in characters. +const TEXT_MAX: usize = 10_000; +/// Longest key or code name. +const KEY_MAX: usize = 32; +/// Farthest one turn of the wheel scrolls, in pixels. +const WHEEL_MAX: f64 = 10_000.0; + +/// Whether an event makes sense on a page of that size. +pub(super) fn fits(input: &BrowserInput, page: Viewport) -> bool { + let on_page = |x: f64, y: f64| page.contains(x, y); + let modifiers_ok = |modifiers: u32| modifiers <= 15; + match input { + BrowserInput::Mouse { + x, + y, + buttons, + clicks, + modifiers, + .. + } => on_page(*x, *y) && *buttons <= 7 && *clicks <= 3 && modifiers_ok(*modifiers), + BrowserInput::Wheel { + x, + y, + dx, + dy, + modifiers, + } => { + on_page(*x, *y) + && dx.abs() <= WHEEL_MAX + && dy.abs() <= WHEEL_MAX + && modifiers_ok(*modifiers) + } + BrowserInput::Key { + key, + code, + modifiers, + } => { + !key.is_empty() + && key.chars().count() <= KEY_MAX + && code.len() <= KEY_MAX + && modifiers_ok(*modifiers) + } + BrowserInput::Text { text } => !text.is_empty() && text.chars().count() <= TEXT_MAX, + } +} impl Session { /// Sends one of the owner's events to the active tab. @@ -179,6 +225,42 @@ mod tests { assert_eq!(word[0]["windowsVirtualKeyCode"], 8); } + #[test] + fn events_off_the_page_or_too_big_are_refused() { + let mouse = |x: f64, y: f64| BrowserInput::Mouse { + action: MouseAction::Down, + x, + y, + button: MouseButton::Left, + buttons: 1, + clicks: 1, + modifiers: 0, + }; + let fits = |input: &BrowserInput| super::fits(input, Viewport::default()); + assert!(fits(&mouse(640.0, 400.0))); + assert!(fits(&mouse(1280.0, 800.0))); + assert!(!fits(&mouse(-1.0, 10.0))); + assert!(!fits(&mouse(f64::NAN, 10.0))); + assert!(!fits(&mouse(10.0, 900.0))); + // A taller page takes points further down. + let tall = Viewport::fitting(640, 700); + assert!(super::fits(&mouse(10.0, 900.0), tall)); + let long = BrowserInput::Text { + text: "x".repeat(TEXT_MAX + 1), + }; + assert!(!fits(&long)); + let empty = BrowserInput::Text { + text: String::new(), + }; + assert!(!fits(&empty)); + let key = BrowserInput::Key { + key: "a".to_owned(), + code: "KeyA".to_owned(), + modifiers: 16, + }; + assert!(!fits(&key)); + } + #[test] fn a_drag_names_the_button_held() { let drag = calls(&BrowserInput::Mouse { diff --git a/crates/botloftd/src/browser/mod.rs b/crates/botloftd/src/browser/mod.rs index 662a3f4b..4db10c74 100644 --- a/crates/botloftd/src/browser/mod.rs +++ b/crates/botloftd/src/browser/mod.rs @@ -10,12 +10,15 @@ mod hands; mod input; mod keys; mod launch; +mod moves; mod page; mod read; mod session; mod settle; pub mod sites; mod sweep; +mod titles; +mod viewport; mod watch; use std::collections::HashMap; @@ -39,6 +42,7 @@ pub use self::page::{Done, Scroll}; pub use self::read::{READ_MAX, Reading}; pub use self::session::Session; pub use self::sweep::run; +pub use self::viewport::Viewport; pub use self::watch::Watching; use crate::clock::Clock; use crate::config::Config; @@ -111,6 +115,8 @@ struct Slot { state: BrowserState, frames: Frames, watchers: usize, + /// The page size the app watching asked for (spec 21.3). + viewport: Viewport, used: Instant, held: Held, /// The bot's open request for the owner's help. @@ -161,6 +167,7 @@ impl Browsers { state: BrowserState::closed(bot.clone(), self.clock.now_ms()), frames: tokio::sync::watch::channel(None).0, watchers: 0, + viewport: Viewport::default(), used: Instant::now(), held: tokio::sync::watch::channel(None).0, asking: None, diff --git a/crates/botloftd/src/browser/moves.rs b/crates/botloftd/src/browser/moves.rs new file mode 100644 index 00000000..2be731f7 --- /dev/null +++ b/crates/botloftd/src/browser/moves.rs @@ -0,0 +1,204 @@ +//! What the owner does in a bot's browser besides using the page (spec +//! 21.10): reloading it, moving between tabs, opening a new one and going +//! to an address. With the browser in their hands these go to the page one +//! by one, in order, along with their mouse and keyboard. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use botloft_core::ids::BotId; +use botloft_core::protocol::{BrowserInput, MouseAction}; +use serde_json::json; +use tokio::sync::mpsc; +use tracing::debug; + +use super::session::{Session, Tab}; +use super::{BrowserError, Browsers}; + +/// How long a new tab may take to show up. +const NEW_TAB_TIMEOUT: Duration = Duration::from_secs(5); +/// What the bot reads after the owner reloaded its page. +const RELOADED: &str = "The owner reloaded the page."; + +/// One thing the owner did with the browser in their hands. +pub(super) enum Move { + Input(BrowserInput), + NewTab, + /// The tab, by its target, that becomes the active one. + SwitchTab(String), + /// A web address for the active tab. + Open(String), +} + +impl Browsers { + /// Reloads the active tab of the bot's browser, in the owner's hands or + /// not. `false` when the browser is not open. + pub fn reload(&self, bot: &BotId) -> bool { + let Some(session) = self.running(bot) else { + return false; + }; + tokio::spawn(async move { + if let Err(err) = session.reload().await { + debug!("browser: the page did not reload: {err}"); + } + }); + true + } +} + +impl Session { + /// Reloads the active tab. The bot reads so with its next result. + async fn reload(&self) -> Result<(), BrowserError> { + let (page, _) = self.page()?; + { + let mut tabs = self.lock(); + if tabs.notes.last().is_none_or(|note| note != RELOADED) { + tabs.notes.push(RELOADED.to_owned()); + } + } + self.cdp.call(Some(&page), "Page.reload", json!({})).await?; + Ok(()) + } + + pub(super) fn has_tab(&self, target: &str) -> bool { + self.lock().list.iter().any(|tab| tab.target == target) + } + + /// Opens a blank tab and waits for it to be the active one, set up. + async fn new_tab(&self) -> Result<(), BrowserError> { + let created = self + .cdp + .call(None, "Target.createTarget", json!({ "url": "about:blank" })) + .await?; + let target = created["targetId"].as_str().unwrap_or_default(); + let deadline = Instant::now() + NEW_TAB_TIMEOUT; + loop { + let notified = self.changed.notified(); + let open = |tab: &Tab| tab.target == target && tab.ready; + if self.lock().active().is_some_and(open) { + return Ok(()); + } + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() || tokio::time::timeout(left, notified).await.is_err() { + return Err(BrowserError::Page("the new tab did not open".to_owned())); + } + } + } + + /// Makes the tab `target` the active one: in front in the browser, with + /// the live frames, and where tools and hands act. + async fn switch_tab(&self, target: &str) -> Result<(), BrowserError> { + let (previous, next, watching) = { + let mut tabs = self.lock(); + let Some(index) = tabs.list.iter().position(|tab| tab.target == target) else { + return Ok(()); + }; + if index + 1 == tabs.list.len() { + return Ok(()); + } + let previous = tabs.active().map(|tab| tab.session.clone()); + let tab = tabs.list.remove(index); + let next = tab.session.clone(); + tabs.list.push(tab); + (previous, next, tabs.watching) + }; + self.moved.notify_one(); + let shown = self + .cdp + .call(None, "Target.activateTarget", json!({ "targetId": target })) + .await; + if watching { + if let Some(previous) = previous { + self.cast(&previous, false).await; + } + self.cast(&next, true).await; + } + shown?; + Ok(()) + } + + /// Takes the active tab to `url`, without waiting for the page. + async fn go(&self, url: &str) -> Result<(), BrowserError> { + let (page, _) = self.page()?; + self.cdp + .call(Some(&page), "Page.navigate", json!({ "url": url })) + .await?; + Ok(()) + } + + /// The owner took the browser: remember the tab the bot was on. + pub(super) fn owner_took(&self) { + let mut tabs = self.lock(); + if tabs.owner_from.is_none() { + tabs.owner_from = tabs.active().map(|tab| tab.target.clone()); + } + } + + /// Whether the owner had the browser and left the bot on another tab + /// than the one it was on; asked once, by the bot's next tool (spec + /// 21.10). + pub fn take_switched(&self) -> bool { + let mut tabs = self.lock(); + let now = tabs.active().map(|tab| tab.target.clone()); + tabs.owner_from + .take() + .is_some_and(|from| now.as_ref() != Some(&from)) + } + + /// Waits until everything the owner did has reached the browser. + pub(super) async fn owner_settled(&self) { + let mut left = self.owner_moves.subscribe(); + let _ = left.wait_for(|left| *left == 0).await; + } + + async fn make(&self, step: &Move) -> Result<(), BrowserError> { + match step { + Move::Input(input) => self.owner_input(input).await, + Move::NewTab => self.new_tab().await, + Move::SwitchTab(target) => self.switch_tab(target).await, + Move::Open(url) => self.go(url).await, + } + } +} + +/// Does what the owner did, one thing at a time, in order. Of several mouse +/// moves in a row only the newest matters. Each thing queued counts in +/// `owner_moves` until it is done or skipped. +pub(super) async fn feed(session: Arc, mut queue: mpsc::UnboundedReceiver) { + let done = || { + session + .owner_moves + .send_modify(|left| *left = left.saturating_sub(1)) + }; + let mut next = queue.recv().await; + while let Some(mut step) = next.take() { + while let Ok(later) = queue.try_recv() { + if glides(&step) && glides(&later) { + step = later; + done(); + } else { + next = Some(later); + break; + } + } + if !session.is_closed() + && let Err(err) = session.make(&step).await + { + debug!("browser: something the owner did failed in the browser: {err}"); + } + done(); + if next.is_none() { + next = queue.recv().await; + } + } +} + +fn glides(step: &Move) -> bool { + matches!( + step, + Move::Input(BrowserInput::Mouse { + action: MouseAction::Move, + .. + }) + ) +} diff --git a/crates/botloftd/src/browser/page.rs b/crates/botloftd/src/browser/page.rs index e0c628aa..2f5baaa2 100644 --- a/crates/botloftd/src/browser/page.rs +++ b/crates/botloftd/src/browser/page.rs @@ -6,7 +6,7 @@ use serde_json::{Value, json}; use super::BrowserError; use super::keys::{self, Key}; -use super::session::{HEIGHT, Session, WIDTH}; +use super::session::Session; /// Where an action happened, for the owner's view, and what the bot should /// know about it. @@ -204,10 +204,15 @@ impl Session { pub async fn scroll(&self, to: Scroll) -> Result { let (session, _) = self.page()?; let marks = self.marks(); - let center = (f64::from(WIDTH) / 2.0, f64::from(HEIGHT) / 2.0); + let screen = self.viewport(); + let center = ( + f64::from(screen.width) / 2.0, + f64::from(screen.height) / 2.0, + ); let done = match to { Scroll::Down | Scroll::Up => { - let delta = f64::from(HEIGHT) * 0.8 * if to == Scroll::Down { 1.0 } else { -1.0 }; + let delta = + f64::from(screen.height) * 0.8 * if to == Scroll::Down { 1.0 } else { -1.0 }; let params = json!({ "type": "mouseWheel", "x": center.0, "y": center.1, "deltaX": 0, "deltaY": delta, }); diff --git a/crates/botloftd/src/browser/read.rs b/crates/botloftd/src/browser/read.rs index 3225de14..28827c03 100644 --- a/crates/botloftd/src/browser/read.rs +++ b/crates/botloftd/src/browser/read.rs @@ -15,7 +15,7 @@ use super::session::Session; /// What reads the page (`reader.js`), and what acts on it (`actions.js`). const READER: &str = include_str!("reader.js"); const ACTIONS: &str = include_str!("actions.js"); -const WORLD: &str = "botloft"; +pub(super) const WORLD: &str = "botloft"; /// Longest page text in one answer, in characters. pub const READ_MAX: usize = 20_000; diff --git a/crates/botloftd/src/browser/session.rs b/crates/botloftd/src/browser/session.rs index 345b8f79..2d59f88c 100644 --- a/crates/botloftd/src/browser/session.rs +++ b/crates/botloftd/src/browser/session.rs @@ -8,7 +8,7 @@ use std::sync::{Arc, Mutex, MutexGuard}; use std::time::{Duration, Instant}; use botloft_core::ids::BotId; -use botloft_core::protocol::BrowserFrame; +use botloft_core::protocol::{BrowserFrame, BrowserTab}; use serde_json::json; use tokio::sync::{Notify, watch}; use tracing::debug; @@ -17,24 +17,23 @@ use super::BrowserError; use super::cdp::Cdp; use super::events::pump; use super::launch::{self, BrowserProcess}; +use super::viewport::{MAX_HEIGHT, Viewport, WIDTH}; -/// The page size bots and the owner see, in CSS pixels (spec 21.3). -pub const WIDTH: u32 = 1280; -pub const HEIGHT: u32 = 800; -/// Most tabs a browser keeps; the oldest inactive one closes past this. +/// Most tabs a browser keeps; past this, the one active longest ago closes. pub(super) const MAX_TABS: usize = 6; /// Frames are confirmed after this, so the browser sends ~15 a second. pub(super) const FRAME_GAP: Duration = Duration::from_millis(66); /// How long the first tab may take to show up. const FIRST_TAB_TIMEOUT: Duration = Duration::from_secs(5); -/// What the app shows about the active tab. +/// What the app shows about the active tab, and the tabs there are. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct PageInfo { pub url: Option, pub title: Option, pub loading: bool, - pub tabs: u32, + /// In the order they opened. + pub tabs: Vec, } /// How a session reports back to the hub. @@ -57,17 +56,27 @@ pub struct Tab { pub inflight: HashSet, /// The last request that started or ended. pub network_at: Instant, + /// Its place among the tabs this browser opened, for the owner's list. + pub opened: u64, + /// It is set up: its events come, its size and user agent are set. + pub ready: bool, } #[derive(Default)] pub struct Tabs { /// In the order they became active; the last one is. pub list: Vec, + /// Tabs opened so far. + pub(super) opened: u64, /// What the bot should read with the next result: dialogs, downloads. pub notes: Vec, pub(super) downloads: HashMap, pub watching: bool, pub closed: bool, + pub viewport: Viewport, + /// The tab the bot was on when the owner took the browser, until the + /// bot's next tool (spec 21.10). + pub(super) owner_from: Option, } impl Tabs { @@ -81,13 +90,23 @@ impl Tabs { pub(super) fn info(&self) -> PageInfo { let tab = self.active(); + let mut open: Vec<&Tab> = self.list.iter().collect(); + open.sort_by_key(|tab| tab.opened); PageInfo { url: tab.map(|tab| tab.url.clone()).filter(|url| !url.is_empty()), title: tab .map(|tab| tab.title.clone()) .filter(|title| !title.is_empty()), loading: tab.is_some_and(|tab| tab.loading), - tabs: u32::try_from(self.list.len()).unwrap_or(u32::MAX), + tabs: open + .into_iter() + .map(|open| BrowserTab { + id: open.target.clone(), + title: open.title.clone(), + url: open.url.clone(), + active: tab.is_some_and(|tab| tab.target == open.target), + }) + .collect(), } } } @@ -97,18 +116,26 @@ pub struct Session { pub tabs: Mutex, /// Wakes whoever waits for the page after any event. pub changed: Notify, + /// The tabs changed without an event from the browser: tell the app. + pub(super) moved: Notify, + /// Things the owner did that have not reached the browser yet. + pub(super) owner_moves: watch::Sender, /// The next element ref, shared by every page of this browser. pub next_ref: AtomicU64, process: Mutex>, pub(super) user_agent: String, + /// One change of what the app wants at a time (`watch.rs`). + pub(super) syncing: tokio::sync::Mutex<()>, } impl Session { - /// Starts the browser and waits for its first tab. + /// Starts the browser, its pages `viewport` in size, and waits for its + /// first tab. pub async fn start( program: &Path, profile: &Path, downloads: &Path, + viewport: Viewport, hooks: Hooks, ) -> Result, BrowserError> { let (process, url) = launch::launch(program, profile).await?; @@ -120,11 +147,17 @@ impl Session { .replace("HeadlessChrome", "Chrome"); let session = Arc::new(Self { cdp, - tabs: Mutex::new(Tabs::default()), + tabs: Mutex::new(Tabs { + viewport, + ..Tabs::default() + }), changed: Notify::new(), + moved: Notify::new(), + owner_moves: watch::channel(0).0, next_ref: AtomicU64::new(1), process: Mutex::new(Some(process)), user_agent, + syncing: tokio::sync::Mutex::new(()), }); tokio::spawn(pump(Arc::clone(&session), events, hooks)); @@ -156,7 +189,8 @@ impl Session { let deadline = Instant::now() + FIRST_TAB_TIMEOUT; let ready = loop { let notified = session.changed.notified(); - if session.lock().active().is_some() { + // Set up too: a page opened before that would load unseen. + if session.lock().active().is_some_and(|tab| tab.ready) { break true; } let left = deadline.saturating_duration_since(Instant::now()); @@ -218,8 +252,9 @@ impl Session { pub(super) async fn cast(&self, session: &str, on: bool) { let result = if on { + // Frames come the size of the page, however tall it is. let params = json!({ - "format": "jpeg", "quality": 60, "maxWidth": WIDTH, "maxHeight": HEIGHT, + "format": "jpeg", "quality": 60, "maxWidth": WIDTH, "maxHeight": MAX_HEIGHT, }); self.cdp .call(Some(session), "Page.startScreencast", params) diff --git a/crates/botloftd/src/browser/sites.rs b/crates/botloftd/src/browser/sites.rs index 637c39d9..c3f87f4d 100644 --- a/crates/botloftd/src/browser/sites.rs +++ b/crates/botloftd/src/browser/sites.rs @@ -58,6 +58,37 @@ pub fn resolve(input: &str, base: &Path, folders: &[PathBuf]) -> Result Option { + let input = input.trim(); + if input.is_empty() || input.len() > TYPED_MAX || input.contains(char::is_whitespace) { + return None; + } + let lower = input.to_ascii_lowercase(); + let url = if lower.starts_with("http://") || lower.starts_with("https://") { + input.to_owned() + } else if has_scheme(input) && !has_port(input) { + return None; + } else { + format!("https://{input}") + }; + site_of(&url).map(|_| url) +} + +/// `localhost:3000/x`: what follows the colon is a port, not the rest of a +/// `scheme:` address. +fn has_port(input: &str) -> bool { + input.split_once(':').is_some_and(|(_, rest)| { + let port = rest.split(['/', '?', '#']).next().unwrap_or_default(); + !port.is_empty() && port.chars().all(|c| c.is_ascii_digit()) + }) +} + /// The site of a web address, or `None` for anything else. pub fn site_of(url: &str) -> Option { let lower = url.to_ascii_lowercase(); diff --git a/crates/botloftd/src/browser/sites_tests.rs b/crates/botloftd/src/browser/sites_tests.rs index 7a72c229..e5704cc7 100644 --- a/crates/botloftd/src/browser/sites_tests.rs +++ b/crates/botloftd/src/browser/sites_tests.rs @@ -115,3 +115,28 @@ fn files_open_only_inside_the_bot_folders() { )); assert!(resolve("missing.html", &bot, &folders).is_err()); } + +#[test] +fn what_the_owner_types_opens_only_as_a_web_address() { + let open = |input: &str| typed(input); + assert_eq!( + open(" example.com "), + Some("https://example.com".to_owned()) + ); + assert_eq!( + open("HTTP://Example.com/a?b=c"), + Some("HTTP://Example.com/a?b=c".to_owned()) + ); + // A host with a port is not a `scheme:` address. + assert_eq!( + open("localhost:3000/orders"), + Some("https://localhost:3000/orders".to_owned()) + ); + assert_eq!(open("javascript:alert(1)"), None); + assert_eq!(open("file:///C:/Users/owner/x.html"), None); + assert_eq!(open("edge://settings"), None); + assert_eq!(open("about:blank"), None); + assert_eq!(open("how to bake bread"), None); + assert_eq!(open("https://"), None); + assert_eq!(open(""), None); +} diff --git a/crates/botloftd/src/browser/titles.rs b/crates/botloftd/src/browser/titles.rs new file mode 100644 index 00000000..a04b9379 --- /dev/null +++ b/crates/botloftd/src/browser/titles.rs @@ -0,0 +1,67 @@ +//! Each tab's title, for the owner's list of tabs (spec 21.7). Edge without +//! a window tells a tab's new address but never its title (spec 19), so the +//! daemon reads `document.title` when the page loads. + +use serde_json::json; +use tracing::debug; + +use super::read::WORLD; +use super::session::Session; + +/// Longest title kept, in characters. +const TITLE_MAX: usize = 200; + +/// Whether a page's event may have come with a new title. +pub(super) fn retitles(method: &str) -> bool { + matches!( + method, + "Page.domContentEventFired" | "Page.loadEventFired" | "Page.navigatedWithinDocument" + ) +} + +impl Session { + /// Reads the title of the tab whose page session is `page`, in the + /// isolated world, where the page's scripts cannot answer for it. + pub(super) async fn retitle(&self, page: &str) { + let target = self.lock().by_session(page).map(|tab| tab.target.clone()); + let Some(target) = target else { + return; + }; + let world = self + .cdp + .call( + Some(page), + "Page.createIsolatedWorld", + json!({ "frameId": target, "worldName": WORLD }), + ) + .await; + let read = match world { + Ok(world) => { + let params = json!({ + "expression": "document.title", + "contextId": world["executionContextId"], + "returnByValue": true, + }); + self.cdp.call(Some(page), "Runtime.evaluate", params).await + } + Err(err) => Err(err), + }; + let title = match &read { + Ok(read) => read["result"]["value"].as_str().unwrap_or_default(), + // Between two documents; the next load reads it. + Err(err) => { + debug!("browser: reading a tab's title: {err}"); + return; + } + }; + let title = botloft_core::chat::one_line(title, TITLE_MAX); + let changed = self.lock().by_session(page).is_some_and(|tab| { + let changed = tab.title != title; + tab.title = title; + changed + }); + if changed { + self.moved.notify_one(); + } + } +} diff --git a/crates/botloftd/src/browser/viewport.rs b/crates/botloftd/src/browser/viewport.rs new file mode 100644 index 00000000..411d9bd8 --- /dev/null +++ b/crates/botloftd/src/browser/viewport.rs @@ -0,0 +1,136 @@ +//! The size of the page (spec 21.3): as wide as a desktop screen always, +//! and as tall as the owner's panel asks while they watch. + +use serde_json::{Value, json}; +use tracing::debug; + +use super::session::{FRAME_GAP, Session}; + +/// The page size bots and the owner see, in CSS pixels. The height is the +/// one nobody asked to change. +pub const WIDTH: u32 = 1280; +pub const HEIGHT: u32 = 800; +/// How short and how tall the owner's panel may make the page. +pub const MIN_HEIGHT: u32 = 600; +pub const MAX_HEIGHT: u32 = 2000; + +/// The size of the page, the same in every tab. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Viewport { + pub width: u32, + pub height: u32, +} + +impl Default for Viewport { + fn default() -> Self { + Self { + width: WIDTH, + height: HEIGHT, + } + } +} + +impl Viewport { + /// The page for a panel with `width` by `height` of room: as wide as + /// ever, so sites keep their desktop layout, and as tall as the room's + /// shape asks, within limits. + pub fn fitting(width: u32, height: u32) -> Self { + let tall = u64::from(WIDTH) * u64::from(height) / u64::from(width.max(1)); + Self { + width: WIDTH, + height: u32::try_from(tall) + .unwrap_or(MAX_HEIGHT) + .clamp(MIN_HEIGHT, MAX_HEIGHT), + } + } + + pub(super) fn metrics(self) -> Value { + json!({ + "width": self.width, "height": self.height, "deviceScaleFactor": 1, "mobile": false, + }) + } + + /// Whether a point is on the page. + pub fn contains(self, x: f64, y: f64) -> bool { + (0.0..=f64::from(self.width)).contains(&x) && (0.0..=f64::from(self.height)).contains(&y) + } +} + +impl Session { + pub fn viewport(&self) -> Viewport { + self.lock().viewport + } + + /// Makes the page `viewport` in size, in every tab, and sends whoever + /// watches a frame of that size. + pub async fn set_viewport(&self, viewport: Viewport) { + let pages: Vec = { + let mut tabs = self.lock(); + if tabs.viewport == viewport { + return; + } + tabs.viewport = viewport; + tabs.list.iter().map(|tab| tab.session.clone()).collect() + }; + for page in pages { + let sized = self + .cdp + .call( + Some(&page), + "Emulation.setDeviceMetricsOverride", + viewport.metrics(), + ) + .await; + if let Err(err) = sized { + debug!("browser: sizing a tab: {err}"); + } + } + // The browser drops the frame of the new size while older ones wait + // to be confirmed, and a page that stands still sends no other + // (spec 19): once those are confirmed, the frames start over. + tokio::time::sleep(FRAME_GAP * 2).await; + let watched = { + let tabs = self.lock(); + tabs.active() + .filter(|_| tabs.watching) + .map(|tab| tab.session.clone()) + }; + if let Some(page) = watched { + self.cast(&page, false).await; + self.cast(&page, true).await; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_page_takes_the_shape_of_the_room_within_limits() { + let page = |width, height| Viewport::fitting(width, height); + // A panel 536 wide with 700 of height: the same shape, 1280 wide. + assert_eq!( + page(536, 700), + Viewport { + width: 1280, + height: 1671 + } + ); + assert_eq!(page(1280, 800), Viewport::default()); + // Too flat or too tall a room stops at the limits. + assert_eq!(page(1600, 300).height, MIN_HEIGHT); + assert_eq!(page(256, 1400).height, MAX_HEIGHT); + assert_eq!(page(0, 0).height, MIN_HEIGHT); + assert_eq!(page(1, u32::MAX).height, MAX_HEIGHT); + } + + #[test] + fn points_are_on_the_page_up_to_its_edges() { + let page = Viewport::fitting(640, 700); + assert!(page.contains(1280.0, 1400.0)); + assert!(!page.contains(10.0, 1400.5)); + assert!(!page.contains(-1.0, 10.0)); + assert!(!page.contains(f64::NAN, 10.0)); + } +} diff --git a/crates/botloftd/src/browser/watch.rs b/crates/botloftd/src/browser/watch.rs index c3bee5ae..d4cccf98 100644 --- a/crates/botloftd/src/browser/watch.rs +++ b/crates/botloftd/src/browser/watch.rs @@ -1,5 +1,6 @@ //! An app connection watching a bot's browser (spec 21.7). While anyone -//! watches, the active tab sends frames; when the last one stops, it stops. +//! watches, the active tab sends frames and the page is the size their +//! panel asks for (spec 21.3); when the last one stops, both stop. use std::sync::Arc; @@ -7,7 +8,7 @@ use botloft_core::ids::BotId; use botloft_core::protocol::BrowserFrame; use tokio::sync::watch; -use super::{Browsers, Slots, lock}; +use super::{Browsers, Slots, Viewport, lock}; /// Held by the connection; dropping it stops watching. pub struct Watching { @@ -25,38 +26,63 @@ impl Browsers { slot.watchers += 1; slot.frames.subscribe() }; - sync(Arc::clone(&self.slots), bot.clone()); + sync_soon(&self.slots, bot); Watching { slots: Arc::clone(&self.slots), bot: bot.clone(), frames, } } + + /// The room a watching app has for the page, `width` by `height`: the + /// page takes its shape, now or when the browser opens (spec 21.3). + pub fn resize(&self, bot: &BotId, width: u32, height: u32) { + { + let mut slots = lock(&self.slots); + self.slot(&mut slots, bot).viewport = Viewport::fitting(width, height); + } + sync_soon(&self.slots, bot); + } } impl Drop for Watching { fn drop(&mut self) { if let Some(slot) = lock(&self.slots).get_mut(&self.bot) { slot.watchers = slot.watchers.saturating_sub(1); + if slot.watchers == 0 { + // No panel to fit anymore. + slot.viewport = Viewport::default(); + } } - sync(Arc::clone(&self.slots), self.bot.clone()); + sync_soon(&self.slots, &self.bot); } } -/// Turns the frames on or off to match whether anyone watches. It reads -/// the count when it runs, so calls that finish out of order still end on -/// the right answer. -fn sync(slots: Slots, bot: BotId) { - let Ok(runtime) = tokio::runtime::Handle::try_current() else { +/// Makes the running browser match what the apps want: frames while anyone +/// watches, and the page the size they asked for. One at a time, each +/// reading what is wanted when its turn comes, so changes that finish out +/// of order still end on the right answer. +pub(super) async fn sync(slots: Slots, bot: BotId) { + let running = lock(&slots).get(&bot).and_then(|slot| { + slot.session + .as_ref() + .map(|(_, session)| Arc::clone(session)) + }); + let Some(session) = running else { return; }; - runtime.spawn(async move { - let found = lock(&slots).get(&bot).and_then(|slot| { - let session = slot.session.as_ref()?.1.clone(); - Some((session, slot.watchers > 0)) - }); - if let Some((session, on)) = found { - session.set_watching(on).await; - } - }); + let _turn = session.syncing.lock().await; + let wanted = lock(&slots) + .get(&bot) + .map(|slot| (slot.watchers > 0, slot.viewport)); + if let Some((on, viewport)) = wanted { + session.set_watching(on).await; + session.set_viewport(viewport).await; + } +} + +fn sync_soon(slots: &Slots, bot: &BotId) { + if let Ok(runtime) = tokio::runtime::Handle::try_current() { + runtime.spawn(sync(Arc::clone(slots), bot.clone())); + } } diff --git a/crates/botloftd/src/rpc/mod.rs b/crates/botloftd/src/rpc/mod.rs index b5105ef1..c8982a46 100644 --- a/crates/botloftd/src/rpc/mod.rs +++ b/crates/botloftd/src/rpc/mod.rs @@ -184,10 +184,16 @@ fn handle( frames.send_replace(None); watch.request(daemon, &request.method, request.params) } - // So is the owner's hold on it (spec 21.10). - method::BROWSER_TAKE | method::BROWSER_RELEASE | method::BROWSER_INPUT => { - watch.request(daemon, &request.method, request.params) - } + // So are the size of its page and what the owner does in it (spec + // 21.10). + method::BROWSER_RESIZE + | method::BROWSER_TAKE + | method::BROWSER_RELEASE + | method::BROWSER_INPUT + | method::BROWSER_RELOAD + | method::BROWSER_NEW_TAB + | method::BROWSER_SWITCH_TAB + | method::BROWSER_OPEN => watch.request(daemon, &request.method, request.params), name => dispatch::dispatch(daemon, name, request.params), }; Some(match result { diff --git a/crates/botloftd/src/rpc/watching.rs b/crates/botloftd/src/rpc/watching.rs index 978db07b..120e7fb7 100644 --- a/crates/botloftd/src/rpc/watching.rs +++ b/crates/botloftd/src/rpc/watching.rs @@ -1,10 +1,13 @@ //! A connection watching one bot's browser (spec 21.7): `browser.watch` -//! and `browser.unwatch`, and the frames that follow. The same connection -//! takes the browser into the owner's hands and gives it back (spec 21.10). +//! and `browser.unwatch`, the frames that follow, and the size its panel +//! gives the page. The same connection reloads the page, takes the browser +//! into the owner's hands, moves between its tabs and gives it back (spec +//! 21.10). +use botloft_core::ids::BotId; use botloft_core::protocol::{ - ApprovalsAnswerParams, BrowserControlParams, BrowserInputParams, BrowserWatchParams, - error_code, method, notification, + ApprovalsAnswerParams, BrowserControlParams, BrowserInputParams, BrowserOpenParams, + BrowserResizeParams, BrowserTabParams, BrowserWatchParams, error_code, method, notification, }; use serde::Serialize; use serde_json::Value; @@ -16,6 +19,9 @@ use crate::browser::{Hands, InputError, Watching}; use crate::service::{ApiError, bots}; use crate::state::Daemon; +/// The most room a panel may say it has, in pixels each way. +const ROOM_MAX: u32 = 10_000; + #[derive(Default)] pub(super) struct Watch { current: Option, @@ -31,9 +37,14 @@ impl Watch { params: Option, ) -> Result { match name { + method::BROWSER_RESIZE => return self.resize(daemon, &parse(params)?), method::BROWSER_TAKE => return self.take(daemon, parse(params)?), method::BROWSER_RELEASE => return self.release(daemon, &parse(params)?), method::BROWSER_INPUT => return self.input(parse(params)?), + method::BROWSER_RELOAD => return self.reload(daemon, &parse(params)?), + method::BROWSER_NEW_TAB => return self.new_tab(&parse(params)?), + method::BROWSER_SWITCH_TAB => return self.switch_tab(&parse(params)?), + method::BROWSER_OPEN => return self.open(&parse(params)?), _ => {} } // Stop the one before, even if the new one is refused; the hands @@ -49,13 +60,34 @@ impl Watch { to_value(&daemon.browsers.view(¶ms.bot_id)) } + fn watches(&self, bot: &BotId) -> bool { + self.current + .as_ref() + .is_some_and(|watching| watching.bot == *bot) + } + + /// The page takes the shape of the room this connection's panel has. + fn resize(&self, daemon: &Daemon, params: &BrowserResizeParams) -> Result { + if !self.watches(¶ms.bot_id) { + return Err( + ApiError::Conflict("watch this browser before sizing it".to_owned()).into(), + ); + } + let fits = |side: u32| (1..=ROOM_MAX).contains(&side); + if !fits(params.width) || !fits(params.height) { + return Err( + ApiError::validation(format!("width and height go from 1 to {ROOM_MAX}")).into(), + ); + } + daemon + .browsers + .resize(¶ms.bot_id, params.width, params.height); + Ok(Value::Null) + } + fn take(&mut self, daemon: &Daemon, params: BrowserControlParams) -> Result { let bot = ¶ms.bot_id; - if self - .current - .as_ref() - .is_none_or(|watching| watching.bot != *bot) - { + if !self.watches(bot) { return Err( ApiError::Conflict("watch this browser before taking it".to_owned()).into(), ); @@ -93,19 +125,44 @@ impl Watch { to_value(&daemon.browsers.view(bot).state) } - fn input(&self, params: BrowserInputParams) -> Result { - let hands = self - .hands + /// The owner's hold on the bot's browser, if this connection has it. + fn hands(&self, bot: &BotId) -> Result<&Hands, RpcError> { + self.hands .as_ref() - .filter(|hands| hands.bot == params.bot_id) - .ok_or_else(|| ApiError::Conflict("take the browser before using it".to_owned()))?; - hands.send(params.input).map_err(|err| match err { - InputError::Invalid => ApiError::validation(err.to_string()), - InputError::NotHeld | InputError::Closed => ApiError::Conflict(err.to_string()), - })?; + .filter(|hands| hands.bot == *bot) + .ok_or_else(|| ApiError::Conflict("take the browser before using it".to_owned()).into()) + } + + fn input(&self, params: BrowserInputParams) -> Result { + made(self.hands(¶ms.bot_id)?.send(params.input)) + } + + /// Reloads the page of the watched browser; it needs no hold. + fn reload(&self, daemon: &Daemon, params: &BrowserControlParams) -> Result { + let bot = ¶ms.bot_id; + if !self.watches(bot) { + return Err( + ApiError::Conflict("watch this browser before reloading it".to_owned()).into(), + ); + } + if !daemon.browsers.reload(bot) { + return Err(ApiError::Conflict("the browser is not open".to_owned()).into()); + } Ok(Value::Null) } + fn new_tab(&self, params: &BrowserControlParams) -> Result { + made(self.hands(¶ms.bot_id)?.new_tab()) + } + + fn switch_tab(&self, params: &BrowserTabParams) -> Result { + made(self.hands(¶ms.bot_id)?.switch_tab(¶ms.tab_id)) + } + + fn open(&self, params: &BrowserOpenParams) -> Result { + made(self.hands(¶ms.bot_id)?.open(¶ms.url)) + } + /// The watched browser's next frame, as a notification. Never resolves /// while nothing is watched. pub(super) async fn next_frame(&mut self) -> Option { @@ -122,6 +179,16 @@ impl Watch { } } +/// What the owner did is on its way to the browser, or why it is not. +fn made(queued: Result<(), InputError>) -> Result { + queued.map_err(|err| match err { + InputError::Invalid | InputError::Address => ApiError::validation(err.to_string()), + InputError::NoTab => ApiError::NotFound(err.to_string()), + InputError::NotHeld | InputError::Closed => ApiError::Conflict(err.to_string()), + })?; + Ok(Value::Null) +} + fn to_value(value: &impl Serialize) -> Result { serde_json::to_value(value) .map_err(|err| RpcError::new(error_code::INTERNAL_ERROR, err.to_string())) diff --git a/crates/botloftd/src/tools/browser.rs b/crates/botloftd/src/tools/browser.rs index 427ce3f0..78b28d47 100644 --- a/crates/botloftd/src/tools/browser.rs +++ b/crates/botloftd/src/tools/browser.rs @@ -1,7 +1,7 @@ //! The browser tools (spec 21.4): each acts in the bot's own browser, asks //! the owner before a new site when the bot's mode asks first (spec 21.5), -//! waits while the owner has it (spec 21.10), and answers with the page as -//! it is after the action. +//! waits while the owner has it and reads first if they left another tab +//! open (spec 21.10), and answers with the page as it is after the action. use std::path::PathBuf; @@ -19,6 +19,14 @@ use crate::browser::{Done, OWNER_WAIT, Scroll, sites}; use crate::service::bots; use crate::state::Daemon; +/// What the bot reads when the owner gave its browser back on another tab +/// than the one it was on (spec 21.10). +const SWITCHED: &str = + "The owner switched tabs while they had your browser. This is the tab that is open now."; +const NOT_DONE: &str = "The owner switched tabs while they had your browser, so this was not \ + done: the page you were on is not the one open now. Call browser_look to read the tab that \ + is open, then decide what to do."; + /// What a browser tool answers. pub(super) enum Reply { Text(String), @@ -76,12 +84,16 @@ async fn run( first.", )?; ask_owner(daemon, id, generation, &session, task).await?; + // What they did last reaches the browser before the bot reads it. + daemon.browsers.wait_for_owner(id, OWNER_WAIT).await; let session = call.running().ok_or( "The owner is done, but your browser closed meanwhile. Open the page again with \ browser_open.", )?; let done = "The owner is done and gave your browser back. This is the page now."; - return answer(&session, &bot, 0, Some(vec![done.to_owned()])).await; + let mut notes = vec![done.to_owned()]; + notes.extend(switched(session.take_switched()).into_iter().flatten()); + return answer(&session, &bot, 0, Some(notes)).await; } // The owner has the browser: wait until they give it back. if !daemon.browsers.wait_for_owner(id, OWNER_WAIT).await { @@ -116,6 +128,7 @@ async fn run( .session(&downloads) .await .map_err(|err| unavailable(&err))?; + let moved = session.take_switched(); let result = session.open(&url).await; report( daemon, @@ -127,12 +140,17 @@ async fn run( }, ); result.map_err(|err| describe(&err))?; - return answer(&session, &bot, 0, None).await; + return answer(&session, &bot, 0, switched(moved)).await; } let session = call .running() .ok_or("Your browser is not open. Open a page with browser_open first.")?; + // On a tab the owner left open, the bot reads before it acts. + let moved = session.take_switched(); + if moved && tool.acts() { + return Err(NOT_DONE.to_owned()); + } let url = session .lock() .active() @@ -140,11 +158,14 @@ async fn run( .unwrap_or_default(); page_allowed(daemon, id, generation, &bot, &session, &url).await?; let (kind, done) = match tool { - Tool::Look { from } => return answer(&session, &bot, from, None).await, + Tool::Look { from } => return answer(&session, &bot, from, switched(moved)).await, Tool::Screenshot => { let data = session.screenshot().await.map_err(|err| describe(&err))?; let reading = session.read(0).await.map_err(|err| describe(&err))?; - let line = format!("Page: {}\nURL: {}", reading.title, reading.url); + let mut line = format!("Page: {}\nURL: {}", reading.title, reading.url); + if moved { + line = format!("{line}\n{SWITCHED}"); + } return Ok(Reply::Picture(data, line)); } Tool::Click { reference } => (BrowserActionKind::Click, session.click(&reference).await), @@ -198,6 +219,11 @@ async fn run( answer(&session, &bot, 0, Some(notes)).await } +/// The note for the bot when the owner left it on another tab. +fn switched(moved: bool) -> Option> { + moved.then(|| vec![SWITCHED.to_owned()]) +} + fn about(daemon: &Daemon, id: &BotId) -> Result { let store = daemon.store(); let (crew, record) = bots::active(&store, id).map_err(explain)?; diff --git a/crates/botloftd/src/tools/browser_args.rs b/crates/botloftd/src/tools/browser_args.rs index d4601074..7bef257c 100644 --- a/crates/botloftd/src/tools/browser_args.rs +++ b/crates/botloftd/src/tools/browser_args.rs @@ -42,6 +42,22 @@ pub(super) enum Tool { }, } +impl Tool { + /// Whether it does something to the page, beyond reading it or going + /// to an address. + pub(super) fn acts(&self) -> bool { + matches!( + self, + Self::Click { .. } + | Self::Type { .. } + | Self::Select { .. } + | Self::Press { .. } + | Self::Scroll { .. } + | Self::Back + ) + } +} + /// Longest request for the owner's help, in characters. const TASK_MAX: usize = 500; diff --git a/crates/botloftd/src/tools/browser_catalog.rs b/crates/botloftd/src/tools/browser_catalog.rs index 262551a2..d84fbf27 100644 --- a/crates/botloftd/src/tools/browser_catalog.rs +++ b/crates/botloftd/src/tools/browser_catalog.rs @@ -156,7 +156,8 @@ pub(super) fn tools() -> Vec { "description": "Asks the owner to do something in your browser themselves and waits \ until they are done: signing in to their account, solving a captcha, typing a \ code sent to their phone, anything only they should or can do. Open the page \ - first; they take over your browser, and it comes back to you on the same page. \ + first; they take over your browser, and it comes back to you with the page as \ + they left it. \ Never ask for passwords, codes or card numbers in the chat: ask with this tool, \ so the owner types them where you never see them.", "inputSchema": { diff --git a/crates/botloftd/tests/browser.rs b/crates/botloftd/tests/browser.rs index 39d4bef6..d039a1a3 100644 --- a/crates/botloftd/tests/browser.rs +++ b/crates/botloftd/tests/browser.rs @@ -181,7 +181,12 @@ async fn new_tabs_and_dialogs_reach_the_bot_and_the_owner_watches() { .expect("new tab"); assert!(tab.contains("Thanks, Tab"), "{tab}"); let open = b.app.call("browser.list", Value::Null).await.expect("list"); - assert_eq!(open[0]["tabs"], 2); + // The new tab is the active one, after the tab that opened it. + let tabs = open[0]["tabs"].as_array().expect("tabs"); + assert_eq!(tabs.len(), 2); + assert_eq!(tabs[0]["active"], false); + assert_eq!(tabs[1]["active"], true); + assert!(tabs[1]["url"].as_str().expect("url").contains("/done")); let picture = b.mcp.tool_result("browser_screenshot", json!({})).await; assert_eq!(picture["content"][0]["type"], "image"); diff --git a/crates/botloftd/tests/browser_panel.rs b/crates/botloftd/tests/browser_panel.rs new file mode 100644 index 00000000..7fd9943f --- /dev/null +++ b/crates/botloftd/tests/browser_panel.rs @@ -0,0 +1,114 @@ +//! The owner's panel on a bot's browser with the real Microsoft Edge (spec +//! 21.3, 21.7): the page taking the shape of the room the panel has. +//! Skipped where Edge is not installed. + +mod common; + +use std::time::Duration; + +use common::Client; +use common::browsing::{Browsing, answer_site, call, setup}; +use serde_json::{Value, json}; + +/// Reads the page until it says its window is `size`. +async fn until_window(b: &mut Browsing, size: &str) -> String { + let mut page = String::new(); + for _ in 0..50 { + page = b + .mcp + .tool_text("browser_look", json!({})) + .await + .expect("look"); + if page.contains(&format!("Window: {size}")) { + return page; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!("the window never became {size}:\n{page}"); +} + +/// The next frame that is `height` tall. +async fn frame_of(app: &mut Client, height: u64) -> Value { + for _ in 0..50 { + let frame = app.notification("browser.frame").await; + if frame["height"] == height { + return frame; + } + } + panic!("no frame came {height} tall"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn the_page_takes_the_shape_of_the_panel_while_the_owner_watches() { + let Some(mut b) = setup().await else { return }; + let id = b.bot["id"].clone(); + let room = |width: u32, height: u32| json!({ "botId": id, "width": width, "height": height }); + + // Only a connection watching the browser sizes it, and with sane numbers. + assert!(b.app.call("browser.resize", room(536, 700)).await.is_err()); + b.app + .call("browser.watch", json!({ "botId": id })) + .await + .expect("watch"); + assert!(b.app.call("browser.resize", room(0, 700)).await.is_err()); + assert!( + b.app + .call("browser.resize", room(536, 20_000)) + .await + .is_err() + ); + + // Asked before the browser opens: it opens that size. + b.app + .call("browser.resize", room(536, 700)) + .await + .expect("resize"); + let opening = call( + &b.mcp, + "browser_open", + json!({ "url": format!("{}/size", b.site) }), + ); + answer_site(&mut b.app, true, None).await; + let page = opening.await.expect("task").expect("page"); + assert!(page.contains("Window: 1280 x 1671"), "{page}"); + let frame = frame_of(&mut b.app, 1671).await; + assert_eq!(frame["width"], 1280); + + // The panel changes: the page follows, and so do the frames. + b.app + .call("browser.resize", room(640, 400)) + .await + .expect("resize"); + until_window(&mut b, "1280 x 800").await; + frame_of(&mut b.app, 800).await; + + // The owner's points go as far as the page does now. + b.app + .call("browser.resize", room(640, 700)) + .await + .expect("resize"); + until_window(&mut b, "1280 x 1400").await; + frame_of(&mut b.app, 1400).await; + b.app + .call("browser.take", json!({ "botId": id })) + .await + .expect("take"); + let click = |y: f64| { + json!({ "botId": id, "input": { + "kind": "mouse", "action": "move", "x": 10.0, "y": y, "button": "none", + "buttons": 0, "clicks": 0, "modifiers": 0, + }}) + }; + b.app + .call("browser.input", click(1390.0)) + .await + .expect("a point on the taller page"); + assert!(b.app.call("browser.input", click(1410.0)).await.is_err()); + + // Nobody watches anymore: back to the size bots work in alone. + b.app + .call("browser.unwatch", Value::Null) + .await + .expect("unwatch"); + until_window(&mut b, "1280 x 800").await; +} diff --git a/crates/botloftd/tests/browser_tabs.rs b/crates/botloftd/tests/browser_tabs.rs new file mode 100644 index 00000000..d95840a4 --- /dev/null +++ b/crates/botloftd/tests/browser_tabs.rs @@ -0,0 +1,229 @@ +//! The owner reloading a bot's page and moving between its tabs with the +//! real Microsoft Edge (spec 21.7, 21.10): the tabs the app sees, a new tab +//! taken to an address, switching, and what the bot reads afterwards. +//! Skipped where Edge is not installed. + +mod common; + +use std::time::Duration; + +use common::Client; +use common::browsing::{Browsing, answer_site, call, pending_approval, setup}; +use serde_json::{Value, json}; + +async fn browser(app: &mut Client) -> Value { + let list = app.call("browser.list", Value::Null).await.expect("list"); + list[0].clone() +} + +fn active(state: &Value) -> Value { + let tabs = state["tabs"].as_array().cloned().unwrap_or_default(); + tabs.into_iter() + .find(|tab| tab["active"] == true) + .unwrap_or(Value::Null) +} + +/// Waits until the bot's browser is as `ready` wants. +async fn until(app: &mut Client, what: &str, ready: impl Fn(&Value) -> bool) -> Value { + for _ in 0..100 { + let state = browser(app).await; + if ready(&state) { + return state; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!("the browser never got to: {what}"); +} + +/// The bot opens the page that counts its loads; the owner allows the site +/// and watches. +async fn visits(b: &mut Browsing) -> Value { + let url = format!("{}/visits", b.site); + let opening = call(&b.mcp, "browser_open", json!({ "url": url })); + answer_site(&mut b.app, true, None).await; + let page = opening.await.expect("task").expect("page"); + assert!(page.contains("Loaded 1 times"), "{page}"); + json!({ "botId": b.bot["id"] }) +} + +async fn look(b: &mut Browsing) -> String { + b.mcp + .tool_text("browser_look", json!({})) + .await + .expect("look") +} + +#[tokio::test(flavor = "multi_thread")] +async fn the_owner_reloads_the_page_without_taking_the_browser() { + let Some(mut b) = setup().await else { return }; + let id = visits(&mut b).await; + + // Only a connection watching the browser reloads it. + assert!(b.app.call("browser.reload", id.clone()).await.is_err()); + b.app + .call("browser.watch", id.clone()) + .await + .expect("watch"); + b.app + .call("browser.reload", id.clone()) + .await + .expect("reload"); + assert_eq!(browser(&mut b.app).await["control"], "bot"); + + // The bot reads the page again, and why it changed. + let mut page = String::new(); + for _ in 0..50 { + page.push_str(&look(&mut b).await); + if page.contains("Loaded 2 times") { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + assert!(page.contains("Loaded 2 times"), "{page}"); + assert!(page.contains("The owner reloaded the page."), "{page}"); + let again = look(&mut b).await; + assert!(!again.contains("reloaded"), "{again}"); + + b.mcp + .tool_text("browser_close", json!({})) + .await + .expect("closed"); + assert!(b.app.call("browser.reload", id).await.is_err()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn the_owner_moves_between_tabs_and_the_bot_reads_where_it_is() { + let Some(mut b) = setup().await else { return }; + let id = visits(&mut b).await; + let bot = b.bot["id"].clone(); + let tab = |tab: &Value| json!({ "botId": bot, "tabId": tab }); + let address = |url: &str| json!({ "botId": bot, "url": url }); + b.app + .call("browser.watch", id.clone()) + .await + .expect("watch"); + + let state = until(&mut b.app, "the page's title", |state| { + state["tabs"][0]["title"] == "Visits" + }) + .await; + assert_eq!(state["tabs"].as_array().map(Vec::len), Some(1)); + assert_eq!(state["tabs"][0]["active"], true); + let first = state["tabs"][0]["id"].clone(); + + // Tabs and addresses are for the owner's hands only. + assert!(b.app.call("browser.newTab", id.clone()).await.is_err()); + assert!(b.app.call("browser.switchTab", tab(&first)).await.is_err()); + assert!(b.app.call("browser.open", address(&b.site)).await.is_err()); + b.app.call("browser.take", id.clone()).await.expect("take"); + + // A new tab, blank and active, after the one the bot opened. + b.app + .call("browser.newTab", id.clone()) + .await + .expect("new tab"); + let state = until(&mut b.app, "a blank second tab", |state| { + active(state)["url"] == "about:blank" + }) + .await; + assert_eq!(state["tabs"].as_array().map(Vec::len), Some(2)); + assert_eq!(state["tabs"][0]["id"], first); + assert_eq!(state["tabs"][1]["active"], true); + + // The owner types where it goes; only web addresses open. + for refused in ["javascript:alert(1)", "file:///C:/x.html", "two words"] { + let error = b.app.call("browser.open", address(refused)).await; + assert!(error.is_err(), "{refused} opened"); + } + let done = format!("{}/done?name=Owner&size=Small", b.site); + b.app + .call("browser.open", address(&done)) + .await + .expect("open"); + let state = until(&mut b.app, "the page the owner opened", |state| { + active(state)["title"] == "Thanks" && state["loading"] == false + }) + .await; + assert!(state["url"].as_str().expect("url").contains("/done")); + let second = active(&state)["id"].clone(); + assert_ne!(second, first); + + // Back to the first tab: it is the active one, and the frames are its. + let gone = b.app.call("browser.switchTab", tab(&json!("nope"))).await; + assert!(gone.expect_err("no such tab").message.contains("not open")); + b.app.forget("browser.frame"); + b.app + .call("browser.switchTab", tab(&first)) + .await + .expect("switch"); + let state = until(&mut b.app, "the first tab again", |state| { + active(state)["id"] == first + }) + .await; + assert_eq!(state["title"], "Visits"); + let frame = b.app.notification("browser.frame").await; + assert_eq!(frame["botId"], bot); + + // Given back on the tab the bot was on: nothing to tell it. + b.app + .call("browser.release", id.clone()) + .await + .expect("release"); + let page = look(&mut b).await; + assert!(page.contains("Loaded 1 times"), "{page}"); + assert!(!page.contains("switched tabs"), "{page}"); + + // Given back on another tab: the bot does not act before it reads. + b.app.call("browser.take", id.clone()).await.expect("take"); + b.app + .call("browser.switchTab", tab(&second)) + .await + .expect("switch"); + b.app + .call("browser.release", id.clone()) + .await + .expect("release"); + let refused = b + .mcp + .tool_text("browser_press", json!({ "key": "Enter" })) + .await + .expect_err("not done"); + assert!(refused.contains("switched tabs"), "{refused}"); + assert!(refused.contains("browser_look"), "{refused}"); + let page = look(&mut b).await; + assert!(page.contains("# Thanks, Owner"), "{page}"); + assert!(!page.contains("switched tabs"), "{page}"); + + // A tool that only reads says so with the page, once. + b.app.call("browser.take", id.clone()).await.expect("take"); + b.app + .call("browser.switchTab", tab(&first)) + .await + .expect("switch"); + b.app + .call("browser.release", id.clone()) + .await + .expect("release"); + let page = look(&mut b).await; + assert!(page.contains("The owner switched tabs"), "{page}"); + assert!(page.contains("Loaded 1 times"), "{page}"); + assert!(!look(&mut b).await.contains("switched tabs")); + + // So does the answer to a request for help. + let asking = call( + &b.mcp, + "browser_ask_owner", + json!({ "task": "Pick the page" }), + ); + pending_approval(&mut b.app).await; + b.app.call("browser.take", id.clone()).await.expect("take"); + b.app + .call("browser.switchTab", tab(&second)) + .await + .expect("switch"); + b.app.call("browser.release", id).await.expect("release"); + let page = asking.await.expect("task").expect("done"); + assert!(page.contains("The owner is done"), "{page}"); + assert!(page.contains("The owner switched tabs"), "{page}"); + assert!(page.contains("# Thanks, Owner"), "{page}"); +} diff --git a/crates/botloftd/tests/common/mod.rs b/crates/botloftd/tests/common/mod.rs index 8d14da67..b6da920b 100644 --- a/crates/botloftd/tests/common/mod.rs +++ b/crates/botloftd/tests/common/mod.rs @@ -279,6 +279,11 @@ impl Client { } } + /// Drops the notifications named `method` that came so far. + pub fn forget(&mut self, method: &str) { + self.notifications.retain(|n| n["method"] != method); + } + /// True when the server closes the connection without sending more. pub async fn closed(&mut self) -> bool { self.recv().await.is_none() diff --git a/crates/botloftd/tests/common/site.rs b/crates/botloftd/tests/common/site.rs index 24d7c222..483f5f58 100644 --- a/crates/botloftd/tests/common/site.rs +++ b/crates/botloftd/tests/common/site.rs @@ -1,10 +1,12 @@ //! A small web site on 127.0.0.1 for the browser tests: a form, the page //! it leads to, a link that opens a new tab, a button that shows a dialog, -//! and a sign-in page with its fields at fixed points, for the owner's -//! hands. +//! a sign-in page with its fields at fixed points, for the owner's hands, +//! a page that says how big its window is, and one that counts its loads. use std::collections::HashMap; use std::net::SocketAddr; +use std::sync::Arc; +use std::sync::atomic::{AtomicU32, Ordering}; use axum::Form; use axum::Router; @@ -30,6 +32,11 @@ const LOGIN: &str = "Sign in