diff --git a/app/src/dev/preview.ts b/app/src/dev/preview.ts index ae9308dd..e397c48a 100644 --- a/app/src/dev/preview.ts +++ b/app/src/dev/preview.ts @@ -10,6 +10,7 @@ import { FakeBotloft } from "../lib/fake"; import { FakeHost } from "../lib/fakeHost"; import type { Host } from "../lib/host"; import { PROTOCOL_VERSION } from "../lib/protocol.gen"; +import { seedBrowser } from "./seedBrowser"; import { seedChats } from "./seedChats"; import { seedChief } from "./seedChief"; import { seedRoutines } from "./seedRoutines"; @@ -111,6 +112,7 @@ function seed(fake: FakeBotloft): void { planner: planner.id, }); seedRoutines(fake, { scout: scout.id, analyst: analyst.id }); + seedBrowser(fake, scout.id); const ops = fake.addCrew("Ops"); const deploy = fake.addBot(ops.id, "Deploy", "Ships the site on Fridays"); const watcher = fake.addBot(ops.id, "Watcher", "Keeps an eye on the error log"); diff --git a/app/src/dev/seedBrowser.ts b/app/src/dev/seedBrowser.ts new file mode 100644 index 00000000..e968486d --- /dev/null +++ b/app/src/dev/seedBrowser.ts @@ -0,0 +1,130 @@ +// Dev only: Scout searching in its browser in the fake preview, so the +// browser panel has something live to show. The "page" is drawn on a +// canvas and sent as JPEG frames, like the daemon's screencast. + +import type { FakeBotloft } from "../lib/fake"; +import type { BotId } from "../lib/protocol.gen"; + +const W = 1280; +const H = 800; + +interface Scene { + url: string; + title: string; + query: string; + results: boolean; +} + +function draw(scene: Scene): string { + const canvas = document.createElement("canvas"); + canvas.width = W; + canvas.height = H; + const g = canvas.getContext("2d"); + if (!g) { + return ""; + } + g.fillStyle = "#ffffff"; + g.fillRect(0, 0, W, H); + g.fillStyle = "#1f2937"; + g.font = "600 34px system-ui, sans-serif"; + g.fillText(scene.results ? "Results" : "Find a recipe", 120, 150); + g.strokeStyle = "#9ca3af"; + g.lineWidth = 2; + g.beginPath(); + g.roundRect(120, 190, 760, 56, 28); + g.stroke(); + g.fillStyle = scene.query ? "#111827" : "#9ca3af"; + g.font = "22px system-ui, sans-serif"; + g.fillText(scene.query || "Search recipes", 150, 226); + g.fillStyle = "#2563eb"; + g.beginPath(); + g.roundRect(900, 190, 150, 56, 28); + g.fill(); + g.fillStyle = "#ffffff"; + g.fillText("Search", 938, 226); + if (scene.results) { + for (let row = 0; row < 4; row++) { + const y = 300 + row * 110; + g.fillStyle = "#1d4ed8"; + g.font = "600 24px system-ui, sans-serif"; + g.fillText( + ["Classic sourdough loaf", "Starter in 5 days", "No-knead rye", "Focaccia"][row] ?? "", + 120, + y, + ); + g.fillStyle = "#e5e7eb"; + g.fillRect(120, y + 18, 820, 14); + g.fillRect(120, y + 42, 560, 14); + } + } + return canvas.toDataURL("image/jpeg", 0.7).split(",")[1] ?? ""; +} + +/** Scout searches for a recipe, over and over. */ +export function seedBrowser(fake: FakeBotloft, scout: BotId): void { + const home: Scene = { + url: "https://recipes.example/", + title: "Recipes", + query: "", + results: false, + }; + const steps: ((scene: Scene) => Scene)[] = [ + (scene) => { + fake.browser.act(scout, "open", { label: "recipes.example" }); + return scene; + }, + (scene) => { + fake.browser.act(scout, "type", { x: 500, y: 218, label: "Search recipes" }); + return { ...scene, query: "sourdough" }; + }, + (scene) => { + fake.browser.act(scout, "click", { x: 975, y: 218, label: "Search" }); + return { ...scene, url: "https://recipes.example/search?q=sourdough", results: true }; + }, + (scene) => { + fake.browser.act(scout, "click", { x: 260, y: 300, label: "Classic sourdough loaf" }); + return scene; + }, + () => { + fake.browser.act(scout, "back"); + return home; + }, + ]; + // How it reads in the chat: the site the owner allowed, then the calls. + const asked = fake.chat.ask( + scout, + "mcp__botloft__browser", + "recipes.example", + JSON.stringify({ site: "recipes.example", url: home.url }), + ); + if (asked.body.kind === "approval") { + void fake.call("approvals.answer", { approvalId: asked.body.approvalId, allow: true }); + } + const calls: [string, string][] = [ + ["browser_open", home.url], + ["browser_type", "e4"], + ["browser_click", "e5"], + ]; + for (const [tool, summary] of calls) { + fake.chat.tool(scout, `mcp__botloft__${tool}`, { summary, status: "done" }); + } + fake.chat.ask( + scout, + "mcp__botloft__browser", + "wikipedia.org", + JSON.stringify({ site: "wikipedia.org", url: "https://en.wikipedia.org/wiki/Sourdough" }), + ); + let scene = home; + let step = 0; + fake.browser.open(scout, home.url, home.title); + fake.browser.frame(scout, draw(scene)); + setInterval(() => { + const next = steps[step % steps.length]; + step += 1; + if (next) { + scene = next(scene); + fake.browser.open(scout, scene.url, scene.title); + fake.browser.frame(scout, draw(scene)); + } + }, 2200); +} diff --git a/app/src/features/bots/BotHeader.tsx b/app/src/features/bots/BotHeader.tsx index d446b10d..b93bbdb4 100644 --- a/app/src/features/bots/BotHeader.tsx +++ b/app/src/features/bots/BotHeader.tsx @@ -4,6 +4,7 @@ import { Ellipsis, Files, FolderOpen, + Globe, PanelRight, Pause, Pencil, @@ -36,6 +37,9 @@ export function BotHeader({ filesOpen, freshFiles, onToggleFiles, + browserOpen, + browsing, + onToggleBrowser, }: { bot: Bot; crew: Crew; @@ -45,6 +49,10 @@ export function BotHeader({ /** Files that showed up since the owner last looked. */ freshFiles: number; onToggleFiles(): void; + browserOpen: boolean; + /** The bot's browser is open (spec 21.8). */ + browsing: boolean; + onToggleBrowser(): void; }) { const t = useT(); const words = t.bots.header; @@ -107,6 +115,28 @@ export function BotHeader({ + + + + + + ); +} diff --git a/app/src/features/browser/browser.test.tsx b/app/src/features/browser/browser.test.tsx new file mode 100644 index 00000000..1188e626 --- /dev/null +++ b/app/src/features/browser/browser.test.tsx @@ -0,0 +1,151 @@ +import { act, cleanup, fireEvent, screen, waitFor, within } from "@testing-library/react"; +import { afterEach, describe, expect, test } from "vitest"; +import { FakeBotloft } from "../../lib/fake"; +import { crewOpened, openBot, renderApp } from "../../test/app"; + +afterEach(cleanup); + +/** A crew "Ops" with @scout, idle. */ +function crew() { + const fake = new FakeBotloft(); + const ops = fake.addCrew("Ops"); + const scout = fake.addBot(ops.id, "Scout", "Finds sources"); + fake.setBotState(scout.id, "idle"); + return { fake, scout }; +} + +async function openScout(fake: FakeBotloft) { + const rendered = renderApp(fake); + await crewOpened("Ops"); + openBot("Scout"); + await screen.findByRole("list", { name: "Messages" }); + return rendered; +} + +const toggle = () => screen.getByRole("button", { name: /Show browser|Hide browser/ }); +const panel = () => screen.getByRole("complementary", { name: "Scout's browser" }); + +describe("browser panel", () => { + test("watches the bot's browser while open, and says when it has not opened one", async () => { + const { fake, scout } = crew(); + await openScout(fake); + fireEvent.click(toggle()); + expect(await within(panel()).findByText("Scout hasn't opened the browser yet")).toBeDefined(); + await waitFor(() => expect(fake.browser.watching).toBe(scout.id)); + + fireEvent.click(within(panel()).getByRole("button", { name: "Close" })); + await waitFor(() => expect(fake.browser.watching).toBeNull()); + expect(fake.browser.watches).toEqual([scout.id, null]); + }); + + test("shows the live page, its address, and the bot's cursor where it clicks", async () => { + const { fake, scout } = crew(); + fake.browser.open(scout.id, "https://example.com/signin", "Sign in"); + await openScout(fake); + fireEvent.click(toggle()); + const address = await within(panel()).findByRole("textbox", { name: "Address" }); + expect((address as HTMLInputElement).value).toBe("https://example.com/signin"); + await waitFor(() => expect(fake.browser.watching).toBe(scout.id)); + + act(() => { + fake.browser.frame(scout.id, "AAAA"); + fake.browser.act(scout.id, "click", { x: 640, y: 400, label: "Sign in" }); + }); + const screenshot = within(panel()).getByRole("figure", { name: "What Scout sees" }); + const image = screenshot.querySelector("img"); + expect(image?.getAttribute("src")).toBe("data:image/jpeg;base64,AAAA"); + const cursor = screenshot.querySelector(".browser-cursor") as HTMLElement; + expect(cursor.style.left).toBe("50%"); + expect(cursor.style.top).toBe("50%"); + expect(within(panel()).getByText("Live")).toBeDefined(); + expect(await within(panel()).findByText("Clicked Sign in")).toBeDefined(); + + // Another bot's actions do not move this cursor. + const other = fake.addBot(scout.crewId, "Writer"); + act(() => { + fake.browser.act(other.id, "open", { label: "elsewhere.com" }); + }); + expect(within(panel()).queryByText("Opened elsewhere.com")).toBeNull(); + }); + + test("the button marks a bot that is browsing while the panel is closed", async () => { + const { fake, scout } = crew(); + await openScout(fake); + expect(toggle().getAttribute("aria-label")).toBe("Show browser"); + act(() => { + fake.browser.open(scout.id, "https://example.com/", "Example"); + }); + await waitFor(() => + expect(toggle().getAttribute("aria-label")).toBe("Show browser: Scout is using the browser"), + ); + fireEvent.click(toggle()); + expect(toggle().getAttribute("aria-label")).toBe("Hide browser"); + }); + + test("a closed browser keeps its last page, faded; one that failed says why", async () => { + const { fake, scout } = crew(); + fake.browser.open(scout.id, "https://example.com/", "Example"); + await openScout(fake); + fireEvent.click(toggle()); + await waitFor(() => expect(fake.browser.watching).toBe(scout.id)); + act(() => { + fake.browser.frame(scout.id, "BBBB"); + }); + act(() => { + fake.browser.close(scout.id); + }); + expect(await within(panel()).findByText("Browser closed")).toBeDefined(); + expect(within(panel()).getByRole("figure").querySelector("img")?.className).toContain( + "grayscale", + ); + + act(() => { + fake.browser.set(scout.id, { + status: "failed", + error: "Microsoft Edge was not found on this computer", + }); + }); + expect(await within(panel()).findByText("The browser couldn't open")).toBeDefined(); + expect( + within(panel()).getByText("Microsoft Edge was not found on this computer"), + ).toBeDefined(); + }); + + test("a browser line in the chat opens the panel", async () => { + const { fake, scout } = crew(); + fake.chat.tool(scout.id, "mcp__botloft__browser_open", { + summary: "https://example.com", + status: "done", + }); + fake.chat.tool(scout.id, "Bash", { summary: "ls", status: "done" }); + await openScout(fake); + expect(screen.getAllByRole("button", { name: /Watch in browser/ })).toHaveLength(1); + fireEvent.click(screen.getByRole("button", { name: "Watch in browser: browser_open" })); + expect(await screen.findByRole("complementary", { name: "Scout's browser" })).toBeDefined(); + }); +}); + +describe("site requests", () => { + test("show the site and its address, and allowing it answers the request", async () => { + const { fake, scout } = crew(); + const item = fake.chat.ask( + scout.id, + "mcp__botloft__browser", + "wikipedia.org", + JSON.stringify({ site: "wikipedia.org", url: "https://wikipedia.org/wiki/Bread" }), + ); + await openScout(fake); + const card = await screen.findByRole("region", { name: "Scout asks to use wikipedia.org" }); + expect(within(card).getByText("https://wikipedia.org/wiki/Bread")).toBeDefined(); + fireEvent.click(within(card).getByRole("button", { name: "Allow" })); + await waitFor(() => + expect(fake.calls).toContainEqual({ + method: "approvals.answer", + params: { + approvalId: item.body.kind === "approval" ? item.body.approvalId : "", + allow: true, + }, + }), + ); + }); +}); diff --git a/app/src/features/browser/showBrowser.ts b/app/src/features/browser/showBrowser.ts new file mode 100644 index 00000000..b6d50262 --- /dev/null +++ b/app/src/features/browser/showBrowser.ts @@ -0,0 +1,11 @@ +// How a browser tool line in the chat opens the browser panel (spec 21.8). +// Absent where there is no panel, so no button shows. + +import { createContext } from "react"; + +export const ShowBrowser = createContext<(() => void) | null>(null); + +/** `mcp__botloft__browser_open` and the other browser tools. */ +export function isBrowserTool(name: string): boolean { + return name.startsWith("mcp__botloft__browser_"); +} diff --git a/app/src/features/browser/useBrowserView.ts b/app/src/features/browser/useBrowserView.ts new file mode 100644 index 00000000..3dbdb72e --- /dev/null +++ b/app/src/features/browser/useBrowserView.ts @@ -0,0 +1,60 @@ +// Watching a bot's browser (spec 21.7): while the panel is open the app +// asks for its live frames and hears what the bot does, and stops asking +// when the panel closes. The browser's state itself lives in the store. + +import { useEffect, useState } from "react"; +import type { Bot, BrowserAction, BrowserFrame } from "../../lib/protocol.gen"; +import { useApi, useApp } from "../../store/context"; + +export interface BrowserView { + /** The newest picture of the page, if any came. */ + frame: BrowserFrame | null; + /** What the bot did last, for the cursor and the caption. */ + action: BrowserAction | null; +} + +export function useBrowserView(bot: Bot, watching: boolean): BrowserView { + const api = useApi(); + const connected = useApp((state) => state.connection.kind === "open"); + const putBrowser = useApp((state) => state.putBrowser); + const [frame, setFrame] = useState(null); + const [action, setAction] = useState(null); + + // biome-ignore lint/correctness/useExhaustiveDependencies: another bot's page is not this bot's + useEffect(() => { + setFrame(null); + setAction(null); + }, [bot.id]); + + useEffect(() => { + if (!watching || !connected) { + return; + } + let alive = true; + const unsubscribe = api.subscribe((event) => { + if (event.name === "browser.frame" && event.params.botId === bot.id) { + setFrame(event.params); + } else if (event.name === "browser.action" && event.params.botId === bot.id) { + setAction(event.params); + } + }); + api.call("browser.watch", { botId: bot.id }).then( + (view) => { + if (alive) { + putBrowser(view.state); + if (view.frame) { + setFrame((current) => current ?? view.frame); + } + } + }, + () => {}, + ); + return () => { + alive = false; + unsubscribe(); + api.call("browser.unwatch").catch(() => {}); + }; + }, [api, bot.id, watching, connected, putBrowser]); + + return { frame, action }; +} diff --git a/app/src/features/chat/ApprovalCard.tsx b/app/src/features/chat/ApprovalCard.tsx index aa64fef3..4c5c3566 100644 --- a/app/src/features/chat/ApprovalCard.tsx +++ b/app/src/features/chat/ApprovalCard.tsx @@ -6,6 +6,7 @@ import { useId } from "react"; import { useT } from "../../i18n"; import type { ApprovalItem, Bot } from "../../lib/protocol.gen"; import { Button } from "../../ui/Button"; +import { SITE_TOOL, SiteCard } from "../browser/SiteCard"; import { PLAN_TOOL, PlanCard } from "./PlanCard"; import { SUGGEST_TOOL, SuggestionCard } from "./SuggestionCard"; import { pretty, toolLabel } from "./ToolLines"; @@ -50,6 +51,8 @@ export function ApprovalCard({ approval, bot }: { approval: ApprovalItem; bot: B return ; case SUGGEST_TOOL: return ; + case SITE_TOOL: + return ; default: return ; } diff --git a/app/src/features/chat/ToolLines.tsx b/app/src/features/chat/ToolLines.tsx index 74cd853d..4602c661 100644 --- a/app/src/features/chat/ToolLines.tsx +++ b/app/src/features/chat/ToolLines.tsx @@ -24,6 +24,7 @@ import { useContext, useState } from "react"; import { useT } from "../../i18n"; import type { ChatItem, ToolItem } from "../../lib/protocol.gen"; import { useArrival } from "../../ui/motion"; +import { isBrowserTool, ShowBrowser } from "../browser/showBrowser"; import { ShowFile } from "../files/showFile"; const ICONS: Record = { @@ -46,6 +47,11 @@ const ICONS: Record = { suggest_bot: UserPlus, }; +/** The browser tools share the globe (spec 21.4). */ +function iconOf(label: string): LucideIcon { + return ICONS[label] ?? (label.startsWith("browser_") ? Globe : Wrench); +} + /** `mcp__botloft__send_message` reads as `send_message`. */ export function toolLabel(name: string): string { return name.split("__").at(-1) ?? name; @@ -82,9 +88,10 @@ function ToolLine({ tool, createdAt }: { tool: ToolItem; createdAt: number }) { const t = useT(); const [open, setOpen] = useState(false); const showFile = useContext(ShowFile); + const showBrowser = useContext(ShowBrowser); const arrival = useArrival(createdAt); const label = toolLabel(tool.name); - const Icon = ICONS[label] ?? Wrench; + const Icon = iconOf(label); const block = "max-h-64 overflow-auto whitespace-pre-wrap break-all rounded-lg border border-line bg-sunken px-2.5 py-1.5 font-mono text-xs leading-relaxed"; return ( @@ -122,6 +129,17 @@ function ToolLine({ tool, createdAt }: { tool: ToolItem; createdAt: number }) { )} + {showBrowser && isBrowserTool(tool.name) && ( + + )} {open && (
diff --git a/app/src/i18n/en/browser.ts b/app/src/i18n/en/browser.ts new file mode 100644 index 00000000..833e6f86 --- /dev/null +++ b/app/src/i18n/en/browser.ts @@ -0,0 +1,50 @@ +// Each bot's own browser (spec 21.8): the panel beside its chat where the +// owner watches it live, and the card where it asks to use a new site. + +export const browser = { + heading: "Browser", + panel: (bot: string) => `${bot}'s browser`, + show: "Show browser", + hide: "Hide browser", + browsing: (bot: string) => `${bot} is using the browser`, + showInPanel: "Watch in browser", + close: "Close", + expand: "Make wider", + shrink: "Make narrower", + openOutside: "Open in my browser", + openFailed: "Could not open the page", + live: "Live", + loading: "Loading…", + address: "Address", + screen: (bot: string) => `What ${bot} sees`, + emptyTitle: (bot: string) => `${bot} hasn't opened the browser yet`, + emptyBody: + "When it searches or uses a site, you'll see it here live: every page it opens, every click.", + starting: "Opening the browser…", + closed: "Browser closed", + closedBody: "It opens again when the bot needs it. Logins stay.", + failedTitle: "The browser couldn't open", + failedBody: + "Botloft uses Microsoft Edge, which comes with Windows. Check that it's installed, then ask the bot to try again.", + details: "Details", + tabs: (count: number) => (count === 1 ? "1 tab" : `${count} tabs`), + did: { + open: (site: string) => `Opened ${site}`, + click: (what: string) => `Clicked ${what}`, + clickSomewhere: "Clicked", + type: (what: string) => `Typed in ${what}`, + typeSomewhere: "Typed", + select: (what: string) => `Chose in ${what}`, + press: (key: string) => `Pressed ${key}`, + scroll: "Scrolled", + back: "Went back", + }, + site: { + wants: (bot: string) => `${bot} wants to use the browser on`, + asks: (bot: string, site: string) => `${bot} asks to use ${site}`, + why: "Once you allow a site, it won't ask again for it.", + allowed: (site: string) => `You allowed ${site}`, + denied: (site: string) => `You declined ${site}`, + expired: (site: string) => `No answer about ${site}`, + }, +}; diff --git a/app/src/i18n/en/index.ts b/app/src/i18n/en/index.ts index c63bbaa0..71f17f73 100644 --- a/app/src/i18n/en/index.ts +++ b/app/src/i18n/en/index.ts @@ -2,6 +2,7 @@ import { account } from "./account"; import { bots } from "./bots"; +import { browser } from "./browser"; import { chat } from "./chat"; import { common } from "./common"; import { crews } from "./crews"; @@ -19,6 +20,7 @@ export const en = { updates, account, bots, + browser, chat, crews, files, diff --git a/app/src/i18n/es/browser.ts b/app/src/i18n/es/browser.ts new file mode 100644 index 00000000..c3dd0576 --- /dev/null +++ b/app/src/i18n/es/browser.ts @@ -0,0 +1,52 @@ +// El navegador de cada bot (spec 21.8): el panel junto al chat donde el +// dueño lo mira en vivo, y la tarjeta en la que pide usar un sitio nuevo. + +import type { Messages } from "../en"; + +export const browser: Messages["browser"] = { + heading: "Navegador", + panel: (bot) => `Navegador de ${bot}`, + show: "Mostrar navegador", + hide: "Ocultar navegador", + browsing: (bot) => `${bot} está usando el navegador`, + showInPanel: "Ver en el navegador", + close: "Cerrar", + expand: "Ensanchar", + shrink: "Estrechar", + openOutside: "Abrir en mi navegador", + openFailed: "No se pudo abrir la página", + live: "En vivo", + loading: "Cargando…", + address: "Dirección", + screen: (bot) => `Lo que ve ${bot}`, + emptyTitle: (bot) => `${bot} todavía no abrió el navegador`, + emptyBody: + "Cuando busque o use un sitio, lo verás aquí en vivo: cada página que abre, cada clic.", + starting: "Abriendo el navegador…", + closed: "Navegador cerrado", + closedBody: "Se abre de nuevo cuando el bot lo necesite. Las sesiones iniciadas siguen.", + failedTitle: "El navegador no se abrió", + failedBody: + "Botloft usa Microsoft Edge, que viene con Windows. Comprueba que esté instalado y pide al bot que lo intente de nuevo.", + details: "Detalles", + tabs: (count) => (count === 1 ? "1 pestaña" : `${count} pestañas`), + did: { + open: (site) => `Abrió ${site}`, + click: (what) => `Hizo clic en ${what}`, + clickSomewhere: "Hizo clic", + type: (what) => `Escribió en ${what}`, + typeSomewhere: "Escribió", + select: (what) => `Eligió en ${what}`, + press: (key) => `Pulsó ${key}`, + scroll: "Desplazó la página", + back: "Volvió atrás", + }, + site: { + wants: (bot) => `${bot} quiere usar el navegador en`, + asks: (bot, site) => `${bot} pide usar ${site}`, + why: "Cuando permites un sitio, no vuelve a preguntar por él.", + allowed: (site) => `Permitiste ${site}`, + denied: (site) => `Rechazaste ${site}`, + expired: (site) => `Sin respuesta sobre ${site}`, + }, +}; diff --git a/app/src/i18n/es/index.ts b/app/src/i18n/es/index.ts index 7bcab64d..2a7af3ae 100644 --- a/app/src/i18n/es/index.ts +++ b/app/src/i18n/es/index.ts @@ -3,6 +3,7 @@ import type { Messages } from "../en"; import { account } from "./account"; import { bots } from "./bots"; +import { browser } from "./browser"; import { chat } from "./chat"; import { common } from "./common"; import { crews } from "./crews"; @@ -20,6 +21,7 @@ export const es: Messages = { updates, account, bots, + browser, chat, crews, files, diff --git a/app/src/i18n/pt-BR/browser.ts b/app/src/i18n/pt-BR/browser.ts new file mode 100644 index 00000000..23a1ad94 --- /dev/null +++ b/app/src/i18n/pt-BR/browser.ts @@ -0,0 +1,52 @@ +// O navegador de cada bot (spec 21.8): o painel ao lado do chat onde o dono +// assiste ao vivo, e o cartão em que o bot pede para usar um site novo. + +import type { Messages } from "../en"; + +export const browser: Messages["browser"] = { + heading: "Navegador", + panel: (bot) => `Navegador de ${bot}`, + show: "Mostrar navegador", + hide: "Ocultar navegador", + browsing: (bot) => `${bot} está usando o navegador`, + showInPanel: "Ver no navegador", + close: "Fechar", + expand: "Alargar", + shrink: "Estreitar", + openOutside: "Abrir no meu navegador", + openFailed: "Não foi possível abrir a página", + live: "Ao vivo", + loading: "Carregando…", + address: "Endereço", + screen: (bot) => `O que ${bot} vê`, + emptyTitle: (bot) => `${bot} ainda não abriu o navegador`, + emptyBody: + "Quando ele pesquisar ou usar um site, você vê aqui ao vivo: cada página que ele abre, cada clique.", + starting: "Abrindo o navegador…", + closed: "Navegador fechado", + closedBody: "Ele abre de novo quando o bot precisar. Os logins continuam.", + failedTitle: "O navegador não abriu", + failedBody: + "O Botloft usa o Microsoft Edge, que vem com o Windows. Confira se ele está instalado e peça ao bot para tentar de novo.", + details: "Detalhes", + tabs: (count) => (count === 1 ? "1 aba" : `${count} abas`), + did: { + open: (site) => `Abriu ${site}`, + click: (what) => `Clicou em ${what}`, + clickSomewhere: "Clicou", + type: (what) => `Escreveu em ${what}`, + typeSomewhere: "Escreveu", + select: (what) => `Escolheu em ${what}`, + press: (key) => `Apertou ${key}`, + scroll: "Rolou a página", + back: "Voltou", + }, + site: { + wants: (bot) => `${bot} quer usar o navegador em`, + asks: (bot, site) => `${bot} pede para usar ${site}`, + why: "Depois que você permite um site, ele não pergunta de novo.", + allowed: (site) => `Você permitiu ${site}`, + denied: (site) => `Você recusou ${site}`, + expired: (site) => `Sem resposta sobre ${site}`, + }, +}; diff --git a/app/src/i18n/pt-BR/index.ts b/app/src/i18n/pt-BR/index.ts index 0a50f8eb..589a0d6f 100644 --- a/app/src/i18n/pt-BR/index.ts +++ b/app/src/i18n/pt-BR/index.ts @@ -3,6 +3,7 @@ import type { Messages } from "../en"; import { account } from "./account"; import { bots } from "./bots"; +import { browser } from "./browser"; import { chat } from "./chat"; import { common } from "./common"; import { crews } from "./crews"; @@ -20,6 +21,7 @@ export const ptBR: Messages = { updates, account, bots, + browser, chat, crews, files, diff --git a/app/src/lib/fake.ts b/app/src/lib/fake.ts index cce2421a..f6709d8e 100644 --- a/app/src/lib/fake.ts +++ b/app/src/lib/fake.ts @@ -4,6 +4,7 @@ import type { BotloftApi } from "./api"; import { botHandlers } from "./fakeBots"; +import { FakeBrowser } from "./fakeBrowser"; import { FakeChat } from "./fakeChat"; import { FakeConversation } from "./fakeConversation"; import { crewHandlers } from "./fakeCrews"; @@ -60,6 +61,7 @@ export class FakeBotloft implements BotloftApi { readonly conversation = new FakeConversation(this); readonly routines = new FakeRoutines(this); readonly files = new FakeFiles(this); + readonly browser = new FakeBrowser(this); /** Clock for created and updated times. */ now = Date.now(); closed = false; @@ -211,6 +213,7 @@ export class FakeBotloft implements BotloftApi { ...this.chat.handlers(), ...this.conversation.handlers(), ...this.files.handlers(), + ...this.browser.handlers(), ...this.routines.handlers(), }; } diff --git a/app/src/lib/fakeBrowser.ts b/app/src/lib/fakeBrowser.ts new file mode 100644 index 00000000..e88b451a --- /dev/null +++ b/app/src/lib/fakeBrowser.ts @@ -0,0 +1,106 @@ +// The fake daemon's browsers (spec 21.7): each bot's state, the live frames +// of the one being watched and what the bot does, for the browser panel. + +import type { FakeBotloft, Handlers } from "./fake"; +import type { + BotId, + BrowserAction, + BrowserActionKind, + BrowserFrame, + BrowserState, +} from "./protocol.gen"; + +export class FakeBrowser { + private readonly states = new Map(); + private readonly frames = new Map(); + /** The bot whose browser the app watches, if any. */ + watching: BotId | null = null; + /** Every watch and unwatch, in order. */ + readonly watches: (BotId | null)[] = []; + + constructor(private readonly fake: FakeBotloft) {} + + state(botId: BotId): BrowserState { + return ( + this.states.get(botId) ?? { + botId, + status: "closed", + url: null, + title: null, + loading: false, + tabs: 0, + error: null, + updatedAt: this.fake.now, + } + ); + } + + /** Changes the bot's browser and tells the app. */ + set(botId: BotId, change: Partial): BrowserState { + this.fake.bot(botId); + const next = { ...this.state(botId), ...change, updatedAt: this.fake.now }; + this.states.set(botId, next); + this.fake.emit({ name: "browser.changed", params: next }); + return next; + } + + /** The bot opened a page. */ + open(botId: BotId, url: string, title: string): BrowserState { + return this.set(botId, { status: "open", url, title, loading: false, tabs: 1, error: null }); + } + + close(botId: BotId): BrowserState { + this.frames.delete(botId); + return this.set(botId, { + status: "closed", + url: null, + title: null, + loading: false, + tabs: 0, + }); + } + + /** A new picture of the page; sent only while the app watches that bot. */ + frame(botId: BotId, data: string, width = 1280, height = 800): void { + const frame: BrowserFrame = { botId, data, width, height }; + this.frames.set(botId, frame); + if (this.watching === botId) { + this.fake.emit({ name: "browser.frame", params: frame }); + } + } + + /** Something the bot did, for the cursor. */ + act( + botId: BotId, + kind: BrowserActionKind, + options: { x?: number; y?: number; label?: string } = {}, + ): BrowserAction { + const action: BrowserAction = { + botId, + kind, + x: options.x ?? null, + y: options.y ?? null, + label: options.label ?? null, + at: this.fake.now, + }; + this.fake.emit({ name: "browser.action", params: action }); + return action; + } + + handlers(): Pick { + return { + "browser.list": () => [...this.states.values()].filter((state) => state.status !== "closed"), + "browser.watch": ({ botId }) => { + this.fake.bot(botId, false); + this.watching = botId; + this.watches.push(botId); + return { state: this.state(botId), frame: this.frames.get(botId) ?? null }; + }, + "browser.unwatch": () => { + this.watching = null; + this.watches.push(null); + return null; + }, + }; + } +} diff --git a/app/src/lib/protocol.gen.ts b/app/src/lib/protocol.gen.ts index 0ac2262b..085e8ee3 100644 --- a/app/src/lib/protocol.gen.ts +++ b/app/src/lib/protocol.gen.ts @@ -779,6 +779,73 @@ export type RoutineIdParams = { routineId: RoutineId, }; export type RoutinesRunsParams = { routineId: RoutineId, before?: RoutineRunId, limit?: number, }; +/** + * Where a bot's browser is. + */ +export type BrowserStatus = "closed" | "starting" | "open" | "failed"; + +export type BrowserState = { botId: BotId, status: BrowserStatus, +/** + * The active tab's address. + */ +url: string | null, title: string | null, +/** + * The active tab is loading a page. + */ +loading: boolean, +/** + * Open tabs. + */ +tabs: number, +/** + * Why it could not start, when `failed`. + */ +error: string | null, +/** + * Unix time in milliseconds. + */ +updatedAt: number, }; + +/** + * One picture of the active tab, sent only to the connection watching it. + */ +export type BrowserFrame = { botId: BotId, +/** + * A JPEG, base64. + */ +data: string, +/** + * Size of the page it shows, in CSS pixels: the space of + * `BrowserAction` points. + */ +width: number, height: number, }; + +export type BrowserActionKind = "open" | "click" | "type" | "select" | "press" | "scroll" | "back"; + +/** + * Something the bot did in its browser, for the cursor in the app. + */ +export type BrowserAction = { botId: BotId, kind: BrowserActionKind, +/** + * Where on the page, in CSS pixels, when the action has a point. + */ +x: number | null, y: number | null, +/** + * The element's name, the key or the site; never the text typed. + */ +label: string | null, +/** + * Unix time in milliseconds. + */ +at: number, }; + +/** + * What `browser.watch` answers: the state and the latest frame. + */ +export type BrowserView = { state: BrowserState, frame: BrowserFrame | null, }; + +export type BrowserWatchParams = { botId: BotId, }; + /** Params and result of every request method. */ export interface RpcMethods { "session.hello": { params: HelloParams; result: HelloResult }; @@ -816,6 +883,9 @@ export interface RpcMethods { "routines.runNow": { params: RoutineIdParams; result: RoutineRun }; "routines.archive": { params: RoutineIdParams; result: Routine }; "routines.runs": { params: RoutinesRunsParams; result: Array }; + "browser.list": { params: undefined; result: Array }; + "browser.watch": { params: BrowserWatchParams; result: BrowserView }; + "browser.unwatch": { params: undefined; result: null }; } /** Params of every server notification. */ @@ -830,6 +900,9 @@ export interface RpcNotifications { "task.changed": Task; "routine.changed": Routine; "routine.run": RoutineRun; + "browser.changed": BrowserState; + "browser.action": BrowserAction; + "browser.frame": BrowserFrame; } export const RpcErrorCode = { diff --git a/app/src/motion.css b/app/src/motion.css index b72ad6b5..065edfae 100644 --- a/app/src/motion.css +++ b/app/src/motion.css @@ -235,6 +235,80 @@ } } +/* The bot's browser, live (features/browser): its cursor glides to each + point it acts on, a ring spreads where it clicks once the cursor gets + there, and a dot pulses while the page is live. */ +.browser-cursor { + position: absolute; + display: flex; + align-items: flex-start; + pointer-events: none; + translate: -4px -3px; + transition: + left 520ms var(--ease-out), + top 520ms var(--ease-out); +} + +.browser-cursor > span { + margin-top: 14px; +} + +.browser-ripple { + position: absolute; + width: 46px; + height: 46px; + margin: -23px 0 0 -23px; + border: 3px solid; + border-radius: 9999px; + pointer-events: none; + animation: ripple 700ms var(--ease-out) 420ms both; +} + +.browser-typing { + letter-spacing: 1px; + animation: caret 0.9s steps(2, jump-none) infinite; +} + +.live-dot { + position: relative; + display: inline-block; + width: 7px; + height: 7px; + border-radius: 9999px; + background: var(--danger); +} + +.live-dot::after { + content: ""; + position: absolute; + inset: 0; + border-radius: inherit; + background: inherit; + animation: live 1.6s ease-out infinite; +} + +@keyframes ripple { + from { + opacity: 0.95; + transform: scale(0.2); + } + to { + opacity: 0; + transform: scale(1.5); + } +} + +@keyframes live { + from { + opacity: 0.7; + transform: scale(1); + } + to { + opacity: 0; + transform: scale(2.6); + } +} + /* Switching bots or crews: the main pane fades out and the new one rises in; the rest of the window stays put (ui/motion.ts). */ ::view-transition-old(root), diff --git a/app/src/store/app.ts b/app/src/store/app.ts index 28a8defd..a3f38eea 100644 --- a/app/src/store/app.ts +++ b/app/src/store/app.ts @@ -7,6 +7,7 @@ import type { BotloftApi, ConnectionState, ServerEvent } from "../lib/api"; import type { Bot, BotId, + BrowserState, Crew, CrewId, Delivery, @@ -35,6 +36,8 @@ export interface AppState { tasks: Record; /** Every routine that is not archived (spec 20), then every change. */ routines: Record; + /** The bots' browsers that are not closed (spec 21.7), then every change. */ + browsers: Record; /** When the owner last had each bot open, for failed routine runs. */ seenAt: Record; selectedCrewId: CrewId | null; @@ -46,6 +49,7 @@ export interface AppState { putBot(bot: Bot): void; putDelivery(delivery: Delivery): void; putRoutine(routine: Routine): void; + putBrowser(browser: BrowserState): void; } export type AppStore = StoreApi; @@ -87,6 +91,7 @@ export function createAppStore(api: BotloftApi): AppStore { deliveries: {}, tasks: {}, routines: {}, + browsers: {}, seenAt: loadSeen(), selectedCrewId: null, selectedBotId: null, @@ -107,6 +112,7 @@ export function createAppStore(api: BotloftApi): AppStore { putCrew: (crew) => set((state) => withCrew(state, crew)), putBot: (bot) => set((state) => withBot(state, bot)), putRoutine: (routine) => set((state) => withRoutine(state, routine)), + putBrowser: (browser) => set((state) => withBrowser(state, browser)), putDelivery: (delivery) => set((state) => ({ deliveries: { ...state.deliveries, [delivery.messageId]: delivery } })), })); @@ -165,6 +171,8 @@ export function applyEvent(state: AppState, event: ServerEvent): Partial { + const { [browser.botId]: _, ...rest } = state.browsers; + return { + browsers: browser.status === "closed" ? rest : { ...rest, [browser.botId]: browser }, + }; +} + /** Adds or replaces a routine; an archived one leaves the store. */ function withRoutine(state: AppState, routine: Routine): Partial { const { [routine.id]: _, ...rest } = state.routines; diff --git a/app/src/store/sync.ts b/app/src/store/sync.ts index dfc315ab..21201dcd 100644 --- a/app/src/store/sync.ts +++ b/app/src/store/sync.ts @@ -54,6 +54,13 @@ export function syncStore(store: AppStore, api: BotloftApi): () => void { alive && store.setState({ routines: Object.fromEntries(list.map((r) => [r.id, r])) }), ) .catch(() => {}); + api + .call("browser.list") + .then( + (list) => + alive && store.setState({ browsers: Object.fromEntries(list.map((b) => [b.botId, b])) }), + ) + .catch(() => {}); api .call("tasks.list", {}) .then( diff --git a/app/src/ui/SidePanel.tsx b/app/src/ui/SidePanel.tsx index 157a2be9..6a6383a3 100644 --- a/app/src/ui/SidePanel.tsx +++ b/app/src/ui/SidePanel.tsx @@ -34,6 +34,7 @@ export function SidePanel({ label, name, defaultWidth, + expanded = false, children, }: { label: string; @@ -41,6 +42,8 @@ export function SidePanel({ name: string; /** Width in px until the owner drags it. */ defaultWidth: number; + /** Takes all the room the main area can spare, until turned off. */ + expanded?: boolean; children: ReactNode; }) { const t = useT(); @@ -98,11 +101,16 @@ export function SidePanel({