diff --git a/app/src/dev/preview.ts b/app/src/dev/preview.ts index c59f0a5c..78c9a345 100644 --- a/app/src/dev/preview.ts +++ b/app/src/dev/preview.ts @@ -13,6 +13,7 @@ import { PROTOCOL_VERSION } from "../lib/protocol.gen"; import { seedBrowser } from "./seedBrowser"; import { seedChats } from "./seedChats"; import { seedChief } from "./seedChief"; +import { seedHands } from "./seedHands"; import { seedRoutines } from "./seedRoutines"; import { seedScreens } from "./seedScreens"; @@ -115,6 +116,7 @@ function seed(fake: FakeBotloft): void { seedRoutines(fake, { scout: scout.id, analyst: analyst.id }); seedBrowser(fake, scout.id); seedScreens(fake, research.id); + seedHands(fake, research.id); const ops = fake.addCrew("Ops"); const deploy = fake.addBot(ops.id, "Deploy", "Ships the site on Fridays"); const watcher = fake.addBot(ops.id, "Watcher", "Keeps an eye on the error log"); diff --git a/app/src/dev/seedHands.ts b/app/src/dev/seedHands.ts new file mode 100644 index 00000000..e55c910a --- /dev/null +++ b/app/src/dev/seedHands.ts @@ -0,0 +1,110 @@ +// Dev only: a Clerk that asks the owner to sign in for it, in the fake +// preview (spec 21.10). The sign-in page is drawn on a canvas and answers +// the owner's clicks and keys, so taking the browser can be tried end to end. + +import type { FakeBotloft } from "../lib/fake"; +import type { BrowserInput, CrewId } from "../lib/protocol.gen"; + +const W = 1280; +const H = 800; +const FIELDS = { user: 300, password: 390 } as const; +type Field = keyof typeof FIELDS; + +interface Form { + user: string; + password: string; + field: Field | null; + signedIn: boolean; +} + +function draw(form: Form): string { + const canvas = document.createElement("canvas"); + canvas.width = W; + canvas.height = H; + const g = canvas.getContext("2d"); + if (!g) { + return ""; + } + g.fillStyle = "#f7f4ee"; + g.fillRect(0, 0, W, H); + g.fillStyle = "#3b2a1a"; + g.font = "600 36px Georgia, serif"; + if (form.signedIn) { + g.fillText(`Welcome back, ${form.user || "baker"}`, 440, 220); + g.font = "20px system-ui, sans-serif"; + const orders = ["#1042 · 25 kg flour", "#1043 · 10 kg rye", "#1044 · 5 kg semolina"]; + for (const [row, line] of orders.entries()) { + g.fillText(`Order ${line}`, 440, 300 + row * 48); + } + return canvas.toDataURL("image/jpeg", 0.7).split(",")[1] ?? ""; + } + g.fillText("Flour Co. · Sign in", 440, 220); + for (const name of ["user", "password"] as const) { + const y = FIELDS[name]; + g.fillStyle = "#ffffff"; + g.strokeStyle = form.field === name ? "#c2410c" : "#b5a896"; + g.lineWidth = form.field === name ? 3 : 2; + g.beginPath(); + g.roundRect(440, y, 400, 52, 12); + g.fill(); + g.stroke(); + const value = name === "password" ? "•".repeat(form.password.length) : form.user; + g.fillStyle = value ? "#1f1409" : "#9a8c7a"; + g.font = "20px system-ui, sans-serif"; + g.fillText(value || (name === "user" ? "E-mail" : "Password"), 460, y + 33); + } + g.fillStyle = "#c2410c"; + g.beginPath(); + g.roundRect(440, 480, 400, 52, 26); + g.fill(); + g.fillStyle = "#ffffff"; + g.font = "600 20px system-ui, sans-serif"; + g.fillText("Sign in", 606, 513); + return canvas.toDataURL("image/jpeg", 0.7).split(",")[1] ?? ""; +} + +/** The page's answer to one of the owner's events; true once they sign in. */ +function handle(form: Form, input: BrowserInput): boolean { + if (input.kind === "mouse" && input.action === "down") { + const inside = (top: number) => + input.x >= 440 && input.x <= 840 && input.y >= top && input.y <= top + 52; + form.field = inside(FIELDS.user) ? "user" : inside(FIELDS.password) ? "password" : null; + return inside(480); + } + const field = form.field; + if (input.kind === "text" && field) { + form[field] += input.text; + } else if (input.kind === "key" && input.key === "Enter") { + return true; + } else if (input.kind === "key" && input.key === "Tab") { + form.field = form.field === "user" ? "password" : "user"; + } else if (input.kind === "key" && input.key === "Backspace" && field) { + form[field] = form[field].slice(0, -1); + } else if (input.kind === "key" && [...input.key].length === 1 && field && input.modifiers < 2) { + form[field] += input.key; + } + return false; +} + +export function seedHands(fake: FakeBotloft, crewId: CrewId): void { + const clerk = fake.addBot(crewId, "Clerk", "Places the bakery's orders"); + fake.setBotState(clerk.id, "needs_approval", 2); + const form: Form = { user: "", password: "", field: null, signedIn: false }; + const login = "https://flour.example/login"; + fake.browser.open(clerk.id, login, "Sign in · Flour Co."); + fake.browser.frame(clerk.id, draw(form)); + fake.chat.tool(clerk.id, "mcp__botloft__browser_open", { summary: login, status: "done" }); + const task = "Sign in to Flour Co. with the bakery's account"; + fake.chat.tool(clerk.id, "mcp__botloft__browser_ask_owner", { summary: task, status: "running" }); + fake.browser.ask(clerk.id, task); + fake.browser.onInput(clerk.id, (input) => { + if (form.signedIn) { + return; + } + if (handle(form, input)) { + form.signedIn = true; + fake.browser.open(clerk.id, "https://flour.example/orders", "Orders · Flour Co."); + } + fake.browser.frame(clerk.id, draw(form)); + }); +} diff --git a/app/src/features/bots/BotHeader.tsx b/app/src/features/bots/BotHeader.tsx index 4da45aff..d92d78c4 100644 --- a/app/src/features/bots/BotHeader.tsx +++ b/app/src/features/bots/BotHeader.tsx @@ -40,6 +40,7 @@ export function BotHeader({ onToggleFiles, browserOpen, browsing, + asking, onToggleBrowser, screensOpen, drawing, @@ -56,6 +57,8 @@ export function BotHeader({ browserOpen: boolean; /** The bot's browser is open (spec 21.8). */ browsing: boolean; + /** The bot asked the owner for a hand in it (spec 21.10). */ + asking: boolean; onToggleBrowser(): void; screensOpen: boolean; /** The bot is writing a screen (spec 22.5). */ @@ -152,18 +155,20 @@ export function BotHeader({ label={ browserOpen ? t.browser.hide - : browsing - ? `${t.browser.show}: ${t.browser.browsing(bot.name)}` - : t.browser.show + : asking + ? `${t.browser.show}: ${t.browser.help.needs(bot.name)}` + : browsing + ? `${t.browser.show}: ${t.browser.browsing(bot.name)}` + : t.browser.show } aria-pressed={browserOpen} onClick={onToggleBrowser} /> - {browsing && !browserOpen && ( + {(browsing || asking) && !browserOpen && ( )} diff --git a/app/src/features/bots/BotView.tsx b/app/src/features/bots/BotView.tsx index bd99629f..968c4de0 100644 --- a/app/src/features/bots/BotView.tsx +++ b/app/src/features/bots/BotView.tsx @@ -76,6 +76,9 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) { return status === "open" || status === "starting"; }); const browserOpen = side === "browser"; + const asking = useApp((state) => Boolean(state.browsers[bot.id]?.ask)); + // Each ask to open the browser in the owner's hands counts up (spec 21.10). + const [takeBrowser, setTakeBrowser] = useState(0); const screens = useScreens(bot); const screensOpen = side === "screens"; const [screenPath, setScreenPath] = useState(null); @@ -106,11 +109,14 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) { } } }, [screens.writing]); - const showBrowser = () => { + const showBrowser = (options?: { take?: boolean }) => { if (filesOpen) { seen(); } setSide("browser"); + if (options?.take) { + setTakeBrowser((count) => count + 1); + } }; const view = stateView(bot, crew.paused, t); const tabs: Tab[] = [ @@ -133,6 +139,7 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) { onToggleFiles={toggleFiles} browserOpen={browserOpen} browsing={browsing} + asking={asking} onToggleBrowser={() => (browserOpen ? setSide(null) : showBrowser())} screensOpen={screensOpen} drawing={screens.writing !== null} @@ -155,7 +162,7 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) { )} {side === "details" &&
setSide(null)} />} - {browserOpen && setSide(null)} />} + {browserOpen && setSide(null)} />} {screensOpen && ( app.browsers[bot.id]) ?? null; const status = state?.status ?? "closed"; - const { frame, action } = useBrowserView(bot, true); + const { frame, action, watched } = useBrowserView(bot, true); const [expanded, setExpanded] = useState(false); const caption = useCaption(bot, action); const live = status === "open" && frame !== null; + const hands = useHands(bot, state); + const [focused, setFocused] = useState(false); + const ask = state?.ask ?? null; + // Taking needs the watch first: the daemon ties the hands to it. + const taken = useRef(0); + useEffect(() => { + if (take > taken.current && watched && status === "open") { + taken.current = take; + if (!hands.held) { + void hands.take(); + } + } + }, [take, watched, status, hands.held, hands.take]); return ( @@ -66,8 +92,27 @@ export function BrowserPanel({ bot, onClose }: { bot: Bot; onClose(): void }) { ) : ( <> + {live && ask && !hands.held && } + {hands.held && }
- + + {hands.held && frame && ( + + )} + {(status !== "open" || !frame) && ( )}
- {status === "open" && caption && ( + {status === "open" && caption && !hands.held && (

{caption}

)} + {live && !hands.held && !ask && } )} diff --git a/app/src/features/browser/HandsBars.tsx b/app/src/features/browser/HandsBars.tsx new file mode 100644 index 00000000..deebcb62 --- /dev/null +++ b/app/src/features/browser/HandsBars.tsx @@ -0,0 +1,98 @@ +// The owner's controls in the browser panel (spec 21.10): taking the +// browser, the bot's request for a hand, and the bar that says the owner +// has it, with the button that gives it back. + +import { Check, Hand, Keyboard, MousePointerClick } from "lucide-react"; +import { useT } from "../../i18n"; +import type { Bot } from "../../lib/protocol.gen"; +import { Button } from "../../ui/Button"; +import { Callout } from "../../ui/Callout"; +import type { Hands } from "./useHands"; + +/** The bot asked for a hand and the owner has not taken the browser yet. */ +export function AskCallout({ bot, task, hands }: { bot: Bot; task: string; hands: Hands }) { + const t = useT().browser; + return ( +
+ +

{task}

+ +
+
+ ); +} + +/** The owner has the browser. */ +export function HeldBar({ + bot, + task, + focused, + hands, +}: { + bot: Bot; + task: string | null; + focused: boolean; + hands: Hands; +}) { + const t = useT().browser; + const Hint = focused ? Keyboard : MousePointerClick; + return ( +
+
+ +
+

{t.hands.holding}

+

+ {task ? t.help.asks(bot.name, task) : t.hands.waits(bot.name)} +

+
+
+
+ + + + {focused ? t.hands.typing : t.hands.clickToType} + +
+
+ ); +} + +/** Under the live screen: the owner may take the browser at any time. */ +export function TakeBar({ bot, hands }: { bot: Bot; hands: Hands }) { + const t = useT().browser.hands; + return ( +
+ +

{t.takeWhy(bot.name)}

+
+ ); +} diff --git a/app/src/features/browser/HandsLayer.tsx b/app/src/features/browser/HandsLayer.tsx new file mode 100644 index 00000000..a8e64e21 --- /dev/null +++ b/app/src/features/browser/HandsLayer.tsx @@ -0,0 +1,236 @@ +// The owner's hands on the live screen (spec 21.10): over the picture of the +// page, their mouse and keyboard become events for the bot's browser. A +// hidden text box takes the keys, so accents, input methods and pasting work +// as in any field. + +import { + type CompositionEvent, + type FormEvent, + type KeyboardEvent, + type MouseEvent, + useEffect, + useRef, +} from "react"; +import type { BrowserInput, MouseButton } from "../../lib/protocol.gen"; + +/** Keys that only change others; they travel as `modifiers`. */ +const MODIFIER_KEYS = new Set([ + "Shift", + "Control", + "Alt", + "AltGraph", + "Meta", + "OS", + "CapsLock", + "NumLock", + "ScrollLock", + "Fn", +]); +/** Keys still composing a character: the text comes when it is done. */ +const COMPOSING = new Set(["Dead", "Process", "Unidentified"]); +const TEXT_MAX = 10_000; +const WHEEL_MAX = 10_000; +/** Pixels in a line and a page of the wheel, for mice that count those. */ +const WHEEL_LINE = 40; +const WHEEL_PAGE = 800; +const BUTTONS: MouseButton[] = ["left", "middle", "right"]; + +interface Modifiable { + altKey: boolean; + ctrlKey: boolean; + metaKey: boolean; + shiftKey: boolean; +} + +/** Alt 1, Ctrl 2, Meta 4, Shift 8, as the daemon takes them. */ +export function modifiersOf(event: Modifiable): number { + return ( + (event.altKey ? 1 : 0) | + (event.ctrlKey ? 2 : 0) | + (event.metaKey ? 4 : 0) | + (event.shiftKey ? 8 : 0) + ); +} + +const clamp = (value: number, max: number) => Math.min(max, Math.max(0, value)); + +export function HandsLayer({ + label, + keysLabel, + width, + height, + send, + onFocus, +}: { + label: string; + keysLabel: string; + /** The page's size, in CSS pixels. */ + width: number; + height: number; + send(input: BrowserInput): void; + /** Whether the keys go to the page. */ + onFocus(focused: boolean): void; +}) { + const area = useRef(null); + const keys = useRef(null); + const composing = useRef(false); + // Moves wait for the next frame; only the newest one goes. + const move = useRef(null); + const tick = useRef(null); + + const point = (event: { clientX: number; clientY: number }) => { + const box = area.current?.getBoundingClientRect(); + if (!box || box.width === 0 || box.height === 0) { + return { x: 0, y: 0 }; + } + return { + x: clamp(((event.clientX - box.left) / box.width) * width, width), + y: clamp(((event.clientY - box.top) / box.height) * height, height), + }; + }; + const flush = () => { + if (tick.current !== null) { + cancelAnimationFrame(tick.current); + tick.current = null; + } + const pending = move.current; + move.current = null; + if (pending) { + send(pending); + } + }; + // Anything but a move goes at once, after the move before it. + const now = (input: BrowserInput) => { + flush(); + send(input); + }; + const mouse = (event: MouseEvent, action: "down" | "up") => { + now({ + kind: "mouse", + action, + ...point(event), + button: BUTTONS[event.button] ?? "left", + buttons: event.buttons, + clicks: Math.min(3, Math.max(1, event.detail)), + modifiers: modifiersOf(event), + }); + }; + const typed = (box: HTMLTextAreaElement) => { + const text = box.value; + box.value = ""; + if (text) { + now({ kind: "text", text: text.slice(0, TEXT_MAX) }); + } + }; + + // The keys go to the page as soon as the owner takes it. + useEffect(() => { + keys.current?.focus({ preventScroll: true }); + return () => { + if (tick.current !== null) { + cancelAnimationFrame(tick.current); + } + }; + }, []); + + // The wheel scrolls the page, not the panel: that needs a listener that + // is not passive. + useEffect(() => { + const node = area.current; + if (!node) { + return; + } + const wheel = (event: WheelEvent) => { + event.preventDefault(); + const scale = event.deltaMode === 1 ? WHEEL_LINE : event.deltaMode === 2 ? WHEEL_PAGE : 1; + const limit = (delta: number) => Math.max(-WHEEL_MAX, Math.min(WHEEL_MAX, delta * scale)); + now({ + kind: "wheel", + ...point(event), + dx: limit(event.deltaX), + dy: limit(event.deltaY), + modifiers: modifiersOf(event), + }); + }; + node.addEventListener("wheel", wheel, { passive: false }); + return () => node.removeEventListener("wheel", wheel); + }); + + const onKeyDown = (event: KeyboardEvent) => { + if (event.nativeEvent.isComposing || COMPOSING.has(event.key) || MODIFIER_KEYS.has(event.key)) { + return; + } + const altGr = event.ctrlKey && event.altKey; + // Ctrl+V pastes the owner's own clipboard, through `paste`. + if ((event.ctrlKey || event.metaKey) && !altGr && event.key.toLowerCase() === "v") { + return; + } + event.preventDefault(); + now({ kind: "key", key: event.key, code: event.code, modifiers: modifiersOf(event) }); + }; + + return ( +
event.currentTarget.setPointerCapture?.(event.pointerId)} + onMouseDown={(event) => { + event.preventDefault(); + keys.current?.focus({ preventScroll: true }); + mouse(event, "down"); + }} + onMouseUp={(event) => mouse(event, "up")} + onMouseMove={(event) => { + move.current = { + kind: "mouse", + action: "move", + ...point(event), + button: "none", + buttons: event.buttons, + clicks: 0, + modifiers: modifiersOf(event), + }; + if (tick.current === null) { + tick.current = requestAnimationFrame(() => { + tick.current = null; + flush(); + }); + } + }} + onContextMenu={(event) => event.preventDefault()} + > +