feat: the owner takes a bot's browser into their own hands (N3)

app/src/dev/preview.ts

@@ -13,6 +13,7 @@ import { PROTOCOL_VERSION } from "../lib/protocol.gen";
 import { seedBrowser } from "./seedBrowser";
 import { seedChats } from "./seedChats";
 import { seedChief } from "./seedChief";
+import { seedHands } from "./seedHands";
 import { seedRoutines } from "./seedRoutines";
 import { seedScreens } from "./seedScreens";
 
@@ -115,6 +116,7 @@ function seed(fake: FakeBotloft): void {
   seedRoutines(fake, { scout: scout.id, analyst: analyst.id });
   seedBrowser(fake, scout.id);
   seedScreens(fake, research.id);
+  seedHands(fake, research.id);
   const ops = fake.addCrew("Ops");
   const deploy = fake.addBot(ops.id, "Deploy", "Ships the site on Fridays");
   const watcher = fake.addBot(ops.id, "Watcher", "Keeps an eye on the error log");

app/src/dev/seedHands.ts

@@ -0,0 +1,110 @@
+// Dev only: a Clerk that asks the owner to sign in for it, in the fake
+// preview (spec 21.10). The sign-in page is drawn on a canvas and answers
+// the owner's clicks and keys, so taking the browser can be tried end to end.
+
+import type { FakeBotloft } from "../lib/fake";
+import type { BrowserInput, CrewId } from "../lib/protocol.gen";
+
+const W = 1280;
+const H = 800;
+const FIELDS = { user: 300, password: 390 } as const;
+type Field = keyof typeof FIELDS;
+
+interface Form {
+  user: string;
+  password: string;
+  field: Field | null;
+  signedIn: boolean;
+}
+
+function draw(form: Form): string {
+  const canvas = document.createElement("canvas");
+  canvas.width = W;
+  canvas.height = H;
+  const g = canvas.getContext("2d");
+  if (!g) {
+    return "";
+  }
+  g.fillStyle = "#f7f4ee";
+  g.fillRect(0, 0, W, H);
+  g.fillStyle = "#3b2a1a";
+  g.font = "600 36px Georgia, serif";
+  if (form.signedIn) {
+    g.fillText(`Welcome back, ${form.user || "baker"}`, 440, 220);
+    g.font = "20px system-ui, sans-serif";
+    const orders = ["#1042 · 25 kg flour", "#1043 · 10 kg rye", "#1044 · 5 kg semolina"];
+    for (const [row, line] of orders.entries()) {
+      g.fillText(`Order ${line}`, 440, 300 + row * 48);
+    }
+    return canvas.toDataURL("image/jpeg", 0.7).split(",")[1] ?? "";
+  }
+  g.fillText("Flour Co. · Sign in", 440, 220);
+  for (const name of ["user", "password"] as const) {
+    const y = FIELDS[name];
+    g.fillStyle = "#ffffff";
+    g.strokeStyle = form.field === name ? "#c2410c" : "#b5a896";
+    g.lineWidth = form.field === name ? 3 : 2;
+    g.beginPath();
+    g.roundRect(440, y, 400, 52, 12);
+    g.fill();
+    g.stroke();
+    const value = name === "password" ? "•".repeat(form.password.length) : form.user;
+    g.fillStyle = value ? "#1f1409" : "#9a8c7a";
+    g.font = "20px system-ui, sans-serif";
+    g.fillText(value || (name === "user" ? "E-mail" : "Password"), 460, y + 33);
+  }
+  g.fillStyle = "#c2410c";
+  g.beginPath();
+  g.roundRect(440, 480, 400, 52, 26);
+  g.fill();
+  g.fillStyle = "#ffffff";
+  g.font = "600 20px system-ui, sans-serif";
+  g.fillText("Sign in", 606, 513);
+  return canvas.toDataURL("image/jpeg", 0.7).split(",")[1] ?? "";
+}
+
+/** The page's answer to one of the owner's events; true once they sign in. */
+function handle(form: Form, input: BrowserInput): boolean {
+  if (input.kind === "mouse" && input.action === "down") {
+    const inside = (top: number) =>
+      input.x >= 440 && input.x <= 840 && input.y >= top && input.y <= top + 52;
+    form.field = inside(FIELDS.user) ? "user" : inside(FIELDS.password) ? "password" : null;
+    return inside(480);
+  }
+  const field = form.field;
+  if (input.kind === "text" && field) {
+    form[field] += input.text;
+  } else if (input.kind === "key" && input.key === "Enter") {
+    return true;
+  } else if (input.kind === "key" && input.key === "Tab") {
+    form.field = form.field === "user" ? "password" : "user";
+  } else if (input.kind === "key" && input.key === "Backspace" && field) {
+    form[field] = form[field].slice(0, -1);
+  } else if (input.kind === "key" && [...input.key].length === 1 && field && input.modifiers < 2) {
+    form[field] += input.key;
+  }
+  return false;
+}
+
+export function seedHands(fake: FakeBotloft, crewId: CrewId): void {
+  const clerk = fake.addBot(crewId, "Clerk", "Places the bakery's orders");
+  fake.setBotState(clerk.id, "needs_approval", 2);
+  const form: Form = { user: "", password: "", field: null, signedIn: false };
+  const login = "https://flour.example/login";
+  fake.browser.open(clerk.id, login, "Sign in · Flour Co.");
+  fake.browser.frame(clerk.id, draw(form));
+  fake.chat.tool(clerk.id, "mcp__botloft__browser_open", { summary: login, status: "done" });
+  const task = "Sign in to Flour Co. with the bakery's account";
+  fake.chat.tool(clerk.id, "mcp__botloft__browser_ask_owner", { summary: task, status: "running" });
+  fake.browser.ask(clerk.id, task);
+  fake.browser.onInput(clerk.id, (input) => {
+    if (form.signedIn) {
+      return;
+    }
+    if (handle(form, input)) {
+      form.signedIn = true;
+      fake.browser.open(clerk.id, "https://flour.example/orders", "Orders · Flour Co.");
+    }
+    fake.browser.frame(clerk.id, draw(form));
+  });
+}

app/src/features/bots/BotHeader.tsx

@@ -40,6 +40,7 @@ export function BotHeader({
   onToggleFiles,
   browserOpen,
   browsing,
+  asking,
   onToggleBrowser,
   screensOpen,
   drawing,
@@ -56,6 +57,8 @@ export function BotHeader({
   browserOpen: boolean;
   /** The bot's browser is open (spec 21.8). */
   browsing: boolean;
+  /** The bot asked the owner for a hand in it (spec 21.10). */
+  asking: boolean;
   onToggleBrowser(): void;
   screensOpen: boolean;
   /** The bot is writing a screen (spec 22.5). */
@@ -152,18 +155,20 @@ export function BotHeader({
           label={
             browserOpen
               ? t.browser.hide
-              : browsing
-                ? `${t.browser.show}: ${t.browser.browsing(bot.name)}`
-                : t.browser.show
+              : asking
+                ? `${t.browser.show}: ${t.browser.help.needs(bot.name)}`
+                : browsing
+                  ? `${t.browser.show}: ${t.browser.browsing(bot.name)}`
+                  : t.browser.show
           }
           aria-pressed={browserOpen}
           onClick={onToggleBrowser}
         />
-        {browsing && !browserOpen && (
+        {(browsing || asking) && !browserOpen && (
           <span
             aria-hidden
             className="live-dot pointer-events-none absolute top-0.5 right-0.5"
-            style={{ background: "var(--work)" }}
+            style={{ background: asking ? "var(--warn)" : "var(--work)" }}
           />
         )}
       </span>

app/src/features/bots/BotView.tsx

@@ -76,6 +76,9 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) {
     return status === "open" || status === "starting";
   });
   const browserOpen = side === "browser";
+  const asking = useApp((state) => Boolean(state.browsers[bot.id]?.ask));
+  // Each ask to open the browser in the owner's hands counts up (spec 21.10).
+  const [takeBrowser, setTakeBrowser] = useState(0);
   const screens = useScreens(bot);
   const screensOpen = side === "screens";
   const [screenPath, setScreenPath] = useState<string | null>(null);
@@ -106,11 +109,14 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) {
       }
     }
   }, [screens.writing]);
-  const showBrowser = () => {
+  const showBrowser = (options?: { take?: boolean }) => {
     if (filesOpen) {
       seen();
     }
     setSide("browser");
+    if (options?.take) {
+      setTakeBrowser((count) => count + 1);
+    }
   };
   const view = stateView(bot, crew.paused, t);
   const tabs: Tab<Pane>[] = [
@@ -133,6 +139,7 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) {
         onToggleFiles={toggleFiles}
         browserOpen={browserOpen}
         browsing={browsing}
+        asking={asking}
         onToggleBrowser={() => (browserOpen ? setSide(null) : showBrowser())}
         screensOpen={screensOpen}
         drawing={screens.writing !== null}
@@ -155,7 +162,7 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) {
           )}
         </div>
         {side === "details" && <Details bot={bot} onClose={() => setSide(null)} />}
-        {browserOpen && <BrowserPanel bot={bot} onClose={() => setSide(null)} />}
+        {browserOpen && <BrowserPanel bot={bot} take={takeBrowser} onClose={() => setSide(null)} />}
         {screensOpen && (
           <ScreensPanel
             bot={bot}

app/src/features/browser/BrowserPanel.tsx

@@ -1,9 +1,10 @@
 // The bot's own browser beside its chat (spec 21.8): the page it is on,
 // live, with its cursor and a line about what it just did. Watching starts
-// when the panel opens and stops when it closes.
+// when the panel opens and stops when it closes. The owner can take it into
+// their own hands and give it back (spec 21.10).
 
 import { ExternalLink, Globe, LoaderCircle, Lock, Maximize2, Minimize2, X } from "lucide-react";
-import { useState } from "react";
+import { useEffect, useRef, useState } from "react";
 import { useT } from "../../i18n";
 import type { Bot, BrowserState } from "../../lib/protocol.gen";
 import { useApp, useHost } from "../../store/context";
@@ -12,17 +13,42 @@ import { Callout } from "../../ui/Callout";
 import { SidePanel } from "../../ui/SidePanel";
 import { attempt } from "../../ui/toast";
 import { BotAvatar } from "../bots/BotAvatar";
+import { AskCallout, HeldBar, TakeBar } from "./HandsBars";
+import { HandsLayer } from "./HandsLayer";
 import { LiveView, useCaption } from "./LiveView";
 import { useBrowserView } from "./useBrowserView";
+import { useHands } from "./useHands";
 
-export function BrowserPanel({ bot, onClose }: { bot: Bot; onClose(): void }) {
+export function BrowserPanel({
+  bot,
+  take = 0,
+  onClose,
+}: {
+  bot: Bot;
+  /** Counts up each time the owner asks to open it in their hands. */
+  take?: number;
+  onClose(): void;
+}) {
   const t = useT().browser;
   const state = useApp((app) => app.browsers[bot.id]) ?? null;
   const status = state?.status ?? "closed";
-  const { frame, action } = useBrowserView(bot, true);
+  const { frame, action, watched } = useBrowserView(bot, true);
   const [expanded, setExpanded] = useState(false);
   const caption = useCaption(bot, action);
   const live = status === "open" && frame !== null;
+  const hands = useHands(bot, state);
+  const [focused, setFocused] = useState(false);
+  const ask = state?.ask ?? null;
+  // Taking needs the watch first: the daemon ties the hands to it.
+  const taken = useRef(0);
+  useEffect(() => {
+    if (take > taken.current && watched && status === "open") {
+      taken.current = take;
+      if (!hands.held) {
+        void hands.take();
+      }
+    }
+  }, [take, watched, status, hands.held, hands.take]);
 
   return (
     <SidePanel label={t.panel(bot.name)} name="browser" defaultWidth={560} expanded={expanded}>
@@ -66,8 +92,27 @@ export function BrowserPanel({ bot, onClose }: { bot: Bot; onClose(): void }) {
           <Empty bot={bot} />
         ) : (
           <>
+            {live && ask && !hands.held && <AskCallout bot={bot} task={ask} hands={hands} />}
+            {hands.held && <HeldBar bot={bot} task={ask} focused={focused} hands={hands} />}
             <div className="relative">
-              <LiveView bot={bot} frame={frame} action={action} dim={status === "closed"} />
+              <LiveView
+                bot={bot}
+                frame={frame}
+                action={action}
+                dim={status === "closed"}
+                held={hands.held}
+              >
+                {hands.held && frame && (
+                  <HandsLayer
+                    label={t.hands.screen(bot.name)}
+                    keysLabel={t.hands.typing}
+                    width={frame.width}
+                    height={frame.height}
+                    send={hands.send}
+                    onFocus={setFocused}
+                  />
+                )}
+              </LiveView>
               {(status !== "open" || !frame) && (
                 <Overlay
                   busy={status === "starting" || status === "open"}
@@ -76,7 +121,7 @@ export function BrowserPanel({ bot, onClose }: { bot: Bot; onClose(): void }) {
                 />
               )}
             </div>
-            {status === "open" && caption && (
+            {status === "open" && caption && !hands.held && (
               <p
                 key={action?.at}
                 aria-live="polite"
@@ -86,6 +131,7 @@ export function BrowserPanel({ bot, onClose }: { bot: Bot; onClose(): void }) {
                 <span className="truncate">{caption}</span>
               </p>
             )}
+            {live && !hands.held && !ask && <TakeBar bot={bot} hands={hands} />}
           </>
         )}
       </div>