feat(app): the desktop panel, live, with Stop

app/src/dev/seedDesktop.ts

@@ -1,13 +1,45 @@
 // Dev only: the preview's desktop (spec 24). Analyst asks to see a
-// spreadsheet in Excel, and Scout may already see Notepad.
+// spreadsheet in Excel, and Scout may already see Notepad, where it just
+// clicked Save.
 
 import type { FakeBotloft } from "../lib/fake";
 import type { BotId } from "../lib/protocol.gen";
 
 const EXCEL = "C:\\Program Files\\Microsoft Office\\root\\Office16\\EXCEL.EXE";
+const NOTES = { id: 42, title: "Shopping list - Notepad", app: "Notepad" };
+
+/** A Notepad window, drawn: a title bar, a menu and a few lines. */
+function drawNotepad(): string {
+  const canvas = document.createElement("canvas");
+  canvas.width = 760;
+  canvas.height = 480;
+  const g = canvas.getContext("2d");
+  if (!g) {
+    return "";
+  }
+  g.fillStyle = "#ffffff";
+  g.fillRect(0, 0, 760, 480);
+  g.fillStyle = "#f3f3f3";
+  g.fillRect(0, 0, 760, 72);
+  g.fillStyle = "#1f1f1f";
+  g.font = "14px system-ui, sans-serif";
+  g.fillText(NOTES.title, 16, 24);
+  g.fillText("File     Edit     View", 16, 58);
+  g.font = "16px Consolas, monospace";
+  ["Flour, 2 kg", "Eggs, 12", "Butter, 500 g", "Sugar, 1 kg"].forEach((line, index) => {
+    g.fillText(line, 20, 112 + index * 26);
+  });
+  return canvas.toDataURL("image/jpeg", 0.8).split(",")[1] ?? "";
+}
 
 export function seedDesktop(fake: FakeBotloft, bots: { scout: BotId; analyst: BotId }) {
   fake.desktop.grant(bots.scout, "C:\\Windows\\notepad.exe", "Notepad");
+  fake.desktop.use(bots.scout, NOTES, { kind: "click", target: "Save", option: null });
+  fake.desktop.paint(bots.scout, drawNotepad(), 760, 480);
+  fake.chat.tool(bots.scout, "mcp__botloft__desktop_click", {
+    summary: "To save the list",
+    status: "done",
+  });
   fake.chat.ask(
     bots.analyst,
     "mcp__botloft__desktop",

app/src/features/bots/BotView.tsx

@@ -11,6 +11,8 @@ import { useStable } from "../../ui/useStable";
 import { BrowserPanel } from "../browser/BrowserPanel";
 import { ShowBrowser } from "../browser/showBrowser";
 import { ChatView } from "../chat/ChatView";
+import { DesktopPanel } from "../desktop/DesktopPanel";
+import { ShowDesktop } from "../desktop/showDesktop";
 import { FilesPanel } from "../files/FilesPanel";
 import { ShowFile } from "../files/showFile";
 import { useBotFiles } from "../files/useBotFiles";
@@ -134,14 +136,22 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) {
     }
     setSide("terminal");
   });
+  const showDesktop = useStable(() => {
+    if (filesOpen) {
+      seen();
+    }
+    setSide("desktop");
+  });
   // The dock of the bot's computer: browser, terminal and files.
   const dock = {
     open: side,
-    pick: (place: "browser" | "terminal" | "files") => {
+    pick: (place: "browser" | "terminal" | "files" | "desktop") => {
       if (place === "browser") {
         showBrowser();
       } else if (place === "terminal") {
         showTerminal();
+      } else if (place === "desktop") {
+        showDesktop();
       } else if (!filesOpen) {
         toggleFiles();
       }
@@ -184,7 +194,9 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) {
               <ShowBrowser.Provider value={showBrowser}>
                 <ShowScreen.Provider value={showScreen}>
                   <ShowTerminal.Provider value={showTerminal}>
-                    <ChatView bot={bot} stopped={bot.paused || crew.paused} />
+                    <ShowDesktop.Provider value={showDesktop}>
+                      <ChatView bot={bot} stopped={bot.paused || crew.paused} />
+                    </ShowDesktop.Provider>
                   </ShowTerminal.Provider>
                 </ShowScreen.Provider>
               </ShowBrowser.Provider>
@@ -221,6 +233,7 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) {
                 />
               )}
               {beside === "terminal" && <TerminalPanel bot={bot} onClose={() => setSide(null)} />}
+              {beside === "desktop" && <DesktopPanel bot={bot} onClose={() => setSide(null)} />}
             </PanelClosing.Provider>
           </PanelRestored.Provider>
         </Dock.Provider>

app/src/features/chat/ToolLines.tsx

@@ -30,6 +30,7 @@ import { useT } from "../../i18n";
 import type { ChatItem, ToolItem } from "../../lib/protocol.gen";
 import { useArrival } from "../../ui/motion";
 import { isBrowserTool, ShowBrowser } from "../browser/showBrowser";
+import { isDesktopTool, ShowDesktop } from "../desktop/showDesktop";
 import { ShowFile } from "../files/showFile";
 import { isScreenFile, ShowScreen } from "../screens/showScreen";
 import { ShowTerminal } from "../terminal/showTerminal";
@@ -106,6 +107,7 @@ function ToolLine({ tool, createdAt }: { tool: ToolItem; createdAt: number }) {
   const showBrowser = useContext(ShowBrowser);
   const showScreen = useContext(ShowScreen);
   const showTerminal = useContext(ShowTerminal);
+  const showDesktop = useContext(ShowDesktop);
   const arrival = useArrival(createdAt);
   const title = toolTitle(tool.name, t.tools);
   // A command reads in the bot's own words, when it gave them; the command
@@ -173,6 +175,17 @@ function ToolLine({ tool, createdAt }: { tool: ToolItem; createdAt: number }) {
             <SquareTerminal aria-hidden size={14} />
           </button>
         )}
+        {showDesktop && isDesktopTool(tool.name) && (
+          <button
+            type="button"
+            title={t.desktop.panel.showInPanel}
+            aria-label={`${t.desktop.panel.showInPanel}: ${title}`}
+            onClick={() => showDesktop()}
+            className="grid size-6 shrink-0 place-items-center rounded-lg text-muted transition-colors hover:bg-sunken hover:text-ink"
+          >
+            <Monitor aria-hidden size={14} />
+          </button>
+        )}
         {showBrowser && isBrowserTool(tool.name) && (
           <button
             type="button"

app/src/features/desktop/DesktopPanel.tsx

@@ -0,0 +1,128 @@
+// A bot's use of the owner's desktop beside its chat (spec 24.9): the
+// window it read or acted in last, live, what it did, and Stop. Stopped,
+// the panel says so and offers to let it go on.
+
+import { Monitor, Play, Square, X } from "lucide-react";
+import type { CSSProperties } from "react";
+import { useT } from "../../i18n";
+import type { Bot, DesktopAction } from "../../lib/protocol.gen";
+import { useApi } from "../../store/context";
+import { Badge } from "../../ui/Badge";
+import { Button } from "../../ui/Button";
+import { Callout } from "../../ui/Callout";
+import { EmptyState } from "../../ui/EmptyState";
+import { SidePanel } from "../../ui/SidePanel";
+import { attempt } from "../../ui/toast";
+import { BotAvatar } from "../bots/BotAvatar";
+import { ComputerFooter } from "../terminal/ComputerDock";
+import { useDesktopView } from "./useDesktopView";
+
+type Words = ReturnType<typeof useT>["desktop"]["panel"];
+
+/** What the bot did, in the owner's language. */
+export function actionText(action: DesktopAction | null, t: Words): string {
+  if (!action) {
+    return t.read;
+  }
+  const target = action.target.trim();
+  switch (action.kind) {
+    case "click":
+      return target ? t.click(target) : t.clickSomething;
+    case "type":
+      return target ? t.type(target) : t.typeSomething;
+    case "select":
+      return t.select(action.option ?? "");
+    case "scroll":
+      return target ? t.scroll(target) : t.scrollSomething;
+  }
+}
+
+export function DesktopPanel({ bot, onClose }: { bot: Bot; onClose(): void }) {
+  const t = useT().desktop.panel;
+  const api = useApi();
+  const { state, frame } = useDesktopView(bot.id);
+  const stopped = state?.stopped === true;
+  const window = state?.window ?? null;
+  const stop = () => attempt(t.stopFailed, () => api.call("desktop.stop", { botId: bot.id }));
+  const resume = () => attempt(t.resumeFailed, () => api.call("desktop.resume", { botId: bot.id }));
+
+  return (
+    <SidePanel label={t.label(bot.name)} name="computer" defaultWidth={600}>
+      <header className="flex h-11 shrink-0 items-center justify-between border-line border-b pr-1.5 pl-4">
+        <h2 className="flex items-center gap-2 font-semibold text-sm">
+          {t.heading}
+          {stopped ? (
+            <Badge tone="warn">{t.stoppedBadge}</Badge>
+          ) : (
+            frame &&
+            window && (
+              <Badge tone="danger">
+                <span aria-hidden className="live-dot" />
+                {t.live}
+              </Badge>
+            )
+          )}
+        </h2>
+        <div className="flex items-center gap-0.5">
+          {state && !stopped && (
+            <Button variant="ghost" size="sm" icon={Square} onClick={() => void stop()}>
+              {t.stop}
+            </Button>
+          )}
+          <Button variant="ghost" size="sm" icon={X} label={t.close} onClick={onClose} />
+        </div>
+      </header>
+      <div className="flex min-h-0 flex-1 flex-col gap-3 overflow-y-auto p-3">
+        {stopped && (
+          <Callout tone="warn" title={t.stoppedTitle(bot.name)}>
+            <p>{t.stoppedBody(bot.name)}</p>
+            <div className="mt-2.5">
+              <Button variant="primary" size="sm" icon={Play} onClick={() => void resume()}>
+                {t.resume}
+              </Button>
+            </div>
+          </Callout>
+        )}
+        {state && !window && (
+          <EmptyState
+            color={bot.color}
+            icon={Monitor}
+            title={t.emptyTitle(bot.name)}
+            body={t.emptyBody}
+          />
+        )}
+        {window && (
+          <>
+            <p className="flex min-w-0 items-baseline gap-2 text-sm">
+              <span className="shrink-0 font-medium">{window.app}</span>
+              <span className="truncate text-muted" data-selectable>
+                {window.title}
+              </span>
+            </p>
+            <div
+              className="bot-stage grid min-h-40 flex-1 place-items-center p-2.5"
+              style={{ "--bot": bot.color } as CSSProperties}
+            >
+              {frame ? (
+                <img
+                  src={`data:image/jpeg;base64,${frame.data}`}
+                  alt={window.title}
+                  draggable={false}
+                  className="max-h-full max-w-full select-none rounded-lg object-contain shadow-lift"
+                />
+              ) : (
+                <p className="text-sm text-white/80">{t.waiting}</p>
+              )}
+            </div>
+            <p aria-live="polite" className="flex items-center gap-2 text-ink-soft text-sm">
+              <BotAvatar color={bot.color} size={18} mood={stopped ? "sleeping" : "working"} />
+              <span className="truncate">{actionText(state?.action ?? null, t)}</span>
+            </p>
+          </>
+        )}
+        <p className="mt-auto text-muted text-xs">{t.shortcut}</p>
+      </div>
+      <ComputerFooter />
+    </SidePanel>
+  );
+}

app/src/features/desktop/panel.test.tsx

@@ -0,0 +1,71 @@
+import { act, cleanup, fireEvent, screen, waitFor, within } from "@testing-library/react";
+import { afterEach, describe, expect, test } from "vitest";
+import { FakeBotloft } from "../../lib/fake";
+import { crewOpened, openBot, renderApp } from "../../test/app";
+
+afterEach(cleanup);
+
+async function openScout() {
+  const fake = new FakeBotloft();
+  const ops = fake.addCrew("Ops");
+  const scout = fake.addBot(ops.id, "Scout", "Finds sources");
+  fake.setBotState(scout.id, "idle");
+  fake.chat.tool(scout.id, "mcp__botloft__desktop_look", {
+    summary: "To read the list",
+    status: "done",
+  });
+  renderApp(fake);
+  await crewOpened("Ops");
+  openBot("Scout");
+  await screen.findByRole("list", { name: "Messages" });
+  return { fake, scout };
+}
+
+const panel = () => screen.getByRole("complementary", { name: "Scout's desktop" });
+const NOTES = { id: 42, title: "Shopping list - Notepad", app: "Notepad" };
+
+describe("the desktop panel", () => {
+  test("opens from a desktop tool in the chat and shows the window the bot uses, live", async () => {
+    const { fake, scout } = await openScout();
+    fireEvent.click(
+      screen.getByRole("button", { name: "Watch on the desktop panel: Read a window" }),
+    );
+    expect(await within(panel()).findByText("Scout has not used your desktop yet")).toBeDefined();
+    expect(fake.desktop.watching).toBe(scout.id);
+
+    act(() => {
+      fake.desktop.use(scout.id, NOTES, { kind: "click", target: "Save", option: null });
+    });
+    expect(await within(panel()).findByText("Shopping list - Notepad")).toBeDefined();
+    expect(within(panel()).getByText('Clicked "Save"')).toBeDefined();
+    expect(within(panel()).getByText("Waiting for the picture…")).toBeDefined();
+
+    act(() => {
+      fake.desktop.paint(scout.id, "AAAA");
+    });
+    const picture = await within(panel()).findByRole("img", { name: "Shopping list - Notepad" });
+    expect(picture.getAttribute("src")).toBe("data:image/jpeg;base64,AAAA");
+    expect(within(panel()).getByText("Live")).toBeDefined();
+
+    // Closed, nobody watches.
+    fireEvent.click(within(panel()).getByRole("button", { name: "Close" }));
+    await waitFor(() => expect(fake.desktop.watching).toBeNull());
+  });
+
+  test("stops the bot on the desktop and lets it go on", async () => {
+    const { fake, scout } = await openScout();
+    fireEvent.click(
+      screen.getByRole("button", { name: "Watch on the desktop panel: Read a window" }),
+    );
+    act(() => {
+      fake.desktop.use(scout.id, NOTES);
+    });
+    expect(await within(panel()).findByText("Read the window")).toBeDefined();
+    fireEvent.click(within(panel()).getByRole("button", { name: "Stop" }));
+    expect(await within(panel()).findByText("You stopped Scout on your desktop")).toBeDefined();
+    expect(fake.desktop.state(scout.id).stopped).toBe(true);
+    fireEvent.click(within(panel()).getByRole("button", { name: "Let it go on" }));
+    await waitFor(() => expect(fake.desktop.state(scout.id).stopped).toBe(false));
+    expect(within(panel()).queryByText("You stopped Scout on your desktop")).toBeNull();
+  });
+});

app/src/features/desktop/showDesktop.ts

@@ -0,0 +1,11 @@
+// How a desktop tool line in the chat opens the bot's desktop panel (spec
+// 24.9). Absent where there is no panel, so no button shows.
+
+import { createContext } from "react";
+
+export const ShowDesktop = createContext<(() => void) | null>(null);
+
+/** `mcp__botloft__desktop_look` and the other desktop tools. */
+export function isDesktopTool(name: string): boolean {
+  return name.startsWith("mcp__botloft__desktop_");
+}

app/src/features/desktop/useDesktopView.ts

@@ -0,0 +1,51 @@
+// What a bot does on the owner's desktop, live (spec 24.9): its state and
+// the picture of the window it is using, while its panel is open.
+
+import { useEffect, useState } from "react";
+import type { BotId, DesktopFrame, DesktopState } from "../../lib/protocol.gen";
+import { useApi } from "../../store/context";
+
+export interface DesktopView {
+  /** `null` until the daemon answers. */
+  state: DesktopState | null;
+  frame: DesktopFrame | null;
+}
+
+export function useDesktopView(botId: BotId): DesktopView {
+  const api = useApi();
+  const [view, setView] = useState<DesktopView>({ state: null, frame: null });
+  useEffect(() => {
+    let alive = true;
+    setView({ state: null, frame: null });
+    const stop = api.subscribe((event) => {
+      if (event.name === "desktop.changed" && event.params.botId === botId) {
+        setView((now) => ({ ...now, state: event.params }));
+      }
+      if (event.name === "desktop.frame" && event.params.botId === botId) {
+        setView((now) => ({ ...now, frame: event.params }));
+      }
+    });
+    api.call("desktop.watch", { botId }).then(
+      // What arrived meanwhile is newer than the answer.
+      (watched) =>
+        alive &&
+        setView((now) => ({
+          state: now.state ?? watched.state,
+          frame: now.frame ?? watched.frame,
+        })),
+      // An older daemon has no desktop: nothing to show.
+      () =>
+        alive &&
+        setView({
+          state: { botId, window: null, action: null, at: null, stopped: false },
+          frame: null,
+        }),
+    );
+    return () => {
+      alive = false;
+      stop();
+      api.call("desktop.unwatch").catch(() => {});
+    };
+  }, [api, botId]);
+  return view;
+}