feat(desktop): bot cursor on the desktop panel, and the panel opens by itself

app/src/dev/seedDesktop.ts

@@ -34,7 +34,13 @@ function drawNotepad(): string {
 
 export function seedDesktop(fake: FakeBotloft, bots: { scout: BotId; analyst: BotId }) {
   fake.desktop.grant(bots.scout, "C:\\Windows\\notepad.exe", "Notepad");
-  fake.desktop.use(bots.scout, NOTES, { kind: "click", target: "Save", option: null });
+  fake.desktop.use(bots.scout, NOTES, {
+    kind: "click",
+    target: "Save",
+    option: null,
+    x: 0.1,
+    y: 0.12,
+  });
   fake.desktop.paint(bots.scout, drawNotepad(), 760, 480);
   fake.chat.tool(bots.scout, "mcp__botloft__desktop_click", {
     summary: "To save the list",

app/src/features/bots/BotView.tsx

@@ -121,15 +121,6 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) {
       setTakeBrowser((count) => count + 1);
     }
   });
-  // The panel follows what the bot starts doing (spec 15.1), over the one
-  // that came back too.
-  useFollowBot({
-    bot,
-    side,
-    writing: screens.writing,
-    open: (panel) => (panel === "browser" ? showBrowser() : showScreen(null)),
-    close: (panel) => side === panel && setSide(null),
-  });
   const showTerminal = useStable(() => {
     if (filesOpen) {
       seen();
@@ -142,6 +133,23 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) {
     }
     setSide("desktop");
   });
+  // The panel follows what the bot starts doing (spec 15.1), over the one
+  // that came back too.
+  useFollowBot({
+    bot,
+    side,
+    writing: screens.writing,
+    open: (panel) => {
+      if (panel === "browser") {
+        showBrowser();
+      } else if (panel === "desktop") {
+        showDesktop();
+      } else {
+        showScreen(null);
+      }
+    },
+    close: (panel) => side === panel && setSide(null),
+  });
   // The dock of the bot's computer: browser, terminal and files.
   const dock = {
     open: side,

app/src/features/bots/useFollowBot.ts

@@ -1,7 +1,9 @@
 // The panel beside the chat follows what the bot starts doing (spec 15.1),
 // so the owner sees it happen: its browser starting or a request for a
 // hand opens the browser (spec 21.8), and a screen it starts writing opens
-// the design area (spec 22.5), whatever panel was open. Each opens once per
+// the design area (spec 22.5), and the bot starting to use the owner's
+// desktop after a pause opens the desktop panel (spec 24.9), whatever panel
+// was open. Each opens once per
 // start; closing it leaves the button's dot. When the browser goes to rest
 // (spec 21.2) its panel closes, and it opens again once the bot wakes it.
 // The browser in the owner's hands is never taken away. The owner can turn this off in Settings
@@ -12,9 +14,12 @@
 import { useEffect, useRef } from "react";
 import type { Bot } from "../../lib/protocol.gen";
 import { prefs } from "../../shell/prefs";
-import { useApp } from "../../store/context";
+import { useApi, useApp } from "../../store/context";
 
-export type Followed = "browser" | "screens";
+export type Followed = "browser" | "screens" | "desktop";
+
+/** A use of the desktop this long after the last one starts a new run. */
+const DESKTOP_PAUSE = 2 * 60_000;
 
 export function useFollowBot({
   bot,
@@ -41,6 +46,10 @@ export function useFollowBot({
   // behind its button, a request for a hand does not.
   const before = useRef<{ browsing: boolean; asking: boolean } | null>(null);
   const drawn = useRef(new Set<string>());
+  const api = useApi();
+  // The latest of what the desktop listener needs, without listening anew.
+  const now = useRef({ side, held, open });
+  now.current = { side, held, open };
 
   // biome-ignore lint/correctness/useExhaustiveDependencies: another bot starts afresh
   useEffect(() => {
@@ -75,4 +84,26 @@ export function useFollowBot({
     }
     open("screens");
   }, [bot.id, writing]);
+
+  useEffect(() => {
+    let last: number | null = null;
+    return api.subscribe((event) => {
+      if (event.name !== "desktop.changed" || event.params.botId !== bot.id) {
+        return;
+      }
+      const { at, window, stopped } = event.params;
+      if (at === null || !window || stopped || at === last) {
+        return;
+      }
+      const was = last;
+      last = at;
+      if (was !== null && at - was < DESKTOP_PAUSE) {
+        return;
+      }
+      const { side, held, open } = now.current;
+      if (side !== "desktop" && !held && prefs.followBot.get()) {
+        open("desktop");
+      }
+    });
+  }, [api, bot.id]);
 }

app/src/features/browser/LiveView.tsx

@@ -1,17 +1,33 @@
 // The bot's screen, live (spec 21.8): the newest picture of its active
 // tab, as large as fits the panel, with its cursor gliding to each point it
-// acts on, a ring where it clicks, and a line about what it just did.
+// acts on, a ring where it clicks, and a line about what it just did. The
+// desktop panel shows the owner's window the same way (spec 24.9).
 
 import { type ReactNode, useEffect, useState } from "react";
 import { useT } from "../../i18n";
-import type { Bot, BrowserAction, BrowserFrame } from "../../lib/protocol.gen";
+import type { Bot, BrowserAction } from "../../lib/protocol.gen";
 import { BotCursor } from "../bots/BotCursor";
 
 /** The page until a picture says its size. */
 const PAGE = { width: 1280, height: 800 };
 
+/** A picture on the stage: a JPEG, base64, and its size. */
+export interface StageFrame {
+  data: string;
+  width: number;
+  height: number;
+}
+
+/** What the bot did, at a point in the picture's pixels when it has one. */
+export interface StageAction {
+  kind: string;
+  x: number | null;
+  y: number | null;
+  at: number;
+}
+
 /** Where on the screen, in percent, an action happened. */
-function spot(action: BrowserAction | null, frame: BrowserFrame | null) {
+function spot(action: StageAction | null, frame: StageFrame | null) {
   if (!action || action.x === null || action.y === null) {
     return null;
   }
@@ -52,13 +68,19 @@ export function LiveView({
   bot,
   frame,
   action,
+  label,
+  alt = "",
   dim = false,
   held = false,
   children,
 }: {
   bot: Bot;
-  frame: BrowserFrame | null;
-  action: BrowserAction | null;
+  frame: StageFrame | null;
+  action: StageAction | null;
+  /** What the stage is, for screen readers; the bot's browser by default. */
+  label?: string;
+  /** The picture's own name, when it has one. */
+  alt?: string;
   /** The browser closed: the last picture stays, faded. */
   dim?: boolean;
   /** The owner has it (spec 21.10): the bot's cursor steps aside. */
@@ -81,7 +103,7 @@ export function LiveView({
 
   return (
     <figure
-      aria-label={t.browser.screen(bot.name)}
+      aria-label={label ?? t.browser.screen(bot.name)}
       className={`relative mx-auto my-0 overflow-hidden rounded-xl border bg-canvas shadow-sm transition-[border-color,box-shadow] duration-200 ${
         held ? "border-accent ring-2 ring-accent/35" : "border-line"
       }`}
@@ -94,7 +116,7 @@ export function LiveView({
       {frame && (
         <img
           src={`data:image/jpeg;base64,${frame.data}`}
-          alt=""
+          alt={alt}
           draggable={false}
           className={`absolute inset-0 h-full w-full select-none object-contain transition-[opacity,filter] duration-500 ${
             dim ? "opacity-40 grayscale" : ""

app/src/features/desktop/DesktopPanel.tsx

@@ -5,7 +5,7 @@
 import { Monitor, Play, Square, X } from "lucide-react";
 import type { CSSProperties } from "react";
 import { useT } from "../../i18n";
-import type { Bot, DesktopAction } from "../../lib/protocol.gen";
+import type { Bot, DesktopAction, DesktopFrame, DesktopState } from "../../lib/protocol.gen";
 import { useApi } from "../../store/context";
 import { Badge } from "../../ui/Badge";
 import { Button } from "../../ui/Button";
@@ -14,6 +14,7 @@ import { EmptyState } from "../../ui/EmptyState";
 import { SidePanel } from "../../ui/SidePanel";
 import { attempt } from "../../ui/toast";
 import { BotAvatar } from "../bots/BotAvatar";
+import { LiveView, type StageAction } from "../browser/LiveView";
 import { ComputerFooter } from "../terminal/ComputerDock";
 import { useDesktopView } from "./useDesktopView";
 
@@ -39,6 +40,20 @@ export function actionText(action: DesktopAction | null, t: Words): string {
   }
 }
 
+/** The bot's last action at its point in the picture, for its cursor. */
+function pointed(state: DesktopState | null, frame: DesktopFrame): StageAction | null {
+  const action = state?.action;
+  if (!action || state?.at == null) {
+    return null;
+  }
+  return {
+    kind: action.kind,
+    x: action.x === null ? null : action.x * frame.width,
+    y: action.y === null ? null : action.y * frame.height,
+    at: state.at,
+  };
+}
+
 export function DesktopPanel({ bot, onClose }: { bot: Bot; onClose(): void }) {
   const t = useT().desktop.panel;
   const api = useApi();
@@ -102,19 +117,28 @@ export function DesktopPanel({ bot, onClose }: { bot: Bot; onClose(): void }) {
               </span>
             </p>
             <div
-              className="bot-stage grid min-h-40 flex-1 place-items-center p-2.5"
+              className="bot-stage flex min-h-40 flex-1 flex-col p-2.5"
               style={{ "--bot": bot.color } as CSSProperties}
             >
-              {frame ? (
-                <img
-                  src={`data:image/jpeg;base64,${frame.data}`}
-                  alt={window.title}
-                  draggable={false}
-                  className="max-h-full max-w-full select-none rounded-lg object-contain shadow-lift"
-                />
-              ) : (
-                <p className="text-sm text-white/80">{t.waiting}</p>
-              )}
+              <div
+                className="grid min-h-0 flex-1 place-items-center"
+                style={{ containerType: "size" }}
+              >
+                {frame ? (
+                  <div className="relative">
+                    <LiveView
+                      bot={bot}
+                      frame={frame}
+                      action={pointed(state, frame)}
+                      label={t.screen(bot.name, window.app)}
+                      alt={window.title}
+                      held={stopped}
+                    />
+                  </div>
+                ) : (
+                  <p className="text-sm text-white/80">{t.waiting}</p>
+                )}
+              </div>
             </div>
             <p aria-live="polite" className="flex items-center gap-2 text-ink-soft text-sm">
               <BotAvatar color={bot.color} size={18} mood={stopped ? "sleeping" : "working"} />

app/src/features/desktop/panel.test.tsx

@@ -34,7 +34,13 @@ describe("the desktop panel", () => {
     expect(fake.desktop.watching).toBe(scout.id);
 
     act(() => {
-      fake.desktop.use(scout.id, NOTES, { kind: "click", target: "Save", option: null });
+      fake.desktop.use(scout.id, NOTES, {
+        kind: "click",
+        target: "Save",
+        option: null,
+        x: 0.5,
+        y: 0.25,
+      });
     });
     expect(await within(panel()).findByText("Shopping list - Notepad")).toBeDefined();
     expect(within(panel()).getByText('Clicked "Save"')).toBeDefined();
@@ -47,6 +53,13 @@ describe("the desktop panel", () => {
     expect(picture.getAttribute("src")).toBe("data:image/jpeg;base64,AAAA");
     expect(within(panel()).getByText("Live")).toBeDefined();
 
+    // The bot's cursor lands where it clicked, with a ring.
+    const stage = within(panel()).getByRole("figure", { name: "Notepad, as Scout sees it" });
+    const cursor = stage.querySelector(".bot-cursor") as HTMLElement;
+    expect(cursor.style.left).toBe("50%");
+    expect(cursor.style.top).toBe("25%");
+    expect(stage.querySelector(".browser-ripple")).not.toBeNull();
+
     // Closed, nobody watches.
     fireEvent.click(within(panel()).getByRole("button", { name: "Close" }));
     await waitFor(() => expect(fake.desktop.watching).toBeNull());
@@ -68,4 +81,31 @@ describe("the desktop panel", () => {
     await waitFor(() => expect(fake.desktop.state(scout.id).stopped).toBe(false));
     expect(within(panel()).queryByText("You stopped Scout on your desktop")).toBeNull();
   });
+
+  test("opens by itself when the bot starts using the desktop, once per run", async () => {
+    const { fake, scout } = await openScout();
+    expect(screen.queryByRole("complementary", { name: "Scout's desktop" })).toBeNull();
+    act(() => {
+      fake.desktop.use(scout.id, NOTES);
+    });
+    expect(await within(panel()).findByText("Shopping list - Notepad")).toBeDefined();
+
+    // Closed, the next action of the same run leaves it closed.
+    fireEvent.click(within(panel()).getByRole("button", { name: "Close" }));
+    await waitFor(() =>
+      expect(screen.queryByRole("complementary", { name: "Scout's desktop" })).toBeNull(),
+    );
+    act(() => {
+      fake.now += 5_000;
+      fake.desktop.use(scout.id, NOTES);
+    });
+    expect(screen.queryByRole("complementary", { name: "Scout's desktop" })).toBeNull();
+
+    // After a pause, a new run opens it again.
+    act(() => {
+      fake.now += 3 * 60_000;
+      fake.desktop.use(scout.id, NOTES);
+    });
+    expect(await within(panel()).findByText("Shopping list - Notepad")).toBeDefined();
+  });
 });

app/src/i18n/en/desktop.ts

@@ -20,6 +20,7 @@ export const desktop = {
   },
   panel: {
     label: (bot: string) => `${bot}'s desktop`,
+    screen: (bot: string, app: string) => `${app}, as ${bot} sees it`,
     heading: "Desktop",
     live: "Live",
     stoppedBadge: "Stopped",

app/src/i18n/es/desktop.ts

@@ -19,6 +19,7 @@ export const desktop: Messages["desktop"] = {
   },
   panel: {
     label: (bot) => `Escritorio de ${bot}`,
+    screen: (bot, app) => `${app}, como lo ve ${bot}`,
     heading: "Escritorio",
     live: "En vivo",
     stoppedBadge: "Detenido",

app/src/i18n/pt-BR/desktop.ts

@@ -19,6 +19,7 @@ export const desktop: Messages["desktop"] = {
   },
   panel: {
     label: (bot) => `Desktop de ${bot}`,
+    screen: (bot, app) => `${app}, como ${bot} vê`,
     heading: "Desktop",
     live: "Ao vivo",
     stoppedBadge: "Parado",

app/src/lib/protocol.gen.ts

@@ -921,7 +921,13 @@ target: string,
 /**
  * The option chosen, for `select`.
  */
-option: string | null, };
+option: string | null, 
+/**
+ * Where in the window, as fractions of its width and height from its
+ * top left corner, when the action has a point: for the bot's cursor
+ * on its panel.
+ */
+x: number | null, y: number | null, };
 
 /**
  * What a bot does on the owner's desktop, for its panel (spec 24.9): the