feat(desktop): bots click, type, choose and scroll in the owner's apps, through accessibility

app/src/features/desktop/DesktopCard.tsx

@@ -1,6 +1,6 @@
-// A bot asking to see an app on the owner's desktop (spec 24.2): the app,
-// why the bot wants it, what seeing it means, and Allow or Deny. Once
-// answered it shrinks to one line.
+// A bot asking to see, or to use, an app on the owner's desktop (spec
+// 24.2): the app, why the bot wants it, what that means, and Allow or
+// Deny. Once answered it shrinks to one line.
 
 import { Ban, Check, Monitor, TimerOff } from "lucide-react";
 import { useT } from "../../i18n";
@@ -15,29 +15,47 @@ export const DESKTOP_TOOL = "mcp__botloft__desktop";
 interface Asked {
   path: string | null;
   why: string | null;
+  /** To use the app, not only see it. */
+  use: boolean;
 }
 
 function askedOf(input: string): Asked {
   try {
-    const parsed = JSON.parse(input) as { path?: unknown; why?: unknown };
+    const parsed = JSON.parse(input) as { path?: unknown; why?: unknown; level?: unknown };
     const text = (value: unknown) => (typeof value === "string" && value.trim() ? value : null);
-    return { path: text(parsed.path), why: text(parsed.why) };
+    return { path: text(parsed.path), why: text(parsed.why), use: parsed.level === "act" };
   } catch {
-    return { path: null, why: null };
+    return { path: null, why: null, use: false };
   }
 }
 
-function Answered({ approval, bot, app }: { approval: ApprovalItem; bot: Bot; app: string }) {
+function Answered({
+  approval,
+  bot,
+  app,
+  use,
+}: {
+  approval: ApprovalItem;
+  bot: Bot;
+  app: string;
+  use: boolean;
+}) {
   const words = useT().desktop.card;
   switch (approval.status) {
     case "allowed":
-      return <SettledLine icon={Check} tone="ok" text={words.allowed(bot.name, app)} />;
+      return (
+        <SettledLine
+          icon={Check}
+          tone="ok"
+          text={(use ? words.allowedUse : words.allowed)(bot.name, app)}
+        />
+      );
     case "denied":
       return (
         <SettledLine
           icon={Ban}
           tone="danger"
-          text={words.denied(bot.name, app)}
+          text={(use ? words.deniedUse : words.denied)(bot.name, app)}
           note={approval.note}
         />
       );
@@ -54,19 +72,20 @@ export function DesktopCard({ approval, bot }: { approval: ApprovalItem; bot: Bo
     deny: t.chat.approval.denyFailed,
   });
   const app = approval.summary;
+  const { path, why, use } = askedOf(approval.input);
   if (approval.status !== "pending") {
-    return <Answered approval={approval} bot={bot} app={app} />;
+    return <Answered approval={approval} bot={bot} app={app} use={use} />;
   }
-  const { path, why } = askedOf(approval.input);
 
   return (
     <RequestCard
-      label={words.asks(bot.name, app)}
+      label={(use ? words.asksUse : words.asks)(bot.name, app)}
       icon={Monitor}
       tone="warn"
       title={
         <span>
-          {words.wants(bot.name)} <span className="font-semibold">{app}</span>
+          {(use ? words.wantsUse : words.wants)(bot.name)}{" "}
+          <span className="font-semibold">{app}</span>
         </span>
       }
     >
@@ -80,7 +99,7 @@ export function DesktopCard({ approval, bot }: { approval: ApprovalItem; bot: Bo
           {path}
         </p>
       )}
-      <p className="mt-1.5 text-muted text-xs">{words.means(bot.name)}</p>
+      <p className="mt-1.5 text-muted text-xs">{(use ? words.meansUse : words.means)(bot.name)}</p>
       <NoteInput
         label={t.chat.approval.noteLabel(bot.name)}
         value={note}

app/src/features/desktop/desktop.test.tsx

@@ -55,6 +55,23 @@ describe("the desktop", () => {
     expect(await screen.findByText("Scout can see Notepad")).toBeDefined();
   });
 
+  test("a request to use an app says what using it means", async () => {
+    const { fake, scout } = await openScout();
+    act(() => {
+      fake.chat.ask(
+        scout.id,
+        "mcp__botloft__desktop",
+        "Notepad",
+        JSON.stringify({ app: "Notepad", path: NOTEPAD, level: "act", why: "To save the list" }),
+      );
+    });
+    const card = screen.getByRole("region", { name: "Scout asks to use Notepad" });
+    expect(within(card).getByText(/without moving your mouse/)).toBeDefined();
+    expect(within(card).getByText(/Never in password fields/)).toBeDefined();
+    fireEvent.click(within(card).getByRole("button", { name: "Deny" }));
+    expect(await screen.findByText("Scout may not use Notepad")).toBeDefined();
+  });
+
   test("the bot's details list what it may see on the desktop, and it can be taken back", async () => {
     const { fake, scout } = await openScout();
     fireEvent.click(screen.getByRole("button", { name: /^Show details/ }));

app/src/i18n/en/desktop.ts

@@ -11,6 +11,12 @@ export const desktop = {
     allowed: (bot: string, app: string) => `${bot} can see ${app}`,
     denied: (bot: string, app: string) => `${bot} may not see ${app}`,
     expired: (app: string) => `No answer about ${app}`,
+    wantsUse: (bot: string) => `${bot} wants to use`,
+    asksUse: (bot: string, app: string) => `${bot} asks to use ${app}`,
+    meansUse: (bot: string) =>
+      `${bot} will click, type and choose in this app's windows as you would, without moving your mouse, while you are at your computer. Every action shows in this chat. Never in password fields. You can take it back in ${bot}'s details.`,
+    allowedUse: (bot: string, app: string) => `${bot} can use ${app}`,
+    deniedUse: (bot: string, app: string) => `${bot} may not use ${app}`,
   },
   grants: {
     title: "Your desktop",

app/src/i18n/en/tools.ts

@@ -55,6 +55,10 @@ export const tools = {
     desktop_windows: "See your windows",
     desktop_look: "Read a window",
     desktop_screenshot: "Look at a window",
+    desktop_click: "Click in a window",
+    desktop_type: "Type in a window",
+    desktop_select: "Choose in a window",
+    desktop_scroll: "Scroll a window",
   },
   /** Where `browser_scroll` went. */
   scroll: { down: "down", up: "up", top: "to the top", bottom: "to the end" },

app/src/i18n/es/desktop.ts

@@ -10,6 +10,12 @@ export const desktop: Messages["desktop"] = {
     allowed: (bot, app) => `${bot} puede ver ${app}`,
     denied: (bot, app) => `${bot} no puede ver ${app}`,
     expired: (app) => `Sin respuesta sobre ${app}`,
+    wantsUse: (bot) => `${bot} quiere usar`,
+    asksUse: (bot, app) => `${bot} pide usar ${app}`,
+    meansUse: (bot) =>
+      `${bot} va a hacer clic, escribir y elegir en las ventanas de esa app como lo harías tú, sin mover tu mouse, mientras estés en la computadora. Cada acción aparece en este chat. Nunca en campos de contraseña. Puedes quitárselo en los detalles de ${bot}.`,
+    allowedUse: (bot, app) => `${bot} puede usar ${app}`,
+    deniedUse: (bot, app) => `${bot} no puede usar ${app}`,
   },
   grants: {
     title: "Tu escritorio",

app/src/i18n/es/tools.ts

@@ -56,6 +56,10 @@ export const tools: Messages["tools"] = {
     desktop_windows: "Ver tus ventanas",
     desktop_look: "Leer una ventana",
     desktop_screenshot: "Mirar una ventana",
+    desktop_click: "Hacer clic en una ventana",
+    desktop_type: "Escribir en una ventana",
+    desktop_select: "Elegir en una ventana",
+    desktop_scroll: "Desplazar una ventana",
   },
   scroll: {
     down: "hacia abajo",

app/src/i18n/pt-BR/desktop.ts

@@ -10,6 +10,12 @@ export const desktop: Messages["desktop"] = {
     allowed: (bot, app) => `${bot} pode ver ${app}`,
     denied: (bot, app) => `${bot} não pode ver ${app}`,
     expired: (app) => `Sem resposta sobre ${app}`,
+    wantsUse: (bot) => `${bot} quer usar`,
+    asksUse: (bot, app) => `${bot} pede para usar ${app}`,
+    meansUse: (bot) =>
+      `${bot} vai clicar, escrever e escolher nas janelas desse app como você faria, sem mexer no seu mouse, enquanto você estiver no computador. Cada ação aparece neste chat. Nunca em campos de senha. Você pode tirar isso nos detalhes de ${bot}.`,
+    allowedUse: (bot, app) => `${bot} pode usar ${app}`,
+    deniedUse: (bot, app) => `${bot} não pode usar ${app}`,
   },
   grants: {
     title: "Seu desktop",

app/src/i18n/pt-BR/tools.ts

@@ -55,6 +55,10 @@ export const tools: Messages["tools"] = {
     desktop_windows: "Ver suas janelas",
     desktop_look: "Ler uma janela",
     desktop_screenshot: "Olhar uma janela",
+    desktop_click: "Clicar numa janela",
+    desktop_type: "Escrever numa janela",
+    desktop_select: "Escolher numa janela",
+    desktop_scroll: "Rolar uma janela",
   },
   scroll: { down: "para baixo", up: "para cima", top: "até o topo", bottom: "até o fim" },
   use: (tool) => `usar ${tool}`,

crates/botloftd/src/desktop.rs

@@ -1,12 +1,17 @@
 //! The owner's desktop as the bots use it (spec 24): one action at a time
-//! across every bot, and whether the owner is there to see it.
+//! across every bot, whether the owner is there to see it, and each bot's
+//! last reading, whose refs its actions name.
 
+use std::collections::HashMap;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::{Arc, Mutex, PoisonError};
 use std::time::Duration;
 
+use botloft_core::ids::BotId;
 use tokio::sync::MutexGuard;
 
+use crate::platform::desktop::Control;
+
 /// How long the owner may go without touching the computer and still count
 /// as there (spec 24.8).
 pub const AWAY_AFTER: Duration = Duration::from_secs(5 * 60);
@@ -15,9 +20,17 @@ pub const AWAY_AFTER: Duration = Duration::from_secs(5 * 60);
 /// that cannot be known.
 pub type OwnerIdle = Arc<dyn Fn() -> Option<Duration> + Send + Sync>;
 
+/// A bot's last reading of a window: its refs are indexes in `controls`.
+#[derive(Debug, Clone)]
+pub struct Reading {
+    pub window: u64,
+    pub controls: Vec<Control>,
+}
+
 pub struct Desktop {
     /// App connections that said hello, open now.
     apps: AtomicUsize,
+    readings: Mutex<HashMap<BotId, Reading>>,
     /// Held for each desktop action: the cursor is one.
     turn: tokio::sync::Mutex<()>,
     idle: Mutex<OwnerIdle>,
@@ -52,6 +65,7 @@ impl Desktop {
     pub fn new(idle: OwnerIdle) -> Self {
         Self {
             apps: AtomicUsize::new(0),
+            readings: Mutex::new(HashMap::new()),
             turn: tokio::sync::Mutex::new(()),
             idle: Mutex::new(idle),
         }
@@ -88,6 +102,23 @@ impl Desktop {
         }
     }
 
+    /// Keeps `controls` as the bot's last reading of `window`.
+    pub fn keep(&self, bot: &BotId, window: u64, controls: Vec<Control>) {
+        self.readings
+            .lock()
+            .unwrap_or_else(PoisonError::into_inner)
+            .insert(bot.clone(), Reading { window, controls });
+    }
+
+    /// The bot's last reading, if it read a window.
+    pub fn reading(&self, bot: &BotId) -> Option<Reading> {
+        self.readings
+            .lock()
+            .unwrap_or_else(PoisonError::into_inner)
+            .get(bot)
+            .cloned()
+    }
+
     /// Waits for the desktop to be free; it is the caller's until dropped.
     pub async fn turn(&self) -> MutexGuard<'_, ()> {
         self.turn.lock().await

crates/botloftd/src/platform/desktop/mod.rs

@@ -64,6 +64,52 @@ pub enum DesktopError {
     Minimized,
     #[error("Windows could not do it: {0}")]
     System(String),
+    /// The control a reading named is not in the window anymore.
+    #[error("that control is not in the window anymore")]
+    NotThere,
+    /// The control does not take that action through accessibility.
+    #[error("that control cannot be {0} through accessibility")]
+    Cannot(&'static str),
+    /// A password field: the owner types there, never a bot (spec 24.3).
+    #[error("that is a password field")]
+    Password,
+    /// A field that cannot be changed.
+    #[error("that field cannot be changed")]
+    ReadOnly,
+    /// No option with that text in the list.
+    #[error("there is no option {0:?} there")]
+    NoOption(String),
+}
+
+/// What a bot does to a control through accessibility (spec 24.5).
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Action {
+    /// Invokes a button or link, ticks a box, picks an item, opens or
+    /// closes what expands.
+    Click,
+    /// Replaces a field's text.
+    Type(String),
+    /// Picks the option with this text in a list or combo box.
+    Select(String),
+    Scroll(Scroll),
+}
+
+/// Where `Action::Scroll` goes.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Scroll {
+    Down,
+    Up,
+    Top,
+    Bottom,
+}
+
+/// How an action went.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Acted {
+    Done,
+    /// The app did not answer in time: it may be showing a dialog, or still
+    /// busy with it.
+    Waiting,
 }
 
 /// The windows open on the owner's desktop: shown ones, minimized too,
@@ -107,6 +153,21 @@ pub fn read(id: u64) -> Result<Vec<Control>, DesktopError> {
     }
 }
 
+/// Does `action` to the control `target` (its `RuntimeId`, from a reading)
+/// in window `id`, through accessibility, without the owner's mouse or
+/// keyboard (spec 24.5).
+pub fn act(id: u64, target: &[i32], action: &Action) -> Result<Acted, DesktopError> {
+    #[cfg(windows)]
+    {
+        windows::act(id, target, action)
+    }
+    #[cfg(not(windows))]
+    {
+        let _ = (id, target, action);
+        Err(DesktopError::Unavailable)
+    }
+}
+
 /// A picture of the window `id`, only of it, even behind other windows, in
 /// logical pixels up to [`PICTURE_MAX`] (spec 24.4).
 pub fn picture(id: u64) -> Result<Picture, DesktopError> {