feat(desktop): the owner turns on the real mouse and keyboard, and sees on screen when a bot has them

app/src/features/desktop/DesktopGrants.tsx

@@ -8,6 +8,7 @@ import type { Bot, DesktopGrant } from "../../lib/protocol.gen";
 import { useApi } from "../../store/context";
 import { Button } from "../../ui/Button";
 import { attempt } from "../../ui/toast";
+import { RealInput } from "./RealInput";
 
 /** The bot's grants, read when shown and kept current by `bot.desktop`. */
 function useDesktopGrants(botId: Bot["id"]): DesktopGrant[] | null {
@@ -60,6 +61,7 @@ export function DesktopGrants({ bot }: { bot: Bot }) {
                     <span className="block text-ink-soft text-xs">
                       {grant.level === "act" ? words.act : words.see}
                     </span>
+                    {grant.level === "act" && <RealInput bot={bot} grant={grant} app={app} />}
                     {grant.appPath && (
                       <span
                         className="block break-all font-mono text-muted text-xs"

app/src/features/desktop/DesktopPanel.tsx

@@ -34,6 +34,8 @@ export function actionText(action: DesktopAction | null, t: Words): string {
       return t.select(action.option ?? "");
     case "scroll":
       return target ? t.scroll(target) : t.scrollSomething;
+    case "press":
+      return t.press(action.option ?? "");
   }
 }
 

app/src/features/desktop/RealInput.tsx

@@ -0,0 +1,64 @@
+// The switch of a desktop grant for the owner's real mouse and keyboard
+// (spec 24.7, 24.10): off from the start; turning it on first says, in a
+// dialog, what changes and how to stop it; turning it off is at once.
+
+import { Check } from "lucide-react";
+import { useState } from "react";
+import { useT } from "../../i18n";
+import type { Bot, DesktopGrant } from "../../lib/protocol.gen";
+import { useApi } from "../../store/context";
+import { Button } from "../../ui/Button";
+import { Dialog } from "../../ui/Dialog";
+import { Switch } from "../../ui/Switch";
+import { attempt } from "../../ui/toast";
+
+export function RealInput({ bot, grant, app }: { bot: Bot; grant: DesktopGrant; app: string }) {
+  const words = useT().desktop.grants;
+  const api = useApi();
+  const [asking, setAsking] = useState(false);
+  const set = (realInput: boolean) =>
+    attempt(words.realFailed, () =>
+      api.call("desktop.setOptions", { grantId: grant.id, realInput }),
+    );
+  return (
+    <>
+      <span className="mt-1 flex items-center gap-2 text-xs">
+        <Switch
+          checked={grant.realInput}
+          label={`${words.real}: ${app}`}
+          onChange={(on) => (on ? setAsking(true) : void set(false))}
+        />
+        <span className="text-ink-soft">{words.real}</span>
+      </span>
+      {asking && (
+        <Dialog
+          title={words.realTitle(bot.name, app)}
+          onClose={() => setAsking(false)}
+          footer={
+            <>
+              <Button variant="ghost" onClick={() => setAsking(false)}>
+                {words.cancel}
+              </Button>
+              <Button
+                variant="primary"
+                icon={Check}
+                onClick={() => {
+                  setAsking(false);
+                  void set(true);
+                }}
+              >
+                {words.realConfirm}
+              </Button>
+            </>
+          }
+        >
+          <ul className="flex list-disc flex-col gap-1.5 pl-5 text-sm">
+            {words.realPoints(bot.name).map((point) => (
+              <li key={point}>{point}</li>
+            ))}
+          </ul>
+        </Dialog>
+      )}
+    </>
+  );
+}

app/src/features/desktop/realInput.test.tsx

@@ -0,0 +1,74 @@
+import { act, cleanup, fireEvent, screen, waitFor, within } from "@testing-library/react";
+import { afterEach, describe, expect, test } from "vitest";
+import { setLocaleChoice } from "../../i18n";
+import { FakeBotloft } from "../../lib/fake";
+import { crewOpened, openBot, renderApp } from "../../test/app";
+
+afterEach(() => {
+  act(() => setLocaleChoice("en"));
+  cleanup();
+});
+
+const NOTEPAD = "C:\\Windows\\notepad.exe";
+
+async function openScout() {
+  const fake = new FakeBotloft();
+  const ops = fake.addCrew("Ops");
+  const scout = fake.addBot(ops.id, "Scout", "Finds sources");
+  fake.setBotState(scout.id, "idle");
+  renderApp(fake);
+  await crewOpened("Ops");
+  openBot("Scout");
+  await screen.findByRole("list", { name: "Messages" });
+  return { fake, scout };
+}
+
+const details = () => screen.getByRole("complementary", { name: "About Scout" });
+
+describe("the real mouse and keyboard", () => {
+  test("are off, and turning them on says first what changes", async () => {
+    const { fake, scout } = await openScout();
+    act(() => {
+      fake.desktop.grant(scout.id, NOTEPAD, "Notepad", "act");
+    });
+    fireEvent.click(screen.getByRole("button", { name: /^Show details/ }));
+    const toggle = await within(details()).findByRole("switch", {
+      name: "Real mouse and keyboard: Notepad",
+    });
+    expect(toggle.getAttribute("aria-checked")).toBe("false");
+
+    fireEvent.click(toggle);
+    const dialog = screen.getByRole("dialog", {
+      name: "Let Scout use your real mouse and keyboard in Notepad?",
+    });
+    expect(within(dialog).getByText(/Move the mouse or press a key and it stops/)).toBeDefined();
+    expect(within(dialog).getByText(/Ctrl\+Alt\+Esc/)).toBeDefined();
+    fireEvent.click(within(dialog).getByRole("button", { name: "Cancel" }));
+    expect(fake.desktop.grants[0]?.realInput).toBe(false);
+
+    fireEvent.click(toggle);
+    fireEvent.click(screen.getByRole("button", { name: "Turn on" }));
+    await waitFor(() => expect(fake.desktop.grants[0]?.realInput).toBe(true));
+    expect(toggle.getAttribute("aria-checked")).toBe("true");
+
+    // Off at once, without asking.
+    fireEvent.click(toggle);
+    await waitFor(() => expect(fake.desktop.grants[0]?.realInput).toBe(false));
+  });
+
+  test("are not offered where the bot may only see", async () => {
+    const { fake, scout } = await openScout();
+    act(() => {
+      fake.desktop.grant(scout.id, NOTEPAD, "Notepad");
+    });
+    fireEvent.click(screen.getByRole("button", { name: /^Show details/ }));
+    await within(details()).findByText("Can see");
+    expect(within(details()).queryByRole("switch", { name: /Real mouse/ })).toBeNull();
+  });
+
+  test("the daemon hears the owner's language when it changes, for its notice", async () => {
+    const { fake } = await openScout();
+    act(() => setLocaleChoice("pt-BR"));
+    await waitFor(() => expect(fake.locale).toBe("pt-BR"));
+  });
+});

app/src/features/desktop/useLocaleToDaemon.ts

@@ -0,0 +1,21 @@
+// The owner's language, told to the daemon when it changes, for the notice
+// it shows on screen while a bot uses the real mouse and keyboard (spec
+// 24.7). The language at connection goes with `session.hello`.
+
+import { useEffect, useRef } from "react";
+import { useLocale } from "../../i18n";
+import { useApi } from "../../store/context";
+
+export function useLocaleToDaemon(): void {
+  const api = useApi();
+  const { locale } = useLocale();
+  const told = useRef(locale);
+  useEffect(() => {
+    if (told.current === locale) {
+      return;
+    }
+    told.current = locale;
+    // An older daemon does not know the method: nothing to tell.
+    api.call("session.setLocale", { locale }).catch(() => {});
+  }, [api, locale]);
+}

app/src/i18n/en/desktop.ts

@@ -41,6 +41,7 @@ export const desktop = {
     select: (option: string) => `Chose "${option}"`,
     scroll: (target: string) => `Scrolled "${target}"`,
     scrollSomething: "Scrolled",
+    press: (keys: string) => `Pressed ${keys}`,
     showInPanel: "Watch on the desktop panel",
     close: "Close",
   },
@@ -53,5 +54,17 @@ export const desktop = {
     whole: "The whole desktop",
     remove: (app: string) => `Take back ${app}`,
     removeFailed: "Could not take it back",
+    real: "Real mouse and keyboard",
+    realFailed: "Could not change it",
+    realTitle: (bot: string, app: string) =>
+      `Let ${bot} use your real mouse and keyboard in ${app}?`,
+    realPoints: (bot: string) => [
+      `When accessibility cannot do something in this app, ${bot} moves your cursor and types as if it were you.`,
+      `While it acts, your mouse and keyboard are its, and a notice on your screen says so.`,
+      `Move the mouse or press a key and it stops at once. Ctrl+Alt+Esc stops every bot on your desktop.`,
+      `It never types in password fields and never presses Windows' own keys, like the Windows key or Alt+Tab.`,
+    ],
+    realConfirm: "Turn on",
+    cancel: "Cancel",
   },
 };

app/src/i18n/es/desktop.ts

@@ -42,6 +42,7 @@ export const desktop: Messages["desktop"] = {
     select: (option) => `Eligió "${option}"`,
     scroll: (target) => `Desplazó "${target}"`,
     scrollSomething: "Desplazó",
+    press: (keys) => `Presionó ${keys}`,
     showInPanel: "Ver en el panel del escritorio",
     close: "Cerrar",
   },
@@ -54,5 +55,16 @@ export const desktop: Messages["desktop"] = {
     whole: "Todo el escritorio",
     remove: (app) => `Quitar ${app}`,
     removeFailed: "No se pudo quitar",
+    real: "Mouse y teclado de verdad",
+    realFailed: "No se pudo cambiar",
+    realTitle: (bot, app) => `¿Dejar que ${bot} use tu mouse y teclado de verdad en ${app}?`,
+    realPoints: (bot) => [
+      `Cuando la accesibilidad no alcanza algo en esa app, ${bot} mueve tu cursor y escribe como si fueras tú.`,
+      `Mientras actúa, el mouse y el teclado son suyos, y un aviso en tu pantalla lo dice.`,
+      `Mueve el mouse o presiona una tecla y se detiene al instante. Ctrl+Alt+Esc detiene a todos los bots en tu escritorio.`,
+      `Nunca escribe en campos de contraseña ni presiona teclas propias de Windows, como la tecla Windows o Alt+Tab.`,
+    ],
+    realConfirm: "Activar",
+    cancel: "Cancelar",
   },
 };

app/src/i18n/pt-BR/desktop.ts

@@ -41,6 +41,7 @@ export const desktop: Messages["desktop"] = {
     select: (option) => `Escolheu "${option}"`,
     scroll: (target) => `Rolou "${target}"`,
     scrollSomething: "Rolou",
+    press: (keys) => `Apertou ${keys}`,
     showInPanel: "Ver no painel do desktop",
     close: "Fechar",
   },
@@ -53,5 +54,16 @@ export const desktop: Messages["desktop"] = {
     whole: "O desktop inteiro",
     remove: (app) => `Tirar ${app}`,
     removeFailed: "Não foi possível tirar",
+    real: "Mouse e teclado de verdade",
+    realFailed: "Não foi possível mudar",
+    realTitle: (bot, app) => `Deixar ${bot} usar seu mouse e teclado de verdade no ${app}?`,
+    realPoints: (bot) => [
+      `Quando a acessibilidade não alcança algo nesse app, ${bot} move o seu cursor e digita como se fosse você.`,
+      `Enquanto ele age, o mouse e o teclado são dele, e um aviso na sua tela diz isso.`,
+      `Mexa o mouse ou aperte uma tecla e ele para na hora. Ctrl+Alt+Esc para todos os bots no seu desktop.`,
+      `Ele nunca digita em campos de senha e nunca aperta teclas do próprio Windows, como a tecla Windows ou Alt+Tab.`,
+    ],
+    realConfirm: "Ligar",
+    cancel: "Cancelar",
   },
 };

app/src/lib/fake.ts

@@ -75,6 +75,8 @@ export class FakeBotloft implements BotloftApi {
   readonly chat = new FakeChat(this);
   readonly allow = new FakeAllow(this);
   readonly desktop = new FakeDesktop(this);
+  /** The language the app last told, as the daemon keeps it (spec 24.7). */
+  locale: string | null = null;
   readonly conversation = new FakeConversation(this);
   readonly routines = new FakeRoutines(this);
   readonly files = new FakeFiles(this);
@@ -239,6 +241,10 @@ export class FakeBotloft implements BotloftApi {
       daemonVersion: this.system.daemonVersion,
       protocol: PROTOCOL_VERSION,
     }),
+    "session.setLocale": ({ locale }) => {
+      this.locale = locale;
+      return null;
+    },
     "system.status": () => this.system,
     "system.refresh": () => {
       this.refreshes += 1;

app/src/lib/fakeDesktop.ts

@@ -104,6 +104,17 @@ export class FakeDesktop {
         this.changed(grant.botId);
         return { botId: grant.botId, grants: this.of(grant.botId) };
       },
+      "desktop.setOptions": ({ grantId, realInput }) => {
+        const grant = this.grants.find((each) => each.id === grantId);
+        if (!grant) {
+          throw notFound(`desktop grant ${grantId}`);
+        }
+        if (realInput !== undefined) {
+          grant.realInput = realInput;
+        }
+        this.changed(grant.botId);
+        return { botId: grant.botId, grants: this.of(grant.botId) };
+      },
       "desktop.watch": ({ botId }) => {
         this.fake.bot(botId);
         this.watching = botId;

app/src/lib/protocol.gen.ts

@@ -265,7 +265,12 @@ archivedAt: number | null, };
 /**
  * Identifies the connecting app in `session.hello`.
  */
-export type ClientInfo = { name: string, version: string, };
+export type ClientInfo = { name: string, version: string, 
+/**
+ * The language the owner reads the app in ("en", "pt-BR", "es"), for
+ * what the daemon itself shows them (spec 24.7).
+ */
+locale?: string, };
 
 /**
  * First request of every connection.
@@ -276,6 +281,11 @@ export type HelloParams = {
  */
 token: string, client: ClientInfo, protocol: number, };
 
+/**
+ * `session.setLocale`: the owner changed the app's language.
+ */
+export type LocaleParams = { locale: string, };
+
 export type HelloResult = { daemonVersion: string, protocol: number, };
 
 /**
@@ -859,6 +869,15 @@ export type DesktopGrantsParams = { botId: BotId, };
 
 export type DesktopGrantIdParams = { grantId: DesktopGrantId, };
 
+/**
+ * `desktop.setOptions`: what the owner changes in a grant (spec 24.10).
+ */
+export type DesktopOptionsParams = { grantId: DesktopGrantId, 
+/**
+ * The bot may use the real mouse and keyboard there (spec 24.7).
+ */
+realInput?: boolean, };
+
 /**
  * A bot's grants after one changed: the result of `desktop.revoke` and
  * the params of the `bot.desktop` notification.
@@ -882,7 +901,7 @@ app: string, };
  * What a bot did in a window (spec 24.5), for its panel, which words it
  * in the owner's language.
  */
-export type DesktopActionKind = "click" | "type" | "select" | "scroll";
+export type DesktopActionKind = "click" | "type" | "select" | "scroll" | "press";
 
 export type DesktopAction = { kind: DesktopActionKind, 
 /**
@@ -1580,6 +1599,7 @@ done: boolean, };
 /** Params and result of every request method. */
 export interface RpcMethods {
   "session.hello": { params: HelloParams; result: HelloResult };
+  "session.setLocale": { params: LocaleParams; result: null };
   "system.status": { params: undefined; result: SystemStatus };
   "system.refresh": { params: undefined; result: SystemStatus };
   "settings.get": { params: undefined; result: Settings };
@@ -1619,6 +1639,7 @@ export interface RpcMethods {
   "rules.delete": { params: RuleIdParams; result: BotRules };
   "desktop.grants": { params: DesktopGrantsParams; result: Array<DesktopGrant> };
   "desktop.revoke": { params: DesktopGrantIdParams; result: BotDesktop };
+  "desktop.setOptions": { params: DesktopOptionsParams; result: BotDesktop };
   "desktop.watch": { params: DesktopBotParams; result: DesktopView };
   "desktop.unwatch": { params: undefined; result: null };
   "desktop.stop": { params: DesktopBotParams; result: DesktopState };

app/src/main.tsx

@@ -4,6 +4,7 @@ import packageJson from "../package.json";
 import { App } from "./App";
 import { startEyeMoments } from "./features/bots/eyeMoments";
 import type { Connect } from "./features/onboarding/link";
+import { currentLocale } from "./i18n";
 import { connect, rpcUrl } from "./lib/client";
 import type { Host } from "./lib/host";
 import { tauriHost } from "./lib/tauriHost";
@@ -22,7 +23,9 @@ async function props(): Promise<{ host: Host; connect: Connect }> {
     const client = { name: "botloft-app", version: packageJson.version };
     return {
       host: tauriHost(),
-      connect: (port, token) => connect({ url: rpcUrl(port), token, client }),
+      // Its language, for what the daemon shows on screen (spec 24.7).
+      connect: (port, token) =>
+        connect({ url: rpcUrl(port), token, client: { ...client, locale: currentLocale() } }),
     };
   }
   if (import.meta.env.DEV) {