feat(desktop): what each bot does on the desktop, live, and stopping it

app/src/lib/fakeDesktop.ts

@@ -1,16 +1,29 @@
-// The fake daemon's desktop grants (spec 24.2, 24.10): what each bot may
-// see and do on the owner's desktop, and `desktop.*`. The bots' desktop
-// tools are not played: grants come from `grant`, as the real daemon makes
-// them when the owner allows a request.
+// The fake daemon's desktop (spec 24): what each bot may see and do on the
+// owner's desktop (spec 24.2, 24.10), what it is doing there for its panel,
+// with pictures while the panel watches, and stopping it (spec 24.9). The
+// bots' desktop tools are not played: grants come from `grant` and use
+// from `use`, as the real daemon makes them.
 
 import type { FakeBotloft, Handlers } from "./fake";
 import { notFound } from "./fakeRules";
-import type { BotId, DesktopGrant, DesktopLevel } from "./protocol.gen";
+import type {
+  BotId,
+  DesktopAction,
+  DesktopFrame,
+  DesktopGrant,
+  DesktopLevel,
+  DesktopState,
+  DesktopWindow,
+} from "./protocol.gen";
 
 type DesktopMethods = Extract<keyof Handlers, `desktop.${string}`>;
 
 export class FakeDesktop {
   readonly grants: DesktopGrant[] = [];
+  private readonly states = new Map<BotId, DesktopState>();
+  private readonly frames = new Map<BotId, DesktopFrame>();
+  /** The bot whose panel the app watches, if any. */
+  watching: BotId | null = null;
 
   constructor(private readonly fake: FakeBotloft) {}
 
@@ -48,6 +61,33 @@ export class FakeDesktop {
     this.fake.emit({ name: "bot.desktop", params: { botId, grants: this.of(botId) } });
   }
 
+  state(botId: BotId): DesktopState {
+    return (
+      this.states.get(botId) ?? { botId, window: null, action: null, at: null, stopped: false }
+    );
+  }
+
+  private set(botId: BotId, change: Partial<DesktopState>): DesktopState {
+    const state = { ...this.state(botId), ...change };
+    this.states.set(botId, state);
+    this.fake.emit({ name: "desktop.changed", params: state });
+    return state;
+  }
+
+  /** The bot read or acted in `window`; `action` says what it did. */
+  use(botId: BotId, window: DesktopWindow, action: DesktopAction | null = null): DesktopState {
+    return this.set(botId, { window, action, at: this.fake.now });
+  }
+
+  /** A new picture of the bot's window: sent while its panel watches. */
+  paint(botId: BotId, data: string, width = 800, height = 600): void {
+    const frame = { botId, data, width, height };
+    this.frames.set(botId, frame);
+    if (this.watching === botId) {
+      this.fake.emit({ name: "desktop.frame", params: frame });
+    }
+  }
+
   handlers(): Pick<Handlers, DesktopMethods> {
     return {
       "desktop.grants": ({ botId }) => {
@@ -64,6 +104,23 @@ export class FakeDesktop {
         this.changed(grant.botId);
         return { botId: grant.botId, grants: this.of(grant.botId) };
       },
+      "desktop.watch": ({ botId }) => {
+        this.fake.bot(botId);
+        this.watching = botId;
+        return { state: this.state(botId), frame: this.frames.get(botId) ?? null };
+      },
+      "desktop.unwatch": () => {
+        this.watching = null;
+        return null;
+      },
+      "desktop.stop": ({ botId }) => {
+        this.fake.bot(botId);
+        return this.set(botId, { stopped: true });
+      },
+      "desktop.resume": ({ botId }) => {
+        this.fake.bot(botId);
+        return this.set(botId, { stopped: false });
+      },
     };
   }
 }

app/src/lib/protocol.gen.ts

@@ -869,6 +869,74 @@ export type BotDesktop = { botId: BotId,
  */
 grants: Array<DesktopGrant>, };
 
+/**
+ * The window a bot is using on the owner's desktop.
+ */
+export type DesktopWindow = { id: number, title: string, 
+/**
+ * The app's name for people.
+ */
+app: string, };
+
+/**
+ * What a bot did in a window (spec 24.5), for its panel, which words it
+ * in the owner's language.
+ */
+export type DesktopActionKind = "click" | "type" | "select" | "scroll";
+
+export type DesktopAction = { kind: DesktopActionKind, 
+/**
+ * The control's name as the window reads it; empty when it has none.
+ */
+target: string, 
+/**
+ * The option chosen, for `select`.
+ */
+option: string | null, };
+
+/**
+ * What a bot does on the owner's desktop, for its panel (spec 24.9): the
+ * params of `desktop.changed`.
+ */
+export type DesktopState = { botId: BotId, 
+/**
+ * The window it read or acted in last.
+ */
+window: DesktopWindow | null, 
+/**
+ * What it did last; `null` after only reading.
+ */
+action: DesktopAction | null, 
+/**
+ * When it last read or acted, Unix ms.
+ */
+at: number | null, 
+/**
+ * The owner stopped it (spec 24.9): its desktop tools refuse until
+ * they let it go on.
+ */
+stopped: boolean, };
+
+/**
+ * A picture of the window a bot is using, for whoever watches its panel.
+ */
+export type DesktopFrame = { botId: BotId, 
+/**
+ * JPEG, base64.
+ */
+data: string, width: number, height: number, };
+
+/**
+ * The result of `desktop.watch`: the bot's state and, when it uses a
+ * window, its picture now.
+ */
+export type DesktopView = { state: DesktopState, frame: DesktopFrame | null, };
+
+/**
+ * `desktop.watch`, `desktop.stop` and `desktop.resume`.
+ */
+export type DesktopBotParams = { botId: BotId, };
+
 export type ApprovalItem = { approvalId: ApprovalId, toolName: string, summary: string, 
 /**
  * What the bot says the command is for, as in `ToolItem`. The bot
@@ -1551,6 +1619,10 @@ export interface RpcMethods {
   "rules.delete": { params: RuleIdParams; result: BotRules };
   "desktop.grants": { params: DesktopGrantsParams; result: Array<DesktopGrant> };
   "desktop.revoke": { params: DesktopGrantIdParams; result: BotDesktop };
+  "desktop.watch": { params: DesktopBotParams; result: DesktopView };
+  "desktop.unwatch": { params: undefined; result: null };
+  "desktop.stop": { params: DesktopBotParams; result: DesktopState };
+  "desktop.resume": { params: DesktopBotParams; result: DesktopState };
   "routines.list": { params: RoutinesListParams; result: Array<Routine> };
   "routines.create": { params: RoutinesCreateParams; result: Routine };
   "routines.update": { params: RoutinesUpdateParams; result: Routine };
@@ -1600,6 +1672,8 @@ export interface RpcNotifications {
   "reaction.changed": ReactionChanged;
   "bot.rules": BotRules;
   "bot.desktop": BotDesktop;
+  "desktop.changed": DesktopState;
+  "desktop.frame": DesktopFrame;
   "browser.changed": BrowserState;
   "browser.action": BrowserAction;
   "browser.frame": BrowserFrame;

crates/botloft-core/src/protocol/desktop.rs

@@ -107,3 +107,85 @@ pub struct BotDesktop {
     /// Oldest first.
     pub grants: Vec<DesktopGrant>,
 }
+
+/// The window a bot is using on the owner's desktop.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+#[cfg_attr(test, derive(ts_rs::TS))]
+pub struct DesktopWindow {
+    pub id: u64,
+    pub title: String,
+    /// The app's name for people.
+    pub app: String,
+}
+
+/// What a bot did in a window (spec 24.5), for its panel, which words it
+/// in the owner's language.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+#[cfg_attr(test, derive(ts_rs::TS))]
+pub enum DesktopActionKind {
+    Click,
+    Type,
+    Select,
+    Scroll,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+#[cfg_attr(test, derive(ts_rs::TS))]
+pub struct DesktopAction {
+    pub kind: DesktopActionKind,
+    /// The control's name as the window reads it; empty when it has none.
+    pub target: String,
+    /// The option chosen, for `select`.
+    pub option: Option<String>,
+}
+
+/// What a bot does on the owner's desktop, for its panel (spec 24.9): the
+/// params of `desktop.changed`.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+#[cfg_attr(test, derive(ts_rs::TS))]
+pub struct DesktopState {
+    pub bot_id: BotId,
+    /// The window it read or acted in last.
+    pub window: Option<DesktopWindow>,
+    /// What it did last; `null` after only reading.
+    pub action: Option<DesktopAction>,
+    /// When it last read or acted, Unix ms.
+    pub at: Option<i64>,
+    /// The owner stopped it (spec 24.9): its desktop tools refuse until
+    /// they let it go on.
+    pub stopped: bool,
+}
+
+/// A picture of the window a bot is using, for whoever watches its panel.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+#[cfg_attr(test, derive(ts_rs::TS))]
+pub struct DesktopFrame {
+    pub bot_id: BotId,
+    /// JPEG, base64.
+    pub data: String,
+    pub width: u32,
+    pub height: u32,
+}
+
+/// The result of `desktop.watch`: the bot's state and, when it uses a
+/// window, its picture now.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+#[cfg_attr(test, derive(ts_rs::TS))]
+pub struct DesktopView {
+    pub state: DesktopState,
+    pub frame: Option<DesktopFrame>,
+}
+
+/// `desktop.watch`, `desktop.stop` and `desktop.resume`.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+#[cfg_attr(test, derive(ts_rs::TS))]
+pub struct DesktopBotParams {
+    pub bot_id: BotId,
+}

crates/botloft-core/src/protocol/export_desktop.rs

@@ -12,6 +12,13 @@ pub(super) fn decls(out: &mut Out) {
     out.decl::<DesktopGrantsParams>();
     out.decl::<DesktopGrantIdParams>();
     out.decl::<BotDesktop>();
+    out.decl::<DesktopWindow>();
+    out.decl::<DesktopActionKind>();
+    out.decl::<DesktopAction>();
+    out.decl::<DesktopState>();
+    out.decl::<DesktopFrame>();
+    out.decl::<DesktopView>();
+    out.decl::<DesktopBotParams>();
 }
 
 pub(super) fn methods(out: &mut Out) {
@@ -25,9 +32,23 @@ pub(super) fn methods(out: &mut Out) {
         &out.name::<DesktopGrantIdParams>(),
         &out.name::<BotDesktop>(),
     );
+    let bot = out.name::<DesktopBotParams>();
+    out.method(method::DESKTOP_WATCH, &bot, &out.name::<DesktopView>());
+    out.method(method::DESKTOP_UNWATCH, "undefined", "null");
+    let state = out.name::<DesktopState>();
+    out.method(method::DESKTOP_STOP, &bot, &state);
+    out.method(method::DESKTOP_RESUME, &bot, &state);
 }
 
 pub(super) fn notifications(out: &mut Out) {
     let desktop = out.name::<BotDesktop>();
     let _ = writeln!(out.text, "  \"{}\": {desktop};", notification::BOT_DESKTOP);
+    let state = out.name::<DesktopState>();
+    let _ = writeln!(
+        out.text,
+        "  \"{}\": {state};",
+        notification::DESKTOP_CHANGED
+    );
+    let frame = out.name::<DesktopFrame>();
+    let _ = writeln!(out.text, "  \"{}\": {frame};", notification::DESKTOP_FRAME);
 }

crates/botloft-core/src/protocol/mod.rs

@@ -125,6 +125,10 @@ pub mod method {
     pub const RULES_DELETE: &str = "rules.delete";
     pub const DESKTOP_GRANTS: &str = "desktop.grants";
     pub const DESKTOP_REVOKE: &str = "desktop.revoke";
+    pub const DESKTOP_WATCH: &str = "desktop.watch";
+    pub const DESKTOP_UNWATCH: &str = "desktop.unwatch";
+    pub const DESKTOP_STOP: &str = "desktop.stop";
+    pub const DESKTOP_RESUME: &str = "desktop.resume";
     pub const MESSAGES_SEND: &str = "messages.send";
     pub const MESSAGES_LIST: &str = "messages.list";
     pub const ATTACHMENTS_READ: &str = "attachments.read";
@@ -175,6 +179,9 @@ pub mod notification {
     pub const BOT_CONTEXT: &str = "bot.context";
     pub const BOT_RULES: &str = "bot.rules";
     pub const BOT_DESKTOP: &str = "bot.desktop";
+    pub const DESKTOP_CHANGED: &str = "desktop.changed";
+    /// Only to the connection watching that bot's desktop panel (spec 24.9).
+    pub const DESKTOP_FRAME: &str = "desktop.frame";
     pub const CHAT_ITEM: &str = "chat.item";
     pub const CHAT_DELTA: &str = "chat.delta";
     pub const MESSAGE_CREATED: &str = "message.created";