feat(desktop): keys, clicks at a point and what accessibility cannot do, with the owner's real hands

app/src/features/desktop/DesktopPanel.tsx

@@ -34,6 +34,8 @@ export function actionText(action: DesktopAction | null, t: Words): string {
       return t.select(action.option ?? "");
     case "scroll":
       return target ? t.scroll(target) : t.scrollSomething;
+    case "press":
+      return t.press(action.option ?? "");
   }
 }
 

app/src/i18n/en/desktop.ts

@@ -41,6 +41,7 @@ export const desktop = {
     select: (option: string) => `Chose "${option}"`,
     scroll: (target: string) => `Scrolled "${target}"`,
     scrollSomething: "Scrolled",
+    press: (keys: string) => `Pressed ${keys}`,
     showInPanel: "Watch on the desktop panel",
     close: "Close",
   },

app/src/i18n/es/desktop.ts

@@ -42,6 +42,7 @@ export const desktop: Messages["desktop"] = {
     select: (option) => `Eligió "${option}"`,
     scroll: (target) => `Desplazó "${target}"`,
     scrollSomething: "Desplazó",
+    press: (keys) => `Presionó ${keys}`,
     showInPanel: "Ver en el panel del escritorio",
     close: "Cerrar",
   },

app/src/i18n/pt-BR/desktop.ts

@@ -41,6 +41,7 @@ export const desktop: Messages["desktop"] = {
     select: (option) => `Escolheu "${option}"`,
     scroll: (target) => `Rolou "${target}"`,
     scrollSomething: "Rolou",
+    press: (keys) => `Apertou ${keys}`,
     showInPanel: "Ver no painel do desktop",
     close: "Fechar",
   },

app/src/lib/fakeDesktop.ts

@@ -104,6 +104,17 @@ export class FakeDesktop {
         this.changed(grant.botId);
         return { botId: grant.botId, grants: this.of(grant.botId) };
       },
+      "desktop.setOptions": ({ grantId, realInput }) => {
+        const grant = this.grants.find((each) => each.id === grantId);
+        if (!grant) {
+          throw notFound(`desktop grant ${grantId}`);
+        }
+        if (realInput !== undefined) {
+          grant.realInput = realInput;
+        }
+        this.changed(grant.botId);
+        return { botId: grant.botId, grants: this.of(grant.botId) };
+      },
       "desktop.watch": ({ botId }) => {
         this.fake.bot(botId);
         this.watching = botId;

app/src/lib/protocol.gen.ts

@@ -859,6 +859,15 @@ export type DesktopGrantsParams = { botId: BotId, };
 
 export type DesktopGrantIdParams = { grantId: DesktopGrantId, };
 
+/**
+ * `desktop.setOptions`: what the owner changes in a grant (spec 24.10).
+ */
+export type DesktopOptionsParams = { grantId: DesktopGrantId, 
+/**
+ * The bot may use the real mouse and keyboard there (spec 24.7).
+ */
+realInput?: boolean, };
+
 /**
  * A bot's grants after one changed: the result of `desktop.revoke` and
  * the params of the `bot.desktop` notification.
@@ -882,7 +891,7 @@ app: string, };
  * What a bot did in a window (spec 24.5), for its panel, which words it
  * in the owner's language.
  */
-export type DesktopActionKind = "click" | "type" | "select" | "scroll";
+export type DesktopActionKind = "click" | "type" | "select" | "scroll" | "press";
 
 export type DesktopAction = { kind: DesktopActionKind, 
 /**
@@ -1619,6 +1628,7 @@ export interface RpcMethods {
   "rules.delete": { params: RuleIdParams; result: BotRules };
   "desktop.grants": { params: DesktopGrantsParams; result: Array<DesktopGrant> };
   "desktop.revoke": { params: DesktopGrantIdParams; result: BotDesktop };
+  "desktop.setOptions": { params: DesktopOptionsParams; result: BotDesktop };
   "desktop.watch": { params: DesktopBotParams; result: DesktopView };
   "desktop.unwatch": { params: undefined; result: null };
   "desktop.stop": { params: DesktopBotParams; result: DesktopState };

crates/botloft-core/src/protocol/desktop.rs

@@ -97,6 +97,18 @@ pub struct DesktopGrantIdParams {
     pub grant_id: DesktopGrantId,
 }
 
+/// `desktop.setOptions`: what the owner changes in a grant (spec 24.10).
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+#[cfg_attr(test, derive(ts_rs::TS))]
+pub struct DesktopOptionsParams {
+    pub grant_id: DesktopGrantId,
+    /// The bot may use the real mouse and keyboard there (spec 24.7).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    #[cfg_attr(test, ts(optional))]
+    pub real_input: Option<bool>,
+}
+
 /// A bot's grants after one changed: the result of `desktop.revoke` and
 /// the params of the `bot.desktop` notification.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -129,6 +141,8 @@ pub enum DesktopActionKind {
     Type,
     Select,
     Scroll,
+    /// Keys pressed with the real keyboard; `option` says which.
+    Press,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]

crates/botloft-core/src/protocol/export_desktop.rs

@@ -11,6 +11,7 @@ pub(super) fn decls(out: &mut Out) {
     out.decl::<DesktopGrant>();
     out.decl::<DesktopGrantsParams>();
     out.decl::<DesktopGrantIdParams>();
+    out.decl::<DesktopOptionsParams>();
     out.decl::<BotDesktop>();
     out.decl::<DesktopWindow>();
     out.decl::<DesktopActionKind>();
@@ -32,6 +33,11 @@ pub(super) fn methods(out: &mut Out) {
         &out.name::<DesktopGrantIdParams>(),
         &out.name::<BotDesktop>(),
     );
+    out.method(
+        method::DESKTOP_SET_OPTIONS,
+        &out.name::<DesktopOptionsParams>(),
+        &out.name::<BotDesktop>(),
+    );
     let bot = out.name::<DesktopBotParams>();
     out.method(method::DESKTOP_WATCH, &bot, &out.name::<DesktopView>());
     out.method(method::DESKTOP_UNWATCH, "undefined", "null");

crates/botloft-core/src/protocol/mod.rs

@@ -125,6 +125,7 @@ pub mod method {
     pub const RULES_DELETE: &str = "rules.delete";
     pub const DESKTOP_GRANTS: &str = "desktop.grants";
     pub const DESKTOP_REVOKE: &str = "desktop.revoke";
+    pub const DESKTOP_SET_OPTIONS: &str = "desktop.setOptions";
     pub const DESKTOP_WATCH: &str = "desktop.watch";
     pub const DESKTOP_UNWATCH: &str = "desktop.unwatch";
     pub const DESKTOP_STOP: &str = "desktop.stop";

crates/botloft-store/src/desktop_grants.rs

@@ -80,6 +80,36 @@ impl Store {
         Ok(grants)
     }
 
+    /// Lets the bot use the owner's real mouse and keyboard in the grant's
+    /// reach, or not (spec 24.7); the bot the grant belongs to, or `None`
+    /// if there is no such grant.
+    pub fn set_desktop_real_input(
+        &self,
+        grant: &DesktopGrantId,
+        on: bool,
+    ) -> Result<Option<BotId>> {
+        Ok(self
+            .conn
+            .query_row(
+                "UPDATE desktop_grants SET real_input = ?2 WHERE id = ?1 RETURNING bot_id",
+                params![grant.as_str(), on],
+                |row| parse_column(row, 0),
+            )
+            .optional()?)
+    }
+
+    /// The bot a grant belongs to, if the grant is there.
+    pub fn desktop_grants_of(&self, grant: &DesktopGrantId) -> Result<Option<BotId>> {
+        Ok(self
+            .conn
+            .query_row(
+                "SELECT bot_id FROM desktop_grants WHERE id = ?1",
+                [grant.as_str()],
+                |row| parse_column(row, 0),
+            )
+            .optional()?)
+    }
+
     /// Takes a grant away; the bot it belonged to, or `None` if there was
     /// none.
     pub fn revoke_desktop_grant(&self, grant: &DesktopGrantId) -> Result<Option<BotId>> {
@@ -131,6 +161,32 @@ mod tests {
         assert!(fx.store.desktop_grants(writer).expect("grants").is_empty());
     }
 
+    #[test]
+    fn the_real_mouse_and_keyboard_are_an_option_of_the_grant() {
+        let fx = Fixture::new();
+        let scout = &fx.bots[0].id;
+        let grant = fx
+            .store
+            .grant_desktop_app(scout, EXCEL, "Microsoft Excel", DesktopLevel::Act, 10)
+            .expect("grant");
+        assert_eq!(
+            fx.store
+                .set_desktop_real_input(&grant.id, true)
+                .expect("on"),
+            Some(scout.clone())
+        );
+        assert!(fx.store.desktop_grants(scout).expect("grants")[0].real_input);
+        // Asking for the app again keeps the option.
+        fx.store
+            .grant_desktop_app(scout, EXCEL, "Microsoft Excel", DesktopLevel::See, 20)
+            .expect("again");
+        assert!(fx.store.desktop_grants(scout).expect("grants")[0].real_input);
+        fx.store
+            .set_desktop_real_input(&grant.id, false)
+            .expect("off");
+        assert!(!fx.store.desktop_grants(scout).expect("grants")[0].real_input);
+    }
+
     #[test]
     fn a_grant_can_be_taken_away() {
         let fx = Fixture::new();
@@ -148,5 +204,11 @@ mod tests {
             None
         );
         assert!(fx.store.desktop_grants(scout).expect("grants").is_empty());
+        assert_eq!(
+            fx.store
+                .set_desktop_real_input(&grant.id, true)
+                .expect("gone"),
+            None
+        );
     }
 }

crates/botloftd/src/desktop/mod.rs

@@ -12,7 +12,7 @@ pub use screen::DesktopWatching;
 use std::collections::HashMap;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::{Arc, Mutex, PoisonError};
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
 use botloft_core::ids::BotId;
 use botloft_core::protocol::DesktopView;
@@ -24,6 +24,12 @@ use crate::platform::desktop::Control;
 /// How long the owner may go without touching the computer and still count
 /// as there (spec 24.8).
 pub const AWAY_AFTER: Duration = Duration::from_secs(5 * 60);
+/// How long the owner must leave the mouse and keyboard alone before a
+/// bot takes them (spec 24.7), and after the owner just took them back.
+pub const QUIET: Duration = Duration::from_secs(2);
+pub const QUIET_AFTER_TAKEOVER: Duration = Duration::from_secs(10);
+/// How long a takeover counts.
+const TAKEOVER_LASTS: Duration = Duration::from_secs(10 * 60);
 
 /// How long ago the owner last used the mouse or keyboard; `None` where
 /// that cannot be known.
@@ -45,6 +51,10 @@ pub struct Desktop {
     idle: Mutex<OwnerIdle>,
     pub activity: Arc<Activity>,
     screens: Arc<Screens>,
+    /// When the owner last took the mouse or keyboard back from a bot.
+    took_over: Mutex<Option<Instant>>,
+    /// Each bot's last picture: the window and its size, for clicks on it.
+    pictures: Mutex<HashMap<BotId, (u64, u32, u32)>>,
 }
 
 /// Why the bot may not use the desktop now.
@@ -81,6 +91,8 @@ impl Desktop {
             idle: Mutex::new(idle),
             activity: Arc::default(),
             screens: Arc::default(),
+            took_over: Mutex::new(None),
+            pictures: Mutex::new(HashMap::new()),
         }
     }
 
@@ -132,6 +144,59 @@ impl Desktop {
             .cloned()
     }
 
+    /// Whether the owner leaves the mouse and keyboard alone, so a bot may
+    /// take them (spec 24.7): a moment's quiet, longer right after the
+    /// owner took them back.
+    pub fn hands_free(&self) -> Result<(), String> {
+        let took_over = *self
+            .took_over
+            .lock()
+            .unwrap_or_else(PoisonError::into_inner);
+        let quiet = match took_over {
+            Some(at) if at.elapsed() < TAKEOVER_LASTS => QUIET_AFTER_TAKEOVER,
+            _ => QUIET,
+        };
+        let idle = self
+            .idle
+            .lock()
+            .unwrap_or_else(PoisonError::into_inner)
+            .clone();
+        match idle() {
+            Some(idle) if idle >= quiet => Ok(()),
+            _ => Err(format!(
+                "The owner is using the mouse or keyboard right now, so you may not take them. \
+                 Wait until they have left them alone for {} seconds, then try again.",
+                quiet.as_secs()
+            )),
+        }
+    }
+
+    /// The owner took the mouse or keyboard back in the middle of an action.
+    pub fn owner_took_over(&self) {
+        *self
+            .took_over
+            .lock()
+            .unwrap_or_else(PoisonError::into_inner) = Some(Instant::now());
+    }
+
+    /// Keeps the size of the bot's last picture of `window`.
+    pub fn pictured(&self, bot: &BotId, window: u64, width: u32, height: u32) {
+        self.pictures
+            .lock()
+            .unwrap_or_else(PoisonError::into_inner)
+            .insert(bot.clone(), (window, width, height));
+    }
+
+    /// The size of the bot's last picture, if it was of `window`.
+    pub fn picture_size(&self, bot: &BotId, window: u64) -> Option<(u32, u32)> {
+        self.pictures
+            .lock()
+            .unwrap_or_else(PoisonError::into_inner)
+            .get(bot)
+            .filter(|(id, _, _)| *id == window)
+            .map(|&(_, width, height)| (width, height))
+    }
+
     /// Starts watching the window the bot is using (spec 24.9).
     pub fn watch(&self, bot: &BotId) -> DesktopWatching {
         self.screens.watch(bot, &self.activity)
@@ -173,4 +238,26 @@ mod tests {
         desktop.app_closed();
         assert_eq!(desktop.owner_here(), Err(Away::NoApp));
     }
+
+    #[test]
+    fn the_hands_are_free_after_a_moments_quiet_and_longer_after_a_takeover() {
+        let desktop = Desktop::new(idle_for(1));
+        assert!(desktop.hands_free().is_err(), "the owner is typing");
+        desktop.set_owner_idle(idle_for(3));
+        assert!(desktop.hands_free().is_ok());
+        desktop.owner_took_over();
+        let busy = desktop.hands_free().expect_err("just took over");
+        assert!(busy.contains("10 seconds"), "{busy}");
+        desktop.set_owner_idle(idle_for(11));
+        assert!(desktop.hands_free().is_ok());
+    }
+
+    #[test]
+    fn a_picture_size_is_for_its_own_window() {
+        let desktop = Desktop::new(idle_for(0));
+        let bot = BotId::generate();
+        desktop.pictured(&bot, 7, 800, 600);
+        assert_eq!(desktop.picture_size(&bot, 7), Some((800, 600)));
+        assert_eq!(desktop.picture_size(&bot, 8), None);
+    }
 }

crates/botloftd/src/rpc/dispatch.rs

@@ -52,6 +52,7 @@ pub fn dispatch(daemon: &Daemon, name: &str, params: Option<Value>) -> Result<Va
         method::DESKTOP_GRANTS => reply(desktop::grants(daemon, parse(params)?)),
         method::DESKTOP_REVOKE => reply(desktop::revoke(daemon, parse(params)?)),
         method::DESKTOP_STOP => reply(desktop::stop(daemon, parse(params)?)),
+        method::DESKTOP_SET_OPTIONS => reply(desktop::set_options(daemon, parse(params)?)),
         method::DESKTOP_RESUME => reply(desktop::resume(daemon, parse(params)?)),
         method::MESSAGES_SEND => reply(messages::send(daemon, parse(params)?)),
         method::MESSAGES_LIST => reply(messages::list(daemon, parse(params)?)),

crates/botloftd/src/service/desktop.rs

@@ -7,7 +7,7 @@
 use botloft_core::ids::BotId;
 use botloft_core::protocol::{
     BotDesktop, DesktopBotParams, DesktopGrant, DesktopGrantIdParams, DesktopGrantsParams,
-    DesktopState,
+    DesktopOptionsParams, DesktopState,
 };
 use tracing::{info, warn};
 
@@ -22,6 +22,23 @@ pub fn grants(daemon: &Daemon, params: DesktopGrantsParams) -> ApiResult<Vec<Des
     Ok(store.desktop_grants(&params.bot_id)?)
 }
 
+/// Changes a grant's options: the real mouse and keyboard.
+pub fn set_options(daemon: &Daemon, params: DesktopOptionsParams) -> ApiResult<BotDesktop> {
+    let Some(on) = params.real_input else {
+        let store = daemon.store();
+        let grant = store
+            .desktop_grants_of(&params.grant_id)?
+            .ok_or_else(|| ApiError::NotFound(format!("desktop grant {}", params.grant_id)))?;
+        drop(store);
+        return changed(daemon, &grant);
+    };
+    let bot_id = daemon
+        .store()
+        .set_desktop_real_input(&params.grant_id, on)?
+        .ok_or_else(|| ApiError::NotFound(format!("desktop grant {}", params.grant_id)))?;
+    changed(daemon, &bot_id)
+}
+
 /// Takes a grant away: the bot asks for that again.
 pub fn revoke(daemon: &Daemon, params: DesktopGrantIdParams) -> ApiResult<BotDesktop> {
     let bot_id = daemon