feat(desktop): the bot's cursor on the owner's screen

The owner asked to see the bot's cursor on their own screen, not only in
the panel. The outline's window now draws it over the window in use: an
arrow in the bot's color with its name in a pill beside it, gliding to
the point of each action, a growing ring where it clicks and dots while
it types. The owner's mouse never moves for it. Reading leaves it where
it was; another window starts without one until the bot acts there.

Author: httpsphl

crates/botloftd/src/desktop/outline.rs

@@ -2,18 +2,19 @@
 //! 24.9): it shows when a bot reads or acts in a window and follows the
 //! window while the bot's turn goes on, until the bot leaves the desktop
 //! alone for a minute, its turn ends or the owner stops it. One window at
-//! a time: the one used last.
+//! a time: the one used last. The bot's cursor goes with it, at the point
+//! of its last action in that window.
 
 use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
 use botloft_core::ids::BotId;
-use botloft_core::protocol::{BotState, DesktopState};
+use botloft_core::protocol::{BotState, DesktopActionKind, DesktopState};
 use tokio::sync::broadcast::error::RecvError;
 
 use super::notice::color;
-use crate::platform::desktop;
+use crate::platform::desktop::{self, ScreenCursor};
 use crate::state::{Daemon, Event};
 
 /// How often the outline catches up with its window.
@@ -29,11 +30,17 @@ struct Outlined {
     bot: BotId,
     window: u64,
     color: (u8, u8, u8),
+    name: String,
+    /// Where the bot acted last in the window.
+    cursor: Option<ScreenCursor>,
     used: Instant,
     /// When the bot's turn ended, if it did.
     turn_over: Option<Instant>,
 }
 
+/// What the outline shows now: the window, the bot's color and cursor.
+type Shown = (u64, (u8, u8, u8), Option<ScreenCursor>);
+
 /// What the outline does, apart from the screen.
 #[derive(Debug, Default)]
 struct Follow {
@@ -44,11 +51,12 @@ struct Follow {
 }
 
 impl Follow {
-    /// A bot read or acted in a window, or the owner stopped it.
+    /// A bot read or acted in a window, or the owner stopped it; `who`
+    /// gives the bot's color and name.
     fn changed(
         &mut self,
         state: &DesktopState,
-        color: impl FnOnce() -> (u8, u8, u8),
+        who: impl FnOnce() -> ((u8, u8, u8), String),
         now: Instant,
     ) {
         let mine = self
@@ -68,14 +76,34 @@ impl Follow {
         if self.last.insert(state.bot_id.clone(), at) == Some(at) {
             return;
         }
-        let color = match &self.outlined {
-            Some(outlined) if mine => outlined.color,
-            _ => color(),
+        let (color, name, before) = match self.outlined.take() {
+            Some(outlined) if mine => {
+                let before = outlined.cursor.filter(|_| outlined.window == window.id);
+                (outlined.color, outlined.name, before)
+            }
+            _ => {
+                let (color, name) = who();
+                (color, name, None)
+            }
+        };
+        // Only reading leaves the cursor where it was.
+        let cursor = match &state.action {
+            Some(action) => action.x.zip(action.y).map(|(x, y)| ScreenCursor {
+                x,
+                y,
+                click: action.kind == DesktopActionKind::Click,
+                typing: action.kind == DesktopActionKind::Type,
+                name: name.clone(),
+                at,
+            }),
+            None => before,
         };
         self.outlined = Some(Outlined {
             bot: state.bot_id.clone(),
             window: window.id,
             color,
+            name,
+            cursor,
             used: now,
             turn_over: None,
         });
@@ -93,7 +121,7 @@ impl Follow {
     }
 
     /// The window to outline now, if any.
-    fn now(&mut self, now: Instant) -> Option<(u64, (u8, u8, u8))> {
+    fn now(&mut self, now: Instant) -> Option<Shown> {
         let outlined = self.outlined.as_ref()?;
         let over = outlined
             .turn_over
@@ -103,7 +131,7 @@ impl Follow {
             self.outlined = None;
             return None;
         }
-        Some((outlined.window, outlined.color))
+        Some((outlined.window, outlined.color, outlined.cursor.clone()))
     }
 }
 
@@ -121,8 +149,8 @@ pub async fn run(daemon: Arc<Daemon>) {
     loop {
         tokio::select! {
             _ = tick.tick() => match follow.now(Instant::now()) {
-                Some((window, color)) => {
-                    desktop::outline_show(window, color);
+                Some((window, color, cursor)) => {
+                    desktop::outline_show(window, color, cursor);
                     shown = true;
                 }
                 None if shown => {
@@ -133,11 +161,17 @@ pub async fn run(daemon: Arc<Daemon>) {
             },
             event = events.recv() => match event {
                 Ok(Event::DesktopChanged(state)) => {
-                    let hex = || {
+                    let who = || {
                         let bot = daemon.store().bot(&state.bot_id).ok().flatten();
-                        color(&bot.map(|bot| bot.color).unwrap_or_default())
+                        let (hex, name) = bot.map(|bot| (bot.color, bot.name)).unwrap_or_default();
+                        (color(&hex), name)
                     };
-                    follow.changed(&state, hex, Instant::now());
+                    follow.changed(&state, who, Instant::now());
+                    // The cursor sets off now, not at the next tick.
+                    if let Some((window, color, cursor)) = follow.now(Instant::now()) {
+                        desktop::outline_show(window, color, cursor);
+                        shown = true;
+                    }
                 }
                 Ok(Event::BotState(change)) => {
                     follow.turn(&change.bot_id, working(change.state), Instant::now());
@@ -151,7 +185,7 @@ pub async fn run(daemon: Arc<Daemon>) {
 
 #[cfg(test)]
 mod tests {
-    use botloft_core::protocol::DesktopWindow;
+    use botloft_core::protocol::{DesktopAction, DesktopWindow};
 
     use super::*;
 
@@ -171,47 +205,90 @@ mod tests {
 
     const BLUE: (u8, u8, u8) = (0x5E, 0xC8, 0xFF);
 
+    /// The window and color outlined at `now`, without the cursor.
+    fn spot(follow: &mut Follow, now: Instant) -> Option<(u64, (u8, u8, u8))> {
+        follow.now(now).map(|(window, color, _)| (window, color))
+    }
+
+    #[test]
+    fn the_cursor_is_at_the_last_action_and_stays_while_the_bot_reads() {
+        let mut follow = Follow::default();
+        let (scout, start) = (BotId::generate(), Instant::now());
+        let mut clicked = used(&scout, 7, 1);
+        clicked.action = Some(DesktopAction {
+            kind: DesktopActionKind::Click,
+            target: "Save".to_owned(),
+            option: None,
+            x: Some(0.25),
+            y: Some(0.5),
+        });
+        follow.changed(&clicked, || (BLUE, "Scout".to_owned()), start);
+        let cursor = follow
+            .now(start)
+            .and_then(|(_, _, cursor)| cursor)
+            .expect("cursor");
+        assert_eq!(
+            (cursor.x, cursor.y, cursor.click, cursor.typing),
+            (0.25, 0.5, true, false)
+        );
+        assert_eq!((cursor.name.as_str(), cursor.at), ("Scout", 1));
+
+        follow.changed(&used(&scout, 7, 2), || (BLUE, "Scout".to_owned()), start);
+        let kept = follow.now(start).and_then(|(_, _, cursor)| cursor);
+        assert_eq!(kept.map(|cursor| cursor.at), Some(1), "reading leaves it");
+        // Another window starts without one.
+        follow.changed(&used(&scout, 8, 3), || (BLUE, "Scout".to_owned()), start);
+        assert_eq!(follow.now(start).and_then(|(_, _, cursor)| cursor), None);
+    }
+
     #[test]
     fn the_outline_follows_the_window_in_use_until_the_turn_ends() {
         let mut follow = Follow::default();
         let (scout, start) = (BotId::generate(), Instant::now());
-        assert_eq!(follow.now(start), None);
-        follow.changed(&used(&scout, 7, 1), || BLUE, start);
-        assert_eq!(follow.now(start), Some((7, BLUE)));
+        assert_eq!(spot(&mut follow, start), None);
+        follow.changed(&used(&scout, 7, 1), || (BLUE, "Scout".to_owned()), start);
+        assert_eq!(spot(&mut follow, start), Some((7, BLUE)));
 
         // Its turn ends: a moment more, then gone.
         follow.turn(&scout, false, start);
-        assert_eq!(follow.now(start + Duration::from_secs(1)), Some((7, BLUE)));
-        assert_eq!(follow.now(start + AFTER_TURN), None);
+        assert_eq!(
+            spot(&mut follow, start + Duration::from_secs(1)),
+            Some((7, BLUE))
+        );
+        assert_eq!(spot(&mut follow, start + AFTER_TURN), None);
     }
 
     #[test]
     fn a_minute_away_from_the_desktop_or_a_stop_takes_it_away() {
         let mut follow = Follow::default();
         let (scout, start) = (BotId::generate(), Instant::now());
-        follow.changed(&used(&scout, 7, 1), || BLUE, start);
-        assert_eq!(follow.now(start + IDLE_FOR), None);
+        follow.changed(&used(&scout, 7, 1), || (BLUE, "Scout".to_owned()), start);
+        assert_eq!(spot(&mut follow, start + IDLE_FOR), None);
 
-        follow.changed(&used(&scout, 7, 2), || BLUE, start);
+        follow.changed(&used(&scout, 7, 2), || (BLUE, "Scout".to_owned()), start);
         let mut stopped = used(&scout, 7, 2);
         stopped.stopped = true;
-        follow.changed(&stopped, || BLUE, start);
-        assert_eq!(follow.now(start), None);
+        follow.changed(&stopped, || (BLUE, "Scout".to_owned()), start);
+        assert_eq!(spot(&mut follow, start), None);
         // Letting it go on is not a use.
         stopped.stopped = false;
-        follow.changed(&stopped, || BLUE, start);
-        assert_eq!(follow.now(start), None);
+        follow.changed(&stopped, || (BLUE, "Scout".to_owned()), start);
+        assert_eq!(spot(&mut follow, start), None);
     }
 
     #[test]
     fn the_window_used_last_is_the_one_outlined() {
         let mut follow = Follow::default();
         let (scout, writer, start) = (BotId::generate(), BotId::generate(), Instant::now());
-        follow.changed(&used(&scout, 7, 1), || BLUE, start);
-        follow.changed(&used(&writer, 9, 2), || (1, 2, 3), start);
-        assert_eq!(follow.now(start), Some((9, (1, 2, 3))));
+        follow.changed(&used(&scout, 7, 1), || (BLUE, "Scout".to_owned()), start);
+        follow.changed(
+            &used(&writer, 9, 2),
+            || ((1, 2, 3), "Writer".to_owned()),
+            start,
+        );
+        assert_eq!(spot(&mut follow, start), Some((9, (1, 2, 3))));
         // Another bot's turn ending leaves it.
         follow.turn(&scout, false, start);
-        assert_eq!(follow.now(start + AFTER_TURN), Some((9, (1, 2, 3))));
+        assert_eq!(spot(&mut follow, start + AFTER_TURN), Some((9, (1, 2, 3))));
     }
 }

crates/botloftd/src/platform/desktop/mod.rs

@@ -16,7 +16,7 @@ mod windows;
 use std::path::PathBuf;
 
 pub use blocked::{Never, never};
-pub use overlay::{notice_hide, notice_show, outline_hide, outline_show};
+pub use overlay::{ScreenCursor, notice_hide, notice_show, outline_hide, outline_show};
 pub use read::{Control, LINES_MAX, by_reference, reference, render};
 
 /// The largest picture of a window the bot gets, in logical pixels (24.4).

crates/botloftd/src/platform/desktop/overlay.rs

@@ -5,6 +5,22 @@
 #[cfg(windows)]
 use super::windows;
 
+/// The bot's cursor on the owner's screen, over the window it uses.
+#[derive(Debug, Clone, PartialEq)]
+pub struct ScreenCursor {
+    /// Where, as fractions of the window's frame from its top left corner.
+    pub x: f64,
+    pub y: f64,
+    /// A ring shows where it clicked.
+    pub click: bool,
+    /// Dots show while it types.
+    pub typing: bool,
+    /// The bot's name, beside the arrow.
+    pub name: String,
+    /// When the bot acted: a new one glides there.
+    pub at: i64,
+}
+
 /// Shows the notice that a bot uses the real mouse and keyboard: `text`
 /// in a pill, a border in `color` around the monitor of window `over`
 /// (spec 24.7).
@@ -21,13 +37,14 @@ pub fn notice_hide() {
     windows::notice_hide();
 }
 
-/// Outlines window `over` in `color`, or moves the outline to where the
-/// window is now; hidden while the window is minimized or gone.
-pub fn outline_show(over: u64, color: (u8, u8, u8)) {
+/// Outlines window `over` in `color`, with the bot's `cursor` in it, or
+/// moves the outline to where the window is now; hidden while the window
+/// is minimized or gone.
+pub fn outline_show(over: u64, color: (u8, u8, u8), cursor: Option<ScreenCursor>) {
     #[cfg(windows)]
-    windows::outline_show(over, color);
+    windows::outline_show(over, color, cursor);
     #[cfg(not(windows))]
-    let _ = (over, color);
+    let _ = (over, color, cursor);
 }
 
 /// Takes the outline away.

crates/botloftd/src/platform/desktop/windows/mod.rs

@@ -9,6 +9,7 @@ mod capture;
 mod input;
 mod notice;
 mod outline;
+mod outline_cursor;
 #[cfg(test)]
 mod outline_tests;
 mod owner;