feat(desktop): the bot's cursor on the owner's screen

crates/botloftd/src/desktop/outline.rs

@@ -2,18 +2,19 @@
 //! 24.9): it shows when a bot reads or acts in a window and follows the
 //! window while the bot's turn goes on, until the bot leaves the desktop
 //! alone for a minute, its turn ends or the owner stops it. One window at
-//! a time: the one used last.
+//! a time: the one used last. The bot's cursor goes with it, at the point
+//! of its last action in that window.
 
 use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
 use botloft_core::ids::BotId;
-use botloft_core::protocol::{BotState, DesktopState};
+use botloft_core::protocol::{BotState, DesktopActionKind, DesktopState};
 use tokio::sync::broadcast::error::RecvError;
 
 use super::notice::color;
-use crate::platform::desktop;
+use crate::platform::desktop::{self, ScreenCursor};
 use crate::state::{Daemon, Event};
 
 /// How often the outline catches up with its window.
@@ -29,11 +30,17 @@ struct Outlined {
     bot: BotId,
     window: u64,
     color: (u8, u8, u8),
+    name: String,
+    /// Where the bot acted last in the window.
+    cursor: Option<ScreenCursor>,
     used: Instant,
     /// When the bot's turn ended, if it did.
     turn_over: Option<Instant>,
 }
 
+/// What the outline shows now: the window, the bot's color and cursor.
+type Shown = (u64, (u8, u8, u8), Option<ScreenCursor>);
+
 /// What the outline does, apart from the screen.
 #[derive(Debug, Default)]
 struct Follow {
@@ -44,11 +51,12 @@ struct Follow {
 }
 
 impl Follow {
-    /// A bot read or acted in a window, or the owner stopped it.
+    /// A bot read or acted in a window, or the owner stopped it; `who`
+    /// gives the bot's color and name.
     fn changed(
         &mut self,
         state: &DesktopState,
-        color: impl FnOnce() -> (u8, u8, u8),
+        who: impl FnOnce() -> ((u8, u8, u8), String),
         now: Instant,
     ) {
         let mine = self
@@ -68,14 +76,34 @@ impl Follow {
         if self.last.insert(state.bot_id.clone(), at) == Some(at) {
             return;
         }
-        let color = match &self.outlined {
-            Some(outlined) if mine => outlined.color,
-            _ => color(),
+        let (color, name, before) = match self.outlined.take() {
+            Some(outlined) if mine => {
+                let before = outlined.cursor.filter(|_| outlined.window == window.id);
+                (outlined.color, outlined.name, before)
+            }
+            _ => {
+                let (color, name) = who();
+                (color, name, None)
+            }
+        };
+        // Only reading leaves the cursor where it was.
+        let cursor = match &state.action {
+            Some(action) => action.x.zip(action.y).map(|(x, y)| ScreenCursor {
+                x,
+                y,
+                click: action.kind == DesktopActionKind::Click,
+                typing: action.kind == DesktopActionKind::Type,
+                name: name.clone(),
+                at,
+            }),
+            None => before,
         };
         self.outlined = Some(Outlined {
             bot: state.bot_id.clone(),
             window: window.id,
             color,
+            name,
+            cursor,
             used: now,
             turn_over: None,
         });
@@ -93,7 +121,7 @@ impl Follow {
     }
 
     /// The window to outline now, if any.
-    fn now(&mut self, now: Instant) -> Option<(u64, (u8, u8, u8))> {
+    fn now(&mut self, now: Instant) -> Option<Shown> {
         let outlined = self.outlined.as_ref()?;
         let over = outlined
             .turn_over
@@ -103,7 +131,7 @@ impl Follow {
             self.outlined = None;
             return None;
         }
-        Some((outlined.window, outlined.color))
+        Some((outlined.window, outlined.color, outlined.cursor.clone()))
     }
 }
 
@@ -121,8 +149,8 @@ pub async fn run(daemon: Arc<Daemon>) {
     loop {
         tokio::select! {
             _ = tick.tick() => match follow.now(Instant::now()) {
-                Some((window, color)) => {
-                    desktop::outline_show(window, color);
+                Some((window, color, cursor)) => {
+                    desktop::outline_show(window, color, cursor);
                     shown = true;
                 }
                 None if shown => {
@@ -133,11 +161,17 @@ pub async fn run(daemon: Arc<Daemon>) {
             },
             event = events.recv() => match event {
                 Ok(Event::DesktopChanged(state)) => {
-                    let hex = || {
+                    let who = || {
                         let bot = daemon.store().bot(&state.bot_id).ok().flatten();
-                        color(&bot.map(|bot| bot.color).unwrap_or_default())
+                        let (hex, name) = bot.map(|bot| (bot.color, bot.name)).unwrap_or_default();
+                        (color(&hex), name)
                     };
-                    follow.changed(&state, hex, Instant::now());
+                    follow.changed(&state, who, Instant::now());
+                    // The cursor sets off now, not at the next tick.
+                    if let Some((window, color, cursor)) = follow.now(Instant::now()) {
+                        desktop::outline_show(window, color, cursor);
+                        shown = true;
+                    }
                 }
                 Ok(Event::BotState(change)) => {
                     follow.turn(&change.bot_id, working(change.state), Instant::now());
@@ -151,7 +185,7 @@ pub async fn run(daemon: Arc<Daemon>) {
 
 #[cfg(test)]
 mod tests {
-    use botloft_core::protocol::DesktopWindow;
+    use botloft_core::protocol::{DesktopAction, DesktopWindow};
 
     use super::*;
 
@@ -171,47 +205,90 @@ mod tests {
 
     const BLUE: (u8, u8, u8) = (0x5E, 0xC8, 0xFF);
 
+    /// The window and color outlined at `now`, without the cursor.
+    fn spot(follow: &mut Follow, now: Instant) -> Option<(u64, (u8, u8, u8))> {
+        follow.now(now).map(|(window, color, _)| (window, color))
+    }
+
+    #[test]
+    fn the_cursor_is_at_the_last_action_and_stays_while_the_bot_reads() {
+        let mut follow = Follow::default();
+        let (scout, start) = (BotId::generate(), Instant::now());
+        let mut clicked = used(&scout, 7, 1);
+        clicked.action = Some(DesktopAction {
+            kind: DesktopActionKind::Click,
+            target: "Save".to_owned(),
+            option: None,
+            x: Some(0.25),
+            y: Some(0.5),
+        });
+        follow.changed(&clicked, || (BLUE, "Scout".to_owned()), start);
+        let cursor = follow
+            .now(start)
+            .and_then(|(_, _, cursor)| cursor)
+            .expect("cursor");
+        assert_eq!(
+            (cursor.x, cursor.y, cursor.click, cursor.typing),
+            (0.25, 0.5, true, false)
+        );
+        assert_eq!((cursor.name.as_str(), cursor.at), ("Scout", 1));
+
+        follow.changed(&used(&scout, 7, 2), || (BLUE, "Scout".to_owned()), start);
+        let kept = follow.now(start).and_then(|(_, _, cursor)| cursor);
+        assert_eq!(kept.map(|cursor| cursor.at), Some(1), "reading leaves it");
+        // Another window starts without one.
+        follow.changed(&used(&scout, 8, 3), || (BLUE, "Scout".to_owned()), start);
+        assert_eq!(follow.now(start).and_then(|(_, _, cursor)| cursor), None);
+    }
+
     #[test]
     fn the_outline_follows_the_window_in_use_until_the_turn_ends() {
         let mut follow = Follow::default();
         let (scout, start) = (BotId::generate(), Instant::now());
-        assert_eq!(follow.now(start), None);
-        follow.changed(&used(&scout, 7, 1), || BLUE, start);
-        assert_eq!(follow.now(start), Some((7, BLUE)));
+        assert_eq!(spot(&mut follow, start), None);
+        follow.changed(&used(&scout, 7, 1), || (BLUE, "Scout".to_owned()), start);
+        assert_eq!(spot(&mut follow, start), Some((7, BLUE)));
 
         // Its turn ends: a moment more, then gone.
         follow.turn(&scout, false, start);
-        assert_eq!(follow.now(start + Duration::from_secs(1)), Some((7, BLUE)));
-        assert_eq!(follow.now(start + AFTER_TURN), None);
+        assert_eq!(
+            spot(&mut follow, start + Duration::from_secs(1)),
+            Some((7, BLUE))
+        );
+        assert_eq!(spot(&mut follow, start + AFTER_TURN), None);
     }
 
     #[test]
     fn a_minute_away_from_the_desktop_or_a_stop_takes_it_away() {
         let mut follow = Follow::default();
         let (scout, start) = (BotId::generate(), Instant::now());
-        follow.changed(&used(&scout, 7, 1), || BLUE, start);
-        assert_eq!(follow.now(start + IDLE_FOR), None);
+        follow.changed(&used(&scout, 7, 1), || (BLUE, "Scout".to_owned()), start);
+        assert_eq!(spot(&mut follow, start + IDLE_FOR), None);
 
-        follow.changed(&used(&scout, 7, 2), || BLUE, start);
+        follow.changed(&used(&scout, 7, 2), || (BLUE, "Scout".to_owned()), start);
         let mut stopped = used(&scout, 7, 2);
         stopped.stopped = true;
-        follow.changed(&stopped, || BLUE, start);
-        assert_eq!(follow.now(start), None);
+        follow.changed(&stopped, || (BLUE, "Scout".to_owned()), start);
+        assert_eq!(spot(&mut follow, start), None);
         // Letting it go on is not a use.
         stopped.stopped = false;
-        follow.changed(&stopped, || BLUE, start);
-        assert_eq!(follow.now(start), None);
+        follow.changed(&stopped, || (BLUE, "Scout".to_owned()), start);
+        assert_eq!(spot(&mut follow, start), None);
     }
 
     #[test]
     fn the_window_used_last_is_the_one_outlined() {
         let mut follow = Follow::default();
         let (scout, writer, start) = (BotId::generate(), BotId::generate(), Instant::now());
-        follow.changed(&used(&scout, 7, 1), || BLUE, start);
-        follow.changed(&used(&writer, 9, 2), || (1, 2, 3), start);
-        assert_eq!(follow.now(start), Some((9, (1, 2, 3))));
+        follow.changed(&used(&scout, 7, 1), || (BLUE, "Scout".to_owned()), start);
+        follow.changed(
+            &used(&writer, 9, 2),
+            || ((1, 2, 3), "Writer".to_owned()),
+            start,
+        );
+        assert_eq!(spot(&mut follow, start), Some((9, (1, 2, 3))));
         // Another bot's turn ending leaves it.
         follow.turn(&scout, false, start);
-        assert_eq!(follow.now(start + AFTER_TURN), Some((9, (1, 2, 3))));
+        assert_eq!(spot(&mut follow, start + AFTER_TURN), Some((9, (1, 2, 3))));
     }
 }

crates/botloftd/src/platform/desktop/mod.rs

@@ -16,7 +16,7 @@ mod windows;
 use std::path::PathBuf;
 
 pub use blocked::{Never, never};
-pub use overlay::{notice_hide, notice_show, outline_hide, outline_show};
+pub use overlay::{ScreenCursor, notice_hide, notice_show, outline_hide, outline_show};
 pub use read::{Control, LINES_MAX, by_reference, reference, render};
 
 /// The largest picture of a window the bot gets, in logical pixels (24.4).

crates/botloftd/src/platform/desktop/overlay.rs

@@ -5,6 +5,22 @@
 #[cfg(windows)]
 use super::windows;
 
+/// The bot's cursor on the owner's screen, over the window it uses.
+#[derive(Debug, Clone, PartialEq)]
+pub struct ScreenCursor {
+    /// Where, as fractions of the window's frame from its top left corner.
+    pub x: f64,
+    pub y: f64,
+    /// A ring shows where it clicked.
+    pub click: bool,
+    /// Dots show while it types.
+    pub typing: bool,
+    /// The bot's name, beside the arrow.
+    pub name: String,
+    /// When the bot acted: a new one glides there.
+    pub at: i64,
+}
+
 /// Shows the notice that a bot uses the real mouse and keyboard: `text`
 /// in a pill, a border in `color` around the monitor of window `over`
 /// (spec 24.7).
@@ -21,13 +37,14 @@ pub fn notice_hide() {
     windows::notice_hide();
 }
 
-/// Outlines window `over` in `color`, or moves the outline to where the
-/// window is now; hidden while the window is minimized or gone.
-pub fn outline_show(over: u64, color: (u8, u8, u8)) {
+/// Outlines window `over` in `color`, with the bot's `cursor` in it, or
+/// moves the outline to where the window is now; hidden while the window
+/// is minimized or gone.
+pub fn outline_show(over: u64, color: (u8, u8, u8), cursor: Option<ScreenCursor>) {
     #[cfg(windows)]
-    windows::outline_show(over, color);
+    windows::outline_show(over, color, cursor);
     #[cfg(not(windows))]
-    let _ = (over, color);
+    let _ = (over, color, cursor);
 }
 
 /// Takes the outline away.

crates/botloftd/src/platform/desktop/windows/mod.rs

@@ -9,6 +9,7 @@ mod capture;
 mod input;
 mod notice;
 mod outline;
+mod outline_cursor;
 #[cfg(test)]
 mod outline_tests;
 mod owner;