feat(desktop): the real mouse and keyboard, stopping when the owner moves

.github/workflows/ci.yml

@@ -27,7 +27,11 @@ jobs:
       - uses: Swatinem/rust-cache@v2
       - run: cargo fmt --all --check
       - run: cargo clippy --workspace --all-targets -- -D warnings
+      # The runner has nobody at its desktop: the bots' real mouse and
+      # keyboard may be tested there (spec 24.7), never on a developer's.
       - run: cargo test --workspace
+        env:
+          BOTLOFT_REAL_INPUT_TESTS: "1"
       - name: Generated TypeScript bindings are up to date
         run: git diff --exit-code -- app/src/lib/protocol.gen.ts
 

crates/botloftd/src/platform/desktop/keys.rs

@@ -0,0 +1,153 @@
+//! Keys a bot presses with the real keyboard (spec 24.7): "Enter",
+//! "Ctrl+S", "Shift+Tab". Never the system's own combinations: the Windows
+//! key, switching or closing what is not the app (Alt+Tab, Alt+Esc,
+//! Ctrl+Esc), or the secure screen and Task Manager (Ctrl+Alt+Del,
+//! Ctrl+Shift+Esc).
+
+/// A key a bot may press, by its name.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Key {
+    Enter,
+    Tab,
+    Escape,
+    Backspace,
+    Delete,
+    Space,
+    Up,
+    Down,
+    Left,
+    Right,
+    Home,
+    End,
+    PageUp,
+    PageDown,
+    Insert,
+    /// F1 to F12.
+    F(u8),
+    /// A letter or digit, upper case.
+    Char(char),
+}
+
+/// A key with the modifiers held while it is pressed.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub struct Keys {
+    pub ctrl: bool,
+    pub alt: bool,
+    pub shift: bool,
+    pub key: Option<Key>,
+}
+
+/// Why keys are refused.
+#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
+pub enum KeysError {
+    #[error("{0:?} is not a key you can press; use names like Enter, Tab, Ctrl+S")]
+    Unknown(String),
+    #[error("{0} belongs to Windows itself, not to the app, so you may never press it")]
+    System(String),
+}
+
+fn key(name: &str) -> Option<Key> {
+    let lower = name.to_ascii_lowercase();
+    Some(match lower.as_str() {
+        "enter" | "return" => Key::Enter,
+        "tab" => Key::Tab,
+        "escape" | "esc" => Key::Escape,
+        "backspace" => Key::Backspace,
+        "delete" | "del" => Key::Delete,
+        "space" => Key::Space,
+        "up" | "arrowup" => Key::Up,
+        "down" | "arrowdown" => Key::Down,
+        "left" | "arrowleft" => Key::Left,
+        "right" | "arrowright" => Key::Right,
+        "home" => Key::Home,
+        "end" => Key::End,
+        "pageup" => Key::PageUp,
+        "pagedown" => Key::PageDown,
+        "insert" | "ins" => Key::Insert,
+        _ => {
+            if let Some(number) = lower.strip_prefix('f')
+                && let Ok(number) = number.parse::<u8>()
+                && (1..=12).contains(&number)
+            {
+                return Some(Key::F(number));
+            }
+            let mut chars = name.chars();
+            match (chars.next(), chars.next()) {
+                (Some(one), None) if one.is_ascii_alphanumeric() => {
+                    Key::Char(one.to_ascii_uppercase())
+                }
+                _ => return None,
+            }
+        }
+    })
+}
+
+/// Reads keys like "Ctrl+Shift+S": modifiers, then one key.
+pub fn parse(text: &str) -> Result<Keys, KeysError> {
+    let mut keys = Keys::default();
+    for part in text.split('+').map(str::trim) {
+        match part.to_ascii_lowercase().as_str() {
+            "ctrl" | "control" => keys.ctrl = true,
+            "alt" => keys.alt = true,
+            "shift" => keys.shift = true,
+            "win" | "windows" | "meta" | "super" | "cmd" => {
+                return Err(KeysError::System(text.to_owned()));
+            }
+            _ if keys.key.is_none() => {
+                keys.key = Some(key(part).ok_or_else(|| KeysError::Unknown(text.to_owned()))?);
+            }
+            _ => return Err(KeysError::Unknown(text.to_owned())),
+        }
+    }
+    let Some(pressed) = keys.key else {
+        return Err(KeysError::Unknown(text.to_owned()));
+    };
+    let system = (keys.alt && matches!(pressed, Key::Tab | Key::Escape))
+        || (keys.ctrl && !keys.alt && pressed == Key::Escape)
+        || (keys.ctrl && keys.alt && pressed == Key::Delete);
+    if system {
+        return Err(KeysError::System(text.to_owned()));
+    }
+    Ok(keys)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn keys_read_by_name_with_their_modifiers() {
+        assert_eq!(parse("Enter").expect("enter").key, Some(Key::Enter));
+        let save = parse("Ctrl+S").expect("save");
+        assert!(save.ctrl && !save.alt && !save.shift);
+        assert_eq!(save.key, Some(Key::Char('S')));
+        let back = parse("shift + tab").expect("back");
+        assert!(back.shift);
+        assert_eq!(back.key, Some(Key::Tab));
+        assert_eq!(parse("F5").expect("f5").key, Some(Key::F(5)));
+        assert_eq!(parse("Alt+F4").expect("close").key, Some(Key::F(4)));
+        assert!(matches!(parse("F13"), Err(KeysError::Unknown(_))));
+        assert!(matches!(parse("Ctrl+"), Err(KeysError::Unknown(_))));
+        assert!(matches!(parse("Ctrl+S+T"), Err(KeysError::Unknown(_))));
+    }
+
+    #[test]
+    fn the_systems_own_combinations_are_never_pressed() {
+        for system in [
+            "Win+R",
+            "Windows+D",
+            "Alt+Tab",
+            "Alt+Shift+Tab",
+            "Alt+Esc",
+            "Ctrl+Esc",
+            "Ctrl+Alt+Delete",
+            "Ctrl+Alt+Del",
+            "Ctrl+Shift+Esc",
+        ] {
+            assert!(
+                matches!(parse(system), Err(KeysError::System(_))),
+                "{system}"
+            );
+        }
+    }
+}

crates/botloftd/src/platform/desktop/mod.rs

@@ -4,6 +4,7 @@
 //! Only on Windows; elsewhere every call says the desktop is not there.
 
 mod blocked;
+pub mod keys;
 mod read;
 // Only Windows takes pictures; the tests run everywhere.
 #[cfg_attr(not(windows), allow(dead_code))]
@@ -79,6 +80,29 @@ pub enum DesktopError {
     /// No option with that text in the list.
     #[error("there is no option {0:?} there")]
     NoOption(String),
+    /// The owner moved the mouse or pressed a key while the bot used them.
+    #[error("the owner took over the mouse and keyboard")]
+    OwnerTookOver,
+    /// Windows would not bring the app's window to the front, or another
+    /// window came over it.
+    #[error("the app's window is not in front")]
+    NotInFront,
+    /// Another window covers the point to click.
+    #[error("another window covers that point")]
+    Covered,
+    /// The screen is locked, or Windows refused the input.
+    #[error("the screen is locked or takes no input now")]
+    Locked,
+}
+
+/// Where a real click lands.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum Spot {
+    /// In real screen pixels, as accessibility gives a control's place.
+    Screen { x: i32, y: i32 },
+    /// In the window's frame, as fractions of its width and height, as a
+    /// point on its picture.
+    Window { x: f64, y: f64 },
 }
 
 /// What a bot does to a control through accessibility (spec 24.5).
@@ -182,6 +206,46 @@ pub fn act(id: u64, target: &[i32], action: &Action) -> Result<Acted, DesktopErr
     }
 }
 
+/// Clicks `spot` in window `id` with the owner's real mouse, the window
+/// brought to the front first (spec 24.7). Stops when the owner moves.
+pub fn real_click(id: u64, spot: Spot) -> Result<(), DesktopError> {
+    #[cfg(windows)]
+    {
+        windows::real_click(id, spot)
+    }
+    #[cfg(not(windows))]
+    {
+        let _ = (id, spot);
+        Err(DesktopError::Unavailable)
+    }
+}
+
+/// Types `text` where window `id` has its focus, with the real keyboard.
+pub fn real_type(id: u64, text: &str) -> Result<(), DesktopError> {
+    #[cfg(windows)]
+    {
+        windows::real_type(id, text)
+    }
+    #[cfg(not(windows))]
+    {
+        let _ = (id, text);
+        Err(DesktopError::Unavailable)
+    }
+}
+
+/// Presses `keys` in window `id` with the real keyboard.
+pub fn real_press(id: u64, keys: &keys::Keys) -> Result<(), DesktopError> {
+    #[cfg(windows)]
+    {
+        windows::real_press(id, keys)
+    }
+    #[cfg(not(windows))]
+    {
+        let _ = (id, keys);
+        Err(DesktopError::Unavailable)
+    }
+}
+
 /// A picture of the window `id`, only of it, even behind other windows, in
 /// logical pixels up to [`PICTURE_MAX`] (spec 24.4).
 pub fn picture(id: u64) -> Result<Picture, DesktopError> {

crates/botloftd/src/platform/desktop/read.rs

@@ -27,6 +27,9 @@ pub struct Control {
     pub selected: bool,
     /// Windows' id for it, to find it again to act on it (D2).
     pub runtime_id: Vec<i32>,
+    /// Where it is on the screen, in real pixels: left, top, width and
+    /// height. For the real mouse (spec 24.7).
+    pub rect: Option<[i32; 4]>,
 }
 
 /// The `ref` of the control at `index` in a reading.

crates/botloftd/src/platform/desktop/windows/input.rs

@@ -1,10 +1,14 @@
-//! The owner's own use of the computer (spec 24.8).
+//! The owner's own use of the computer (spec 24.8), leaving out what a bot
+//! typed or clicked with the real mouse and keyboard.
 
+use std::sync::atomic::Ordering;
 use std::time::Duration;
 
 use windows::Win32::System::SystemInformation::GetTickCount;
 use windows::Win32::UI::Input::KeyboardAndMouse::{GetLastInputInfo, LASTINPUTINFO};
 
+use super::owner::{BOT_TICK, OWNER_TICK};
+
 /// How long ago the session last had mouse or keyboard input.
 pub fn owner_idle() -> Option<Duration> {
     let mut last = LASTINPUTINFO {
@@ -15,10 +19,14 @@ pub fn owner_idle() -> Option<Duration> {
     if !unsafe { GetLastInputInfo(&mut last) }.as_bool() {
         return None;
     }
+    // The last input was the bot's: the owner's came before it.
+    let last = if last.dwTime == BOT_TICK.load(Ordering::SeqCst) {
+        OWNER_TICK.load(Ordering::SeqCst)
+    } else {
+        last.dwTime
+    };
     // SAFETY: a plain query. Both count milliseconds since boot and wrap
     // together, so the difference wraps right.
     let now = unsafe { GetTickCount() };
-    Some(Duration::from_millis(u64::from(
-        now.wrapping_sub(last.dwTime),
-    )))
+    Some(Duration::from_millis(u64::from(now.wrapping_sub(last))))
 }

crates/botloftd/src/platform/desktop/windows/mod.rs

@@ -7,7 +7,11 @@ mod act;
 mod act_tests;
 mod capture;
 mod input;
+mod owner;
 mod process;
+mod real;
+#[cfg(test)]
+mod real_tests;
 mod stop_key;
 #[cfg(test)]
 mod test_window;
@@ -32,6 +36,7 @@ use windows::core::{BOOL, w};
 pub use act::act;
 pub use capture::capture;
 pub use input::owner_idle;
+pub use real::{click as real_click, press as real_press, type_text as real_type};
 pub use stop_key::on_stop_key;
 pub use uia::read;