From c6b6aaf4a1fe793e389917bd7d780f46d1aed69e Mon Sep 17 00:00:00 2001 From: Phelipe Lorran <112786693+httpsphl@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:02:26 -0300 Subject: [PATCH 1/2] feat(desktop): the real mouse and keyboard, stopping when the owner moves The platform part of D4 (spec 24.7), not used by the tools yet: - real_click, real_type and real_press with SendInput. The app's window comes to the front first (an empty mouse move makes Windows let the daemon do it); before every step the window in front must be the app's, and a click must land on it, not on a window over it; otherwise nothing is sent. Typing is Unicode, so the keyboard layout does not matter. Clicks go to a control's center, from accessibility's BoundingRectangle (sessions are now per-monitor DPI aware), or to a point of the window's picture. - Low-level mouse and keyboard hooks, only during the action, see input that is not injected: the owner's. The bot stops at once. - Keys by name with Ctrl, Alt and Shift, never the system's own combinations (the Windows key, Alt+Tab, Alt+Esc, Ctrl+Esc, Ctrl+Shift+Esc, Ctrl+Alt+Del), in pure code with tests. - The owner's idle time leaves the bot's own input out. - Readings keep each control's place on the screen. The tests that move the cursor and type run only with BOTLOFT_REAL_INPUT_TESTS=1, which CI sets on its Windows runner, where nobody sits at the desktop; never on a developer's computer. --- .github/workflows/ci.yml | 4 + crates/botloftd/src/platform/desktop/keys.rs | 153 ++++++++++ crates/botloftd/src/platform/desktop/mod.rs | 64 ++++ crates/botloftd/src/platform/desktop/read.rs | 3 + .../src/platform/desktop/windows/input.rs | 16 +- .../src/platform/desktop/windows/mod.rs | 5 + .../src/platform/desktop/windows/owner.rs | 113 +++++++ .../src/platform/desktop/windows/real.rs | 286 ++++++++++++++++++ .../platform/desktop/windows/real_tests.rs | 64 ++++ .../src/platform/desktop/windows/uia.rs | 12 +- .../platform/desktop/windows/uia_control.rs | 31 +- crates/botloftd/src/tools/desktop_grant.rs | 13 + docs/spec.md | 9 +- 13 files changed, 762 insertions(+), 11 deletions(-) create mode 100644 crates/botloftd/src/platform/desktop/keys.rs create mode 100644 crates/botloftd/src/platform/desktop/windows/owner.rs create mode 100644 crates/botloftd/src/platform/desktop/windows/real.rs create mode 100644 crates/botloftd/src/platform/desktop/windows/real_tests.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 096f4d53..77815f04 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,11 @@ jobs: - uses: Swatinem/rust-cache@v2 - run: cargo fmt --all --check - run: cargo clippy --workspace --all-targets -- -D warnings + # The runner has nobody at its desktop: the bots' real mouse and + # keyboard may be tested there (spec 24.7), never on a developer's. - run: cargo test --workspace + env: + BOTLOFT_REAL_INPUT_TESTS: "1" - name: Generated TypeScript bindings are up to date run: git diff --exit-code -- app/src/lib/protocol.gen.ts diff --git a/crates/botloftd/src/platform/desktop/keys.rs b/crates/botloftd/src/platform/desktop/keys.rs new file mode 100644 index 00000000..7fcf3c29 --- /dev/null +++ b/crates/botloftd/src/platform/desktop/keys.rs @@ -0,0 +1,153 @@ +//! Keys a bot presses with the real keyboard (spec 24.7): "Enter", +//! "Ctrl+S", "Shift+Tab". Never the system's own combinations: the Windows +//! key, switching or closing what is not the app (Alt+Tab, Alt+Esc, +//! Ctrl+Esc), or the secure screen and Task Manager (Ctrl+Alt+Del, +//! Ctrl+Shift+Esc). + +/// A key a bot may press, by its name. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Key { + Enter, + Tab, + Escape, + Backspace, + Delete, + Space, + Up, + Down, + Left, + Right, + Home, + End, + PageUp, + PageDown, + Insert, + /// F1 to F12. + F(u8), + /// A letter or digit, upper case. + Char(char), +} + +/// A key with the modifiers held while it is pressed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct Keys { + pub ctrl: bool, + pub alt: bool, + pub shift: bool, + pub key: Option, +} + +/// Why keys are refused. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum KeysError { + #[error("{0:?} is not a key you can press; use names like Enter, Tab, Ctrl+S")] + Unknown(String), + #[error("{0} belongs to Windows itself, not to the app, so you may never press it")] + System(String), +} + +fn key(name: &str) -> Option { + let lower = name.to_ascii_lowercase(); + Some(match lower.as_str() { + "enter" | "return" => Key::Enter, + "tab" => Key::Tab, + "escape" | "esc" => Key::Escape, + "backspace" => Key::Backspace, + "delete" | "del" => Key::Delete, + "space" => Key::Space, + "up" | "arrowup" => Key::Up, + "down" | "arrowdown" => Key::Down, + "left" | "arrowleft" => Key::Left, + "right" | "arrowright" => Key::Right, + "home" => Key::Home, + "end" => Key::End, + "pageup" => Key::PageUp, + "pagedown" => Key::PageDown, + "insert" | "ins" => Key::Insert, + _ => { + if let Some(number) = lower.strip_prefix('f') + && let Ok(number) = number.parse::() + && (1..=12).contains(&number) + { + return Some(Key::F(number)); + } + let mut chars = name.chars(); + match (chars.next(), chars.next()) { + (Some(one), None) if one.is_ascii_alphanumeric() => { + Key::Char(one.to_ascii_uppercase()) + } + _ => return None, + } + } + }) +} + +/// Reads keys like "Ctrl+Shift+S": modifiers, then one key. +pub fn parse(text: &str) -> Result { + let mut keys = Keys::default(); + for part in text.split('+').map(str::trim) { + match part.to_ascii_lowercase().as_str() { + "ctrl" | "control" => keys.ctrl = true, + "alt" => keys.alt = true, + "shift" => keys.shift = true, + "win" | "windows" | "meta" | "super" | "cmd" => { + return Err(KeysError::System(text.to_owned())); + } + _ if keys.key.is_none() => { + keys.key = Some(key(part).ok_or_else(|| KeysError::Unknown(text.to_owned()))?); + } + _ => return Err(KeysError::Unknown(text.to_owned())), + } + } + let Some(pressed) = keys.key else { + return Err(KeysError::Unknown(text.to_owned())); + }; + let system = (keys.alt && matches!(pressed, Key::Tab | Key::Escape)) + || (keys.ctrl && !keys.alt && pressed == Key::Escape) + || (keys.ctrl && keys.alt && pressed == Key::Delete); + if system { + return Err(KeysError::System(text.to_owned())); + } + Ok(keys) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keys_read_by_name_with_their_modifiers() { + assert_eq!(parse("Enter").expect("enter").key, Some(Key::Enter)); + let save = parse("Ctrl+S").expect("save"); + assert!(save.ctrl && !save.alt && !save.shift); + assert_eq!(save.key, Some(Key::Char('S'))); + let back = parse("shift + tab").expect("back"); + assert!(back.shift); + assert_eq!(back.key, Some(Key::Tab)); + assert_eq!(parse("F5").expect("f5").key, Some(Key::F(5))); + assert_eq!(parse("Alt+F4").expect("close").key, Some(Key::F(4))); + assert!(matches!(parse("F13"), Err(KeysError::Unknown(_)))); + assert!(matches!(parse("Ctrl+"), Err(KeysError::Unknown(_)))); + assert!(matches!(parse("Ctrl+S+T"), Err(KeysError::Unknown(_)))); + } + + #[test] + fn the_systems_own_combinations_are_never_pressed() { + for system in [ + "Win+R", + "Windows+D", + "Alt+Tab", + "Alt+Shift+Tab", + "Alt+Esc", + "Ctrl+Esc", + "Ctrl+Alt+Delete", + "Ctrl+Alt+Del", + "Ctrl+Shift+Esc", + ] { + assert!( + matches!(parse(system), Err(KeysError::System(_))), + "{system}" + ); + } + } +} diff --git a/crates/botloftd/src/platform/desktop/mod.rs b/crates/botloftd/src/platform/desktop/mod.rs index 73096570..8b10e1d4 100644 --- a/crates/botloftd/src/platform/desktop/mod.rs +++ b/crates/botloftd/src/platform/desktop/mod.rs @@ -4,6 +4,7 @@ //! Only on Windows; elsewhere every call says the desktop is not there. mod blocked; +pub mod keys; mod read; // Only Windows takes pictures; the tests run everywhere. #[cfg_attr(not(windows), allow(dead_code))] @@ -79,6 +80,29 @@ pub enum DesktopError { /// No option with that text in the list. #[error("there is no option {0:?} there")] NoOption(String), + /// The owner moved the mouse or pressed a key while the bot used them. + #[error("the owner took over the mouse and keyboard")] + OwnerTookOver, + /// Windows would not bring the app's window to the front, or another + /// window came over it. + #[error("the app's window is not in front")] + NotInFront, + /// Another window covers the point to click. + #[error("another window covers that point")] + Covered, + /// The screen is locked, or Windows refused the input. + #[error("the screen is locked or takes no input now")] + Locked, +} + +/// Where a real click lands. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Spot { + /// In real screen pixels, as accessibility gives a control's place. + Screen { x: i32, y: i32 }, + /// In the window's frame, as fractions of its width and height, as a + /// point on its picture. + Window { x: f64, y: f64 }, } /// What a bot does to a control through accessibility (spec 24.5). @@ -182,6 +206,46 @@ pub fn act(id: u64, target: &[i32], action: &Action) -> Result Result<(), DesktopError> { + #[cfg(windows)] + { + windows::real_click(id, spot) + } + #[cfg(not(windows))] + { + let _ = (id, spot); + Err(DesktopError::Unavailable) + } +} + +/// Types `text` where window `id` has its focus, with the real keyboard. +pub fn real_type(id: u64, text: &str) -> Result<(), DesktopError> { + #[cfg(windows)] + { + windows::real_type(id, text) + } + #[cfg(not(windows))] + { + let _ = (id, text); + Err(DesktopError::Unavailable) + } +} + +/// Presses `keys` in window `id` with the real keyboard. +pub fn real_press(id: u64, keys: &keys::Keys) -> Result<(), DesktopError> { + #[cfg(windows)] + { + windows::real_press(id, keys) + } + #[cfg(not(windows))] + { + let _ = (id, keys); + Err(DesktopError::Unavailable) + } +} + /// A picture of the window `id`, only of it, even behind other windows, in /// logical pixels up to [`PICTURE_MAX`] (spec 24.4). pub fn picture(id: u64) -> Result { diff --git a/crates/botloftd/src/platform/desktop/read.rs b/crates/botloftd/src/platform/desktop/read.rs index 003005db..9f77a08a 100644 --- a/crates/botloftd/src/platform/desktop/read.rs +++ b/crates/botloftd/src/platform/desktop/read.rs @@ -27,6 +27,9 @@ pub struct Control { pub selected: bool, /// Windows' id for it, to find it again to act on it (D2). pub runtime_id: Vec, + /// Where it is on the screen, in real pixels: left, top, width and + /// height. For the real mouse (spec 24.7). + pub rect: Option<[i32; 4]>, } /// The `ref` of the control at `index` in a reading. diff --git a/crates/botloftd/src/platform/desktop/windows/input.rs b/crates/botloftd/src/platform/desktop/windows/input.rs index 784af704..544f15ea 100644 --- a/crates/botloftd/src/platform/desktop/windows/input.rs +++ b/crates/botloftd/src/platform/desktop/windows/input.rs @@ -1,10 +1,14 @@ -//! The owner's own use of the computer (spec 24.8). +//! The owner's own use of the computer (spec 24.8), leaving out what a bot +//! typed or clicked with the real mouse and keyboard. +use std::sync::atomic::Ordering; use std::time::Duration; use windows::Win32::System::SystemInformation::GetTickCount; use windows::Win32::UI::Input::KeyboardAndMouse::{GetLastInputInfo, LASTINPUTINFO}; +use super::owner::{BOT_TICK, OWNER_TICK}; + /// How long ago the session last had mouse or keyboard input. pub fn owner_idle() -> Option { let mut last = LASTINPUTINFO { @@ -15,10 +19,14 @@ pub fn owner_idle() -> Option { if !unsafe { GetLastInputInfo(&mut last) }.as_bool() { return None; } + // The last input was the bot's: the owner's came before it. + let last = if last.dwTime == BOT_TICK.load(Ordering::SeqCst) { + OWNER_TICK.load(Ordering::SeqCst) + } else { + last.dwTime + }; // SAFETY: a plain query. Both count milliseconds since boot and wrap // together, so the difference wraps right. let now = unsafe { GetTickCount() }; - Some(Duration::from_millis(u64::from( - now.wrapping_sub(last.dwTime), - ))) + Some(Duration::from_millis(u64::from(now.wrapping_sub(last)))) } diff --git a/crates/botloftd/src/platform/desktop/windows/mod.rs b/crates/botloftd/src/platform/desktop/windows/mod.rs index c7c82c76..af6e91ef 100644 --- a/crates/botloftd/src/platform/desktop/windows/mod.rs +++ b/crates/botloftd/src/platform/desktop/windows/mod.rs @@ -7,7 +7,11 @@ mod act; mod act_tests; mod capture; mod input; +mod owner; mod process; +mod real; +#[cfg(test)] +mod real_tests; mod stop_key; #[cfg(test)] mod test_window; @@ -32,6 +36,7 @@ use windows::core::{BOOL, w}; pub use act::act; pub use capture::capture; pub use input::owner_idle; +pub use real::{click as real_click, press as real_press, type_text as real_type}; pub use stop_key::on_stop_key; pub use uia::read; diff --git a/crates/botloftd/src/platform/desktop/windows/owner.rs b/crates/botloftd/src/platform/desktop/windows/owner.rs new file mode 100644 index 00000000..3cc2e69f --- /dev/null +++ b/crates/botloftd/src/platform/desktop/windows/owner.rs @@ -0,0 +1,113 @@ +//! The owner's own hands while a bot uses the real mouse and keyboard +//! (spec 24.7): low-level hooks see every input, and what the bot sends +//! comes marked as injected. Any other input is the owner's, and the bot +//! stops at once. + +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::mpsc; +use std::thread::JoinHandle; + +use windows::Win32::Foundation::{LPARAM, LRESULT, WPARAM}; +use windows::Win32::System::Threading::GetCurrentThreadId; +use windows::Win32::UI::WindowsAndMessaging::{ + CallNextHookEx, GetMessageW, HHOOK, KBDLLHOOKSTRUCT, LLKHF_INJECTED, LLMHF_INJECTED, MSG, + MSLLHOOKSTRUCT, PostThreadMessageW, SetWindowsHookExW, UnhookWindowsHookEx, WH_KEYBOARD_LL, + WH_MOUSE_LL, WM_QUIT, +}; + +use super::super::DesktopError; + +/// Set by the hooks when input that is not the bot's arrives. +static OWNER_MOVED: AtomicBool = AtomicBool::new(false); +/// The tick of the last input the bot sent, and of the owner's last one +/// before it: the owner's idle time leaves the bot's input out (spec 24.8). +pub(super) static BOT_TICK: AtomicU32 = AtomicU32::new(0); +pub(super) static OWNER_TICK: AtomicU32 = AtomicU32::new(0); + +unsafe extern "system" fn mouse(code: i32, wparam: WPARAM, lparam: LPARAM) -> LRESULT { + if code >= 0 { + // SAFETY: for a mouse hook, `lparam` is an MSLLHOOKSTRUCT. + let event = unsafe { &*(lparam.0 as *const MSLLHOOKSTRUCT) }; + if event.flags & LLMHF_INJECTED == 0 { + OWNER_MOVED.store(true, Ordering::SeqCst); + } + } + // SAFETY: passes the event on, as every hook must. + unsafe { CallNextHookEx(None, code, wparam, lparam) } +} + +unsafe extern "system" fn keyboard(code: i32, wparam: WPARAM, lparam: LPARAM) -> LRESULT { + if code >= 0 { + // SAFETY: for a keyboard hook, `lparam` is a KBDLLHOOKSTRUCT. + let event = unsafe { &*(lparam.0 as *const KBDLLHOOKSTRUCT) }; + if event.flags.0 & LLKHF_INJECTED.0 == 0 { + OWNER_MOVED.store(true, Ordering::SeqCst); + } + } + // SAFETY: passes the event on, as every hook must. + unsafe { CallNextHookEx(None, code, wparam, lparam) } +} + +/// Watches the owner's hands while it lives: the hooks run on a thread of +/// their own, which pumps their messages. +pub struct OwnerWatch { + thread: Option>, + id: u32, +} + +impl OwnerWatch { + pub fn start() -> Result { + OWNER_MOVED.store(false, Ordering::SeqCst); + let (told, started) = mpsc::channel(); + let thread = std::thread::spawn(move || { + // SAFETY: hooks for this process, removed before the thread ends. + let hooks = unsafe { + ( + SetWindowsHookExW(WH_MOUSE_LL, Some(mouse), None, 0), + SetWindowsHookExW(WH_KEYBOARD_LL, Some(keyboard), None, 0), + ) + }; + let ok = hooks.0.is_ok() && hooks.1.is_ok(); + // SAFETY: a plain query. + let id = unsafe { GetCurrentThreadId() }; + let _ = told.send(ok.then_some(id)); + if ok { + let mut message = MSG::default(); + // SAFETY: this thread's own queue, until WM_QUIT. + while unsafe { GetMessageW(&mut message, None, 0, 0) }.as_bool() {} + } + for hook in [hooks.0, hooks.1].into_iter().flatten() { + let hook: HHOOK = hook; + // SAFETY: a hook this thread set. + let _ = unsafe { UnhookWindowsHookEx(hook) }; + } + }); + match started.recv() { + Ok(Some(id)) => Ok(Self { + thread: Some(thread), + id, + }), + _ => { + let _ = thread.join(); + Err(DesktopError::System( + "Botloft could not watch the owner's mouse and keyboard".to_owned(), + )) + } + } + } + + /// Whether the owner touched the mouse or keyboard since it started. + pub fn moved(&self) -> bool { + OWNER_MOVED.load(Ordering::SeqCst) + } +} + +impl Drop for OwnerWatch { + fn drop(&mut self) { + // SAFETY: ends the hooks' thread's message loop. + let _ = unsafe { PostThreadMessageW(self.id, WM_QUIT, WPARAM(0), LPARAM(0)) }; + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} diff --git a/crates/botloftd/src/platform/desktop/windows/real.rs b/crates/botloftd/src/platform/desktop/windows/real.rs new file mode 100644 index 00000000..6512005a --- /dev/null +++ b/crates/botloftd/src/platform/desktop/windows/real.rs @@ -0,0 +1,286 @@ +//! The real mouse and keyboard (spec 24.7), for what accessibility cannot +//! do: `SendInput`, with the app's window brought to the front first. +//! Before every step the window must still be in front, a click must land +//! on it and not on another window over it, and the owner must not have +//! touched the mouse or keyboard: then it stops at once. + +use std::sync::atomic::Ordering; +use std::time::Duration; + +use windows::Win32::Foundation::{HWND, POINT, RECT}; +use windows::Win32::Graphics::Dwm::{DWMWA_EXTENDED_FRAME_BOUNDS, DwmGetWindowAttribute}; +use windows::Win32::UI::HiDpi::{ + DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2, SetThreadDpiAwarenessContext, +}; +use windows::Win32::UI::Input::KeyboardAndMouse::{ + GetLastInputInfo, INPUT, INPUT_0, INPUT_KEYBOARD, INPUT_MOUSE, KEYBD_EVENT_FLAGS, KEYBDINPUT, + KEYEVENTF_KEYUP, KEYEVENTF_UNICODE, LASTINPUTINFO, MOUSE_EVENT_FLAGS, MOUSEEVENTF_ABSOLUTE, + MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP, MOUSEEVENTF_MOVE, MOUSEEVENTF_VIRTUALDESK, + MOUSEINPUT, SendInput, VIRTUAL_KEY, VK_BACK, VK_CONTROL, VK_DELETE, VK_DOWN, VK_END, VK_ESCAPE, + VK_F1, VK_HOME, VK_INSERT, VK_LEFT, VK_MENU, VK_NEXT, VK_PRIOR, VK_RETURN, VK_RIGHT, VK_SHIFT, + VK_SPACE, VK_TAB, VK_UP, +}; +use windows::Win32::UI::WindowsAndMessaging::{ + GA_ROOT, GetAncestor, GetForegroundWindow, GetSystemMetrics, IsIconic, IsWindow, + SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN, SM_XVIRTUALSCREEN, SM_YVIRTUALSCREEN, SW_RESTORE, + SetForegroundWindow, ShowWindow, WindowFromPoint, +}; + +use super::super::keys::{Key, Keys}; +use super::super::{DesktopError, Spot}; +use super::handle; +use super::owner::{BOT_TICK, OWNER_TICK, OwnerWatch}; + +/// Time for the window to come to the front, and between steps. +const BREATH: Duration = Duration::from_millis(60); +/// Characters typed between checks. +const CHUNK: usize = 8; + +fn last_input() -> u32 { + let mut last = LASTINPUTINFO { + cbSize: size_of::() as u32, + dwTime: 0, + }; + // SAFETY: `last` has its size set. + let _ = unsafe { GetLastInputInfo(&mut last) }; + last.dwTime +} + +/// One real action: the window in front, the owner watched, the input +/// sent in steps, each checked first. +struct Hands { + hwnd: HWND, + owner: OwnerWatch, +} + +impl Hands { + fn take(id: u64) -> Result { + let hwnd = handle(id); + // SAFETY: plain calls on a window handle. + unsafe { + if !IsWindow(Some(hwnd)).as_bool() { + return Err(DesktopError::Gone); + } + SetThreadDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2); + } + // The owner's last input before the bot's, for their idle time. + if BOT_TICK.load(Ordering::SeqCst) != last_input() { + OWNER_TICK.store(last_input(), Ordering::SeqCst); + } + let owner = OwnerWatch::start()?; + let hands = Self { hwnd, owner }; + hands.front()?; + Ok(hands) + } + + /// Brings the window to the front: Windows lets a process do that when + /// it sent the last input, so an empty mouse move goes first. + fn front(&self) -> Result<(), DesktopError> { + // SAFETY: plain calls on a window handle. + unsafe { + if IsIconic(self.hwnd).as_bool() { + let _ = ShowWindow(self.hwnd, SW_RESTORE); + } + if GetForegroundWindow() != self.hwnd { + send(&[mouse_input(0, 0, MOUSEEVENTF_MOVE)])?; + let _ = SetForegroundWindow(self.hwnd); + std::thread::sleep(BREATH); + } + } + self.check() + } + + /// The window is in front and the owner did not move. + fn check(&self) -> Result<(), DesktopError> { + if self.owner.moved() { + return Err(DesktopError::OwnerTookOver); + } + // SAFETY: a plain query. + let front = unsafe { GetForegroundWindow() }; + if front.is_invalid() { + return Err(DesktopError::Locked); + } + // SAFETY: a plain query on a window handle. + if unsafe { GetAncestor(front, GA_ROOT) } != self.hwnd { + return Err(DesktopError::NotInFront); + } + Ok(()) + } + + fn sent(&self, inputs: &[INPUT]) -> Result<(), DesktopError> { + self.check()?; + send(inputs)?; + BOT_TICK.store(last_input(), Ordering::SeqCst); + Ok(()) + } + + /// The point on the screen, in real pixels, of `spot` in the window. + fn screen(&self, spot: Spot) -> Result { + match spot { + Spot::Screen { x, y } => Ok(POINT { x, y }), + Spot::Window { x, y } => { + let mut frame = RECT::default(); + // SAFETY: `frame` is a RECT of the size given. + unsafe { + DwmGetWindowAttribute( + self.hwnd, + DWMWA_EXTENDED_FRAME_BOUNDS, + std::ptr::from_mut(&mut frame).cast(), + size_of::() as u32, + ) + } + .map_err(|_| DesktopError::Gone)?; + let width = f64::from(frame.right - frame.left); + let height = f64::from(frame.bottom - frame.top); + Ok(POINT { + x: frame.left + (x.clamp(0.0, 1.0) * width) as i32, + y: frame.top + (y.clamp(0.0, 1.0) * height) as i32, + }) + } + } + } +} + +fn send(inputs: &[INPUT]) -> Result<(), DesktopError> { + // SAFETY: the inputs are well formed and live through the call. + let sent = unsafe { SendInput(inputs, size_of::() as i32) }; + if sent as usize != inputs.len() { + return Err(DesktopError::Locked); + } + Ok(()) +} + +fn mouse_input(x: i32, y: i32, flags: MOUSE_EVENT_FLAGS) -> INPUT { + INPUT { + r#type: INPUT_MOUSE, + Anonymous: INPUT_0 { + mi: MOUSEINPUT { + dx: x, + dy: y, + dwFlags: flags, + ..Default::default() + }, + }, + } +} + +fn key_input(key: VIRTUAL_KEY, unicode: u16, flags: KEYBD_EVENT_FLAGS) -> INPUT { + INPUT { + r#type: INPUT_KEYBOARD, + Anonymous: INPUT_0 { + ki: KEYBDINPUT { + wVk: key, + wScan: unicode, + dwFlags: flags, + ..Default::default() + }, + }, + } +} + +/// Clicks `spot` in window `id` with the real mouse. +pub fn click(id: u64, spot: Spot) -> Result<(), DesktopError> { + let hands = Hands::take(id)?; + let point = hands.screen(spot)?; + // SAFETY: a plain query. + let under = unsafe { GetAncestor(WindowFromPoint(point), GA_ROOT) }; + if under != hands.hwnd { + return Err(DesktopError::Covered); + } + // Absolute moves go from 0 to 65535 across all the screens. + // SAFETY: plain queries. + let (left, top, width, height) = unsafe { + ( + GetSystemMetrics(SM_XVIRTUALSCREEN), + GetSystemMetrics(SM_YVIRTUALSCREEN), + GetSystemMetrics(SM_CXVIRTUALSCREEN).max(1), + GetSystemMetrics(SM_CYVIRTUALSCREEN).max(1), + ) + }; + let x = ((i64::from(point.x - left) * 65535) / i64::from(width)) as i32; + let y = ((i64::from(point.y - top) * 65535) / i64::from(height)) as i32; + let moved = MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK; + hands.sent(&[mouse_input(x, y, moved)])?; + std::thread::sleep(BREATH); + hands.sent(&[ + mouse_input(0, 0, MOUSEEVENTF_LEFTDOWN), + mouse_input(0, 0, MOUSEEVENTF_LEFTUP), + ]) +} + +/// Types `text` into window `id`, where its focus is, with the real +/// keyboard. +pub fn type_text(id: u64, text: &str) -> Result<(), DesktopError> { + let hands = Hands::take(id)?; + let units: Vec = text.replace("\r\n", "\n").encode_utf16().collect(); + for chunk in units.chunks(CHUNK) { + let mut inputs = Vec::with_capacity(chunk.len() * 2); + for &unit in chunk { + if unit == u16::from(b'\n') { + inputs.push(key_input(VK_RETURN, 0, KEYBD_EVENT_FLAGS(0))); + inputs.push(key_input(VK_RETURN, 0, KEYEVENTF_KEYUP)); + } else { + inputs.push(key_input(VIRTUAL_KEY(0), unit, KEYEVENTF_UNICODE)); + inputs.push(key_input( + VIRTUAL_KEY(0), + unit, + KEYEVENTF_UNICODE | KEYEVENTF_KEYUP, + )); + } + } + hands.sent(&inputs)?; + } + Ok(()) +} + +fn virtual_key(key: Key) -> VIRTUAL_KEY { + match key { + Key::Enter => VK_RETURN, + Key::Tab => VK_TAB, + Key::Escape => VK_ESCAPE, + Key::Backspace => VK_BACK, + Key::Delete => VK_DELETE, + Key::Space => VK_SPACE, + Key::Up => VK_UP, + Key::Down => VK_DOWN, + Key::Left => VK_LEFT, + Key::Right => VK_RIGHT, + Key::Home => VK_HOME, + Key::End => VK_END, + Key::PageUp => VK_PRIOR, + Key::PageDown => VK_NEXT, + Key::Insert => VK_INSERT, + Key::F(number) => VIRTUAL_KEY(VK_F1.0 + u16::from(number) - 1), + // Letters and digits have their ASCII code as virtual key. + Key::Char(char) => VIRTUAL_KEY(char as u16), + } +} + +/// Presses `keys` in window `id` with the real keyboard: the modifiers +/// down, the key, all up again. +pub fn press(id: u64, keys: &Keys) -> Result<(), DesktopError> { + let hands = Hands::take(id)?; + let Some(key) = keys.key else { + return Ok(()); + }; + let held: Vec = [ + (keys.ctrl, VK_CONTROL), + (keys.alt, VK_MENU), + (keys.shift, VK_SHIFT), + ] + .into_iter() + .filter_map(|(on, key)| on.then_some(key)) + .collect(); + let mut inputs: Vec = held + .iter() + .map(|&key| key_input(key, 0, KEYBD_EVENT_FLAGS(0))) + .collect(); + inputs.push(key_input(virtual_key(key), 0, KEYBD_EVENT_FLAGS(0))); + inputs.push(key_input(virtual_key(key), 0, KEYEVENTF_KEYUP)); + inputs.extend( + held.iter() + .rev() + .map(|&key| key_input(key, 0, KEYEVENTF_KEYUP)), + ); + hands.sent(&inputs) +} diff --git a/crates/botloftd/src/platform/desktop/windows/real_tests.rs b/crates/botloftd/src/platform/desktop/windows/real_tests.rs new file mode 100644 index 00000000..3caa93fb --- /dev/null +++ b/crates/botloftd/src/platform/desktop/windows/real_tests.rs @@ -0,0 +1,64 @@ +//! The real mouse and keyboard in the tests' own window (spec 24.7). They +//! move the cursor and type on the computer they run on, so they run only +//! with BOTLOFT_REAL_INPUT_TESTS=1, as CI's Windows runner sets. + +use super::super::keys::parse; +use super::super::{Control, Spot, read, real_click, real_press, real_type, windows as listed}; +use super::test_window::TestWindow; + +fn allowed() -> bool { + let on = std::env::var("BOTLOFT_REAL_INPUT_TESTS").is_ok_and(|value| value == "1"); + if !on { + eprintln!("real input tests are off: set BOTLOFT_REAL_INPUT_TESTS=1"); + } + on +} + +fn center(control: &Control) -> Spot { + let [left, top, width, height] = control.rect.expect("a place on the screen"); + Spot::Screen { + x: left + width / 2, + y: top + height / 2, + } +} + +fn find(window: &TestWindow, what: impl Fn(&Control) -> bool) -> Control { + read(window.id) + .expect("read") + .into_iter() + .find(|control| what(control)) + .expect("the control") +} + +#[test] +fn a_real_click_and_real_typing_reach_the_window() { + if !allowed() { + return; + } + let window = TestWindow::open("Botloft desktop test: real"); + let field = find(&window, |control| control.value.as_deref() == Some("hello")); + real_click(window.id, center(&field)).expect("click the field"); + real_press(window.id, &parse("Ctrl+A").expect("keys")).expect("select all"); + real_type(window.id, "typed for real").expect("type"); + find(&window, |control| { + control.value.as_deref() == Some("typed for real") + }); + + let save = find(&window, |control| control.name == "Save"); + real_click(window.id, center(&save)).expect("click Save"); + std::thread::sleep(std::time::Duration::from_millis(300)); + let title = listed() + .expect("windows") + .into_iter() + .find(|found| found.id == window.id) + .map(|found| found.title); + assert_eq!(title.as_deref(), Some("Saved")); +} + +#[test] +fn every_control_shown_has_its_place_on_the_screen() { + let window = TestWindow::open("Botloft desktop test: places"); + let save = find(&window, |control| control.name == "Save"); + let [_, _, width, height] = save.rect.expect("a place"); + assert!(width > 0 && height > 0); +} diff --git a/crates/botloftd/src/platform/desktop/windows/uia.rs b/crates/botloftd/src/platform/desktop/windows/uia.rs index 08ebb59e..244b96e2 100644 --- a/crates/botloftd/src/platform/desktop/windows/uia.rs +++ b/crates/botloftd/src/platform/desktop/windows/uia.rs @@ -13,7 +13,8 @@ use windows::Win32::System::Com::{ use windows::Win32::System::Variant::VARIANT; use windows::Win32::UI::Accessibility::{ CUIAutomation, IUIAutomation, IUIAutomation2, IUIAutomationCacheRequest, - IUIAutomationCondition, IUIAutomationElement, TreeScope_Children, UIA_ControlTypePropertyId, + IUIAutomationCondition, IUIAutomationElement, TreeScope_Children, + UIA_BoundingRectanglePropertyId, UIA_ControlTypePropertyId, UIA_ExpandCollapseExpandCollapseStatePropertyId, UIA_IsEnabledPropertyId, UIA_IsOffscreenPropertyId, UIA_IsPasswordPropertyId, UIA_IsValuePatternAvailablePropertyId, UIA_NamePropertyId, UIA_PROPERTY_ID, UIA_RuntimeIdPropertyId, @@ -25,6 +26,9 @@ use super::super::DesktopError; use super::super::read::Control; use super::handle; use super::uia_control::{control, kind_of}; +use windows::Win32::UI::HiDpi::{ + DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2, SetThreadDpiAwarenessContext, +}; use windows::core::Interface; /// The most controls one walk gathers, how deep it goes and how long it @@ -36,7 +40,7 @@ const WALK_MAX: Duration = Duration::from_secs(15); /// busy with a dialog, or stuck, would otherwise hold the call for long. const CALL_MS: u32 = 5000; -const PROPERTIES: [UIA_PROPERTY_ID; 11] = [ +const PROPERTIES: [UIA_PROPERTY_ID; 12] = [ UIA_NamePropertyId, UIA_ControlTypePropertyId, UIA_IsPasswordPropertyId, @@ -48,6 +52,7 @@ const PROPERTIES: [UIA_PROPERTY_ID; 11] = [ UIA_ExpandCollapseExpandCollapseStatePropertyId, UIA_SelectionItemIsSelectedPropertyId, UIA_RuntimeIdPropertyId, + UIA_BoundingRectanglePropertyId, ]; /// COM on this thread while it lives, multithreaded, as UI Automation @@ -95,6 +100,9 @@ pub(super) struct Session { impl Session { pub(super) fn start() -> Result { let one = ONE.lock().unwrap_or_else(PoisonError::into_inner); + // Places in real pixels, as the real mouse needs them. + // SAFETY: changes only this thread's DPI awareness. + unsafe { SetThreadDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2) }; let com = Com::start(); // SAFETY: COM calls on interfaces made here, on this thread. unsafe { diff --git a/crates/botloftd/src/platform/desktop/windows/uia_control.rs b/crates/botloftd/src/platform/desktop/windows/uia_control.rs index 8e335e17..f6792584 100644 --- a/crates/botloftd/src/platform/desktop/windows/uia_control.rs +++ b/crates/botloftd/src/platform/desktop/windows/uia_control.rs @@ -1,11 +1,11 @@ //! One control as a line of a reading (spec 24.5): its properties, fetched //! with the walk, read out of UI Automation's variants. -use windows::Win32::System::Variant::{VARIANT, VT_ARRAY, VT_BSTR, VT_I4}; +use windows::Win32::System::Variant::{VARIANT, VT_ARRAY, VT_BSTR, VT_I4, VT_R8}; use windows::Win32::UI::Accessibility::{ ExpandCollapseState_Collapsed, ExpandCollapseState_Expanded, - ExpandCollapseState_PartiallyExpanded, IUIAutomationElement, UIA_CONTROLTYPE_ID, - UIA_ExpandCollapseExpandCollapseStatePropertyId, UIA_IsEnabledPropertyId, + ExpandCollapseState_PartiallyExpanded, IUIAutomationElement, UIA_BoundingRectanglePropertyId, + UIA_CONTROLTYPE_ID, UIA_ExpandCollapseExpandCollapseStatePropertyId, UIA_IsEnabledPropertyId, UIA_IsPasswordPropertyId, UIA_IsValuePatternAvailablePropertyId, UIA_PROPERTY_ID, UIA_RuntimeIdPropertyId, UIA_SelectionItemIsSelectedPropertyId, UIA_ToggleToggleStatePropertyId, UIA_ValueValuePropertyId, @@ -65,6 +65,7 @@ pub(super) unsafe fn control( runtime_id: property(element, UIA_RuntimeIdPropertyId) .map(|value| runtime_id(&value)) .unwrap_or_default(), + rect: property(element, UIA_BoundingRectanglePropertyId).and_then(|value| rect(&value)), } } } @@ -96,6 +97,30 @@ fn runtime_id(value: &VARIANT) -> Vec { } } +/// A place on the screen: four doubles, left, top, width and height; none +/// for what has no size. +fn rect(value: &VARIANT) -> Option<[i32; 4]> { + // SAFETY: the union is read as an array only when its type says so, + // and the array's own bounds limit the read. + unsafe { + let inner = &value.Anonymous.Anonymous; + if inner.vt != (VT_ARRAY | VT_R8) { + return None; + } + let array = inner.Anonymous.parray; + if array.is_null() || (*array).cDims != 1 || (*array).pvData.is_null() { + return None; + } + if (*array).rgsabound[0].cElements != 4 { + return None; + } + let numbers = std::slice::from_raw_parts((*array).pvData.cast::(), 4); + let rect = + [numbers[0], numbers[1], numbers[2], numbers[3]].map(|number| number.round() as i32); + (rect[2] > 0 && rect[3] > 0).then_some(rect) + } +} + /// A control type in words; `None` for what is left out. pub(super) fn kind_of(kind: UIA_CONTROLTYPE_ID) -> Option<&'static str> { Some(match kind.0 { diff --git a/crates/botloftd/src/tools/desktop_grant.rs b/crates/botloftd/src/tools/desktop_grant.rs index 51350ba8..da031724 100644 --- a/crates/botloftd/src/tools/desktop_grant.rs +++ b/crates/botloftd/src/tools/desktop_grant.rs @@ -40,6 +40,19 @@ pub(super) fn describe(err: &DesktopError) -> String { DesktopError::NoOption(option) => format!( "There is no option \"{option}\" there. Read the window again to see the options." ), + DesktopError::OwnerTookOver => "The owner moved the mouse or pressed a key, so you \ + stopped at once: they are using their computer. Wait, then try again; do not fight \ + them for the mouse." + .to_owned(), + DesktopError::NotInFront => "Windows did not bring the app's window to the front, or \ + another window came over it, so nothing was sent. Try again in a moment." + .to_owned(), + DesktopError::Covered => "Another window covers that point, so the click was not sent. \ + Read the windows again." + .to_owned(), + DesktopError::Locked => "The screen is locked or takes no input now, so the real mouse \ + and keyboard cannot be used. Accessibility may still work." + .to_owned(), } } diff --git a/docs/spec.md b/docs/spec.md index 273c2c37..31d16d6b 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1742,10 +1742,15 @@ No servidor `botloft` (11), como as do navegador. ### 24.7 Mouse e teclado de verdade -- **Entrada**: `SendInput`, com a janela do app trazida para a frente antes (`SetForegroundWindow`, que o Windows só deixa quando o processo tem permissão; senão a tool diz que não conseguiu). -- **O dono por perto manda**: enquanto o bot age com mouse e teclado, um gancho de baixo nível (`WH_MOUSE_LL`, `WH_KEYBOARD_LL`) olha a entrada que não foi injetada (sem `LLMHF_INJECTED`). Se o dono mexe o mouse ou aperta uma tecla, a ação para na hora, e as tools de mouse e teclado daquele bot esperam 10 s sem o dono mexer antes de voltar. A tool diz ao bot que o dono assumiu. +- **Entrada**: `SendInput`, com a janela do app trazida para a frente antes (`SetForegroundWindow`, que o Windows só deixa ao processo que mandou a última entrada: antes vai um movimento vazio do mouse, que não mexe nada; restaurada, se estava minimizada). Antes de cada passo (o movimento até o ponto, o clique, cada 8 caracteres digitados, as teclas): a janela de cima (`GetForegroundWindow`, subindo até a raiz) tem de ser a do app, senão nada vai e a tool diz que a janela não está na frente; num clique, a janela no ponto (`WindowFromPoint`) também, senão diz que outra janela cobre o ponto. Sem janela de cima, ou `SendInput` mandando menos do que devia, a tela está bloqueada ou não aceita entrada. +- **Onde clica**: no centro do controle, pelo lugar que a acessibilidade dá (`BoundingRectangle`, em pixels reais: a sessão da UI Automation fica com DPI por monitor), ou num ponto da foto da janela, como frações da moldura (`DWMWA_EXTENDED_FRAME_BOUNDS`), levado aos pixels reais. O movimento é absoluto sobre todas as telas (`MOUSEEVENTF_VIRTUALDESK`). O cursor fica onde o bot o deixou. +- **Digitar**: cada caractere como Unicode (`KEYEVENTF_UNICODE`), sem depender do layout do teclado; quebra de linha é Enter. +- **Teclas** (`desktop_press`): nomes como `Enter`, `Tab`, `Ctrl+S`, `Shift+Tab`, `F5`, letras e números, com Ctrl, Alt e Shift. Nunca a tecla Windows (sozinha ou com outras), Alt+Tab, Alt+Esc, Ctrl+Esc, Ctrl+Shift+Esc nem Ctrl+Alt+Del: são do Windows, não do app (`platform/desktop/keys.rs`, com testes). Alt+F4 vale: fecha a janela do app, como para quem usa. +- **O dono por perto manda**: enquanto o bot age com mouse e teclado, ganchos de baixo nível (`WH_MOUSE_LL`, `WH_KEYBOARD_LL`, num thread com fila própria, só durante a ação) olham a entrada que não foi injetada (sem `LLMHF_INJECTED` nem `LLKHF_INJECTED`). Se o dono mexe o mouse ou aperta uma tecla, a ação para na hora, e as tools de mouse e teclado daquele bot esperam 10 s sem o dono mexer antes de voltar. A tool diz ao bot que o dono assumiu. - **Aviso na tela**: enquanto o bot age assim, o daemon mostra uma borda fina na cor do bot em volta da tela e uma pílula no alto, " está usando o seu mouse e teclado. Mexa o mouse para parar." (uma janela própria em camadas, sempre no topo, que não recebe clique). O texto vem nos três idiomas (15.6). - Com a tela bloqueada, não há mouse e teclado de verdade: o Windows não entrega a entrada. A tool diz isso; a acessibilidade continua (24.8). +- **O tempo sem o dono** (24.8) não conta o que o bot mandou: guardada a marca da última entrada do bot e a do dono antes dela, quando a última entrada da sessão é a do bot vale a do dono. +- **Testes**: mexem no cursor e digitam no computador em que rodam, então só rodam com `BOTLOFT_REAL_INPUT_TESTS=1`, que o CI liga no runner do Windows, onde ninguém está na frente. No computador de quem desenvolve, ficam de fora. ### 24.8 Sem você na frente From b062db68eb8877732fc1ac6fac59666b6ee76f20 Mon Sep 17 00:00:00 2001 From: Phelipe Lorran <112786693+httpsphl@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:12:27 -0300 Subject: [PATCH 2/2] test(desktop): select a field's text from start to end, as every classic field takes it --- .../src/platform/desktop/windows/real_tests.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/crates/botloftd/src/platform/desktop/windows/real_tests.rs b/crates/botloftd/src/platform/desktop/windows/real_tests.rs index 3caa93fb..4fa757f0 100644 --- a/crates/botloftd/src/platform/desktop/windows/real_tests.rs +++ b/crates/botloftd/src/platform/desktop/windows/real_tests.rs @@ -38,11 +38,20 @@ fn a_real_click_and_real_typing_reach_the_window() { let window = TestWindow::open("Botloft desktop test: real"); let field = find(&window, |control| control.value.as_deref() == Some("hello")); real_click(window.id, center(&field)).expect("click the field"); - real_press(window.id, &parse("Ctrl+A").expect("keys")).expect("select all"); + // Ctrl+A is not in every classic field; from the start to the end is. + real_press(window.id, &parse("Ctrl+Home").expect("keys")).expect("to the start"); + real_press(window.id, &parse("Ctrl+Shift+End").expect("keys")).expect("select to the end"); real_type(window.id, "typed for real").expect("type"); - find(&window, |control| { - control.value.as_deref() == Some("typed for real") - }); + std::thread::sleep(std::time::Duration::from_millis(200)); + let values: Vec> = read(window.id) + .expect("read") + .into_iter() + .map(|control| control.value) + .collect(); + assert!( + values.contains(&Some("typed for real".to_owned())), + "{values:?}" + ); let save = find(&window, |control| control.name == "Save"); real_click(window.id, center(&save)).expect("click Save");