From 6489d427f87e65519c30acdb3c78d2705c37aa48 Mon Sep 17 00:00:00 2001 From: Phelipe Lorran <112786693+httpsphl@users.noreply.github.com> Date: Sun, 4 Oct 2026 18:05:06 -0300 Subject: [PATCH 1/3] feat(desktop): find the owner's windows, what is never granted, and a picture of one The first part of D1 (spec 24.3, 24.4), in the platform layer: the top-level windows of the owner's session (shown or minimized, with a title, not tool windows nor cloaked ones), the program each one is from (its executable, named by its file description; a Store app by the core window inside its frame) and whether it runs as administrator. The list of what is never granted (Botloft, elevated windows, terminals and command boxes with the Run box, Task Manager, password managers and the Credential Manager) is pure code with tests. A picture of a window comes from PrintWindow with full content, in real pixels, cropped to the DWM frame, shrunk to logical pixels up to 1600 x 1200 and encoded as a JPEG (jpeg-encoder). Other systems say the desktop is not there yet. Tested with a real window of the test's own on Windows 11: listed with its app, pictured red behind others, minimized, closed. --- Cargo.lock | 7 + crates/botloftd/Cargo.toml | 3 +- .../botloftd/src/platform/desktop/blocked.rs | 208 ++++++++++++++++++ crates/botloftd/src/platform/desktop/mod.rs | 100 +++++++++ .../botloftd/src/platform/desktop/shrink.rs | 111 ++++++++++ .../src/platform/desktop/windows/capture.rs | 178 +++++++++++++++ .../src/platform/desktop/windows/mod.rs | 144 ++++++++++++ .../src/platform/desktop/windows/process.rs | 170 ++++++++++++++ .../src/platform/desktop/windows/tests.rs | 180 +++++++++++++++ crates/botloftd/src/platform/mod.rs | 1 + docs/spec.md | 10 +- 11 files changed, 1106 insertions(+), 6 deletions(-) create mode 100644 crates/botloftd/src/platform/desktop/blocked.rs create mode 100644 crates/botloftd/src/platform/desktop/mod.rs create mode 100644 crates/botloftd/src/platform/desktop/shrink.rs create mode 100644 crates/botloftd/src/platform/desktop/windows/capture.rs create mode 100644 crates/botloftd/src/platform/desktop/windows/mod.rs create mode 100644 crates/botloftd/src/platform/desktop/windows/process.rs create mode 100644 crates/botloftd/src/platform/desktop/windows/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 65ff8bd2..7503b8c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -483,6 +483,7 @@ dependencies = [ "getrandom 0.4.3", "hex", "jiff", + "jpeg-encoder", "libc", "serde", "serde_json", @@ -2319,6 +2320,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "jpeg-encoder" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0370574b86f7eca156b9f298392b5e69a23f8c86f3f865add60bbc2e79467a6" + [[package]] name = "js-sys" version = "0.3.106" diff --git a/crates/botloftd/Cargo.toml b/crates/botloftd/Cargo.toml index 8fd1be16..f20604f3 100644 --- a/crates/botloftd/Cargo.toml +++ b/crates/botloftd/Cargo.toml @@ -20,6 +20,7 @@ futures-util = "0.3.34" getrandom = "0.4.3" hex = "0.4.3" jiff = "0.2.37" +jpeg-encoder = "0.7.1" serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.151" sha2 = "0.11.0" @@ -36,7 +37,7 @@ tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } workspace = true [target."cfg(windows)".dependencies] -windows = { version = "0.62.2", features = ["Win32_Foundation", "Win32_Security", "Win32_Security_Authorization", "Win32_System_Threading", "Win32_System_Memory", "Win32_System_JobObjects", "Win32_System_Environment", "Win32_System_Power", "Win32_System_Console", "Win32_System_Com", "Win32_System_Ole", "Win32_System_Variant", "Win32_System_TaskScheduler", "Win32_System_RemoteDesktop", "Win32_Security_Authentication_Identity", "Win32_UI_Shell", "Win32_UI_Shell_Common"] } +windows = { version = "0.62.2", features = ["Win32_Foundation", "Win32_Security", "Win32_Security_Authorization", "Win32_System_Threading", "Win32_System_Memory", "Win32_System_JobObjects", "Win32_System_Environment", "Win32_System_Power", "Win32_System_Console", "Win32_System_Com", "Win32_System_Ole", "Win32_System_Variant", "Win32_System_TaskScheduler", "Win32_System_RemoteDesktop", "Win32_Security_Authentication_Identity", "Win32_UI_Shell", "Win32_UI_Shell_Common", "Win32_UI_WindowsAndMessaging", "Win32_UI_HiDpi", "Win32_Graphics_Gdi", "Win32_Graphics_Dwm", "Win32_Storage_FileSystem", "Win32_Storage_Xps", "Win32_System_LibraryLoader"] } # Named by the code that `windows::core::implement` writes. windows-core = "0.62.2" diff --git a/crates/botloftd/src/platform/desktop/blocked.rs b/crates/botloftd/src/platform/desktop/blocked.rs new file mode 100644 index 00000000..656c5f96 --- /dev/null +++ b/crates/botloftd/src/platform/desktop/blocked.rs @@ -0,0 +1,208 @@ +//! What is never granted on the desktop, whatever the owner allows (spec +//! 24.3): Botloft itself, windows running as administrator, terminals and +//! command boxes, and password managers. The secure desktop (UAC, the lock +//! screen) has no windows here at all. Password fields are left out where +//! a window is read. + +use super::Window; + +/// Why a window is never granted. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Never { + /// A bot never approves itself. + Botloft, + /// It runs with more rights than the daemon. + Elevated, + /// Typing there would skip the bot's rules for commands. + Terminal, + /// It keeps passwords or the computer's security. + Passwords, +} + +impl Never { + /// What the bot reads about it. + pub fn why(self) -> &'static str { + match self { + Self::Botloft => "it is Botloft itself", + Self::Elevated => "it runs as administrator", + Self::Terminal => { + "it runs commands, and you have your own tools for that, under your rules" + } + Self::Passwords => "it keeps passwords or the computer's security", + } + } +} + +const BOTLOFT: &[&str] = &[ + "botloft.exe", + "botloftd.exe", + "botloft-app.exe", + "botloft-setup.exe", +]; + +/// Terminals, command boxes and the tools that change the system under +/// them. +const TERMINALS: &[&str] = &[ + "cmd.exe", + "powershell.exe", + "powershell_ise.exe", + "pwsh.exe", + "windowsterminal.exe", + "wt.exe", + "openconsole.exe", + "conhost.exe", + "wsl.exe", + "wslhost.exe", + "bash.exe", + "mintty.exe", + "alacritty.exe", + "wezterm-gui.exe", + "regedit.exe", + "mmc.exe", + "taskmgr.exe", +]; + +const PASSWORDS: &[&str] = &[ + "1password.exe", + "bitwarden.exe", + "keepass.exe", + "keepassxc.exe", + "lastpass.exe", + "dashlane.exe", + "credentialuibroker.exe", + "sechealthui.exe", + "securityhealthsystray.exe", + "consent.exe", + "logonui.exe", +]; + +/// The Credential Manager is a page of the Control Panel, an Explorer +/// window: known by its title, in the languages Botloft speaks. +const CREDENTIALS: &[&str] = &[ + "credential manager", + "gerenciador de credenciais", + "administrador de credenciales", +]; + +/// The class of a dialog box: Explorer's are the Run box (Win+R). +const DIALOG: &str = "#32770"; + +/// Why `window` is never granted, if it is not. +pub fn never(window: &Window) -> Option { + let file = window + .app + .path + .file_name() + .map(|name| name.to_string_lossy().to_lowercase()) + .unwrap_or_default(); + let named = |list: &[&str]| list.contains(&file.as_str()); + if named(BOTLOFT) { + return Some(Never::Botloft); + } + if window.elevated { + return Some(Never::Elevated); + } + if named(TERMINALS) || (file == "explorer.exe" && window.class == DIALOG) { + return Some(Never::Terminal); + } + let title = window.title.to_lowercase(); + if named(PASSWORDS) + || (file == "explorer.exe" && CREDENTIALS.iter().any(|name| title.contains(name))) + { + return Some(Never::Passwords); + } + None +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::super::App; + use super::*; + + fn window(path: &str, title: &str, class: &str) -> Window { + Window { + id: 1, + title: title.to_owned(), + class: class.to_owned(), + app: App { + path: PathBuf::from(path), + name: String::new(), + }, + minimized: false, + elevated: false, + } + } + + #[test] + fn everyday_apps_may_be_granted() { + for path in [ + r"C:\Program Files\Microsoft Office\root\Office16\EXCEL.EXE", + r"C:\Windows\System32\notepad.exe", + r"C:\Users\ana\AppData\Local\Clinic\clinic.exe", + ] { + assert_eq!(never(&window(path, "Book1", "XLMAIN")), None, "{path}"); + } + // An ordinary Explorer folder window too. + let folder = window(r"C:\Windows\explorer.exe", "Downloads", "CabinetWClass"); + assert_eq!(never(&folder), None); + } + + #[test] + fn botloft_terminals_and_passwords_never_are() { + let cases = [ + ( + r"C:\Users\ana\AppData\Local\Botloft\Botloft.exe", + Never::Botloft, + ), + (r"C:\Windows\System32\cmd.exe", Never::Terminal), + ( + r"C:\Program Files\WindowsApps\Microsoft.WindowsTerminal\WindowsTerminal.exe", + Never::Terminal, + ), + (r"C:\Program Files\PowerShell\7\pwsh.exe", Never::Terminal), + (r"C:\Windows\regedit.exe", Never::Terminal), + (r"C:\Windows\System32\Taskmgr.exe", Never::Terminal), + ( + r"C:\Program Files\KeePassXC\KeePassXC.exe", + Never::Passwords, + ), + ( + r"C:\Program Files\Bitwarden\Bitwarden.exe", + Never::Passwords, + ), + ( + r"C:\Windows\SystemApps\Microsoft.SecHealthUI\SecHealthUI.exe", + Never::Passwords, + ), + ]; + for (path, why) in cases { + assert_eq!(never(&window(path, "Any", "Any")), Some(why), "{path}"); + } + } + + #[test] + fn the_run_box_and_the_credential_manager_never_are() { + let explorer = r"C:\Windows\explorer.exe"; + assert_eq!( + never(&window(explorer, "Executar", DIALOG)), + Some(Never::Terminal) + ); + assert_eq!( + never(&window( + explorer, + "Gerenciador de Credenciais", + "CabinetWClass" + )), + Some(Never::Passwords) + ); + } + + #[test] + fn a_window_running_as_administrator_never_is() { + let mut admin = window(r"C:\Windows\System32\notepad.exe", "Notes", "Notepad"); + admin.elevated = true; + assert_eq!(never(&admin), Some(Never::Elevated)); + } +} diff --git a/crates/botloftd/src/platform/desktop/mod.rs b/crates/botloftd/src/platform/desktop/mod.rs new file mode 100644 index 00000000..84b09f18 --- /dev/null +++ b/crates/botloftd/src/platform/desktop/mod.rs @@ -0,0 +1,100 @@ +//! The owner's desktop (spec 24): the windows open on their screen, which +//! app each one is from, what is never granted, and a picture of a window. +//! Only on Windows; elsewhere every call says the desktop is not there. + +mod blocked; +// Only Windows takes pictures; the tests run everywhere. +#[cfg_attr(not(windows), allow(dead_code))] +mod shrink; +#[cfg(windows)] +mod windows; + +use std::path::PathBuf; + +pub use blocked::{Never, never}; + +/// The largest picture of a window the bot gets, in logical pixels (24.4). +pub const PICTURE_MAX: (u32, u32) = (1600, 1200); +/// The JPEG quality of a picture, as `browser_screenshot`'s. +#[cfg_attr(not(windows), allow(dead_code))] +const QUALITY: u8 = 70; + +/// A program on the owner's computer, by its executable (spec 24.2). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct App { + pub path: PathBuf, + /// The description in the file's version info ("Microsoft Excel"), or + /// the file's name without `.exe`. + pub name: String, +} + +/// A window open on the owner's desktop. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Window { + /// The window's handle, as the number the bot names it by. + pub id: u64, + pub title: String, + /// The window class, which tells some system windows apart. + pub class: String, + pub app: App, + pub minimized: bool, + /// Its process runs with more rights than the daemon (as administrator), + /// or its rights could not be read. + pub elevated: bool, +} + +/// A picture of a window, as a JPEG. +#[derive(Debug, Clone)] +pub struct Picture { + pub jpeg: Vec, + pub width: u32, + pub height: u32, +} + +#[derive(Debug, thiserror::Error)] +pub enum DesktopError { + #[error("the desktop is not available on this system yet")] + Unavailable, + #[error("that window is not open anymore")] + Gone, + #[error("that window is minimized, so there is nothing to see in it")] + Minimized, + #[error("Windows could not do it: {0}")] + System(String), +} + +/// The windows open on the owner's desktop: shown ones, minimized too, +/// with a title, from every app (spec 24.4). What the bot may see of them +/// is the caller's to decide. +pub fn windows() -> Result, DesktopError> { + #[cfg(windows)] + { + windows::list() + } + #[cfg(not(windows))] + { + Err(DesktopError::Unavailable) + } +} + +/// A picture of the window `id`, only of it, even behind other windows, in +/// logical pixels up to [`PICTURE_MAX`] (spec 24.4). +pub fn picture(id: u64) -> Result { + #[cfg(windows)] + { + let (rgb, width, height, scale) = windows::capture(id)?; + Ok(shrink::picture( + &rgb, + width, + height, + scale, + PICTURE_MAX, + QUALITY, + )) + } + #[cfg(not(windows))] + { + let _ = id; + Err(DesktopError::Unavailable) + } +} diff --git a/crates/botloftd/src/platform/desktop/shrink.rs b/crates/botloftd/src/platform/desktop/shrink.rs new file mode 100644 index 00000000..b8331cbc --- /dev/null +++ b/crates/botloftd/src/platform/desktop/shrink.rs @@ -0,0 +1,111 @@ +//! A window's pixels as the picture the bot gets (spec 24.4): in logical +//! pixels, without Windows' display scale, no larger than a limit, as a +//! JPEG. + +use super::Picture; + +/// The size `width` × `height` takes to fit within `max`, keeping its +/// shape; never larger than it was, never below one pixel. +pub fn fit(width: u32, height: u32, max: (u32, u32)) -> (u32, u32) { + let (width, height) = (width.max(1), height.max(1)); + let scale = (f64::from(max.0) / f64::from(width)) + .min(f64::from(max.1) / f64::from(height)) + .min(1.0); + let side = |length: u32| ((f64::from(length) * scale).round() as u32).max(1); + (side(width), side(height)) +} + +/// `rgb`, `width` × `height`, shrunk to `to`: each pixel is the average of +/// the ones it covers. +pub fn shrink(rgb: &[u8], width: u32, height: u32, to: (u32, u32)) -> Vec { + let (width, height) = (width as usize, height as usize); + let (to_width, to_height) = (to.0 as usize, to.1 as usize); + if (to_width, to_height) == (width, height) { + return rgb.to_vec(); + } + let mut out = Vec::with_capacity(to_width * to_height * 3); + for y in 0..to_height { + let top = y * height / to_height; + let bottom = ((y + 1) * height / to_height).max(top + 1); + for x in 0..to_width { + let left = x * width / to_width; + let right = ((x + 1) * width / to_width).max(left + 1); + let mut sum = [0u64; 3]; + for row in top..bottom { + for column in left..right { + let at = (row * width + column) * 3; + for (channel, total) in sum.iter_mut().enumerate() { + *total += u64::from(rgb[at + channel]); + } + } + } + let count = ((bottom - top) * (right - left)) as u64; + out.extend(sum.map(|total| (total / count) as u8)); + } + } + out +} + +/// The picture of a window whose pixels are `rgb`, `width` × `height`, on +/// a display scaled `scale` times (1.5 at 150%). +pub fn picture( + rgb: &[u8], + width: u32, + height: u32, + scale: f64, + max: (u32, u32), + quality: u8, +) -> Picture { + let logical = |length: u32| ((f64::from(length) / scale.max(1.0)).round() as u32).max(1); + let (to_width, to_height) = fit(logical(width), logical(height), max); + let pixels = shrink(rgb, width, height, (to_width, to_height)); + let mut jpeg = Vec::new(); + let encoder = jpeg_encoder::Encoder::new(&mut jpeg, quality); + // A picture this size always encodes; an empty one means nothing to see. + if encoder + .encode( + &pixels, + to_width as u16, + to_height as u16, + jpeg_encoder::ColorType::Rgb, + ) + .is_err() + { + jpeg.clear(); + } + Picture { + jpeg, + width: to_width, + height: to_height, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_window_fits_the_limit_in_its_shape_and_never_grows() { + assert_eq!(fit(3200, 1800, (1600, 1200)), (1600, 900)); + assert_eq!(fit(1000, 2400, (1600, 1200)), (500, 1200)); + assert_eq!(fit(800, 600, (1600, 1200)), (800, 600)); + assert_eq!(fit(0, 0, (1600, 1200)), (1, 1)); + } + + #[test] + fn shrinking_averages_the_pixels_it_covers() { + // Two pixels wide, one tall: black and white become grey. + let rgb = [0, 0, 0, 255, 255, 255]; + assert_eq!(shrink(&rgb, 2, 1, (1, 1)), [127, 127, 127]); + assert_eq!(shrink(&rgb, 2, 1, (2, 1)), rgb); + } + + #[test] + fn the_picture_drops_the_display_scale_and_is_a_jpeg() { + let rgb = vec![200u8; 300 * 150 * 3]; + // A window 300 × 150 on a display at 150%: 200 × 100 to the bot. + let picture = picture(&rgb, 300, 150, 1.5, (1600, 1200), 70); + assert_eq!((picture.width, picture.height), (200, 100)); + assert_eq!(&picture.jpeg[..2], [0xFF, 0xD8]); + } +} diff --git a/crates/botloftd/src/platform/desktop/windows/capture.rs b/crates/botloftd/src/platform/desktop/windows/capture.rs new file mode 100644 index 00000000..2a2636d2 --- /dev/null +++ b/crates/botloftd/src/platform/desktop/windows/capture.rs @@ -0,0 +1,178 @@ +//! A picture of one window with `PrintWindow` (spec 24.4): only the +//! window, even behind others, without the invisible resize border Windows +//! keeps around it. + +use std::ffi::c_void; + +use windows::Win32::Foundation::RECT; +use windows::Win32::Graphics::Dwm::{DWMWA_EXTENDED_FRAME_BOUNDS, DwmGetWindowAttribute}; +use windows::Win32::Graphics::Gdi::{ + BI_RGB, BITMAPINFO, BITMAPINFOHEADER, CreateCompatibleBitmap, CreateCompatibleDC, + DIB_RGB_COLORS, DeleteDC, DeleteObject, GetDC, GetDIBits, HBITMAP, HDC, HGDIOBJ, ReleaseDC, + SelectObject, +}; +use windows::Win32::Storage::Xps::{PRINT_WINDOW_FLAGS, PrintWindow}; +use windows::Win32::UI::HiDpi::{ + DPI_AWARENESS_CONTEXT, DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2, GetDpiForWindow, + SetThreadDpiAwarenessContext, +}; +use windows::Win32::UI::WindowsAndMessaging::{GetWindowRect, IsIconic, IsWindow}; + +use super::super::DesktopError; +use super::handle; + +/// Asks the window to draw itself fully, as DirectX and browser windows +/// need. +const RENDER_FULL_CONTENT: PRINT_WINDOW_FLAGS = PRINT_WINDOW_FLAGS(2); +/// Windows' unscaled DPI. +const BASE_DPI: f64 = 96.0; + +/// Sizes in real pixels on this thread while it lives, then back. +struct RealPixels(DPI_AWARENESS_CONTEXT); + +impl RealPixels { + fn new() -> Self { + // SAFETY: changes only this thread's DPI awareness, restored on drop. + Self(unsafe { SetThreadDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2) }) + } +} + +impl Drop for RealPixels { + fn drop(&mut self) { + if !self.0.is_invalid() { + // SAFETY: puts back the awareness this thread had. + unsafe { SetThreadDpiAwarenessContext(self.0) }; + } + } +} + +/// Frees the drawing objects in the right order when it goes. +struct Canvas { + screen: HDC, + memory: HDC, + bitmap: HBITMAP, + previous: HGDIOBJ, +} + +impl Drop for Canvas { + fn drop(&mut self) { + // SAFETY: each object was made by `capture` and is freed once. + unsafe { + SelectObject(self.memory, self.previous); + let _ = DeleteObject(self.bitmap.into()); + let _ = DeleteDC(self.memory); + ReleaseDC(None, self.screen); + } + } +} + +fn system(what: &str) -> DesktopError { + DesktopError::System(what.to_owned()) +} + +/// The window `id`'s pixels as RGB, its width and height, and the display +/// scale it is on (1.5 at 150%). +pub fn capture(id: u64) -> Result<(Vec, u32, u32, f64), DesktopError> { + let hwnd = handle(id); + // SAFETY: plain queries on a window handle. + unsafe { + if !IsWindow(Some(hwnd)).as_bool() { + return Err(DesktopError::Gone); + } + if IsIconic(hwnd).as_bool() { + return Err(DesktopError::Minimized); + } + } + let _real = RealPixels::new(); + let mut outer = RECT::default(); + // SAFETY: `outer` is a valid out-pointer. + unsafe { GetWindowRect(hwnd, &mut outer) }.map_err(|_| DesktopError::Gone)?; + let mut frame = outer; + // SAFETY: `frame` is a RECT of the size given. Without DWM the window + // rectangle is the frame. + let _ = unsafe { + DwmGetWindowAttribute( + hwnd, + DWMWA_EXTENDED_FRAME_BOUNDS, + std::ptr::from_mut(&mut frame).cast::(), + size_of::() as u32, + ) + }; + let full = (outer.right - outer.left, outer.bottom - outer.top); + if full.0 <= 0 || full.1 <= 0 { + return Err(DesktopError::Minimized); + } + // SAFETY: plain query on a window handle. + let dpi = unsafe { GetDpiForWindow(hwnd) }; + let scale = if dpi == 0 { + 1.0 + } else { + f64::from(dpi) / BASE_DPI + }; + + // SAFETY: each object made here is freed by `Canvas`. + let canvas = unsafe { + let screen = GetDC(None); + let memory = CreateCompatibleDC(Some(screen)); + let bitmap = CreateCompatibleBitmap(screen, full.0, full.1); + if memory.is_invalid() || bitmap.is_invalid() { + ReleaseDC(None, screen); + return Err(system("no memory for the picture")); + } + let previous = SelectObject(memory, bitmap.into()); + Canvas { + screen, + memory, + bitmap, + previous, + } + }; + // SAFETY: the window draws into our bitmap. + if !unsafe { PrintWindow(hwnd, canvas.memory, RENDER_FULL_CONTENT) }.as_bool() { + return Err(system("the window did not draw itself")); + } + let mut info = BITMAPINFO { + bmiHeader: BITMAPINFOHEADER { + biSize: size_of::() as u32, + biWidth: full.0, + // Negative: rows from the top down. + biHeight: -full.1, + biPlanes: 1, + biBitCount: 32, + biCompression: BI_RGB.0, + ..Default::default() + }, + ..Default::default() + }; + let mut bgra = vec![0u8; full.0 as usize * full.1 as usize * 4]; + // SAFETY: the buffer holds the whole bitmap at 32 bits a pixel. + let rows = unsafe { + GetDIBits( + canvas.memory, + canvas.bitmap, + 0, + full.1 as u32, + Some(bgra.as_mut_ptr().cast::()), + &mut info, + DIB_RGB_COLORS, + ) + }; + drop(canvas); + if rows == 0 { + return Err(system("the picture could not be read")); + } + // Only the frame, without the invisible border. + let left = (frame.left - outer.left).clamp(0, full.0) as usize; + let top = (frame.top - outer.top).clamp(0, full.1) as usize; + let right = (frame.right - outer.left).clamp(left as i32 + 1, full.0) as usize; + let bottom = (frame.bottom - outer.top).clamp(top as i32 + 1, full.1) as usize; + let mut rgb = Vec::with_capacity((right - left) * (bottom - top) * 3); + for row in top..bottom { + let start = (row * full.0 as usize + left) * 4; + let (pixels, _) = bgra[start..start + (right - left) * 4].as_chunks::<4>(); + for [blue, green, red, _] in pixels { + rgb.extend([*red, *green, *blue]); + } + } + Ok((rgb, (right - left) as u32, (bottom - top) as u32, scale)) +} diff --git a/crates/botloftd/src/platform/desktop/windows/mod.rs b/crates/botloftd/src/platform/desktop/windows/mod.rs new file mode 100644 index 00000000..2d61ee97 --- /dev/null +++ b/crates/botloftd/src/platform/desktop/windows/mod.rs @@ -0,0 +1,144 @@ +//! The owner's windows on Windows (spec 24.4): the top-level windows of +//! their session, which program each one is from, and whether it runs as +//! administrator. + +mod capture; +mod process; +#[cfg(test)] +mod tests; + +use std::collections::HashMap; +use std::ffi::c_void; +use std::path::PathBuf; + +use windows::Win32::Foundation::{HWND, LPARAM}; +use windows::Win32::Graphics::Dwm::{DWMWA_CLOAKED, DwmGetWindowAttribute}; +use windows::Win32::UI::WindowsAndMessaging::{ + EnumWindows, FindWindowExW, GWL_EXSTYLE, GetClassNameW, GetWindowLongPtrW, + GetWindowTextLengthW, GetWindowTextW, GetWindowThreadProcessId, IsIconic, IsWindowVisible, + WS_EX_TOOLWINDOW, +}; +use windows::core::{BOOL, w}; + +pub use capture::capture; + +use super::{App, DesktopError, Window}; + +/// The process that frames the windows of Store apps: the app itself is +/// the process of the frame's core window. +const FRAME_HOST: &str = "applicationframehost.exe"; + +pub fn list() -> Result, DesktopError> { + let mut handles: Vec = Vec::new(); + // SAFETY: the callback only pushes into `handles`, which outlives the + // call. + unsafe { + EnumWindows( + Some(collect), + LPARAM(std::ptr::from_mut(&mut handles) as isize), + ) + } + .map_err(|err| DesktopError::System(err.message()))?; + let mut apps: HashMap = HashMap::new(); + let mut found = Vec::new(); + for hwnd in handles { + if !shown(hwnd) { + continue; + } + let title = text(hwnd); + if title.trim().is_empty() { + continue; + } + let class = class_of(hwnd); + let Some((path, elevated)) = process::of(app_window(hwnd)) else { + continue; + }; + let app = apps + .entry(path.clone()) + .or_insert_with(|| process::app(&path)) + .clone(); + found.push(Window { + id: hwnd.0 as usize as u64, + title, + class, + app, + // SAFETY: a plain query on a window handle. + minimized: unsafe { IsIconic(hwnd) }.as_bool(), + elevated, + }); + } + Ok(found) +} + +unsafe extern "system" fn collect(hwnd: HWND, found: LPARAM) -> BOOL { + // SAFETY: `found` is the `Vec` `list` passed in, alive for the call. + let handles = unsafe { &mut *(found.0 as *mut Vec) }; + handles.push(hwnd); + BOOL(1) +} + +/// A window the owner can see on the taskbar or the screen: shown, not a +/// tool window, and not cloaked (Store apps kept hidden, other virtual +/// desktops). +fn shown(hwnd: HWND) -> bool { + // SAFETY: plain queries on a window handle. + unsafe { + if !IsWindowVisible(hwnd).as_bool() { + return false; + } + if GetWindowLongPtrW(hwnd, GWL_EXSTYLE) & WS_EX_TOOLWINDOW.0 as isize != 0 { + return false; + } + let mut cloaked = 0u32; + let read = DwmGetWindowAttribute( + hwnd, + DWMWA_CLOAKED, + std::ptr::from_mut(&mut cloaked).cast::(), + size_of::() as u32, + ); + read.is_err() || cloaked == 0 + } +} + +pub(super) fn handle(id: u64) -> HWND { + HWND(id as usize as *mut c_void) +} + +fn text(hwnd: HWND) -> String { + // SAFETY: the buffer is as long as the length asked for, plus the end. + unsafe { + let length = GetWindowTextLengthW(hwnd).max(0) as usize; + let mut buffer = vec![0u16; length + 1]; + let copied = GetWindowTextW(hwnd, &mut buffer).max(0) as usize; + String::from_utf16_lossy(&buffer[..copied]) + } +} + +fn class_of(hwnd: HWND) -> String { + let mut buffer = [0u16; 256]; + // SAFETY: the buffer is valid for its length. + let copied = unsafe { GetClassNameW(hwnd, &mut buffer) }.max(0) as usize; + String::from_utf16_lossy(&buffer[..copied]) +} + +fn process_id(hwnd: HWND) -> u32 { + let mut id = 0u32; + // SAFETY: `id` is a valid out-pointer. + unsafe { GetWindowThreadProcessId(hwnd, Some(&mut id)) }; + id +} + +/// The process a window belongs to, for its app: a Store app's frame +/// belongs to the frame host, and the app is its core window's process. +fn app_window(hwnd: HWND) -> u32 { + let id = process_id(hwnd); + if !process::is_named(id, FRAME_HOST) { + return id; + } + // SAFETY: a plain query on a window handle and a constant class name. + let core = unsafe { FindWindowExW(Some(hwnd), None, w!("Windows.UI.Core.CoreWindow"), None) }; + match core { + Ok(core) if !core.is_invalid() => process_id(core), + _ => id, + } +} diff --git a/crates/botloftd/src/platform/desktop/windows/process.rs b/crates/botloftd/src/platform/desktop/windows/process.rs new file mode 100644 index 00000000..e00b26c7 --- /dev/null +++ b/crates/botloftd/src/platform/desktop/windows/process.rs @@ -0,0 +1,170 @@ +//! The program behind a window: its executable, its name for people, and +//! whether it runs with more rights than the daemon. + +use std::ffi::c_void; +use std::os::windows::ffi::OsStrExt; +use std::path::{Path, PathBuf}; + +use windows::Win32::Foundation::{CloseHandle, HANDLE}; +use windows::Win32::Security::{GetTokenInformation, TOKEN_ELEVATION, TOKEN_QUERY, TokenElevation}; +use windows::Win32::Storage::FileSystem::{ + GetFileVersionInfoSizeW, GetFileVersionInfoW, VerQueryValueW, +}; +use windows::Win32::System::Threading::{ + GetCurrentProcess, OpenProcess, OpenProcessToken, PROCESS_NAME_WIN32, + PROCESS_QUERY_LIMITED_INFORMATION, QueryFullProcessImageNameW, +}; +use windows::core::{PCWSTR, PWSTR}; + +use super::super::App; + +/// Closes a handle when it goes. +struct Owned(HANDLE); + +impl Drop for Owned { + fn drop(&mut self) { + // SAFETY: the handle was opened by this module and is closed once. + let _ = unsafe { CloseHandle(self.0) }; + } +} + +fn open(id: u32) -> Option { + // SAFETY: a plain open; the handle is closed by `Owned`. + unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, id) } + .ok() + .map(Owned) +} + +fn path_of(process: &Owned) -> Option { + let mut buffer = vec![0u16; 1024]; + let mut length = buffer.len() as u32; + // SAFETY: the buffer and its length match. + unsafe { + QueryFullProcessImageNameW( + process.0, + PROCESS_NAME_WIN32, + PWSTR(buffer.as_mut_ptr()), + &mut length, + ) + } + .ok()?; + Some(PathBuf::from(String::from_utf16_lossy( + &buffer[..length as usize], + ))) +} + +/// Whether the process runs elevated; `None` when its rights cannot be +/// read, which a process running as administrator also causes. +fn elevated(process: HANDLE) -> Option { + let mut token = HANDLE::default(); + // SAFETY: `token` is a valid out-pointer, closed by `Owned`. + unsafe { OpenProcessToken(process, TOKEN_QUERY, &mut token) }.ok()?; + let token = Owned(token); + let mut elevation = TOKEN_ELEVATION::default(); + let mut length = 0u32; + // SAFETY: the buffer is a `TOKEN_ELEVATION` of the size given. + unsafe { + GetTokenInformation( + token.0, + TokenElevation, + Some(std::ptr::from_mut(&mut elevation).cast::()), + size_of::() as u32, + &mut length, + ) + } + .ok()?; + Some(elevation.TokenIsElevated != 0) +} + +/// The executable of process `id` and whether it runs with more rights +/// than the daemon; `None` when the process is gone. +pub fn of(id: u32) -> Option<(PathBuf, bool)> { + let process = open(id)?; + let path = path_of(&process)?; + // SAFETY: the pseudo handle of this process needs no closing. + let ours = elevated(unsafe { GetCurrentProcess() }).unwrap_or(false); + let theirs = elevated(process.0).unwrap_or(true); + Some((path, theirs && !ours)) +} + +/// Whether process `id` runs the executable named `file` (lower case). +pub fn is_named(id: u32, file: &str) -> bool { + open(id) + .and_then(|process| path_of(&process)) + .and_then(|path| { + path.file_name() + .map(|name| name.to_string_lossy().to_lowercase()) + }) + .is_some_and(|name| name == file) +} + +/// The app an executable is: named by its file description. +pub fn app(path: &Path) -> App { + let fallback = || { + path.file_stem() + .map(|stem| stem.to_string_lossy().into_owned()) + .unwrap_or_default() + }; + App { + path: path.to_owned(), + name: description(path).unwrap_or_else(fallback), + } +} + +fn wide(text: &str) -> Vec { + text.encode_utf16().chain(Some(0)).collect() +} + +/// The `FileDescription` in the executable's version info, in its first +/// language. +fn description(path: &Path) -> Option { + let file: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); + // SAFETY: `file` ends with a zero. + let size = unsafe { GetFileVersionInfoSizeW(PCWSTR(file.as_ptr()), None) }; + if size == 0 { + return None; + } + let mut block = vec![0u8; size as usize]; + // SAFETY: the block is as large as asked for. + unsafe { + GetFileVersionInfoW( + PCWSTR(file.as_ptr()), + None, + size, + block.as_mut_ptr().cast::(), + ) + } + .ok()?; + let query = |name: &str| -> Option<(*const u8, usize)> { + let name = wide(name); + let mut found: *mut c_void = std::ptr::null_mut(); + let mut length = 0u32; + // SAFETY: `block` holds the version info read above. + let ok = unsafe { + VerQueryValueW( + block.as_ptr().cast::(), + PCWSTR(name.as_ptr()), + &mut found, + &mut length, + ) + }; + (ok.as_bool() && !found.is_null() && length > 0) + .then_some((found.cast::().cast_const(), length as usize)) + }; + let (languages, _) = query(r"\VarFileInfo\Translation")?; + // SAFETY: the translation table starts with two u16: language, code page. + let (language, page) = unsafe { + let pair = languages.cast::(); + (pair.read_unaligned(), pair.add(1).read_unaligned()) + }; + let (text, length) = query(&format!( + r"\StringFileInfo\{language:04x}{page:04x}\FileDescription" + ))?; + // SAFETY: the value is `length` u16, the last one possibly the end. + let units = unsafe { std::slice::from_raw_parts(text.cast::(), length) }; + let name = String::from_utf16_lossy(units) + .trim_end_matches('\0') + .trim() + .to_owned(); + (!name.is_empty()).then_some(name) +} diff --git a/crates/botloftd/src/platform/desktop/windows/tests.rs b/crates/botloftd/src/platform/desktop/windows/tests.rs new file mode 100644 index 00000000..fe7255be --- /dev/null +++ b/crates/botloftd/src/platform/desktop/windows/tests.rs @@ -0,0 +1,180 @@ +//! With real windows: a window of the test's own, painted red, found in +//! the list, pictured, minimized and closed. + +use std::sync::Once; +use std::sync::mpsc; +use std::thread::JoinHandle; +use std::time::Duration; + +use windows::Win32::Foundation::{COLORREF, HWND, LPARAM, LRESULT, WPARAM}; +use windows::Win32::Graphics::Gdi::{CreateSolidBrush, HBRUSH}; +use windows::Win32::System::LibraryLoader::GetModuleHandleW; +use windows::Win32::UI::WindowsAndMessaging::{ + CW_USEDEFAULT, CreateWindowExW, DefWindowProcW, DispatchMessageW, GetMessageW, MSG, + PostMessageW, PostQuitMessage, RegisterClassW, SW_MINIMIZE, ShowWindow, TranslateMessage, + WINDOW_EX_STYLE, WM_CLOSE, WM_DESTROY, WNDCLASSW, WS_OVERLAPPEDWINDOW, WS_VISIBLE, +}; +use windows::core::{PCWSTR, w}; + +use super::super::{DesktopError, never, windows as listed}; +use super::capture; + +const CLASS: PCWSTR = w!("BotloftDesktopTest"); + +unsafe extern "system" fn procedure( + hwnd: HWND, + message: u32, + wparam: WPARAM, + lparam: LPARAM, +) -> LRESULT { + if message == WM_DESTROY { + // SAFETY: ends this thread's message loop. + unsafe { PostQuitMessage(0) }; + return LRESULT(0); + } + // SAFETY: everything else as Windows does by default. + unsafe { DefWindowProcW(hwnd, message, wparam, lparam) } +} + +/// A red window of the test's own, on a thread with its message loop. +struct TestWindow { + id: u64, + thread: Option>, +} + +impl TestWindow { + fn open(title: &str) -> Self { + static CLASS_ONCE: Once = Once::new(); + CLASS_ONCE.call_once(|| { + // SAFETY: a window class with a static name and procedure. + unsafe { + let class = WNDCLASSW { + lpfnWndProc: Some(procedure), + hInstance: GetModuleHandleW(None).expect("module").into(), + lpszClassName: CLASS, + // Pure red: 0x00BBGGRR. + hbrBackground: HBRUSH(CreateSolidBrush(COLORREF(0x0000_00FF)).0), + ..Default::default() + }; + assert_ne!(RegisterClassW(&class), 0, "register the class"); + } + }); + let title: Vec = title.encode_utf16().chain(Some(0)).collect(); + let (sent, opened) = mpsc::channel(); + let thread = std::thread::spawn(move || { + // SAFETY: a window on this thread, with its message loop. + unsafe { + let hwnd = CreateWindowExW( + WINDOW_EX_STYLE::default(), + CLASS, + PCWSTR(title.as_ptr()), + WS_OVERLAPPEDWINDOW | WS_VISIBLE, + CW_USEDEFAULT, + CW_USEDEFAULT, + 420, + 320, + None, + None, + None, + None, + ) + .expect("create the window"); + sent.send(hwnd.0 as usize as u64).expect("send"); + let mut message = MSG::default(); + while GetMessageW(&mut message, None, 0, 0).as_bool() { + let _ = TranslateMessage(&message); + DispatchMessageW(&message); + } + } + }); + let id = opened + .recv_timeout(Duration::from_secs(5)) + .expect("the window opened"); + // Time to paint. + std::thread::sleep(Duration::from_millis(300)); + Self { + id, + thread: Some(thread), + } + } + + fn close(&mut self) { + // SAFETY: asks the window's own thread to close it. + let _ = + unsafe { PostMessageW(Some(super::handle(self.id)), WM_CLOSE, WPARAM(0), LPARAM(0)) }; + if let Some(thread) = self.thread.take() { + thread.join().expect("the window's thread"); + } + } +} + +impl Drop for TestWindow { + fn drop(&mut self) { + self.close(); + } +} + +#[test] +fn the_list_has_the_window_with_its_app() { + let window = TestWindow::open("Botloft desktop test: list"); + let found = listed() + .expect("windows") + .into_iter() + .find(|found| found.id == window.id) + .expect("our window is in the list"); + assert_eq!(found.title, "Botloft desktop test: list"); + assert_eq!(found.class, "BotloftDesktopTest"); + let exe = std::env::current_exe().expect("exe"); + assert_eq!( + found.app.path.to_string_lossy().to_lowercase(), + exe.to_string_lossy().to_lowercase() + ); + assert!(!found.app.name.is_empty()); + assert!(!found.minimized); + assert!(!found.elevated, "the test is not running as administrator"); + assert_eq!(never(&found), None); +} + +#[test] +fn a_picture_is_only_the_window_in_its_own_pixels() { + let window = TestWindow::open("Botloft desktop test: picture"); + let (rgb, width, height, scale) = capture(window.id).expect("picture"); + assert!(scale >= 1.0); + // 420 × 320 logical pixels, the title bar included, without the + // invisible border: within a few pixels of it at any display scale. + let logical = |pixels: u32| f64::from(pixels) / scale; + assert!((logical(width) - 420.0).abs() < 20.0, "{width} at {scale}"); + assert!( + (logical(height) - 320.0).abs() < 20.0, + "{height} at {scale}" + ); + // The middle of the window is its red background. + let middle = ((height / 2 * width + width / 2) * 3) as usize; + assert_eq!(&rgb[middle..middle + 3], [255, 0, 0]); + let picture = super::super::picture(window.id).expect("jpeg"); + assert_eq!(&picture.jpeg[..2], [0xFF, 0xD8]); +} + +#[test] +fn a_minimized_window_is_listed_but_has_nothing_to_see_and_a_closed_one_is_gone() { + let mut window = TestWindow::open("Botloft desktop test: minimized"); + // SAFETY: a plain request on our own window. + let _ = unsafe { ShowWindow(super::handle(window.id), SW_MINIMIZE) }; + std::thread::sleep(Duration::from_millis(300)); + let found = listed() + .expect("windows") + .into_iter() + .find(|found| found.id == window.id) + .expect("still listed"); + assert!(found.minimized); + assert!(matches!(capture(window.id), Err(DesktopError::Minimized))); + let id = window.id; + window.close(); + assert!(matches!(capture(id), Err(DesktopError::Gone))); + assert!( + listed() + .expect("windows") + .iter() + .all(|found| found.id != id) + ); +} diff --git a/crates/botloftd/src/platform/mod.rs b/crates/botloftd/src/platform/mod.rs index f7376042..efa1344e 100644 --- a/crates/botloftd/src/platform/mod.rs +++ b/crates/botloftd/src/platform/mod.rs @@ -1,6 +1,7 @@ //! Everything that talks to the operating system directly. The rest of the //! daemon calls these functions and never Win32. +pub mod desktop; #[cfg(unix)] mod unix; #[cfg(windows)] diff --git a/docs/spec.md b/docs/spec.md index 0882962f..0f34af4a 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1106,7 +1106,7 @@ Conferência na documentação oficial (code.claude.com/docs) em 2026-09-28; esf | Memória automática do Claude Code nos bots | 7.5 | Confirmado (`memory`, `settings-reference`): ligada por padrão; o Claude Code grava `MEMORY.md` e um arquivo por nota em `%USERPROFILE%\.claude\projects\\memory\` e carrega o índice em toda sessão; `` vem do repositório git ou, fora de um, do caminho do cwd; `autoMemoryEnabled: false` vale em qualquer settings. **Testado com 2.1.284** (`-p --setting-sources project,local`, Haiku): ligada também assim, com cerca de 3 mil tokens a mais no prompt de sistema (6457 contra 3495). A pasta `memory\` é criada quando a sessão sobe. Pedido para lembrar de uma coisa, o bot gravou a nota e o `MEMORY.md` nessa pasta, fora do workspace, sem pedido de permissão no modo `default`, e não tocou no `CLAUDE.md` dele. O nome de `` é o caminho com `-` no lugar de tudo que não é letra nem número (`C--Users-ana-Botloft-site-revisor`); por essa regra `site-web\dev` e `site\web-dev` dariam a mesma pasta (não testado). Com o cwd dentro de um repositório git, a sessão carregou como `AutoMem` o `MEMORY.md` que o Claude Code guarda para o dono naquele repositório. Com `autoMemoryEnabled: false` no `settings.json` do projeto, o prompt encolhe e nada disso é carregado, nem dentro do repositório. **Pelo daemon de dev** (Haiku, "Aceitar edições"): pedido para lembrar, o bot editou o `CLAUDE.md` do workspace, e nenhuma pasta `memory\` foi criada. Mover a memória automática para dentro do workspace (`autoMemoryDirectory`) não foi testado | feito | | Telas no WebView2 do app | 22.5 | Não visto: a CSP do Tauri com `frame-src http://127.0.0.1:*` e as telas carregando no `iframe` | manual (PR): `pnpm tauri dev`, pedir uma página HTML a um bot e ver a área de design | | UI Automation com a tela bloqueada | 24.8 | **A verificar** (Windows 11): ler a árvore e usar `Invoke` e `Value` numa janela comum com a sessão bloqueada | D5 | -| `PrintWindow` com `PW_RENDERFULLCONTENT` | 24.4 | **A verificar**: foto de janelas comuns, de apps com GPU (Chromium, Electron) e com a tela bloqueada | D1, D5 | +| `PrintWindow` com `PW_RENDERFULLCONTENT` | 24.4 | **Visto no Windows 11** (2026-10-04) com uma janela Win32 comum do teste: a foto atrás de outras, o fundo na cor certa, o tamanho da moldura; minimizada, nada a fotografar. **A verificar**: apps com GPU (Chromium, Electron) e com a tela bloqueada | D1 (feito em parte), D5 | | `SetForegroundWindow` vindo do daemon | 24.7 | **A verificar**: o daemon roda como tarefa de logon, sem janela; o Windows pode recusar trazer outra janela para a frente | D4 | | Entrada do dono no gancho de baixo nível | 24.7 | **A verificar**: `LLMHF_INJECTED` e `LLKHF_INJECTED` separam a entrada do `SendInput` da do dono | D4 | @@ -1689,16 +1689,16 @@ Nem com o desktop inteiro, nem com o dono pedindo: - O próprio Botloft (o app e qualquer janela do daemon): um bot não aprova a si mesmo. - Janelas de processos com nível de integridade maior que o do daemon (rodando como administrador): o Windows já recusa a entrada (UIPI); o bot também não as vê. - A área de trabalho segura (UAC, Ctrl+Alt+Del, tela de bloqueio): o Windows não deixa, e o daemon nem tenta. -- Terminais e caixas de comando: Prompt de Comando, PowerShell, Windows Terminal, a caixa "Executar" (Win+R), o Editor do Registro, o Agendador de Tarefas. Digitar num terminal passaria por cima das regras de permissão do bot para comandos (10.1); para isso o bot já tem as tools dele. -- Gerenciadores de senha e cofres: Gerenciador de Credenciais, Segurança do Windows, 1Password, Bitwarden, KeePass, KeePassXC, LastPass, Dashlane. +- Terminais e caixas de comando: Prompt de Comando, PowerShell (e o ISE), Windows Terminal, WSL, Git Bash, a caixa "Executar" (Win+R, uma caixa de diálogo do Explorer), o Editor do Registro, os consoles do `mmc.exe` (Agendador de Tarefas, Serviços e outros) e o Gerenciador de Tarefas, que encerra qualquer programa, o Botloft inclusive. Digitar num terminal passaria por cima das regras de permissão do bot para comandos (10.1); para isso o bot já tem as tools dele. +- Gerenciadores de senha e cofres: Gerenciador de Credenciais (uma página do Painel de Controle, numa janela do Explorer, reconhecida pelo título em inglês, português e espanhol), Segurança do Windows, a janela de credenciais do Windows, 1Password, Bitwarden, KeePass, KeePassXC, LastPass, Dashlane. - Campos de senha (`IsPassword` da acessibilidade) em qualquer app: o bot não lê o valor e não digita neles. A tool diz que ali é com o dono. -A lista fica no código (`platform/desktop/blocked.rs`), com testes, e entra na tela de riscos (24.10) para o dono saber. +Os programas são reconhecidos pelo nome do executável, sem diferença de maiúsculas, e um Store app pelo processo da janela de dentro do quadro (`ApplicationFrameHost.exe`). Uma janela cujo processo não deixa ler os próprios direitos conta como de administrador. A lista fica no código (`platform/desktop/blocked.rs`), com testes, e entra na tela de riscos (24.10) para o dono saber. ### 24.4 O que o bot vê - **Janelas**: as de nível superior, visíveis ou minimizadas, com dono (não ferramentas nem janelas sem título). De um app fora do alcance, o bot vê só o nome do app, nunca o título da janela: um título diz o assunto de um e-mail ou o nome de um arquivo. -- **Foto de uma janela**: `PrintWindow` com `PW_RENDERFULLCONTENT`, que pega a janela mesmo atrás de outras, só a janela, sem o que estiver por cima dela. Uma janela minimizada não tem o que fotografar: a tool diz isso. +- **Foto de uma janela**: `PrintWindow` com `PW_RENDERFULLCONTENT`, que pega a janela mesmo atrás de outras, só a janela, sem o que estiver por cima dela, recortada na moldura que o DWM diz (`DWMWA_EXTENDED_FRAME_BOUNDS`), sem a borda invisível de redimensionar que o Windows deixa em volta. A medida é em pixels reais (o thread fica com DPI por monitor enquanto fotografa) e a foto é reduzida pela escala da tela da janela (`GetDpiForWindow`). Uma janela minimizada não tem o que fotografar: a tool diz isso. - **Foto do desktop inteiro** (só com esse alcance): a tela toda, com as janelas que nunca são liberadas (24.3) cobertas de preto. - **Tamanho**: a foto vai ao bot no tamanho da janela em pixels lógicos (sem a escala do Windows), até 1600 × 1200, reduzida mantendo a proporção. JPEG, qualidade 70, como `browser_screenshot`. From f45c59249ebf89cdc59d4c1357034230c8442d18 Mon Sep 17 00:00:00 2001 From: Phelipe Lorran <112786693+httpsphl@users.noreply.github.com> Date: Sun, 4 Oct 2026 18:13:08 -0300 Subject: [PATCH 2/3] feat(desktop): read a window through its accessibility tree The second part of D1 (spec 24.5): UI Automation walks the window a level at a time, with the properties fetched in the same call, and keeps the controls a person sees (the control view, nothing offscreen). Each becomes a line with a ref, its kind, name, value and state (checked, open, selected, disabled); unnamed panes and groups give way to what is in them, and scroll bars, title bars, tips and separators are left out. A password field never shows its value. Readings go on from where they stopped past 600 lines, and the walk stops at 3000 controls or 40 levels. Each control keeps its RuntimeId to be found again to act on it. Tested with real controls in the test's own window: a button, a field, a password field (its text never read) and a ticked check box. --- crates/botloftd/Cargo.toml | 2 +- crates/botloftd/src/platform/desktop/mod.rs | 19 +- crates/botloftd/src/platform/desktop/read.rs | 176 +++++++++++ .../src/platform/desktop/windows/mod.rs | 2 + .../src/platform/desktop/windows/tests.rs | 80 ++++- .../src/platform/desktop/windows/uia.rs | 279 ++++++++++++++++++ docs/spec.md | 2 +- 7 files changed, 553 insertions(+), 7 deletions(-) create mode 100644 crates/botloftd/src/platform/desktop/read.rs create mode 100644 crates/botloftd/src/platform/desktop/windows/uia.rs diff --git a/crates/botloftd/Cargo.toml b/crates/botloftd/Cargo.toml index f20604f3..2948891e 100644 --- a/crates/botloftd/Cargo.toml +++ b/crates/botloftd/Cargo.toml @@ -37,7 +37,7 @@ tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } workspace = true [target."cfg(windows)".dependencies] -windows = { version = "0.62.2", features = ["Win32_Foundation", "Win32_Security", "Win32_Security_Authorization", "Win32_System_Threading", "Win32_System_Memory", "Win32_System_JobObjects", "Win32_System_Environment", "Win32_System_Power", "Win32_System_Console", "Win32_System_Com", "Win32_System_Ole", "Win32_System_Variant", "Win32_System_TaskScheduler", "Win32_System_RemoteDesktop", "Win32_Security_Authentication_Identity", "Win32_UI_Shell", "Win32_UI_Shell_Common", "Win32_UI_WindowsAndMessaging", "Win32_UI_HiDpi", "Win32_Graphics_Gdi", "Win32_Graphics_Dwm", "Win32_Storage_FileSystem", "Win32_Storage_Xps", "Win32_System_LibraryLoader"] } +windows = { version = "0.62.2", features = ["Win32_Foundation", "Win32_Security", "Win32_Security_Authorization", "Win32_System_Threading", "Win32_System_Memory", "Win32_System_JobObjects", "Win32_System_Environment", "Win32_System_Power", "Win32_System_Console", "Win32_System_Com", "Win32_System_Ole", "Win32_System_Variant", "Win32_System_TaskScheduler", "Win32_System_RemoteDesktop", "Win32_Security_Authentication_Identity", "Win32_UI_Shell", "Win32_UI_Shell_Common", "Win32_UI_WindowsAndMessaging", "Win32_UI_HiDpi", "Win32_Graphics_Gdi", "Win32_Graphics_Dwm", "Win32_Storage_FileSystem", "Win32_Storage_Xps", "Win32_System_LibraryLoader", "Win32_UI_Accessibility"] } # Named by the code that `windows::core::implement` writes. windows-core = "0.62.2" diff --git a/crates/botloftd/src/platform/desktop/mod.rs b/crates/botloftd/src/platform/desktop/mod.rs index 84b09f18..97d540e7 100644 --- a/crates/botloftd/src/platform/desktop/mod.rs +++ b/crates/botloftd/src/platform/desktop/mod.rs @@ -1,8 +1,10 @@ //! The owner's desktop (spec 24): the windows open on their screen, which -//! app each one is from, what is never granted, and a picture of a window. +//! app each one is from, what is never granted, a window read through its +//! accessibility tree, and a picture of a window. //! Only on Windows; elsewhere every call says the desktop is not there. mod blocked; +mod read; // Only Windows takes pictures; the tests run everywhere. #[cfg_attr(not(windows), allow(dead_code))] mod shrink; @@ -12,6 +14,7 @@ mod windows; use std::path::PathBuf; pub use blocked::{Never, never}; +pub use read::{Control, LINES_MAX, by_reference, reference, render}; /// The largest picture of a window the bot gets, in logical pixels (24.4). pub const PICTURE_MAX: (u32, u32) = (1600, 1200); @@ -77,6 +80,20 @@ pub fn windows() -> Result, DesktopError> { } } +/// The controls of window `id` through its accessibility tree, in the +/// order they sit in it, the ones shown on screen (spec 24.5). +pub fn read(id: u64) -> Result, DesktopError> { + #[cfg(windows)] + { + windows::read(id) + } + #[cfg(not(windows))] + { + let _ = id; + Err(DesktopError::Unavailable) + } +} + /// A picture of the window `id`, only of it, even behind other windows, in /// logical pixels up to [`PICTURE_MAX`] (spec 24.4). pub fn picture(id: u64) -> Result { diff --git a/crates/botloftd/src/platform/desktop/read.rs b/crates/botloftd/src/platform/desktop/read.rs new file mode 100644 index 00000000..003005db --- /dev/null +++ b/crates/botloftd/src/platform/desktop/read.rs @@ -0,0 +1,176 @@ +//! A window read as text (spec 24.5): one control a line, indented by its +//! place in the window, with a `ref` the bot names it by, its kind, name, +//! value and state. A password field never shows its value. + +/// The most lines one reading gives; `from` goes on from where it stopped. +pub const LINES_MAX: usize = 600; +/// The longest value shown, in characters. +const VALUE_MAX: usize = 200; + +/// A control of a window, as the accessibility tree has it. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Control { + /// How deep it sits in the window: 0 for the window's own children. + pub depth: usize, + /// Its kind, in words: "button", "edit", "check box". + pub kind: &'static str, + pub name: String, + /// What it holds (a field's text, a cell's value), when it has a value. + pub value: Option, + /// A password field: its value is never read. + pub password: bool, + pub enabled: bool, + /// For a check box or toggle button. + pub checked: Option, + /// For what opens and closes: a combo box, a tree item, a menu. + pub expanded: Option, + pub selected: bool, + /// Windows' id for it, to find it again to act on it (D2). + pub runtime_id: Vec, +} + +/// The `ref` of the control at `index` in a reading. +pub fn reference(index: usize) -> String { + format!("d{}", index + 1) +} + +/// The control a `ref` names in `controls`, if it is one. +pub fn by_reference<'a>(controls: &'a [Control], reference: &str) -> Option<&'a Control> { + let number: usize = reference.strip_prefix('d')?.parse().ok()?; + controls.get(number.checked_sub(1)?) +} + +fn clip(text: &str) -> String { + let line: String = text.replace(['\r', '\n'], " "); + if line.chars().count() <= VALUE_MAX { + return line; + } + let cut: String = line.chars().take(VALUE_MAX).collect(); + format!("{cut}…") +} + +fn line(index: usize, control: &Control) -> String { + let mut text = format!( + "{}[{}] {}", + " ".repeat(control.depth), + reference(index), + control.kind + ); + if !control.name.is_empty() { + text.push_str(&format!(" \"{}\"", clip(&control.name))); + } + if control.password { + text.push_str(" (password: the owner types it, you never read or type it)"); + } else if let Some(value) = &control.value { + text.push_str(&format!(" = \"{}\"", clip(value))); + } + let mut states = Vec::new(); + match control.checked { + Some(true) => states.push("checked"), + Some(false) => states.push("not checked"), + None => {} + } + match control.expanded { + Some(true) => states.push("open"), + Some(false) => states.push("closed"), + None => {} + } + if control.selected { + states.push("selected"); + } + if !control.enabled { + states.push("disabled"); + } + if !states.is_empty() { + text.push_str(&format!(" ({})", states.join(", "))); + } + text +} + +/// The window's controls as text, from line `from`, at most +/// [`LINES_MAX`] lines; and where the next reading goes on, when it stopped +/// short. +pub fn render(controls: &[Control], from: usize) -> (String, Option) { + let end = controls.len().min(from.saturating_add(LINES_MAX)); + let lines: Vec = controls + .iter() + .enumerate() + .skip(from) + .take(end.saturating_sub(from)) + .map(|(index, control)| line(index, control)) + .collect(); + let next = (end < controls.len()).then_some(end); + (lines.join("\n"), next) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn control(kind: &'static str, name: &str) -> Control { + Control { + kind, + name: name.to_owned(), + enabled: true, + ..Control::default() + } + } + + #[test] + fn each_control_is_a_line_with_its_ref_kind_name_value_and_state() { + let controls = vec![ + Control { + value: Some("Ana\nLima".to_owned()), + ..control("edit", "Name") + }, + Control { + depth: 1, + checked: Some(true), + ..control("check box", "Remember me") + }, + Control { + enabled: false, + ..control("button", "Save") + }, + ]; + let (text, next) = render(&controls, 0); + assert_eq!( + text, + "[d1] edit \"Name\" = \"Ana Lima\"\n [d2] check box \"Remember me\" (checked)\n\ + [d3] button \"Save\" (disabled)" + ); + assert_eq!(next, None); + assert_eq!( + by_reference(&controls, "d2").map(|found| found.kind), + Some("check box") + ); + assert!(by_reference(&controls, "d0").is_none()); + assert!(by_reference(&controls, "x2").is_none()); + } + + #[test] + fn a_password_field_never_shows_its_value() { + let field = Control { + password: true, + value: Some("hunter2".to_owned()), + ..control("edit", "Password") + }; + let (text, _) = render(&[field], 0); + assert!(!text.contains("hunter2"), "{text}"); + assert!(text.contains("the owner types it"), "{text}"); + } + + #[test] + fn a_long_window_goes_on_from_where_it_stopped() { + let controls: Vec = (0..LINES_MAX + 5) + .map(|_| control("list item", "Row")) + .collect(); + let (first, next) = render(&controls, 0); + assert_eq!(first.lines().count(), LINES_MAX); + assert_eq!(next, Some(LINES_MAX)); + let (rest, next) = render(&controls, LINES_MAX); + assert_eq!(rest.lines().count(), 5); + assert!(rest.starts_with(&format!("[d{}]", LINES_MAX + 1))); + assert_eq!(next, None); + } +} diff --git a/crates/botloftd/src/platform/desktop/windows/mod.rs b/crates/botloftd/src/platform/desktop/windows/mod.rs index 2d61ee97..03a90d28 100644 --- a/crates/botloftd/src/platform/desktop/windows/mod.rs +++ b/crates/botloftd/src/platform/desktop/windows/mod.rs @@ -6,6 +6,7 @@ mod capture; mod process; #[cfg(test)] mod tests; +mod uia; use std::collections::HashMap; use std::ffi::c_void; @@ -21,6 +22,7 @@ use windows::Win32::UI::WindowsAndMessaging::{ use windows::core::{BOOL, w}; pub use capture::capture; +pub use uia::read; use super::{App, DesktopError, Window}; diff --git a/crates/botloftd/src/platform/desktop/windows/tests.rs b/crates/botloftd/src/platform/desktop/windows/tests.rs index fe7255be..d9fefe98 100644 --- a/crates/botloftd/src/platform/desktop/windows/tests.rs +++ b/crates/botloftd/src/platform/desktop/windows/tests.rs @@ -10,13 +10,15 @@ use windows::Win32::Foundation::{COLORREF, HWND, LPARAM, LRESULT, WPARAM}; use windows::Win32::Graphics::Gdi::{CreateSolidBrush, HBRUSH}; use windows::Win32::System::LibraryLoader::GetModuleHandleW; use windows::Win32::UI::WindowsAndMessaging::{ - CW_USEDEFAULT, CreateWindowExW, DefWindowProcW, DispatchMessageW, GetMessageW, MSG, - PostMessageW, PostQuitMessage, RegisterClassW, SW_MINIMIZE, ShowWindow, TranslateMessage, - WINDOW_EX_STYLE, WM_CLOSE, WM_DESTROY, WNDCLASSW, WS_OVERLAPPEDWINDOW, WS_VISIBLE, + BM_SETCHECK, BS_AUTOCHECKBOX, BS_PUSHBUTTON, CW_USEDEFAULT, CreateWindowExW, DefWindowProcW, + DispatchMessageW, ES_PASSWORD, GetMessageW, HMENU, MSG, PostMessageW, PostQuitMessage, + RegisterClassW, SW_MINIMIZE, SendMessageW, ShowWindow, TranslateMessage, WINDOW_EX_STYLE, + WINDOW_STYLE, WM_CLOSE, WM_DESTROY, WNDCLASSW, WS_BORDER, WS_CHILD, WS_OVERLAPPEDWINDOW, + WS_VISIBLE, }; use windows::core::{PCWSTR, w}; -use super::super::{DesktopError, never, windows as listed}; +use super::super::{DesktopError, never, read, render, windows as listed}; use super::capture; const CLASS: PCWSTR = w!("BotloftDesktopTest"); @@ -36,6 +38,53 @@ unsafe extern "system" fn procedure( unsafe { DefWindowProcW(hwnd, message, wparam, lparam) } } +/// A button, a field with text, a password field and a ticked check box, +/// on the left of the window: its middle stays red. +/// +/// # Safety +/// +/// `parent` is a window of this thread. +unsafe fn controls(parent: HWND) { + let child = |class: PCWSTR, text: PCWSTR, style: u32, top: i32, width: i32| { + // SAFETY: a child of `parent`, on its thread. + unsafe { + CreateWindowExW( + WINDOW_EX_STYLE::default(), + class, + text, + WS_CHILD | WS_VISIBLE | WINDOW_STYLE(style), + 10, + top, + width, + 24, + Some(parent), + Some(HMENU(std::ptr::null_mut())), + None, + None, + ) + .expect("a control") + } + }; + child(w!("BUTTON"), w!("Save"), BS_PUSHBUTTON as u32, 10, 120); + child(w!("EDIT"), w!("hello"), WS_BORDER.0, 40, 160); + child( + w!("EDIT"), + w!("secret"), + WS_BORDER.0 | ES_PASSWORD as u32, + 70, + 160, + ); + let check = child( + w!("BUTTON"), + w!("Remember"), + BS_AUTOCHECKBOX as u32, + 100, + 160, + ); + // SAFETY: ticks our own check box. + unsafe { SendMessageW(check, BM_SETCHECK, Some(WPARAM(1)), None) }; +} + /// A red window of the test's own, on a thread with its message loop. struct TestWindow { id: u64, @@ -79,6 +128,7 @@ impl TestWindow { None, ) .expect("create the window"); + controls(hwnd); sent.send(hwnd.0 as usize as u64).expect("send"); let mut message = MSG::default(); while GetMessageW(&mut message, None, 0, 0).as_bool() { @@ -178,3 +228,25 @@ fn a_minimized_window_is_listed_but_has_nothing_to_see_and_a_closed_one_is_gone( .all(|found| found.id != id) ); } + +#[test] +fn a_window_reads_as_its_controls_and_never_shows_a_password() { + let window = TestWindow::open("Botloft desktop test: read"); + let controls = read(window.id).expect("read"); + let (text, next) = render(&controls, 0); + assert_eq!(next, None); + assert!(text.contains(r#"button "Save""#), "{text}"); + assert!(text.contains(r#"= "hello""#), "{text}"); + assert!(text.contains(r#"check box "Remember" (checked)"#), "{text}"); + assert!(text.contains("the owner types it"), "{text}"); + assert!(!text.contains("secret"), "{text}"); + let field = controls + .iter() + .find(|control| control.value.as_deref() == Some("hello")) + .expect("the field"); + assert!(!field.runtime_id.is_empty()); + // A window that closed has nothing to read. + let id = window.id; + drop(window); + assert!(read(id).is_err()); +} diff --git a/crates/botloftd/src/platform/desktop/windows/uia.rs b/crates/botloftd/src/platform/desktop/windows/uia.rs new file mode 100644 index 00000000..246465f0 --- /dev/null +++ b/crates/botloftd/src/platform/desktop/windows/uia.rs @@ -0,0 +1,279 @@ +//! A window's controls through UI Automation (spec 24.5), the interface +//! screen readers use: it works with the window behind others and never +//! moves the owner's cursor. One walk down the tree, a level at a time, +//! with the properties fetched in the same call. + +use windows::Win32::System::Com::{ + CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED, CoCreateInstance, CoInitializeEx, CoUninitialize, +}; +use windows::Win32::System::Variant::{VARIANT, VT_ARRAY, VT_BSTR, VT_I4}; +use windows::Win32::UI::Accessibility::{ + CUIAutomation, ExpandCollapseState_Collapsed, ExpandCollapseState_Expanded, + ExpandCollapseState_PartiallyExpanded, IUIAutomation, IUIAutomationCacheRequest, + IUIAutomationCondition, IUIAutomationElement, TreeScope_Children, UIA_CONTROLTYPE_ID, + UIA_ControlTypePropertyId, UIA_ExpandCollapseExpandCollapseStatePropertyId, + UIA_IsEnabledPropertyId, UIA_IsOffscreenPropertyId, UIA_IsPasswordPropertyId, + UIA_IsValuePatternAvailablePropertyId, UIA_NamePropertyId, UIA_PROPERTY_ID, + UIA_RuntimeIdPropertyId, UIA_SelectionItemIsSelectedPropertyId, + UIA_ToggleToggleStatePropertyId, UIA_ValueValuePropertyId, +}; + +use super::super::DesktopError; +use super::super::read::Control; +use super::handle; + +/// The most controls one walk gathers, and how deep it goes: a huge +/// window is cut short, not read for minutes. +const CONTROLS_MAX: usize = 3000; +const DEPTH_MAX: usize = 40; + +const PROPERTIES: [UIA_PROPERTY_ID; 11] = [ + UIA_NamePropertyId, + UIA_ControlTypePropertyId, + UIA_IsPasswordPropertyId, + UIA_IsEnabledPropertyId, + UIA_IsOffscreenPropertyId, + UIA_IsValuePatternAvailablePropertyId, + UIA_ValueValuePropertyId, + UIA_ToggleToggleStatePropertyId, + UIA_ExpandCollapseExpandCollapseStatePropertyId, + UIA_SelectionItemIsSelectedPropertyId, + UIA_RuntimeIdPropertyId, +]; + +/// COM on this thread while it lives, multithreaded, as UI Automation +/// clients should be. +struct Com { + started: bool, +} + +impl Com { + fn start() -> Self { + // SAFETY: balanced by `CoUninitialize` on drop when it started here. + let started = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) }.is_ok(); + Self { started } + } +} + +impl Drop for Com { + fn drop(&mut self) { + if self.started { + // SAFETY: this thread started COM in `start`. + unsafe { CoUninitialize() }; + } + } +} + +fn system(err: &windows::core::Error) -> DesktopError { + DesktopError::System(err.message()) +} + +/// Everything a walk needs, made once. +struct Walk { + cache: IUIAutomationCacheRequest, + shown: IUIAutomationCondition, + found: Vec, +} + +/// The controls of window `id`, in the order they sit in it. +pub fn read(id: u64) -> Result, DesktopError> { + let _com = Com::start(); + // SAFETY: COM calls on interfaces made here, on this thread. + unsafe { + let automation: IUIAutomation = + CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) + .map_err(|err| system(&err))?; + let cache = automation + .CreateCacheRequest() + .map_err(|err| system(&err))?; + for property in PROPERTIES { + cache.AddProperty(property).map_err(|err| system(&err))?; + } + // What a person sees: the control view, nothing scrolled away. + let offscreen = automation + .CreatePropertyCondition(UIA_IsOffscreenPropertyId, &VARIANT::from(true)) + .map_err(|err| system(&err))?; + let shown = automation + .CreateAndCondition( + &automation + .ControlViewCondition() + .map_err(|err| system(&err))?, + &automation + .CreateNotCondition(&offscreen) + .map_err(|err| system(&err))?, + ) + .map_err(|err| system(&err))?; + let root = automation + .ElementFromHandleBuildCache(handle(id), &cache) + .map_err(|_| DesktopError::Gone)?; + let mut walk = Walk { + cache, + shown, + found: Vec::new(), + }; + walk.children(&root, 0); + Ok(walk.found) + } +} + +impl Walk { + /// Adds the shown children of `element`, and theirs. + unsafe fn children(&mut self, element: &IUIAutomationElement, depth: usize) { + if depth > DEPTH_MAX || self.found.len() >= CONTROLS_MAX { + return; + } + // SAFETY: COM calls on live interfaces of this walk. + let Ok(children) = + (unsafe { element.FindAllBuildCache(TreeScope_Children, &self.shown, &self.cache) }) + else { + return; + }; + let count = unsafe { children.Length() }.unwrap_or(0); + for index in 0..count { + if self.found.len() >= CONTROLS_MAX { + return; + } + let Ok(child) = (unsafe { children.GetElement(index) }) else { + continue; + }; + let kind = kind_of(unsafe { child.CachedControlType() }.unwrap_or_default()); + let Some(kind) = kind else { + // Window furniture: scroll bars, the title bar. + continue; + }; + let control = unsafe { control(&child, depth, kind) }; + // Unnamed panes and groups only hold others: their children + // take their place. + let holder = matches!(kind, "pane" | "group" | "custom") && control.name.is_empty(); + let next = if holder { + depth + } else { + self.found.push(control); + depth + 1 + }; + unsafe { self.children(&child, next) }; + } + } +} + +unsafe fn property(element: &IUIAutomationElement, id: UIA_PROPERTY_ID) -> Option { + // SAFETY: a cached property of a live element. + unsafe { element.GetCachedPropertyValue(id) }.ok() +} + +unsafe fn control(element: &IUIAutomationElement, depth: usize, kind: &'static str) -> Control { + // SAFETY: cached properties of a live element. + unsafe { + let flag = |id| property(element, id).and_then(|value| bool::try_from(&value).ok()); + let number = |id| property(element, id).and_then(|value| i32::try_from(&value).ok()); + let password = flag(UIA_IsPasswordPropertyId).unwrap_or(false); + let value = if !password && flag(UIA_IsValuePatternAvailablePropertyId) == Some(true) { + property(element, UIA_ValueValuePropertyId).and_then(|value| text(&value)) + } else { + None + }; + let expanded = number(UIA_ExpandCollapseExpandCollapseStatePropertyId).and_then(|state| { + if state == ExpandCollapseState_Expanded.0 + || state == ExpandCollapseState_PartiallyExpanded.0 + { + Some(true) + } else if state == ExpandCollapseState_Collapsed.0 { + Some(false) + } else { + None + } + }); + Control { + depth, + kind, + name: element + .CachedName() + .map(|name| name.to_string()) + .unwrap_or_default(), + value, + password, + enabled: flag(UIA_IsEnabledPropertyId).unwrap_or(true), + // On, off; "indeterminate" says nothing. + checked: number(UIA_ToggleToggleStatePropertyId).and_then(|state| match state { + 0 => Some(false), + 1 => Some(true), + _ => None, + }), + expanded, + selected: flag(UIA_SelectionItemIsSelectedPropertyId).unwrap_or(false), + runtime_id: property(element, UIA_RuntimeIdPropertyId) + .map(|value| runtime_id(&value)) + .unwrap_or_default(), + } + } +} + +/// A string value. +fn text(value: &VARIANT) -> Option { + // SAFETY: the union is read as a string only when its type says so. + unsafe { + let inner = &value.Anonymous.Anonymous; + (inner.vt == VT_BSTR).then(|| inner.Anonymous.bstrVal.to_string()) + } +} + +/// The numbers of a runtime id: a one-dimension array of 32-bit integers. +fn runtime_id(value: &VARIANT) -> Vec { + // SAFETY: the union is read as an array only when its type says so, + // and the array's own bounds limit the read. + unsafe { + let inner = &value.Anonymous.Anonymous; + if inner.vt != (VT_ARRAY | VT_I4) { + return Vec::new(); + } + let array = inner.Anonymous.parray; + if array.is_null() || (*array).cDims != 1 || (*array).pvData.is_null() { + return Vec::new(); + } + let length = (*array).rgsabound[0].cElements as usize; + std::slice::from_raw_parts((*array).pvData.cast::(), length).to_vec() + } +} + +/// A control type in words; `None` for what is left out. +fn kind_of(kind: UIA_CONTROLTYPE_ID) -> Option<&'static str> { + Some(match kind.0 { + 50000 => "button", + 50001 => "calendar", + 50002 => "check box", + 50003 => "combo box", + 50004 => "edit", + 50005 => "link", + 50006 => "image", + 50007 => "list item", + 50008 => "list", + 50009 => "menu", + 50010 => "menu bar", + 50011 => "menu item", + 50012 => "progress bar", + 50013 => "radio button", + 50015 => "slider", + 50016 => "spinner", + 50017 => "status bar", + 50018 => "tab list", + 50019 => "tab", + 50020 => "text", + 50021 => "toolbar", + 50023 => "tree", + 50024 => "tree item", + 50026 => "group", + 50028 => "grid", + 50029 => "cell", + 50030 => "document", + 50031 => "split button", + 50032 => "window", + 50033 => "pane", + 50034 => "header", + 50035 => "header item", + 50036 => "table", + 50040 => "app bar", + // Scroll bars, tool tips, thumbs, title bars, separators. + 50014 | 50022 | 50027 | 50037 | 50038 => return None, + _ => "custom", + }) +} diff --git a/docs/spec.md b/docs/spec.md index 0f34af4a..2cd0212c 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1706,7 +1706,7 @@ Os programas são reconhecidos pelo nome do executável, sem diferença de maiú O bot lê e usa as janelas pela **UI Automation** do Windows (`IUIAutomation`, COM), a mesma que os leitores de tela usam. Ela funciona com a janela atrás de outras, sem mexer no cursor do dono. -- **Ler**: a árvore de controles da janela vira texto, como a leitura do navegador (21.6): um controle por linha, com uma `ref` (`d12`), o tipo (botão, campo, caixa de seleção, item de lista, célula, menu), o nome, o valor e o estado (marcado, desativado, expandido, selecionado). Só os controles visíveis na janela, até 600 linhas; `from` continua de onde parou. O valor de campo de senha nunca aparece. +- **Ler**: a árvore de controles da janela vira texto, como a leitura do navegador (21.6): um controle por linha, com uma `ref` (`d12`), o tipo (botão, campo, caixa de seleção, item de lista, célula, menu), o nome, o valor e o estado (marcado, desativado, expandido, selecionado). Só os controles visíveis na janela (a visão de controles da UI Automation, sem os que estão fora da tela), até 600 linhas; `from` continua de onde parou. Painéis e grupos sem nome só guardam outros controles: não ganham linha, e os de dentro sobem um nível. Barras de rolagem, de título, dicas e separadores ficam de fora. O valor de campo de senha nunca aparece; a linha diz que ali é com o dono. A leitura desce um nível por vez, com as propriedades vindo na mesma chamada (`FindAllBuildCache`), e para em 3000 controles ou 40 níveis: uma janela enorme é cortada, não lida por minutos. Cada controle guarda o `RuntimeId` da UI Automation, para achá-lo de novo na hora de agir (D2). - **Agir**, pelos padrões da UI Automation: - clicar: `Invoke`, senão `Toggle`, senão `SelectionItem.Select`, senão `ExpandCollapse`; - digitar: `Value.SetValue` (troca o texto; o bot lê o que ficou); From 4eb391508b9f733fcf585204f348b04da22a49eb Mon Sep 17 00:00:00 2001 From: Phelipe Lorran <112786693+httpsphl@users.noreply.github.com> Date: Sun, 4 Oct 2026 18:15:03 -0300 Subject: [PATCH 3/3] fix(desktop): name the executable by hand, so Windows paths read the same on every system --- crates/botloftd/src/platform/desktop/blocked.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/botloftd/src/platform/desktop/blocked.rs b/crates/botloftd/src/platform/desktop/blocked.rs index 656c5f96..269f9bcb 100644 --- a/crates/botloftd/src/platform/desktop/blocked.rs +++ b/crates/botloftd/src/platform/desktop/blocked.rs @@ -89,12 +89,13 @@ const DIALOG: &str = "#32770"; /// Why `window` is never granted, if it is not. pub fn never(window: &Window) -> Option { - let file = window - .app - .path - .file_name() - .map(|name| name.to_string_lossy().to_lowercase()) - .unwrap_or_default(); + // By hand: a Windows path read on another system has no separators. + let path = window.app.path.to_string_lossy().to_lowercase(); + let file = path + .rsplit(['\\', '/']) + .next() + .unwrap_or_default() + .to_owned(); let named = |list: &[&str]| list.contains(&file.as_str()); if named(BOTLOFT) { return Some(Never::Botloft);