From a9aa9101e3f38a61e38945789c1e2ccf8fece00e Mon Sep 17 00:00:00 2001 From: Phelipe Lorran <112786693+httpsphl@users.noreply.github.com> Date: Sun, 4 Oct 2026 22:31:55 -0300 Subject: [PATCH 1/2] feat(desktop): D6, the whole desktop, given only by the owner desktop.grantWhole gives a bot the whole desktop at a level, only with the risks accepted, and keeps when (spec 24.10). With it, every window shows with its title and is read without asking, and desktop_screenshot without a window is a picture of the whole screen: every monitor, copied with BitBlt, with every shown window of a process never granted covered in black, tool and untitled windows included. The daemon's outline and notice are excluded from capture. Away from the computer it needs the grant's "While you are away", and the app's notice on return says "Desktop". --- app/src/lib/protocol.gen.ts | 11 ++ crates/botloft-core/src/protocol/desktop.rs | 12 ++ .../src/protocol/export_desktop.rs | 6 + crates/botloft-core/src/protocol/mod.rs | 1 + crates/botloft-store/src/desktop_grants.rs | 60 +++++++++ crates/botloftd/src/platform/desktop/mod.rs | 2 + .../botloftd/src/platform/desktop/screen.rs | 72 +++++++++++ .../src/platform/desktop/windows/capture.rs | 16 +-- .../src/platform/desktop/windows/mod.rs | 29 ++++- .../src/platform/desktop/windows/notice.rs | 8 +- .../src/platform/desktop/windows/outline.rs | 8 +- .../src/platform/desktop/windows/screen.rs | 120 ++++++++++++++++++ crates/botloftd/src/rpc/dispatch.rs | 1 + crates/botloftd/src/service/desktop.rs | 21 ++- crates/botloftd/src/tools/desktop.rs | 25 ++-- crates/botloftd/src/tools/desktop_act.rs | 3 +- crates/botloftd/src/tools/desktop_catalog.rs | 12 +- crates/botloftd/src/tools/desktop_grant.rs | 17 ++- crates/botloftd/src/tools/desktop_screen.rs | 58 +++++++++ crates/botloftd/src/tools/mod.rs | 1 + crates/botloftd/tests/desktop_whole.rs | 71 +++++++++++ docs/spec.md | 9 +- 22 files changed, 515 insertions(+), 48 deletions(-) create mode 100644 crates/botloftd/src/platform/desktop/screen.rs create mode 100644 crates/botloftd/src/platform/desktop/windows/screen.rs create mode 100644 crates/botloftd/src/tools/desktop_screen.rs create mode 100644 crates/botloftd/tests/desktop_whole.rs diff --git a/app/src/lib/protocol.gen.ts b/app/src/lib/protocol.gen.ts index a1863f0a..f4393228 100644 --- a/app/src/lib/protocol.gen.ts +++ b/app/src/lib/protocol.gen.ts @@ -992,6 +992,16 @@ from: number, until: number, */ itemId: ChatItemId | null, }; +/** + * `desktop.grantWhole`: the owner gives the bot the whole desktop (spec + * 24.10), never asked for in the chat. + */ +export type DesktopWholeParams = { botId: BotId, level: DesktopLevel, +/** + * The owner checked "I understand the risks"; without it, refused. + */ +acceptedRisks: boolean, }; + export type ApprovalItem = { approvalId: ApprovalId, toolName: string, summary: string, /** * What the bot says the command is for, as in `ToolItem`. The bot @@ -1676,6 +1686,7 @@ export interface RpcMethods { "desktop.grants": { params: DesktopGrantsParams; result: Array }; "desktop.revoke": { params: DesktopGrantIdParams; result: BotDesktop }; "desktop.setOptions": { params: DesktopOptionsParams; result: BotDesktop }; + "desktop.grantWhole": { params: DesktopWholeParams; result: BotDesktop }; "desktop.watch": { params: DesktopBotParams; result: DesktopView }; "desktop.unwatch": { params: undefined; result: null }; "desktop.stop": { params: DesktopBotParams; result: DesktopState }; diff --git a/crates/botloft-core/src/protocol/desktop.rs b/crates/botloft-core/src/protocol/desktop.rs index 41fc4b15..9ce7052b 100644 --- a/crates/botloft-core/src/protocol/desktop.rs +++ b/crates/botloft-core/src/protocol/desktop.rs @@ -119,6 +119,18 @@ pub struct DesktopOptionsParams { pub accepted_risks: Option, } +/// `desktop.grantWhole`: the owner gives the bot the whole desktop (spec +/// 24.10), never asked for in the chat. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct DesktopWholeParams { + pub bot_id: BotId, + pub level: DesktopLevel, + /// The owner checked "I understand the risks"; without it, refused. + pub accepted_risks: bool, +} + /// A bot used an app while the owner was away (spec 24.8): what the app /// tells them when they are back, until they dismiss it. The result of /// `desktop.awayUses` and the params of `desktop.away` are a list of these. diff --git a/crates/botloft-core/src/protocol/export_desktop.rs b/crates/botloft-core/src/protocol/export_desktop.rs index be663315..6931dc4b 100644 --- a/crates/botloft-core/src/protocol/export_desktop.rs +++ b/crates/botloft-core/src/protocol/export_desktop.rs @@ -21,6 +21,7 @@ pub(super) fn decls(out: &mut Out) { out.decl::(); out.decl::(); out.decl::(); + out.decl::(); } pub(super) fn methods(out: &mut Out) { @@ -39,6 +40,11 @@ pub(super) fn methods(out: &mut Out) { &out.name::(), &out.name::(), ); + out.method( + method::DESKTOP_GRANT_WHOLE, + &out.name::(), + &out.name::(), + ); let bot = out.name::(); out.method(method::DESKTOP_WATCH, &bot, &out.name::()); out.method(method::DESKTOP_UNWATCH, "undefined", "null"); diff --git a/crates/botloft-core/src/protocol/mod.rs b/crates/botloft-core/src/protocol/mod.rs index 4e5fd62a..7d3c4022 100644 --- a/crates/botloft-core/src/protocol/mod.rs +++ b/crates/botloft-core/src/protocol/mod.rs @@ -127,6 +127,7 @@ pub mod method { pub const DESKTOP_GRANTS: &str = "desktop.grants"; pub const DESKTOP_REVOKE: &str = "desktop.revoke"; pub const DESKTOP_SET_OPTIONS: &str = "desktop.setOptions"; + pub const DESKTOP_GRANT_WHOLE: &str = "desktop.grantWhole"; pub const DESKTOP_WATCH: &str = "desktop.watch"; pub const DESKTOP_UNWATCH: &str = "desktop.unwatch"; pub const DESKTOP_STOP: &str = "desktop.stop"; diff --git a/crates/botloft-store/src/desktop_grants.rs b/crates/botloft-store/src/desktop_grants.rs index 2899a502..248b79c2 100644 --- a/crates/botloft-store/src/desktop_grants.rs +++ b/crates/botloft-store/src/desktop_grants.rs @@ -69,6 +69,36 @@ impl Store { )?) } + /// Gives `bot` the whole desktop at `level` (spec 24.2), the owner + /// having accepted the risks at `now` (spec 24.10); a grant of it + /// already there takes the new level and keeps its options. + pub fn grant_desktop_whole( + &self, + bot: &BotId, + level: DesktopLevel, + now: i64, + ) -> Result { + self.conn.execute( + "INSERT INTO desktop_grants (id, bot_id, scope, level, accepted_risks_at, created_at) \ + VALUES (?1, ?2, 'desktop', ?3, ?4, ?4) \ + ON CONFLICT (bot_id, scope, lower(coalesce(app_path, ''))) DO UPDATE SET \ + level = excluded.level, accepted_risks_at = excluded.accepted_risks_at", + params![ + DesktopGrantId::generate().as_str(), + bot.as_str(), + level.as_str(), + now, + ], + )?; + Ok(self.conn.query_row( + &format!( + "SELECT {COLUMNS} FROM desktop_grants WHERE bot_id = ?1 AND scope = 'desktop'" + ), + [bot.as_str()], + from_row, + )?) + } + /// The grants of `bot`, oldest first. pub fn desktop_grants(&self, bot: &BotId) -> Result> { let mut statement = self.conn.prepare_cached(&format!( @@ -235,6 +265,36 @@ mod tests { assert_eq!(off.accepted_risks_at, Some(50)); } + #[test] + fn the_whole_desktop_is_one_grant_whose_level_the_owner_sets() { + let fx = Fixture::new(); + let scout = &fx.bots[0].id; + let see = fx + .store + .grant_desktop_whole(scout, DesktopLevel::Act, 10) + .expect("whole"); + assert_eq!(see.scope, DesktopScope::Desktop); + assert_eq!((see.app_path.clone(), see.app_name.clone()), (None, None)); + assert_eq!(see.accepted_risks_at, Some(10)); + fx.store + .set_desktop_real_input(&see.id, true) + .expect("real"); + // Given again, it can go down and keeps its options. + let again = fx + .store + .grant_desktop_whole(scout, DesktopLevel::See, 20) + .expect("again"); + assert_eq!(again.id, see.id); + assert_eq!(again.level, DesktopLevel::See); + assert!(again.real_input); + assert_eq!(again.accepted_risks_at, Some(20)); + // An app grant is another one. + fx.store + .grant_desktop_app(scout, EXCEL, "Microsoft Excel", DesktopLevel::See, 30) + .expect("app"); + assert_eq!(fx.store.desktop_grants(scout).expect("grants").len(), 2); + } + #[test] fn a_grant_can_be_taken_away() { let fx = Fixture::new(); diff --git a/crates/botloftd/src/platform/desktop/mod.rs b/crates/botloftd/src/platform/desktop/mod.rs index 7cd6c548..92db33a0 100644 --- a/crates/botloftd/src/platform/desktop/mod.rs +++ b/crates/botloftd/src/platform/desktop/mod.rs @@ -7,6 +7,7 @@ mod blocked; pub mod keys; mod overlay; mod read; +mod screen; // Only Windows takes pictures; the tests run everywhere. #[cfg_attr(not(windows), allow(dead_code))] mod shrink; @@ -18,6 +19,7 @@ use std::path::PathBuf; pub use blocked::{Never, never}; pub use overlay::{ScreenCursor, notice_hide, notice_show, outline_hide, outline_show}; pub use read::{Control, LINES_MAX, by_reference, reference, render}; +pub use screen::{cover, screen_picture}; /// The largest picture of a window the bot gets, in logical pixels (24.4). pub const PICTURE_MAX: (u32, u32) = (1600, 1200); diff --git a/crates/botloftd/src/platform/desktop/screen.rs b/crates/botloftd/src/platform/desktop/screen.rs new file mode 100644 index 00000000..df6bfc24 --- /dev/null +++ b/crates/botloftd/src/platform/desktop/screen.rs @@ -0,0 +1,72 @@ +//! A picture of the owner's whole screen for a bot with the whole desktop +//! (spec 24.4, D6): every window as the owner sees it, the ones that are +//! never granted (spec 24.3) covered in black. + +use super::{DesktopError, Picture}; + +/// Paints each of `covered` (left, top, width and height on the screen) +/// black in `rgb`, a picture `width` × `height` of the screen from +/// `origin`; what falls outside the picture is left out. +pub fn cover(rgb: &mut [u8], width: u32, height: u32, origin: (i32, i32), covered: &[[i32; 4]]) { + let (width, height) = (i64::from(width), i64::from(height)); + for &[left, top, across, down] in covered { + let from_x = (i64::from(left) - i64::from(origin.0)).clamp(0, width); + let to_x = (i64::from(left) + i64::from(across) - i64::from(origin.0)).clamp(0, width); + let from_y = (i64::from(top) - i64::from(origin.1)).clamp(0, height); + let to_y = (i64::from(top) + i64::from(down) - i64::from(origin.1)).clamp(0, height); + for row in from_y..to_y { + let start = ((row * width + from_x) * 3) as usize; + let end = ((row * width + to_x) * 3) as usize; + rgb[start..end].fill(0); + } + } +} + +/// The whole screen, with every window never granted covered, in logical +/// pixels up to [`super::PICTURE_MAX`]. +pub fn screen_picture() -> Result { + #[cfg(windows)] + { + use super::{QUALITY, never, shrink, windows}; + let covered: Vec<[i32; 4]> = windows::every_shown()? + .iter() + .filter(|window| !window.minimized && never(window).is_some()) + .filter_map(|window| windows::frame(window.id).ok()) + .collect(); + let (rgb, width, height, scale) = windows::capture_screen(&covered)?; + Ok(shrink::picture( + &rgb, + width, + height, + scale, + super::PICTURE_MAX, + QUALITY, + )) + } + #[cfg(not(windows))] + { + Err(DesktopError::Unavailable) + } +} + +#[cfg(test)] +mod tests { + use super::cover; + + #[test] + fn covered_windows_turn_black_and_only_inside_the_picture() { + // A 4 × 3 screen from (-2, 10), all white. + let mut rgb = vec![255u8; 4 * 3 * 3]; + // A window at (-1, 11), 2 × 5: columns 1 and 2, rows 1 and 2. + cover(&mut rgb, 4, 3, (-2, 10), &[[-1, 11, 2, 5]]); + let black: Vec<(usize, usize)> = (0..3) + .flat_map(|row| (0..4).map(move |column| (row, column))) + .filter(|&(row, column)| rgb[(row * 4 + column) * 3] == 0) + .collect(); + assert_eq!(black, vec![(1, 1), (1, 2), (2, 1), (2, 2)]); + // Off the screen, nothing. + let mut rgb = vec![255u8; 4 * 3 * 3]; + cover(&mut rgb, 4, 3, (0, 0), &[[10, 10, 5, 5], [-9, -9, 2, 2]]); + assert!(rgb.iter().all(|&byte| byte == 255)); + } +} diff --git a/crates/botloftd/src/platform/desktop/windows/capture.rs b/crates/botloftd/src/platform/desktop/windows/capture.rs index ce880a47..e620ed26 100644 --- a/crates/botloftd/src/platform/desktop/windows/capture.rs +++ b/crates/botloftd/src/platform/desktop/windows/capture.rs @@ -28,10 +28,10 @@ const RENDER_FULL_CONTENT: PRINT_WINDOW_FLAGS = PRINT_WINDOW_FLAGS(2); const BASE_DPI: f64 = 96.0; /// Sizes in real pixels on this thread while it lives, then back. -struct RealPixels(DPI_AWARENESS_CONTEXT); +pub(super) struct RealPixels(DPI_AWARENESS_CONTEXT); impl RealPixels { - fn new() -> Self { + pub(super) fn new() -> Self { // SAFETY: changes only this thread's DPI awareness, restored on drop. Self(unsafe { SetThreadDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2) }) } @@ -47,11 +47,11 @@ impl Drop for RealPixels { } /// Frees the drawing objects in the right order when it goes. -struct Canvas { - screen: HDC, - memory: HDC, - bitmap: HBITMAP, - previous: HGDIOBJ, +pub(super) struct Canvas { + pub(super) screen: HDC, + pub(super) memory: HDC, + pub(super) bitmap: HBITMAP, + pub(super) previous: HGDIOBJ, } impl Drop for Canvas { @@ -66,7 +66,7 @@ impl Drop for Canvas { } } -fn system(what: &str) -> DesktopError { +pub(super) fn system(what: &str) -> DesktopError { DesktopError::System(what.to_owned()) } diff --git a/crates/botloftd/src/platform/desktop/windows/mod.rs b/crates/botloftd/src/platform/desktop/windows/mod.rs index b740f640..210cb156 100644 --- a/crates/botloftd/src/platform/desktop/windows/mod.rs +++ b/crates/botloftd/src/platform/desktop/windows/mod.rs @@ -17,6 +17,7 @@ mod process; mod real; #[cfg(test)] mod real_tests; +mod screen; mod stop_key; #[cfg(test)] mod test_window; @@ -44,6 +45,7 @@ pub use input::owner_idle; pub use notice::{hide as notice_hide, show as notice_show}; pub use outline::{hide as outline_hide, show as outline_show}; pub use real::{click as real_click, press as real_press, type_text as real_type}; +pub use screen::capture_screen; pub use stop_key::on_stop_key; pub use uia::read; @@ -54,6 +56,17 @@ use super::{App, DesktopError, Window}; const FRAME_HOST: &str = "applicationframehost.exe"; pub fn list() -> Result, DesktopError> { + gather(false) +} + +/// Every window shown on the screen, of other processes: with tool windows +/// and windows without a title too, to cover the ones never granted in a +/// picture of the whole screen (spec 24.4). +pub fn every_shown() -> Result, DesktopError> { + gather(true) +} + +fn gather(all: bool) -> Result, DesktopError> { let mut handles: Vec = Vec::new(); // SAFETY: the callback only pushes into `handles`, which outlives the // call. @@ -67,11 +80,15 @@ pub fn list() -> Result, DesktopError> { let mut apps: HashMap = HashMap::new(); let mut found = Vec::new(); for hwnd in handles { - if !shown(hwnd) { + if !shown(hwnd, all) { continue; } let title = text(hwnd); - if title.trim().is_empty() { + if !all && title.trim().is_empty() { + continue; + } + // The daemon's own outline and notice stay out of every picture. + if all && process_id(hwnd) == std::process::id() { continue; } let class = class_of(hwnd); @@ -103,15 +120,15 @@ unsafe extern "system" fn collect(hwnd: HWND, found: LPARAM) -> BOOL { } /// A window the owner can see on the taskbar or the screen: shown, not a -/// tool window, and not cloaked (Store apps kept hidden, other virtual -/// desktops). -fn shown(hwnd: HWND) -> bool { +/// tool window (unless `all`), and not cloaked (Store apps kept hidden, +/// other virtual desktops). +fn shown(hwnd: HWND, all: bool) -> bool { // SAFETY: plain queries on a window handle. unsafe { if !IsWindowVisible(hwnd).as_bool() { return false; } - if GetWindowLongPtrW(hwnd, GWL_EXSTYLE) & WS_EX_TOOLWINDOW.0 as isize != 0 { + if !all && GetWindowLongPtrW(hwnd, GWL_EXSTYLE) & WS_EX_TOOLWINDOW.0 as isize != 0 { return false; } let mut cloaked = 0u32; diff --git a/crates/botloftd/src/platform/desktop/windows/notice.rs b/crates/botloftd/src/platform/desktop/windows/notice.rs index 2559cd68..34643aec 100644 --- a/crates/botloftd/src/platform/desktop/windows/notice.rs +++ b/crates/botloftd/src/platform/desktop/windows/notice.rs @@ -22,9 +22,9 @@ use windows::Win32::UI::HiDpi::{ use windows::Win32::UI::WindowsAndMessaging::{ CreateWindowExW, DefWindowProcW, DispatchMessageW, GetMessageW, HWND_TOPMOST, LWA_COLORKEY, MSG, PostMessageW, RegisterClassW, SW_HIDE, SW_SHOWNOACTIVATE, SWP_NOACTIVATE, - SetLayeredWindowAttributes, SetWindowPos, ShowWindow, TranslateMessage, WM_APP, WM_PAINT, - WNDCLASSW, WS_EX_LAYERED, WS_EX_NOACTIVATE, WS_EX_TOOLWINDOW, WS_EX_TOPMOST, WS_EX_TRANSPARENT, - WS_POPUP, + SetLayeredWindowAttributes, SetWindowDisplayAffinity, SetWindowPos, ShowWindow, + TranslateMessage, WDA_EXCLUDEFROMCAPTURE, WM_APP, WM_PAINT, WNDCLASSW, WS_EX_LAYERED, + WS_EX_NOACTIVATE, WS_EX_TOOLWINDOW, WS_EX_TOPMOST, WS_EX_TRANSPARENT, WS_POPUP, }; use windows::core::{PCWSTR, w}; @@ -103,6 +103,8 @@ fn run(made: &Sender>) { return; }; let _ = SetLayeredWindowAttributes(hwnd, SEE_THROUGH, 255, LWA_COLORKEY); + // Never in a picture of the whole screen (spec 24.4). + let _ = SetWindowDisplayAffinity(hwnd, WDA_EXCLUDEFROMCAPTURE); let _ = made.send(Some(hwnd.0 as isize)); let mut message = MSG::default(); while GetMessageW(&mut message, None, 0, 0).as_bool() { diff --git a/crates/botloftd/src/platform/desktop/windows/outline.rs b/crates/botloftd/src/platform/desktop/windows/outline.rs index 77047df8..ff89984a 100644 --- a/crates/botloftd/src/platform/desktop/windows/outline.rs +++ b/crates/botloftd/src/platform/desktop/windows/outline.rs @@ -22,9 +22,9 @@ use windows::Win32::UI::WindowsAndMessaging::{ CreateWindowExW, DefWindowProcW, DispatchMessageW, GW_HWNDPREV, GetClientRect, GetMessageW, GetWindow, HWND_TOP, IsIconic, IsWindow, IsWindowVisible, KillTimer, LWA_COLORKEY, MSG, PostMessageW, RegisterClassW, SW_HIDE, SW_SHOWNOACTIVATE, SWP_NOACTIVATE, - SetLayeredWindowAttributes, SetTimer, SetWindowPos, ShowWindow, TranslateMessage, WM_APP, - WM_PAINT, WM_TIMER, WNDCLASSW, WS_EX_LAYERED, WS_EX_NOACTIVATE, WS_EX_TOOLWINDOW, - WS_EX_TRANSPARENT, WS_POPUP, + SetLayeredWindowAttributes, SetTimer, SetWindowDisplayAffinity, SetWindowPos, ShowWindow, + TranslateMessage, WDA_EXCLUDEFROMCAPTURE, WM_APP, WM_PAINT, WM_TIMER, WNDCLASSW, WS_EX_LAYERED, + WS_EX_NOACTIVATE, WS_EX_TOOLWINDOW, WS_EX_TRANSPARENT, WS_POPUP, }; use windows::core::{PCWSTR, w}; @@ -110,6 +110,8 @@ fn run(made: &Sender>) { return; }; let _ = SetLayeredWindowAttributes(hwnd, SEE_THROUGH, 255, LWA_COLORKEY); + // Never in a picture of the whole screen (spec 24.4). + let _ = SetWindowDisplayAffinity(hwnd, WDA_EXCLUDEFROMCAPTURE); let _ = made.send(Some(hwnd.0 as isize)); let mut message = MSG::default(); while GetMessageW(&mut message, None, 0, 0).as_bool() { diff --git a/crates/botloftd/src/platform/desktop/windows/screen.rs b/crates/botloftd/src/platform/desktop/windows/screen.rs new file mode 100644 index 00000000..bf4438e7 --- /dev/null +++ b/crates/botloftd/src/platform/desktop/windows/screen.rs @@ -0,0 +1,120 @@ +//! A picture of the owner's whole screen (spec 24.4): every monitor, as the +//! owner sees it, with the windows that are never granted covered in black. +//! The daemon's own outline and notice are kept out of it (they are +//! excluded from capture). + +use std::ffi::c_void; + +use windows::Win32::Graphics::Gdi::{ + BI_RGB, BITMAPINFO, BITMAPINFOHEADER, BitBlt, CAPTUREBLT, CreateCompatibleBitmap, + CreateCompatibleDC, DIB_RGB_COLORS, GetDC, GetDIBits, ReleaseDC, SRCCOPY, SelectObject, +}; +use windows::Win32::UI::HiDpi::GetDpiForSystem; +use windows::Win32::UI::WindowsAndMessaging::{ + GetSystemMetrics, SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN, SM_XVIRTUALSCREEN, SM_YVIRTUALSCREEN, +}; + +use super::super::{DesktopError, cover}; +use super::capture::{Canvas, RealPixels, system}; + +/// Windows' unscaled DPI. +const BASE_DPI: f64 = 96.0; + +/// The whole screen's pixels as RGB, its width and height, and the display +/// scale, with each of `covered` (left, top, width and height, in real +/// pixels on the screen) painted black. +pub fn capture_screen(covered: &[[i32; 4]]) -> Result<(Vec, u32, u32, f64), DesktopError> { + let _real = RealPixels::new(); + // SAFETY: plain queries. + let (left, top, width, height) = unsafe { + ( + GetSystemMetrics(SM_XVIRTUALSCREEN), + GetSystemMetrics(SM_YVIRTUALSCREEN), + GetSystemMetrics(SM_CXVIRTUALSCREEN), + GetSystemMetrics(SM_CYVIRTUALSCREEN), + ) + }; + if width <= 0 || height <= 0 { + return Err(DesktopError::Locked); + } + // SAFETY: each object made here is freed by `Canvas`. + let canvas = unsafe { + let screen = GetDC(None); + let memory = CreateCompatibleDC(Some(screen)); + let bitmap = CreateCompatibleBitmap(screen, width, height); + if memory.is_invalid() || bitmap.is_invalid() { + ReleaseDC(None, screen); + return Err(system("no memory for the picture")); + } + let previous = SelectObject(memory, bitmap.into()); + Canvas { + screen, + memory, + bitmap, + previous, + } + }; + // SAFETY: copies the screen into our bitmap of the same size. A locked + // screen or the secure desktop refuses. + let copied = unsafe { + BitBlt( + canvas.memory, + 0, + 0, + width, + height, + Some(canvas.screen), + left, + top, + SRCCOPY | CAPTUREBLT, + ) + }; + if copied.is_err() { + return Err(DesktopError::Locked); + } + let mut info = BITMAPINFO { + bmiHeader: BITMAPINFOHEADER { + biSize: size_of::() as u32, + biWidth: width, + // Negative: rows from the top down. + biHeight: -height, + biPlanes: 1, + biBitCount: 32, + biCompression: BI_RGB.0, + ..Default::default() + }, + ..Default::default() + }; + let mut bgra = vec![0u8; width as usize * height as usize * 4]; + // SAFETY: the buffer holds the whole bitmap at 32 bits a pixel. + let rows = unsafe { + GetDIBits( + canvas.memory, + canvas.bitmap, + 0, + height as u32, + Some(bgra.as_mut_ptr().cast::()), + &mut info, + DIB_RGB_COLORS, + ) + }; + drop(canvas); + if rows == 0 { + return Err(system("the picture could not be read")); + } + let (pixels, _) = bgra.as_chunks::<4>(); + let mut rgb = Vec::with_capacity(pixels.len() * 3); + for [blue, green, red, _] in pixels { + rgb.extend([*red, *green, *blue]); + } + let (width, height) = (width as u32, height as u32); + cover(&mut rgb, width, height, (left, top), covered); + // SAFETY: a plain query. + let dpi = unsafe { GetDpiForSystem() }; + let scale = if dpi == 0 { + 1.0 + } else { + f64::from(dpi) / BASE_DPI + }; + Ok((rgb, width, height, scale)) +} diff --git a/crates/botloftd/src/rpc/dispatch.rs b/crates/botloftd/src/rpc/dispatch.rs index 9a076731..a9733b11 100644 --- a/crates/botloftd/src/rpc/dispatch.rs +++ b/crates/botloftd/src/rpc/dispatch.rs @@ -58,6 +58,7 @@ pub fn dispatch(daemon: &Daemon, name: &str, params: Option) -> Result reply(desktop::revoke(daemon, parse(params)?)), method::DESKTOP_STOP => reply(desktop::stop(daemon, parse(params)?)), method::DESKTOP_SET_OPTIONS => reply(desktop::set_options(daemon, parse(params)?)), + method::DESKTOP_GRANT_WHOLE => reply(desktop::grant_whole(daemon, parse(params)?)), method::DESKTOP_AWAY_USES => reply(desktop::away_uses(daemon)), method::DESKTOP_DISMISS_AWAY => reply(desktop::dismiss_away(daemon)), method::DESKTOP_RESUME => reply(desktop::resume(daemon, parse(params)?)), diff --git a/crates/botloftd/src/service/desktop.rs b/crates/botloftd/src/service/desktop.rs index 4781d776..b6aaefc4 100644 --- a/crates/botloftd/src/service/desktop.rs +++ b/crates/botloftd/src/service/desktop.rs @@ -7,7 +7,7 @@ use botloft_core::ids::BotId; use botloft_core::protocol::{ BotDesktop, DesktopAwayUse, DesktopBotParams, DesktopGrant, DesktopGrantIdParams, - DesktopGrantsParams, DesktopOptionsParams, DesktopState, + DesktopGrantsParams, DesktopOptionsParams, DesktopState, DesktopWholeParams, }; use tracing::{info, warn}; @@ -46,6 +46,25 @@ pub fn set_options(daemon: &Daemon, params: DesktopOptionsParams) -> ApiResult ApiResult { + if !params.accepted_risks { + return Err(ApiError::Conflict( + "the whole desktop needs the risks accepted first".to_owned(), + )); + } + super::bots::find(&daemon.store(), ¶ms.bot_id)?; + daemon + .store() + .grant_desktop_whole(¶ms.bot_id, params.level, daemon.clock.now_ms())?; + info!( + level = params.level.as_str(), + "the owner gave a bot the whole desktop" + ); + changed(daemon, ¶ms.bot_id) +} + /// What bots did while the owner was away, for the app to tell them. pub fn away_uses(daemon: &Daemon) -> ApiResult> { Ok(daemon.desktop.away.list()) diff --git a/crates/botloftd/src/tools/desktop.rs b/crates/botloftd/src/tools/desktop.rs index f08c50a9..593b2b6a 100644 --- a/crates/botloftd/src/tools/desktop.rs +++ b/crates/botloftd/src/tools/desktop.rs @@ -8,19 +8,19 @@ use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; use botloft_core::ids::BotId; -use botloft_core::protocol::{DesktopAction, DesktopLevel}; +use botloft_core::protocol::DesktopLevel; use serde_json::{Value, json}; use tracing::debug; use super::calls::explain; use super::desktop_act::{Ask, act, answered}; -use super::desktop_grant::{blocking, granted, heading}; +use super::desktop_grant::{blocking, granted, heading, used}; use super::desktop_list::{in_reach, list}; use super::desktop_real::{click_at, press}; +use super::desktop_screen::whole; use crate::desktop::STOPPED; -use crate::platform::desktop::Window; use crate::platform::desktop::{self as platform, Action, Scroll, render}; -use crate::state::{Daemon, Event}; +use crate::state::Daemon; pub(super) const PREFIX: &str = "desktop_"; @@ -31,8 +31,9 @@ enum Tool { from: usize, why: String, }, + /// A window's picture, or the whole screen's without one (D6). Screenshot { - window: u64, + window: Option, why: String, }, Act(Ask), @@ -90,7 +91,7 @@ impl Tool { why: why()?, }, Some("screenshot") => Self::Screenshot { - window: window()?, + window: arguments["window"].as_u64(), why: why()?, }, Some("click") => acting(Action::Click)?, @@ -215,6 +216,11 @@ async fn run(daemon: &Daemon, bot: &BotId, generation: u64, tool: Tool) -> Resul Ok(Reply::Text(reading)) } Tool::Screenshot { window, why } => { + let Some(window) = window else { + return whole(daemon, bot, &grants) + .await + .map(|(data, text)| Reply::Picture(data, text)); + }; let window = granted( daemon, bot, @@ -287,10 +293,3 @@ async fn run(daemon: &Daemon, bot: &BotId, generation: u64, tool: Tool) -> Resul .map(Reply::Text), } } - -/// The bot read or acted in `window`: its panel shows it (spec 24.9). -pub(super) fn used(daemon: &Daemon, bot: &BotId, window: &Window, action: Option) { - let now = daemon.clock.now_ms(); - let state = daemon.desktop.activity.used(bot, window, action, now); - daemon.emit(Event::DesktopChanged(state)); -} diff --git a/crates/botloftd/src/tools/desktop_act.rs b/crates/botloftd/src/tools/desktop_act.rs index 0cc23efd..6b2b6c28 100644 --- a/crates/botloftd/src/tools/desktop_act.rs +++ b/crates/botloftd/src/tools/desktop_act.rs @@ -10,8 +10,7 @@ use std::time::Duration; use botloft_core::ids::BotId; use botloft_core::protocol::{DesktopAction, DesktopActionKind, DesktopGrant, DesktopLevel}; -use super::desktop::used; -use super::desktop_grant::{blocking, describe, granted, heading}; +use super::desktop_grant::{blocking, describe, granted, heading, used}; use super::desktop_real::{enter, instead, needs_real, real_allowed}; use crate::desktop::STOPPED; use crate::platform::desktop::{ diff --git a/crates/botloftd/src/tools/desktop_catalog.rs b/crates/botloftd/src/tools/desktop_catalog.rs index 7400dfa1..0e21a97c 100644 --- a/crates/botloftd/src/tools/desktop_catalog.rs +++ b/crates/botloftd/src/tools/desktop_catalog.rs @@ -81,15 +81,21 @@ pub(super) fn tools() -> Vec { "Shows you a picture of one window on the owner's computer, only that window, \ even behind others: for what its text does not tell, like a chart or a \ layout. Prefer desktop_look, which costs you less. A minimized window has \ - nothing to show. {ASKS} {UNTRUSTED}" + nothing to show. Without window, the owner's whole screen, only if they gave \ + you their whole desktop, with the windows you may never use covered. {ASKS} \ + {UNTRUSTED}" ), "inputSchema": { "type": "object", "properties": { - "window": window(), + "window": { + "type": "integer", + "description": "The window's number from desktop_windows; leave it \ + out for the owner's whole screen.", + }, "why": why(), }, - "required": ["window", "why"], + "required": ["why"], "additionalProperties": false, }, "annotations": { "readOnlyHint": true }, diff --git a/crates/botloftd/src/tools/desktop_grant.rs b/crates/botloftd/src/tools/desktop_grant.rs index 2fe42ea1..3c68ea5f 100644 --- a/crates/botloftd/src/tools/desktop_grant.rs +++ b/crates/botloftd/src/tools/desktop_grant.rs @@ -4,7 +4,7 @@ use botloft_core::chat::DESKTOP_TOOL; use botloft_core::ids::BotId; -use botloft_core::protocol::{DesktopGrant, DesktopLevel}; +use botloft_core::protocol::{DesktopAction, DesktopGrant, DesktopLevel}; use serde_json::json; use super::calls::explain; @@ -81,9 +81,9 @@ fn verb(level: DesktopLevel) -> &'static str { } } -/// The bot used `window` with the owner away: the app tells them when they +/// The bot used `app` with the owner away: the app tells them when they /// are back, with the chat where it began. -fn used_away(daemon: &Daemon, bot: &BotId, window: &Window) { +pub(super) fn used_away(daemon: &Daemon, bot: &BotId, app: &str) { let item = daemon .store() .chat_history(bot, None, 1) @@ -91,7 +91,7 @@ fn used_away(daemon: &Daemon, bot: &BotId, window: &Window) { .and_then(|items| items.into_iter().next()) .map(|item| item.id); let now = daemon.clock.now_ms(); - let uses = daemon.desktop.away.record(bot, &window.app.name, now, item); + let uses = daemon.desktop.away.record(bot, app, now, item); daemon.emit(Event::DesktopAway(uses)); } @@ -125,7 +125,7 @@ pub(super) async fn granted( if !covering(grants, &window, level).is_some_and(|grant| grant.unattended) { return Err(away.why().to_owned()); } - used_away(daemon, bot, &window); + used_away(daemon, bot, &window.app.name); return Ok(window); } if covering(grants, &window, level).is_some() { @@ -171,3 +171,10 @@ pub(super) async fn granted( None => Err("Botloft could not ask the owner.".to_owned()), } } + +/// The bot read or acted in `window`: its panel shows it (spec 24.9). +pub(super) fn used(daemon: &Daemon, bot: &BotId, window: &Window, action: Option) { + let now = daemon.clock.now_ms(); + let state = daemon.desktop.activity.used(bot, window, action, now); + daemon.emit(Event::DesktopChanged(state)); +} diff --git a/crates/botloftd/src/tools/desktop_screen.rs b/crates/botloftd/src/tools/desktop_screen.rs new file mode 100644 index 00000000..4553993b --- /dev/null +++ b/crates/botloftd/src/tools/desktop_screen.rs @@ -0,0 +1,58 @@ +//! The picture of the owner's whole screen (spec 24.4, D6): only for a bot +//! the owner gave the whole desktop, which is never asked for in the chat +//! (spec 24.2), with the windows never granted covered in black. + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use botloft_core::ids::BotId; +use botloft_core::protocol::{DesktopGrant, DesktopScope}; + +use super::desktop_grant::{blocking, used_away}; +use crate::desktop::STOPPED; +use crate::platform::desktop as platform; +use crate::state::Daemon; + +/// What the owner reads when a bot used the whole screen while away. +const WHOLE: &str = "Desktop"; + +/// A JPEG of the whole screen, base64, and a line about it. +pub(super) async fn whole( + daemon: &Daemon, + bot: &BotId, + grants: &[DesktopGrant], +) -> Result<(String, String), String> { + let Some(grant) = grants + .iter() + .find(|grant| grant.scope == DesktopScope::Desktop) + else { + return Err( + "The owner has not given you their whole desktop, so pass window: the \ + number of one window from desktop_windows. Only the owner gives the whole desktop, \ + in your details in Botloft." + .to_owned(), + ); + }; + if let Err(away) = daemon.desktop.owner_here() { + if !grant.unattended { + return Err(away.why().to_owned()); + } + used_away(daemon, bot, WHOLE); + } + let _turn = daemon.desktop.turn().await; + // The owner may have stopped it while it waited. + if daemon.desktop.activity.stopped(bot) { + return Err(STOPPED.to_owned()); + } + let picture = blocking(platform::screen_picture).await?; + if picture.jpeg.is_empty() { + return Err("The picture of the screen came out empty.".to_owned()); + } + Ok(( + BASE64.encode(&picture.jpeg), + format!( + "The owner's whole screen, {} by {}. Windows you may never use are covered in \ + black. To read or use a window, call desktop_windows and pass its number.", + picture.width, picture.height + ), + )) +} diff --git a/crates/botloftd/src/tools/mod.rs b/crates/botloftd/src/tools/mod.rs index 920a6d56..eddef66d 100644 --- a/crates/botloftd/src/tools/mod.rs +++ b/crates/botloftd/src/tools/mod.rs @@ -20,6 +20,7 @@ mod desktop_catalog; mod desktop_grant; mod desktop_list; mod desktop_real; +mod desktop_screen; mod era; mod question; mod routine; diff --git a/crates/botloftd/tests/desktop_whole.rs b/crates/botloftd/tests/desktop_whole.rs new file mode 100644 index 00000000..0e616fe7 --- /dev/null +++ b/crates/botloftd/tests/desktop_whole.rs @@ -0,0 +1,71 @@ +//! A bot with the owner's whole desktop (spec 24.2, 24.4, D6): only the +//! owner gives it, after accepting the risks; then every window shows with +//! its title, a window is read without asking, and the whole screen comes +//! as one picture. Windows only. +#![cfg(windows)] + +mod common; + +use common::desktop::{TITLE, TestWindow, reading, setup}; +use serde_json::json; + +#[tokio::test(flavor = "multi_thread")] +async fn the_owner_gives_the_whole_desktop_and_the_bot_sees_the_screen() { + let window = TestWindow::open(); + let mut s = setup().await; + + // Without it, no whole screen, and a window's title stays hidden. + let refused = s + .mcp + .tool_text("desktop_screenshot", json!({ "why": "To see the screen" })) + .await; + assert!( + reading(&refused).contains("not given you their whole desktop"), + "{}", + reading(&refused) + ); + let listed = s + .mcp + .tool_text("desktop_windows", json!({})) + .await + .expect("windows"); + assert!(!listed.contains(TITLE), "{listed}"); + + // Only with the risks accepted. + let unaccepted = json!({ "botId": s.bot["id"], "level": "see", "acceptedRisks": false }); + assert!(s.app.call("desktop.grantWhole", unaccepted).await.is_err()); + let accepted = json!({ "botId": s.bot["id"], "level": "see", "acceptedRisks": true }); + let given = s + .app + .call("desktop.grantWhole", accepted) + .await + .expect("given"); + assert_eq!(given["grants"][0]["scope"], "desktop"); + assert!(given["grants"][0]["acceptedRisksAt"].is_i64()); + let changed = s.app.notification("bot.desktop").await; + assert_eq!(changed["grants"][0]["scope"], "desktop"); + + // Every window with its title; one is read without asking. + let listed = s + .mcp + .tool_text("desktop_windows", json!({})) + .await + .expect("windows"); + assert!(listed.contains(TITLE), "{listed}"); + let look = json!({ "window": window.id, "why": "To read the visit" }); + let text = s.mcp.tool_text("desktop_look", look).await.expect("read"); + assert!(text.contains(r#"button "Save""#), "{text}"); + + // And the whole screen as one picture. + let picture = s + .mcp + .tool_result("desktop_screenshot", json!({ "why": "To see the screen" })) + .await; + assert_eq!(picture["content"][0]["type"], "image", "{picture}"); + assert!( + picture["content"][1]["text"] + .as_str() + .is_some_and(|text| text.contains("whole screen")), + "{picture}" + ); +} diff --git a/docs/spec.md b/docs/spec.md index 3471e603..4c28d336 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1656,7 +1656,7 @@ Tabela `questions`: `id, crew_id, bot_id, chat_item_id, text, options (JSON), st ## 24. Desktop -Status: **D1 a D5 implementados** (24.12). É o item 10 da seção 18. Decisões e riscos em `docs/adr/0002-desktop-use.md`. +Status: **D1 a D6 implementados** (24.12). É o item 10 da seção 18. Decisões e riscos em `docs/adr/0002-desktop-use.md`. ### 24.1 O que é @@ -1672,7 +1672,7 @@ Cada permissão é de um bot. Ela tem um **alcance**, um **nível** e duas opç - **Alcance**, escolhido pelo dono: - **Um app**: as janelas de um programa, identificado pelo executável (caminho completo e nome do produto da versão do arquivo, por exemplo `C:\Program Files\Microsoft Office\root\Office16\EXCEL.EXE`, "Microsoft Excel"). Um executável com o mesmo nome em outra pasta é outro app. - - **O desktop inteiro**: todas as janelas de todos os apps, menos as que nunca são liberadas (24.3). + - **O desktop inteiro**: todas as janelas de todos os apps, menos as que nunca são liberadas (24.3). Uma permissão só (`scope: desktop`, sem executável), no nível que o dono escolhe, e que ele pode baixar de Ver e usar para Ver; as opções (mouse e teclado, sem você na frente) ficam. Com ela, o bot vê o título de toda janela, lê e fotografa qualquer uma sem perguntar e, no nível Ver e usar, age em qualquer uma; no nível Ver, usar um app ainda pede por ele no chat. - **Nível**, separado (o dono libera um, depois o outro): - **Ver**: o bot lista as janelas do alcance, lê o que há nelas (a árvore de acessibilidade, 24.5) e tira fotos delas. - **Mexer**: além de ver, clica, digita, escolhe e rola por acessibilidade (24.5). Pede "Ver" antes. @@ -1702,7 +1702,7 @@ Os programas são reconhecidos pelo nome do executável, sem diferença de maiú - **Janelas**: as de nível superior, visíveis ou minimizadas, com dono (não ferramentas nem janelas sem título). De um app fora do alcance, o bot vê só o nome do app, nunca o título da janela: um título diz o assunto de um e-mail ou o nome de um arquivo. - **Foto de uma janela**: `PrintWindow` com `PW_RENDERFULLCONTENT`, que pega a janela mesmo atrás de outras, só a janela, sem o que estiver por cima dela, recortada na moldura que o DWM diz (`DWMWA_EXTENDED_FRAME_BOUNDS`), sem a borda invisível de redimensionar que o Windows deixa em volta. A medida é em pixels reais (o thread fica com DPI por monitor enquanto fotografa) e a foto é reduzida pela escala da tela da janela (`GetDpiForWindow`). Uma janela minimizada não tem o que fotografar: a tool diz isso. -- **Foto do desktop inteiro** (só com esse alcance): a tela toda, com as janelas que nunca são liberadas (24.3) cobertas de preto. +- **Foto do desktop inteiro** (só com esse alcance): a tela toda, com as janelas que nunca são liberadas (24.3) cobertas de preto. Todos os monitores (a tela virtual, `SM_XVIRTUALSCREEN`...), copiados da tela com `BitBlt` (`SRCCOPY | CAPTUREBLT`), como o dono vê. As janelas cobertas são todas as mostradas na tela de um processo que nunca é liberado, inclusive as sem título e as de ferramenta (uma janelinha de um gerenciador de senhas também conta), cobertas pela moldura do DWM, mesmo com outra janela por cima (o preto sobra, nunca falta). O contorno e o aviso do próprio daemon ficam fora de qualquer foto (`WDA_EXCLUDEFROMCAPTURE`). Reduzida como a de uma janela, pela escala do sistema (`GetDpiForSystem`), até 1600 × 1200. Com a tela bloqueada, a cópia falha e a tool diz isso. O painel do bot (24.9) segue a última janela, não a tela inteira. - **Tamanho**: a foto vai ao bot no tamanho da janela em pixels lógicos (sem a escala do Windows), até 1600 × 1200, reduzida mantendo a proporção. JPEG, qualidade 70, como `browser_screenshot`. ### 24.5 Acessibilidade (UI Automation) @@ -1730,7 +1730,7 @@ No servidor `botloft` (11), como as do navegador. |---|---|---| | `desktop_windows` | | as janelas abertas: as do alcance com título e `window` (id), as outras só pelo nome do app | | `desktop_look` | `window`, `why`, `from?` | lê a janela (24.5) | -| `desktop_screenshot` | `window`, `why` | a foto da janela; a do desktop inteiro vem com esse alcance (D6) | +| `desktop_screenshot` | `window?`, `why` | a foto da janela; sem `window`, a da tela inteira, só com o desktop inteiro (sem ele, a tool diz que só o dono o dá); com o dono fora, só se essa permissão tiver "Sem você na frente", e o aviso na volta diz "Desktop" | | `desktop_click` | `ref`, `why?` | clica no controle (24.5) | | `desktop_type` | `ref`, `text`, `submit?`, `why?` | troca o texto do campo; com `submit`, aperta Enter depois, com o teclado de verdade (só onde ele está ligado) | | `desktop_select` | `ref`, `option`, `why?` | escolhe uma opção pelo texto | @@ -1778,6 +1778,7 @@ No servidor `botloft` (11), como as do navegador. ### 24.10 Permissões no app - Nos detalhes do bot ("Sobre "), a seção **Seu desktop** lista as permissões: o app (ou "O desktop inteiro"), o caminho do executável, Pode ver ou Pode ver e usar, e as opções. Cada uma com um X que tira a permissão; o bot pede de novo na próxima vez. Sem nenhuma, a seção diz que o bot pede no chat na primeira vez que precisar. +- **Dar o desktop inteiro**: um botão na seção, que abre a tela de riscos abaixo com a escolha do nível (Ver, ou Ver e usar). Gravado, ele aparece na lista como "O desktop inteiro", com o X e as opções de qualquer permissão. O app chama `desktop.grantWhole {botId, level, acceptedRisks}`, que devolve as permissões do bot (`BotDesktop`) e manda `bot.desktop`; sem `acceptedRisks: true` é `conflict`. - **Dar o desktop inteiro** e **ligar "Sem você na frente"** abrem uma tela de riscos, em palavras simples, antes de gravar: - o bot vê tudo o que estiver aberto no alcance, inclusive dados pessoais, e-mails, conversas e dados de clientes; - ele pode errar: apagar, enviar ou mudar algo no app como se fosse você; From 34c579d0345ac8ccfdb2d1b58bccc85b0c2a06f8 Mon Sep 17 00:00:00 2001 From: Phelipe Lorran <112786693+httpsphl@users.noreply.github.com> Date: Sun, 4 Oct 2026 22:35:59 -0300 Subject: [PATCH 2/2] feat(desktop): give the whole desktop from the bot's details "Give the whole desktop" in the bot's "Your desktop" section opens the risks screen with the level to give (see, or see and use); the confirm button lights only once the owner checks that they understand the risks. The grant then shows as "The whole desktop" with its options. The fake daemon answers desktop.grantWhole, which the app check needs. --- app/src/features/desktop/DesktopGrants.tsx | 6 ++ app/src/features/desktop/WholeDesktop.tsx | 85 ++++++++++++++++++++++ app/src/features/desktop/whole.test.tsx | 60 +++++++++++++++ app/src/i18n/en/desktop.ts | 12 +++ app/src/i18n/es/desktop.ts | 12 +++ app/src/i18n/pt-BR/desktop.ts | 12 +++ app/src/lib/fakeDesktop.ts | 28 +++++++ 7 files changed, 215 insertions(+) create mode 100644 app/src/features/desktop/WholeDesktop.tsx create mode 100644 app/src/features/desktop/whole.test.tsx diff --git a/app/src/features/desktop/DesktopGrants.tsx b/app/src/features/desktop/DesktopGrants.tsx index d966a064..6a7f0726 100644 --- a/app/src/features/desktop/DesktopGrants.tsx +++ b/app/src/features/desktop/DesktopGrants.tsx @@ -10,6 +10,7 @@ import { Button } from "../../ui/Button"; import { attempt } from "../../ui/toast"; import { RealInput } from "./RealInput"; import { Unattended } from "./Unattended"; +import { WholeDesktop } from "./WholeDesktop"; /** The bot's grants, read when shown and kept current by `bot.desktop`. */ export function useDesktopGrants(botId: Bot["id"]): DesktopGrant[] | null { @@ -89,6 +90,11 @@ export function DesktopGrants({ bot }: { bot: Bot }) { })} )} + {grants !== null && !grants.some((grant) => grant.scope === "desktop") && ( +
+ +
+ )} ); diff --git a/app/src/features/desktop/WholeDesktop.tsx b/app/src/features/desktop/WholeDesktop.tsx new file mode 100644 index 00000000..edd20e10 --- /dev/null +++ b/app/src/features/desktop/WholeDesktop.tsx @@ -0,0 +1,85 @@ +// Giving a bot the owner's whole desktop (spec 24.2, 24.10): a button in +// the bot's details that opens the risks screen, with the level to give; +// the confirm button lights only once the owner checks that they +// understand the risks. Never asked for in the chat. + +import { Check, Monitor } from "lucide-react"; +import { useState } from "react"; +import { useT } from "../../i18n"; +import type { Bot, DesktopLevel } from "../../lib/protocol.gen"; +import { useApi } from "../../store/context"; +import { Button } from "../../ui/Button"; +import { Choices } from "../../ui/Choices"; +import { Dialog } from "../../ui/Dialog"; +import { attempt } from "../../ui/toast"; + +export function WholeDesktop({ bot }: { bot: Bot }) { + const words = useT().desktop.grants; + const api = useApi(); + const [asking, setAsking] = useState(false); + const [level, setLevel] = useState("see"); + const [understood, setUnderstood] = useState(false); + const close = () => { + setAsking(false); + setUnderstood(false); + setLevel("see"); + }; + const give = () => + attempt(words.wholeFailed, () => + api.call("desktop.grantWhole", { botId: bot.id, level, acceptedRisks: true }), + ); + return ( + <> + + {asking && ( + + + + + } + > + + label={words.wholeLevel} + value={level} + options={[ + { value: "see", label: words.see }, + { value: "act", label: words.act }, + ]} + onChange={setLevel} + /> +
    + {words.wholeRisks(bot.name).map((risk) => ( +
  • {risk}
    • + ))} +
+ + + )} + + ); +} diff --git a/app/src/features/desktop/whole.test.tsx b/app/src/features/desktop/whole.test.tsx new file mode 100644 index 00000000..55cae956 --- /dev/null +++ b/app/src/features/desktop/whole.test.tsx @@ -0,0 +1,60 @@ +import { act, cleanup, fireEvent, screen, waitFor, within } from "@testing-library/react"; +import { afterEach, describe, expect, test } from "vitest"; +import { setLocaleChoice } from "../../i18n"; +import { FakeBotloft } from "../../lib/fake"; +import { crewOpened, openBot, renderApp } from "../../test/app"; + +afterEach(() => { + act(() => setLocaleChoice("en")); + cleanup(); +}); + +async function details() { + const fake = new FakeBotloft(); + const ops = fake.addCrew("Ops"); + const scout = fake.addBot(ops.id, "Scout", "Finds sources"); + fake.setBotState(scout.id, "idle"); + renderApp(fake); + await crewOpened("Ops"); + openBot("Scout"); + await screen.findByRole("list", { name: "Messages" }); + fireEvent.click(screen.getByRole("button", { name: /^Show details/ })); + const about = screen.getByRole("complementary", { name: "About Scout" }); + return { fake, scout, about }; +} + +describe("the whole desktop", () => { + test("is given only after the owner chooses a level and accepts the risks", async () => { + const { fake, about } = await details(); + fireEvent.click(await within(about).findByRole("button", { name: "Give the whole desktop" })); + const dialog = screen.getByRole("dialog", { name: "Give Scout your whole desktop?" }); + expect(within(dialog).getByText(/without asking you first/)).toBeDefined(); + expect(within(dialog).getByText(/covered in black/)).toBeDefined(); + const confirm = within(dialog).getByRole("button", { name: "Give it" }) as HTMLButtonElement; + expect(confirm.disabled).toBe(true); + + fireEvent.click(within(dialog).getByRole("radio", { name: "Can see and use" })); + fireEvent.click(within(dialog).getByRole("checkbox", { name: "I understand the risks" })); + expect(confirm.disabled).toBe(false); + fireEvent.click(confirm); + await waitFor(() => expect(fake.desktop.grants[0]?.scope).toBe("desktop")); + expect(fake.desktop.grants[0]?.level).toBe("act"); + expect(fake.desktop.grants[0]?.acceptedRisksAt).not.toBeNull(); + + // Listed like any grant, with its options; the button goes. + expect(await within(about).findByText("The whole desktop")).toBeDefined(); + expect( + within(about).getByRole("switch", { name: "Real mouse and keyboard: The whole desktop" }), + ).toBeDefined(); + expect(within(about).queryByRole("button", { name: "Give the whole desktop" })).toBeNull(); + }, 15_000); + + test("speaks the owner's language", async () => { + const { about } = await details(); + act(() => setLocaleChoice("pt-BR")); + fireEvent.click(await within(about).findByRole("button", { name: "Dar o desktop inteiro" })); + expect( + screen.getByRole("dialog", { name: "Dar a Scout o seu desktop inteiro?" }), + ).toBeDefined(); + }); +}); diff --git a/app/src/i18n/en/desktop.ts b/app/src/i18n/en/desktop.ts index 8cafa537..fb36ee0d 100644 --- a/app/src/i18n/en/desktop.ts +++ b/app/src/i18n/en/desktop.ts @@ -79,6 +79,18 @@ export const desktop = { ], awayUnderstood: "I understand the risks", awayConfirm: "Turn on", + giveWhole: "Give the whole desktop", + wholeFailed: "Could not give the whole desktop", + wholeTitle: (bot: string) => `Give ${bot} your whole desktop?`, + wholeLevel: "What it may do", + wholeRisks: (bot: string) => [ + `${bot} sees every window open on your computer, personal data, emails, chats and client data included, and pictures of the whole screen.`, + `It reads any app without asking you first. If you choose "Can see and use", it also clicks and types in any of them.`, + `${bot} can make mistakes: delete, send or change something as if it were you.`, + `Text in an app or a message can try to trick ${bot} into doing something else.`, + "Never, not even then: Botloft itself, apps running as administrator, terminals and command boxes, the Task Manager, password managers and password fields. They are covered in black in its pictures.", + ], + wholeConfirm: "Give it", }, away: { label: "While you were away", diff --git a/app/src/i18n/es/desktop.ts b/app/src/i18n/es/desktop.ts index 2725640e..6d952ee8 100644 --- a/app/src/i18n/es/desktop.ts +++ b/app/src/i18n/es/desktop.ts @@ -79,6 +79,18 @@ export const desktop: Messages["desktop"] = { ], awayUnderstood: "Entiendo los riesgos", awayConfirm: "Activar", + giveWhole: "Dar todo el escritorio", + wholeFailed: "No se pudo dar todo el escritorio", + wholeTitle: (bot) => `¿Dar a ${bot} todo tu escritorio?`, + wholeLevel: "Lo que puede hacer", + wholeRisks: (bot) => [ + `${bot} ve toda ventana abierta en tu computadora, incluidos datos personales, correos, conversaciones y datos de clientes, y fotos de toda la pantalla.`, + `Lee cualquier app sin preguntarte antes. Si eliges "Puede ver y usar", también hace clic y escribe en cualquiera.`, + `${bot} puede equivocarse: borrar, enviar o cambiar algo como si fueras tú.`, + `Un texto en una app o en un mensaje puede intentar engañar a ${bot} para que haga otra cosa.`, + "Nunca, ni así: el propio Botloft, apps que se ejecutan como administrador, terminales y ventanas de comandos, el Administrador de tareas, gestores de contraseñas y campos de contraseña. En sus fotos quedan cubiertos de negro.", + ], + wholeConfirm: "Dar", }, away: { label: "Mientras no estabas", diff --git a/app/src/i18n/pt-BR/desktop.ts b/app/src/i18n/pt-BR/desktop.ts index e7fccead..ed782212 100644 --- a/app/src/i18n/pt-BR/desktop.ts +++ b/app/src/i18n/pt-BR/desktop.ts @@ -78,6 +78,18 @@ export const desktop: Messages["desktop"] = { ], awayUnderstood: "Entendi os riscos", awayConfirm: "Ligar", + giveWhole: "Dar o desktop inteiro", + wholeFailed: "Não foi possível dar o desktop inteiro", + wholeTitle: (bot) => `Dar a ${bot} o seu desktop inteiro?`, + wholeLevel: "O que ele pode fazer", + wholeRisks: (bot) => [ + `${bot} vê toda janela aberta no seu computador, inclusive dados pessoais, e-mails, conversas e dados de clientes, e fotos da tela inteira.`, + `Ele lê qualquer app sem te perguntar antes. Escolhendo "Pode ver e usar", também clica e digita em qualquer um.`, + `${bot} pode errar: apagar, enviar ou mudar algo como se fosse você.`, + `Um texto num app ou numa mensagem pode tentar enganar ${bot} para fazer outra coisa.`, + "Nunca, nem assim: o próprio Botloft, apps rodando como administrador, terminais e caixas de comando, o Gerenciador de Tarefas, gerenciadores de senha e campos de senha. Nas fotos dele, eles ficam cobertos de preto.", + ], + wholeConfirm: "Dar", }, away: { label: "Enquanto você estava fora", diff --git a/app/src/lib/fakeDesktop.ts b/app/src/lib/fakeDesktop.ts index 9a71714e..04616faa 100644 --- a/app/src/lib/fakeDesktop.ts +++ b/app/src/lib/fakeDesktop.ts @@ -154,6 +154,34 @@ export class FakeDesktop { this.fake.bot(botId); return this.set(botId, { stopped: false }); }, + "desktop.grantWhole": ({ botId, level, acceptedRisks }) => { + this.fake.bot(botId); + if (!acceptedRisks) { + throw new Error("the whole desktop needs the risks accepted first"); + } + const same = this.grants.find( + (grant) => grant.botId === botId && grant.scope === "desktop", + ); + if (same) { + same.level = level; + same.acceptedRisksAt = this.fake.now; + } else { + this.grants.push({ + id: this.fake.id("dsk"), + botId, + scope: "desktop", + appPath: null, + appName: null, + level, + realInput: false, + unattended: false, + acceptedRisksAt: this.fake.now, + createdAt: this.fake.now, + }); + } + this.changed(botId); + return { botId, grants: this.of(botId) }; + }, "desktop.awayUses": () => this.away, "desktop.dismissAway": () => { this.away = [];