diff --git a/app/src/features/desktop/DesktopGrants.tsx b/app/src/features/desktop/DesktopGrants.tsx index d966a064..6a7f0726 100644 --- a/app/src/features/desktop/DesktopGrants.tsx +++ b/app/src/features/desktop/DesktopGrants.tsx @@ -10,6 +10,7 @@ import { Button } from "../../ui/Button"; import { attempt } from "../../ui/toast"; import { RealInput } from "./RealInput"; import { Unattended } from "./Unattended"; +import { WholeDesktop } from "./WholeDesktop"; /** The bot's grants, read when shown and kept current by `bot.desktop`. */ export function useDesktopGrants(botId: Bot["id"]): DesktopGrant[] | null { @@ -89,6 +90,11 @@ export function DesktopGrants({ bot }: { bot: Bot }) { })} )} + {grants !== null && !grants.some((grant) => grant.scope === "desktop") && ( +
+ +
+ )} ); diff --git a/app/src/features/desktop/WholeDesktop.tsx b/app/src/features/desktop/WholeDesktop.tsx new file mode 100644 index 00000000..edd20e10 --- /dev/null +++ b/app/src/features/desktop/WholeDesktop.tsx @@ -0,0 +1,85 @@ +// Giving a bot the owner's whole desktop (spec 24.2, 24.10): a button in +// the bot's details that opens the risks screen, with the level to give; +// the confirm button lights only once the owner checks that they +// understand the risks. Never asked for in the chat. + +import { Check, Monitor } from "lucide-react"; +import { useState } from "react"; +import { useT } from "../../i18n"; +import type { Bot, DesktopLevel } from "../../lib/protocol.gen"; +import { useApi } from "../../store/context"; +import { Button } from "../../ui/Button"; +import { Choices } from "../../ui/Choices"; +import { Dialog } from "../../ui/Dialog"; +import { attempt } from "../../ui/toast"; + +export function WholeDesktop({ bot }: { bot: Bot }) { + const words = useT().desktop.grants; + const api = useApi(); + const [asking, setAsking] = useState(false); + const [level, setLevel] = useState("see"); + const [understood, setUnderstood] = useState(false); + const close = () => { + setAsking(false); + setUnderstood(false); + setLevel("see"); + }; + const give = () => + attempt(words.wholeFailed, () => + api.call("desktop.grantWhole", { botId: bot.id, level, acceptedRisks: true }), + ); + return ( + <> + + {asking && ( + + + + + } + > + + label={words.wholeLevel} + value={level} + options={[ + { value: "see", label: words.see }, + { value: "act", label: words.act }, + ]} + onChange={setLevel} + /> +
    + {words.wholeRisks(bot.name).map((risk) => ( +
  • {risk}
    • + ))} +
+ + + )} + + ); +} diff --git a/app/src/features/desktop/whole.test.tsx b/app/src/features/desktop/whole.test.tsx new file mode 100644 index 00000000..55cae956 --- /dev/null +++ b/app/src/features/desktop/whole.test.tsx @@ -0,0 +1,60 @@ +import { act, cleanup, fireEvent, screen, waitFor, within } from "@testing-library/react"; +import { afterEach, describe, expect, test } from "vitest"; +import { setLocaleChoice } from "../../i18n"; +import { FakeBotloft } from "../../lib/fake"; +import { crewOpened, openBot, renderApp } from "../../test/app"; + +afterEach(() => { + act(() => setLocaleChoice("en")); + cleanup(); +}); + +async function details() { + const fake = new FakeBotloft(); + const ops = fake.addCrew("Ops"); + const scout = fake.addBot(ops.id, "Scout", "Finds sources"); + fake.setBotState(scout.id, "idle"); + renderApp(fake); + await crewOpened("Ops"); + openBot("Scout"); + await screen.findByRole("list", { name: "Messages" }); + fireEvent.click(screen.getByRole("button", { name: /^Show details/ })); + const about = screen.getByRole("complementary", { name: "About Scout" }); + return { fake, scout, about }; +} + +describe("the whole desktop", () => { + test("is given only after the owner chooses a level and accepts the risks", async () => { + const { fake, about } = await details(); + fireEvent.click(await within(about).findByRole("button", { name: "Give the whole desktop" })); + const dialog = screen.getByRole("dialog", { name: "Give Scout your whole desktop?" }); + expect(within(dialog).getByText(/without asking you first/)).toBeDefined(); + expect(within(dialog).getByText(/covered in black/)).toBeDefined(); + const confirm = within(dialog).getByRole("button", { name: "Give it" }) as HTMLButtonElement; + expect(confirm.disabled).toBe(true); + + fireEvent.click(within(dialog).getByRole("radio", { name: "Can see and use" })); + fireEvent.click(within(dialog).getByRole("checkbox", { name: "I understand the risks" })); + expect(confirm.disabled).toBe(false); + fireEvent.click(confirm); + await waitFor(() => expect(fake.desktop.grants[0]?.scope).toBe("desktop")); + expect(fake.desktop.grants[0]?.level).toBe("act"); + expect(fake.desktop.grants[0]?.acceptedRisksAt).not.toBeNull(); + + // Listed like any grant, with its options; the button goes. + expect(await within(about).findByText("The whole desktop")).toBeDefined(); + expect( + within(about).getByRole("switch", { name: "Real mouse and keyboard: The whole desktop" }), + ).toBeDefined(); + expect(within(about).queryByRole("button", { name: "Give the whole desktop" })).toBeNull(); + }, 15_000); + + test("speaks the owner's language", async () => { + const { about } = await details(); + act(() => setLocaleChoice("pt-BR")); + fireEvent.click(await within(about).findByRole("button", { name: "Dar o desktop inteiro" })); + expect( + screen.getByRole("dialog", { name: "Dar a Scout o seu desktop inteiro?" }), + ).toBeDefined(); + }); +}); diff --git a/app/src/i18n/en/desktop.ts b/app/src/i18n/en/desktop.ts index 8cafa537..fb36ee0d 100644 --- a/app/src/i18n/en/desktop.ts +++ b/app/src/i18n/en/desktop.ts @@ -79,6 +79,18 @@ export const desktop = { ], awayUnderstood: "I understand the risks", awayConfirm: "Turn on", + giveWhole: "Give the whole desktop", + wholeFailed: "Could not give the whole desktop", + wholeTitle: (bot: string) => `Give ${bot} your whole desktop?`, + wholeLevel: "What it may do", + wholeRisks: (bot: string) => [ + `${bot} sees every window open on your computer, personal data, emails, chats and client data included, and pictures of the whole screen.`, + `It reads any app without asking you first. If you choose "Can see and use", it also clicks and types in any of them.`, + `${bot} can make mistakes: delete, send or change something as if it were you.`, + `Text in an app or a message can try to trick ${bot} into doing something else.`, + "Never, not even then: Botloft itself, apps running as administrator, terminals and command boxes, the Task Manager, password managers and password fields. They are covered in black in its pictures.", + ], + wholeConfirm: "Give it", }, away: { label: "While you were away", diff --git a/app/src/i18n/es/desktop.ts b/app/src/i18n/es/desktop.ts index 2725640e..6d952ee8 100644 --- a/app/src/i18n/es/desktop.ts +++ b/app/src/i18n/es/desktop.ts @@ -79,6 +79,18 @@ export const desktop: Messages["desktop"] = { ], awayUnderstood: "Entiendo los riesgos", awayConfirm: "Activar", + giveWhole: "Dar todo el escritorio", + wholeFailed: "No se pudo dar todo el escritorio", + wholeTitle: (bot) => `¿Dar a ${bot} todo tu escritorio?`, + wholeLevel: "Lo que puede hacer", + wholeRisks: (bot) => [ + `${bot} ve toda ventana abierta en tu computadora, incluidos datos personales, correos, conversaciones y datos de clientes, y fotos de toda la pantalla.`, + `Lee cualquier app sin preguntarte antes. Si eliges "Puede ver y usar", también hace clic y escribe en cualquiera.`, + `${bot} puede equivocarse: borrar, enviar o cambiar algo como si fueras tú.`, + `Un texto en una app o en un mensaje puede intentar engañar a ${bot} para que haga otra cosa.`, + "Nunca, ni así: el propio Botloft, apps que se ejecutan como administrador, terminales y ventanas de comandos, el Administrador de tareas, gestores de contraseñas y campos de contraseña. En sus fotos quedan cubiertos de negro.", + ], + wholeConfirm: "Dar", }, away: { label: "Mientras no estabas", diff --git a/app/src/i18n/pt-BR/desktop.ts b/app/src/i18n/pt-BR/desktop.ts index e7fccead..ed782212 100644 --- a/app/src/i18n/pt-BR/desktop.ts +++ b/app/src/i18n/pt-BR/desktop.ts @@ -78,6 +78,18 @@ export const desktop: Messages["desktop"] = { ], awayUnderstood: "Entendi os riscos", awayConfirm: "Ligar", + giveWhole: "Dar o desktop inteiro", + wholeFailed: "Não foi possível dar o desktop inteiro", + wholeTitle: (bot) => `Dar a ${bot} o seu desktop inteiro?`, + wholeLevel: "O que ele pode fazer", + wholeRisks: (bot) => [ + `${bot} vê toda janela aberta no seu computador, inclusive dados pessoais, e-mails, conversas e dados de clientes, e fotos da tela inteira.`, + `Ele lê qualquer app sem te perguntar antes. Escolhendo "Pode ver e usar", também clica e digita em qualquer um.`, + `${bot} pode errar: apagar, enviar ou mudar algo como se fosse você.`, + `Um texto num app ou numa mensagem pode tentar enganar ${bot} para fazer outra coisa.`, + "Nunca, nem assim: o próprio Botloft, apps rodando como administrador, terminais e caixas de comando, o Gerenciador de Tarefas, gerenciadores de senha e campos de senha. Nas fotos dele, eles ficam cobertos de preto.", + ], + wholeConfirm: "Dar", }, away: { label: "Enquanto você estava fora", diff --git a/app/src/lib/fakeDesktop.ts b/app/src/lib/fakeDesktop.ts index 9a71714e..04616faa 100644 --- a/app/src/lib/fakeDesktop.ts +++ b/app/src/lib/fakeDesktop.ts @@ -154,6 +154,34 @@ export class FakeDesktop { this.fake.bot(botId); return this.set(botId, { stopped: false }); }, + "desktop.grantWhole": ({ botId, level, acceptedRisks }) => { + this.fake.bot(botId); + if (!acceptedRisks) { + throw new Error("the whole desktop needs the risks accepted first"); + } + const same = this.grants.find( + (grant) => grant.botId === botId && grant.scope === "desktop", + ); + if (same) { + same.level = level; + same.acceptedRisksAt = this.fake.now; + } else { + this.grants.push({ + id: this.fake.id("dsk"), + botId, + scope: "desktop", + appPath: null, + appName: null, + level, + realInput: false, + unattended: false, + acceptedRisksAt: this.fake.now, + createdAt: this.fake.now, + }); + } + this.changed(botId); + return { botId, grants: this.of(botId) }; + }, "desktop.awayUses": () => this.away, "desktop.dismissAway": () => { this.away = []; diff --git a/app/src/lib/protocol.gen.ts b/app/src/lib/protocol.gen.ts index a1863f0a..f4393228 100644 --- a/app/src/lib/protocol.gen.ts +++ b/app/src/lib/protocol.gen.ts @@ -992,6 +992,16 @@ from: number, until: number, */ itemId: ChatItemId | null, }; +/** + * `desktop.grantWhole`: the owner gives the bot the whole desktop (spec + * 24.10), never asked for in the chat. + */ +export type DesktopWholeParams = { botId: BotId, level: DesktopLevel, +/** + * The owner checked "I understand the risks"; without it, refused. + */ +acceptedRisks: boolean, }; + export type ApprovalItem = { approvalId: ApprovalId, toolName: string, summary: string, /** * What the bot says the command is for, as in `ToolItem`. The bot @@ -1676,6 +1686,7 @@ export interface RpcMethods { "desktop.grants": { params: DesktopGrantsParams; result: Array }; "desktop.revoke": { params: DesktopGrantIdParams; result: BotDesktop }; "desktop.setOptions": { params: DesktopOptionsParams; result: BotDesktop }; + "desktop.grantWhole": { params: DesktopWholeParams; result: BotDesktop }; "desktop.watch": { params: DesktopBotParams; result: DesktopView }; "desktop.unwatch": { params: undefined; result: null }; "desktop.stop": { params: DesktopBotParams; result: DesktopState }; diff --git a/crates/botloft-core/src/protocol/desktop.rs b/crates/botloft-core/src/protocol/desktop.rs index 41fc4b15..9ce7052b 100644 --- a/crates/botloft-core/src/protocol/desktop.rs +++ b/crates/botloft-core/src/protocol/desktop.rs @@ -119,6 +119,18 @@ pub struct DesktopOptionsParams { pub accepted_risks: Option, } +/// `desktop.grantWhole`: the owner gives the bot the whole desktop (spec +/// 24.10), never asked for in the chat. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct DesktopWholeParams { + pub bot_id: BotId, + pub level: DesktopLevel, + /// The owner checked "I understand the risks"; without it, refused. + pub accepted_risks: bool, +} + /// A bot used an app while the owner was away (spec 24.8): what the app /// tells them when they are back, until they dismiss it. The result of /// `desktop.awayUses` and the params of `desktop.away` are a list of these. diff --git a/crates/botloft-core/src/protocol/export_desktop.rs b/crates/botloft-core/src/protocol/export_desktop.rs index be663315..6931dc4b 100644 --- a/crates/botloft-core/src/protocol/export_desktop.rs +++ b/crates/botloft-core/src/protocol/export_desktop.rs @@ -21,6 +21,7 @@ pub(super) fn decls(out: &mut Out) { out.decl::(); out.decl::(); out.decl::(); + out.decl::(); } pub(super) fn methods(out: &mut Out) { @@ -39,6 +40,11 @@ pub(super) fn methods(out: &mut Out) { &out.name::(), &out.name::(), ); + out.method( + method::DESKTOP_GRANT_WHOLE, + &out.name::(), + &out.name::(), + ); let bot = out.name::(); out.method(method::DESKTOP_WATCH, &bot, &out.name::()); out.method(method::DESKTOP_UNWATCH, "undefined", "null"); diff --git a/crates/botloft-core/src/protocol/mod.rs b/crates/botloft-core/src/protocol/mod.rs index 4e5fd62a..7d3c4022 100644 --- a/crates/botloft-core/src/protocol/mod.rs +++ b/crates/botloft-core/src/protocol/mod.rs @@ -127,6 +127,7 @@ pub mod method { pub const DESKTOP_GRANTS: &str = "desktop.grants"; pub const DESKTOP_REVOKE: &str = "desktop.revoke"; pub const DESKTOP_SET_OPTIONS: &str = "desktop.setOptions"; + pub const DESKTOP_GRANT_WHOLE: &str = "desktop.grantWhole"; pub const DESKTOP_WATCH: &str = "desktop.watch"; pub const DESKTOP_UNWATCH: &str = "desktop.unwatch"; pub const DESKTOP_STOP: &str = "desktop.stop"; diff --git a/crates/botloft-store/src/desktop_grants.rs b/crates/botloft-store/src/desktop_grants.rs index 2899a502..248b79c2 100644 --- a/crates/botloft-store/src/desktop_grants.rs +++ b/crates/botloft-store/src/desktop_grants.rs @@ -69,6 +69,36 @@ impl Store { )?) } + /// Gives `bot` the whole desktop at `level` (spec 24.2), the owner + /// having accepted the risks at `now` (spec 24.10); a grant of it + /// already there takes the new level and keeps its options. + pub fn grant_desktop_whole( + &self, + bot: &BotId, + level: DesktopLevel, + now: i64, + ) -> Result { + self.conn.execute( + "INSERT INTO desktop_grants (id, bot_id, scope, level, accepted_risks_at, created_at) \ + VALUES (?1, ?2, 'desktop', ?3, ?4, ?4) \ + ON CONFLICT (bot_id, scope, lower(coalesce(app_path, ''))) DO UPDATE SET \ + level = excluded.level, accepted_risks_at = excluded.accepted_risks_at", + params![ + DesktopGrantId::generate().as_str(), + bot.as_str(), + level.as_str(), + now, + ], + )?; + Ok(self.conn.query_row( + &format!( + "SELECT {COLUMNS} FROM desktop_grants WHERE bot_id = ?1 AND scope = 'desktop'" + ), + [bot.as_str()], + from_row, + )?) + } + /// The grants of `bot`, oldest first. pub fn desktop_grants(&self, bot: &BotId) -> Result> { let mut statement = self.conn.prepare_cached(&format!( @@ -235,6 +265,36 @@ mod tests { assert_eq!(off.accepted_risks_at, Some(50)); } + #[test] + fn the_whole_desktop_is_one_grant_whose_level_the_owner_sets() { + let fx = Fixture::new(); + let scout = &fx.bots[0].id; + let see = fx + .store + .grant_desktop_whole(scout, DesktopLevel::Act, 10) + .expect("whole"); + assert_eq!(see.scope, DesktopScope::Desktop); + assert_eq!((see.app_path.clone(), see.app_name.clone()), (None, None)); + assert_eq!(see.accepted_risks_at, Some(10)); + fx.store + .set_desktop_real_input(&see.id, true) + .expect("real"); + // Given again, it can go down and keeps its options. + let again = fx + .store + .grant_desktop_whole(scout, DesktopLevel::See, 20) + .expect("again"); + assert_eq!(again.id, see.id); + assert_eq!(again.level, DesktopLevel::See); + assert!(again.real_input); + assert_eq!(again.accepted_risks_at, Some(20)); + // An app grant is another one. + fx.store + .grant_desktop_app(scout, EXCEL, "Microsoft Excel", DesktopLevel::See, 30) + .expect("app"); + assert_eq!(fx.store.desktop_grants(scout).expect("grants").len(), 2); + } + #[test] fn a_grant_can_be_taken_away() { let fx = Fixture::new(); diff --git a/crates/botloftd/src/platform/desktop/mod.rs b/crates/botloftd/src/platform/desktop/mod.rs index 7cd6c548..92db33a0 100644 --- a/crates/botloftd/src/platform/desktop/mod.rs +++ b/crates/botloftd/src/platform/desktop/mod.rs @@ -7,6 +7,7 @@ mod blocked; pub mod keys; mod overlay; mod read; +mod screen; // Only Windows takes pictures; the tests run everywhere. #[cfg_attr(not(windows), allow(dead_code))] mod shrink; @@ -18,6 +19,7 @@ use std::path::PathBuf; pub use blocked::{Never, never}; pub use overlay::{ScreenCursor, notice_hide, notice_show, outline_hide, outline_show}; pub use read::{Control, LINES_MAX, by_reference, reference, render}; +pub use screen::{cover, screen_picture}; /// The largest picture of a window the bot gets, in logical pixels (24.4). pub const PICTURE_MAX: (u32, u32) = (1600, 1200); diff --git a/crates/botloftd/src/platform/desktop/screen.rs b/crates/botloftd/src/platform/desktop/screen.rs new file mode 100644 index 00000000..df6bfc24 --- /dev/null +++ b/crates/botloftd/src/platform/desktop/screen.rs @@ -0,0 +1,72 @@ +//! A picture of the owner's whole screen for a bot with the whole desktop +//! (spec 24.4, D6): every window as the owner sees it, the ones that are +//! never granted (spec 24.3) covered in black. + +use super::{DesktopError, Picture}; + +/// Paints each of `covered` (left, top, width and height on the screen) +/// black in `rgb`, a picture `width` × `height` of the screen from +/// `origin`; what falls outside the picture is left out. +pub fn cover(rgb: &mut [u8], width: u32, height: u32, origin: (i32, i32), covered: &[[i32; 4]]) { + let (width, height) = (i64::from(width), i64::from(height)); + for &[left, top, across, down] in covered { + let from_x = (i64::from(left) - i64::from(origin.0)).clamp(0, width); + let to_x = (i64::from(left) + i64::from(across) - i64::from(origin.0)).clamp(0, width); + let from_y = (i64::from(top) - i64::from(origin.1)).clamp(0, height); + let to_y = (i64::from(top) + i64::from(down) - i64::from(origin.1)).clamp(0, height); + for row in from_y..to_y { + let start = ((row * width + from_x) * 3) as usize; + let end = ((row * width + to_x) * 3) as usize; + rgb[start..end].fill(0); + } + } +} + +/// The whole screen, with every window never granted covered, in logical +/// pixels up to [`super::PICTURE_MAX`]. +pub fn screen_picture() -> Result { + #[cfg(windows)] + { + use super::{QUALITY, never, shrink, windows}; + let covered: Vec<[i32; 4]> = windows::every_shown()? + .iter() + .filter(|window| !window.minimized && never(window).is_some()) + .filter_map(|window| windows::frame(window.id).ok()) + .collect(); + let (rgb, width, height, scale) = windows::capture_screen(&covered)?; + Ok(shrink::picture( + &rgb, + width, + height, + scale, + super::PICTURE_MAX, + QUALITY, + )) + } + #[cfg(not(windows))] + { + Err(DesktopError::Unavailable) + } +} + +#[cfg(test)] +mod tests { + use super::cover; + + #[test] + fn covered_windows_turn_black_and_only_inside_the_picture() { + // A 4 × 3 screen from (-2, 10), all white. + let mut rgb = vec![255u8; 4 * 3 * 3]; + // A window at (-1, 11), 2 × 5: columns 1 and 2, rows 1 and 2. + cover(&mut rgb, 4, 3, (-2, 10), &[[-1, 11, 2, 5]]); + let black: Vec<(usize, usize)> = (0..3) + .flat_map(|row| (0..4).map(move |column| (row, column))) + .filter(|&(row, column)| rgb[(row * 4 + column) * 3] == 0) + .collect(); + assert_eq!(black, vec![(1, 1), (1, 2), (2, 1), (2, 2)]); + // Off the screen, nothing. + let mut rgb = vec![255u8; 4 * 3 * 3]; + cover(&mut rgb, 4, 3, (0, 0), &[[10, 10, 5, 5], [-9, -9, 2, 2]]); + assert!(rgb.iter().all(|&byte| byte == 255)); + } +} diff --git a/crates/botloftd/src/platform/desktop/windows/capture.rs b/crates/botloftd/src/platform/desktop/windows/capture.rs index ce880a47..e620ed26 100644 --- a/crates/botloftd/src/platform/desktop/windows/capture.rs +++ b/crates/botloftd/src/platform/desktop/windows/capture.rs @@ -28,10 +28,10 @@ const RENDER_FULL_CONTENT: PRINT_WINDOW_FLAGS = PRINT_WINDOW_FLAGS(2); const BASE_DPI: f64 = 96.0; /// Sizes in real pixels on this thread while it lives, then back. -struct RealPixels(DPI_AWARENESS_CONTEXT); +pub(super) struct RealPixels(DPI_AWARENESS_CONTEXT); impl RealPixels { - fn new() -> Self { + pub(super) fn new() -> Self { // SAFETY: changes only this thread's DPI awareness, restored on drop. Self(unsafe { SetThreadDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2) }) } @@ -47,11 +47,11 @@ impl Drop for RealPixels { } /// Frees the drawing objects in the right order when it goes. -struct Canvas { - screen: HDC, - memory: HDC, - bitmap: HBITMAP, - previous: HGDIOBJ, +pub(super) struct Canvas { + pub(super) screen: HDC, + pub(super) memory: HDC, + pub(super) bitmap: HBITMAP, + pub(super) previous: HGDIOBJ, } impl Drop for Canvas { @@ -66,7 +66,7 @@ impl Drop for Canvas { } } -fn system(what: &str) -> DesktopError { +pub(super) fn system(what: &str) -> DesktopError { DesktopError::System(what.to_owned()) } diff --git a/crates/botloftd/src/platform/desktop/windows/mod.rs b/crates/botloftd/src/platform/desktop/windows/mod.rs index b740f640..210cb156 100644 --- a/crates/botloftd/src/platform/desktop/windows/mod.rs +++ b/crates/botloftd/src/platform/desktop/windows/mod.rs @@ -17,6 +17,7 @@ mod process; mod real; #[cfg(test)] mod real_tests; +mod screen; mod stop_key; #[cfg(test)] mod test_window; @@ -44,6 +45,7 @@ pub use input::owner_idle; pub use notice::{hide as notice_hide, show as notice_show}; pub use outline::{hide as outline_hide, show as outline_show}; pub use real::{click as real_click, press as real_press, type_text as real_type}; +pub use screen::capture_screen; pub use stop_key::on_stop_key; pub use uia::read; @@ -54,6 +56,17 @@ use super::{App, DesktopError, Window}; const FRAME_HOST: &str = "applicationframehost.exe"; pub fn list() -> Result, DesktopError> { + gather(false) +} + +/// Every window shown on the screen, of other processes: with tool windows +/// and windows without a title too, to cover the ones never granted in a +/// picture of the whole screen (spec 24.4). +pub fn every_shown() -> Result, DesktopError> { + gather(true) +} + +fn gather(all: bool) -> Result, DesktopError> { let mut handles: Vec = Vec::new(); // SAFETY: the callback only pushes into `handles`, which outlives the // call. @@ -67,11 +80,15 @@ pub fn list() -> Result, DesktopError> { let mut apps: HashMap = HashMap::new(); let mut found = Vec::new(); for hwnd in handles { - if !shown(hwnd) { + if !shown(hwnd, all) { continue; } let title = text(hwnd); - if title.trim().is_empty() { + if !all && title.trim().is_empty() { + continue; + } + // The daemon's own outline and notice stay out of every picture. + if all && process_id(hwnd) == std::process::id() { continue; } let class = class_of(hwnd); @@ -103,15 +120,15 @@ unsafe extern "system" fn collect(hwnd: HWND, found: LPARAM) -> BOOL { } /// A window the owner can see on the taskbar or the screen: shown, not a -/// tool window, and not cloaked (Store apps kept hidden, other virtual -/// desktops). -fn shown(hwnd: HWND) -> bool { +/// tool window (unless `all`), and not cloaked (Store apps kept hidden, +/// other virtual desktops). +fn shown(hwnd: HWND, all: bool) -> bool { // SAFETY: plain queries on a window handle. unsafe { if !IsWindowVisible(hwnd).as_bool() { return false; } - if GetWindowLongPtrW(hwnd, GWL_EXSTYLE) & WS_EX_TOOLWINDOW.0 as isize != 0 { + if !all && GetWindowLongPtrW(hwnd, GWL_EXSTYLE) & WS_EX_TOOLWINDOW.0 as isize != 0 { return false; } let mut cloaked = 0u32; diff --git a/crates/botloftd/src/platform/desktop/windows/notice.rs b/crates/botloftd/src/platform/desktop/windows/notice.rs index 2559cd68..34643aec 100644 --- a/crates/botloftd/src/platform/desktop/windows/notice.rs +++ b/crates/botloftd/src/platform/desktop/windows/notice.rs @@ -22,9 +22,9 @@ use windows::Win32::UI::HiDpi::{ use windows::Win32::UI::WindowsAndMessaging::{ CreateWindowExW, DefWindowProcW, DispatchMessageW, GetMessageW, HWND_TOPMOST, LWA_COLORKEY, MSG, PostMessageW, RegisterClassW, SW_HIDE, SW_SHOWNOACTIVATE, SWP_NOACTIVATE, - SetLayeredWindowAttributes, SetWindowPos, ShowWindow, TranslateMessage, WM_APP, WM_PAINT, - WNDCLASSW, WS_EX_LAYERED, WS_EX_NOACTIVATE, WS_EX_TOOLWINDOW, WS_EX_TOPMOST, WS_EX_TRANSPARENT, - WS_POPUP, + SetLayeredWindowAttributes, SetWindowDisplayAffinity, SetWindowPos, ShowWindow, + TranslateMessage, WDA_EXCLUDEFROMCAPTURE, WM_APP, WM_PAINT, WNDCLASSW, WS_EX_LAYERED, + WS_EX_NOACTIVATE, WS_EX_TOOLWINDOW, WS_EX_TOPMOST, WS_EX_TRANSPARENT, WS_POPUP, }; use windows::core::{PCWSTR, w}; @@ -103,6 +103,8 @@ fn run(made: &Sender>) { return; }; let _ = SetLayeredWindowAttributes(hwnd, SEE_THROUGH, 255, LWA_COLORKEY); + // Never in a picture of the whole screen (spec 24.4). + let _ = SetWindowDisplayAffinity(hwnd, WDA_EXCLUDEFROMCAPTURE); let _ = made.send(Some(hwnd.0 as isize)); let mut message = MSG::default(); while GetMessageW(&mut message, None, 0, 0).as_bool() { diff --git a/crates/botloftd/src/platform/desktop/windows/outline.rs b/crates/botloftd/src/platform/desktop/windows/outline.rs index 77047df8..ff89984a 100644 --- a/crates/botloftd/src/platform/desktop/windows/outline.rs +++ b/crates/botloftd/src/platform/desktop/windows/outline.rs @@ -22,9 +22,9 @@ use windows::Win32::UI::WindowsAndMessaging::{ CreateWindowExW, DefWindowProcW, DispatchMessageW, GW_HWNDPREV, GetClientRect, GetMessageW, GetWindow, HWND_TOP, IsIconic, IsWindow, IsWindowVisible, KillTimer, LWA_COLORKEY, MSG, PostMessageW, RegisterClassW, SW_HIDE, SW_SHOWNOACTIVATE, SWP_NOACTIVATE, - SetLayeredWindowAttributes, SetTimer, SetWindowPos, ShowWindow, TranslateMessage, WM_APP, - WM_PAINT, WM_TIMER, WNDCLASSW, WS_EX_LAYERED, WS_EX_NOACTIVATE, WS_EX_TOOLWINDOW, - WS_EX_TRANSPARENT, WS_POPUP, + SetLayeredWindowAttributes, SetTimer, SetWindowDisplayAffinity, SetWindowPos, ShowWindow, + TranslateMessage, WDA_EXCLUDEFROMCAPTURE, WM_APP, WM_PAINT, WM_TIMER, WNDCLASSW, WS_EX_LAYERED, + WS_EX_NOACTIVATE, WS_EX_TOOLWINDOW, WS_EX_TRANSPARENT, WS_POPUP, }; use windows::core::{PCWSTR, w}; @@ -110,6 +110,8 @@ fn run(made: &Sender>) { return; }; let _ = SetLayeredWindowAttributes(hwnd, SEE_THROUGH, 255, LWA_COLORKEY); + // Never in a picture of the whole screen (spec 24.4). + let _ = SetWindowDisplayAffinity(hwnd, WDA_EXCLUDEFROMCAPTURE); let _ = made.send(Some(hwnd.0 as isize)); let mut message = MSG::default(); while GetMessageW(&mut message, None, 0, 0).as_bool() { diff --git a/crates/botloftd/src/platform/desktop/windows/screen.rs b/crates/botloftd/src/platform/desktop/windows/screen.rs new file mode 100644 index 00000000..bf4438e7 --- /dev/null +++ b/crates/botloftd/src/platform/desktop/windows/screen.rs @@ -0,0 +1,120 @@ +//! A picture of the owner's whole screen (spec 24.4): every monitor, as the +//! owner sees it, with the windows that are never granted covered in black. +//! The daemon's own outline and notice are kept out of it (they are +//! excluded from capture). + +use std::ffi::c_void; + +use windows::Win32::Graphics::Gdi::{ + BI_RGB, BITMAPINFO, BITMAPINFOHEADER, BitBlt, CAPTUREBLT, CreateCompatibleBitmap, + CreateCompatibleDC, DIB_RGB_COLORS, GetDC, GetDIBits, ReleaseDC, SRCCOPY, SelectObject, +}; +use windows::Win32::UI::HiDpi::GetDpiForSystem; +use windows::Win32::UI::WindowsAndMessaging::{ + GetSystemMetrics, SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN, SM_XVIRTUALSCREEN, SM_YVIRTUALSCREEN, +}; + +use super::super::{DesktopError, cover}; +use super::capture::{Canvas, RealPixels, system}; + +/// Windows' unscaled DPI. +const BASE_DPI: f64 = 96.0; + +/// The whole screen's pixels as RGB, its width and height, and the display +/// scale, with each of `covered` (left, top, width and height, in real +/// pixels on the screen) painted black. +pub fn capture_screen(covered: &[[i32; 4]]) -> Result<(Vec, u32, u32, f64), DesktopError> { + let _real = RealPixels::new(); + // SAFETY: plain queries. + let (left, top, width, height) = unsafe { + ( + GetSystemMetrics(SM_XVIRTUALSCREEN), + GetSystemMetrics(SM_YVIRTUALSCREEN), + GetSystemMetrics(SM_CXVIRTUALSCREEN), + GetSystemMetrics(SM_CYVIRTUALSCREEN), + ) + }; + if width <= 0 || height <= 0 { + return Err(DesktopError::Locked); + } + // SAFETY: each object made here is freed by `Canvas`. + let canvas = unsafe { + let screen = GetDC(None); + let memory = CreateCompatibleDC(Some(screen)); + let bitmap = CreateCompatibleBitmap(screen, width, height); + if memory.is_invalid() || bitmap.is_invalid() { + ReleaseDC(None, screen); + return Err(system("no memory for the picture")); + } + let previous = SelectObject(memory, bitmap.into()); + Canvas { + screen, + memory, + bitmap, + previous, + } + }; + // SAFETY: copies the screen into our bitmap of the same size. A locked + // screen or the secure desktop refuses. + let copied = unsafe { + BitBlt( + canvas.memory, + 0, + 0, + width, + height, + Some(canvas.screen), + left, + top, + SRCCOPY | CAPTUREBLT, + ) + }; + if copied.is_err() { + return Err(DesktopError::Locked); + } + let mut info = BITMAPINFO { + bmiHeader: BITMAPINFOHEADER { + biSize: size_of::() as u32, + biWidth: width, + // Negative: rows from the top down. + biHeight: -height, + biPlanes: 1, + biBitCount: 32, + biCompression: BI_RGB.0, + ..Default::default() + }, + ..Default::default() + }; + let mut bgra = vec![0u8; width as usize * height as usize * 4]; + // SAFETY: the buffer holds the whole bitmap at 32 bits a pixel. + let rows = unsafe { + GetDIBits( + canvas.memory, + canvas.bitmap, + 0, + height as u32, + Some(bgra.as_mut_ptr().cast::()), + &mut info, + DIB_RGB_COLORS, + ) + }; + drop(canvas); + if rows == 0 { + return Err(system("the picture could not be read")); + } + let (pixels, _) = bgra.as_chunks::<4>(); + let mut rgb = Vec::with_capacity(pixels.len() * 3); + for [blue, green, red, _] in pixels { + rgb.extend([*red, *green, *blue]); + } + let (width, height) = (width as u32, height as u32); + cover(&mut rgb, width, height, (left, top), covered); + // SAFETY: a plain query. + let dpi = unsafe { GetDpiForSystem() }; + let scale = if dpi == 0 { + 1.0 + } else { + f64::from(dpi) / BASE_DPI + }; + Ok((rgb, width, height, scale)) +} diff --git a/crates/botloftd/src/rpc/dispatch.rs b/crates/botloftd/src/rpc/dispatch.rs index 9a076731..a9733b11 100644 --- a/crates/botloftd/src/rpc/dispatch.rs +++ b/crates/botloftd/src/rpc/dispatch.rs @@ -58,6 +58,7 @@ pub fn dispatch(daemon: &Daemon, name: &str, params: Option) -> Result reply(desktop::revoke(daemon, parse(params)?)), method::DESKTOP_STOP => reply(desktop::stop(daemon, parse(params)?)), method::DESKTOP_SET_OPTIONS => reply(desktop::set_options(daemon, parse(params)?)), + method::DESKTOP_GRANT_WHOLE => reply(desktop::grant_whole(daemon, parse(params)?)), method::DESKTOP_AWAY_USES => reply(desktop::away_uses(daemon)), method::DESKTOP_DISMISS_AWAY => reply(desktop::dismiss_away(daemon)), method::DESKTOP_RESUME => reply(desktop::resume(daemon, parse(params)?)), diff --git a/crates/botloftd/src/service/desktop.rs b/crates/botloftd/src/service/desktop.rs index 4781d776..b6aaefc4 100644 --- a/crates/botloftd/src/service/desktop.rs +++ b/crates/botloftd/src/service/desktop.rs @@ -7,7 +7,7 @@ use botloft_core::ids::BotId; use botloft_core::protocol::{ BotDesktop, DesktopAwayUse, DesktopBotParams, DesktopGrant, DesktopGrantIdParams, - DesktopGrantsParams, DesktopOptionsParams, DesktopState, + DesktopGrantsParams, DesktopOptionsParams, DesktopState, DesktopWholeParams, }; use tracing::{info, warn}; @@ -46,6 +46,25 @@ pub fn set_options(daemon: &Daemon, params: DesktopOptionsParams) -> ApiResult ApiResult { + if !params.accepted_risks { + return Err(ApiError::Conflict( + "the whole desktop needs the risks accepted first".to_owned(), + )); + } + super::bots::find(&daemon.store(), ¶ms.bot_id)?; + daemon + .store() + .grant_desktop_whole(¶ms.bot_id, params.level, daemon.clock.now_ms())?; + info!( + level = params.level.as_str(), + "the owner gave a bot the whole desktop" + ); + changed(daemon, ¶ms.bot_id) +} + /// What bots did while the owner was away, for the app to tell them. pub fn away_uses(daemon: &Daemon) -> ApiResult> { Ok(daemon.desktop.away.list()) diff --git a/crates/botloftd/src/tools/desktop.rs b/crates/botloftd/src/tools/desktop.rs index f08c50a9..593b2b6a 100644 --- a/crates/botloftd/src/tools/desktop.rs +++ b/crates/botloftd/src/tools/desktop.rs @@ -8,19 +8,19 @@ use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; use botloft_core::ids::BotId; -use botloft_core::protocol::{DesktopAction, DesktopLevel}; +use botloft_core::protocol::DesktopLevel; use serde_json::{Value, json}; use tracing::debug; use super::calls::explain; use super::desktop_act::{Ask, act, answered}; -use super::desktop_grant::{blocking, granted, heading}; +use super::desktop_grant::{blocking, granted, heading, used}; use super::desktop_list::{in_reach, list}; use super::desktop_real::{click_at, press}; +use super::desktop_screen::whole; use crate::desktop::STOPPED; -use crate::platform::desktop::Window; use crate::platform::desktop::{self as platform, Action, Scroll, render}; -use crate::state::{Daemon, Event}; +use crate::state::Daemon; pub(super) const PREFIX: &str = "desktop_"; @@ -31,8 +31,9 @@ enum Tool { from: usize, why: String, }, + /// A window's picture, or the whole screen's without one (D6). Screenshot { - window: u64, + window: Option, why: String, }, Act(Ask), @@ -90,7 +91,7 @@ impl Tool { why: why()?, }, Some("screenshot") => Self::Screenshot { - window: window()?, + window: arguments["window"].as_u64(), why: why()?, }, Some("click") => acting(Action::Click)?, @@ -215,6 +216,11 @@ async fn run(daemon: &Daemon, bot: &BotId, generation: u64, tool: Tool) -> Resul Ok(Reply::Text(reading)) } Tool::Screenshot { window, why } => { + let Some(window) = window else { + return whole(daemon, bot, &grants) + .await + .map(|(data, text)| Reply::Picture(data, text)); + }; let window = granted( daemon, bot, @@ -287,10 +293,3 @@ async fn run(daemon: &Daemon, bot: &BotId, generation: u64, tool: Tool) -> Resul .map(Reply::Text), } } - -/// The bot read or acted in `window`: its panel shows it (spec 24.9). -pub(super) fn used(daemon: &Daemon, bot: &BotId, window: &Window, action: Option) { - let now = daemon.clock.now_ms(); - let state = daemon.desktop.activity.used(bot, window, action, now); - daemon.emit(Event::DesktopChanged(state)); -} diff --git a/crates/botloftd/src/tools/desktop_act.rs b/crates/botloftd/src/tools/desktop_act.rs index 0cc23efd..6b2b6c28 100644 --- a/crates/botloftd/src/tools/desktop_act.rs +++ b/crates/botloftd/src/tools/desktop_act.rs @@ -10,8 +10,7 @@ use std::time::Duration; use botloft_core::ids::BotId; use botloft_core::protocol::{DesktopAction, DesktopActionKind, DesktopGrant, DesktopLevel}; -use super::desktop::used; -use super::desktop_grant::{blocking, describe, granted, heading}; +use super::desktop_grant::{blocking, describe, granted, heading, used}; use super::desktop_real::{enter, instead, needs_real, real_allowed}; use crate::desktop::STOPPED; use crate::platform::desktop::{ diff --git a/crates/botloftd/src/tools/desktop_catalog.rs b/crates/botloftd/src/tools/desktop_catalog.rs index 7400dfa1..0e21a97c 100644 --- a/crates/botloftd/src/tools/desktop_catalog.rs +++ b/crates/botloftd/src/tools/desktop_catalog.rs @@ -81,15 +81,21 @@ pub(super) fn tools() -> Vec { "Shows you a picture of one window on the owner's computer, only that window, \ even behind others: for what its text does not tell, like a chart or a \ layout. Prefer desktop_look, which costs you less. A minimized window has \ - nothing to show. {ASKS} {UNTRUSTED}" + nothing to show. Without window, the owner's whole screen, only if they gave \ + you their whole desktop, with the windows you may never use covered. {ASKS} \ + {UNTRUSTED}" ), "inputSchema": { "type": "object", "properties": { - "window": window(), + "window": { + "type": "integer", + "description": "The window's number from desktop_windows; leave it \ + out for the owner's whole screen.", + }, "why": why(), }, - "required": ["window", "why"], + "required": ["why"], "additionalProperties": false, }, "annotations": { "readOnlyHint": true }, diff --git a/crates/botloftd/src/tools/desktop_grant.rs b/crates/botloftd/src/tools/desktop_grant.rs index 2fe42ea1..3c68ea5f 100644 --- a/crates/botloftd/src/tools/desktop_grant.rs +++ b/crates/botloftd/src/tools/desktop_grant.rs @@ -4,7 +4,7 @@ use botloft_core::chat::DESKTOP_TOOL; use botloft_core::ids::BotId; -use botloft_core::protocol::{DesktopGrant, DesktopLevel}; +use botloft_core::protocol::{DesktopAction, DesktopGrant, DesktopLevel}; use serde_json::json; use super::calls::explain; @@ -81,9 +81,9 @@ fn verb(level: DesktopLevel) -> &'static str { } } -/// The bot used `window` with the owner away: the app tells them when they +/// The bot used `app` with the owner away: the app tells them when they /// are back, with the chat where it began. -fn used_away(daemon: &Daemon, bot: &BotId, window: &Window) { +pub(super) fn used_away(daemon: &Daemon, bot: &BotId, app: &str) { let item = daemon .store() .chat_history(bot, None, 1) @@ -91,7 +91,7 @@ fn used_away(daemon: &Daemon, bot: &BotId, window: &Window) { .and_then(|items| items.into_iter().next()) .map(|item| item.id); let now = daemon.clock.now_ms(); - let uses = daemon.desktop.away.record(bot, &window.app.name, now, item); + let uses = daemon.desktop.away.record(bot, app, now, item); daemon.emit(Event::DesktopAway(uses)); } @@ -125,7 +125,7 @@ pub(super) async fn granted( if !covering(grants, &window, level).is_some_and(|grant| grant.unattended) { return Err(away.why().to_owned()); } - used_away(daemon, bot, &window); + used_away(daemon, bot, &window.app.name); return Ok(window); } if covering(grants, &window, level).is_some() { @@ -171,3 +171,10 @@ pub(super) async fn granted( None => Err("Botloft could not ask the owner.".to_owned()), } } + +/// The bot read or acted in `window`: its panel shows it (spec 24.9). +pub(super) fn used(daemon: &Daemon, bot: &BotId, window: &Window, action: Option) { + let now = daemon.clock.now_ms(); + let state = daemon.desktop.activity.used(bot, window, action, now); + daemon.emit(Event::DesktopChanged(state)); +} diff --git a/crates/botloftd/src/tools/desktop_screen.rs b/crates/botloftd/src/tools/desktop_screen.rs new file mode 100644 index 00000000..4553993b --- /dev/null +++ b/crates/botloftd/src/tools/desktop_screen.rs @@ -0,0 +1,58 @@ +//! The picture of the owner's whole screen (spec 24.4, D6): only for a bot +//! the owner gave the whole desktop, which is never asked for in the chat +//! (spec 24.2), with the windows never granted covered in black. + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use botloft_core::ids::BotId; +use botloft_core::protocol::{DesktopGrant, DesktopScope}; + +use super::desktop_grant::{blocking, used_away}; +use crate::desktop::STOPPED; +use crate::platform::desktop as platform; +use crate::state::Daemon; + +/// What the owner reads when a bot used the whole screen while away. +const WHOLE: &str = "Desktop"; + +/// A JPEG of the whole screen, base64, and a line about it. +pub(super) async fn whole( + daemon: &Daemon, + bot: &BotId, + grants: &[DesktopGrant], +) -> Result<(String, String), String> { + let Some(grant) = grants + .iter() + .find(|grant| grant.scope == DesktopScope::Desktop) + else { + return Err( + "The owner has not given you their whole desktop, so pass window: the \ + number of one window from desktop_windows. Only the owner gives the whole desktop, \ + in your details in Botloft." + .to_owned(), + ); + }; + if let Err(away) = daemon.desktop.owner_here() { + if !grant.unattended { + return Err(away.why().to_owned()); + } + used_away(daemon, bot, WHOLE); + } + let _turn = daemon.desktop.turn().await; + // The owner may have stopped it while it waited. + if daemon.desktop.activity.stopped(bot) { + return Err(STOPPED.to_owned()); + } + let picture = blocking(platform::screen_picture).await?; + if picture.jpeg.is_empty() { + return Err("The picture of the screen came out empty.".to_owned()); + } + Ok(( + BASE64.encode(&picture.jpeg), + format!( + "The owner's whole screen, {} by {}. Windows you may never use are covered in \ + black. To read or use a window, call desktop_windows and pass its number.", + picture.width, picture.height + ), + )) +} diff --git a/crates/botloftd/src/tools/mod.rs b/crates/botloftd/src/tools/mod.rs index 920a6d56..eddef66d 100644 --- a/crates/botloftd/src/tools/mod.rs +++ b/crates/botloftd/src/tools/mod.rs @@ -20,6 +20,7 @@ mod desktop_catalog; mod desktop_grant; mod desktop_list; mod desktop_real; +mod desktop_screen; mod era; mod question; mod routine; diff --git a/crates/botloftd/tests/desktop_whole.rs b/crates/botloftd/tests/desktop_whole.rs new file mode 100644 index 00000000..0e616fe7 --- /dev/null +++ b/crates/botloftd/tests/desktop_whole.rs @@ -0,0 +1,71 @@ +//! A bot with the owner's whole desktop (spec 24.2, 24.4, D6): only the +//! owner gives it, after accepting the risks; then every window shows with +//! its title, a window is read without asking, and the whole screen comes +//! as one picture. Windows only. +#![cfg(windows)] + +mod common; + +use common::desktop::{TITLE, TestWindow, reading, setup}; +use serde_json::json; + +#[tokio::test(flavor = "multi_thread")] +async fn the_owner_gives_the_whole_desktop_and_the_bot_sees_the_screen() { + let window = TestWindow::open(); + let mut s = setup().await; + + // Without it, no whole screen, and a window's title stays hidden. + let refused = s + .mcp + .tool_text("desktop_screenshot", json!({ "why": "To see the screen" })) + .await; + assert!( + reading(&refused).contains("not given you their whole desktop"), + "{}", + reading(&refused) + ); + let listed = s + .mcp + .tool_text("desktop_windows", json!({})) + .await + .expect("windows"); + assert!(!listed.contains(TITLE), "{listed}"); + + // Only with the risks accepted. + let unaccepted = json!({ "botId": s.bot["id"], "level": "see", "acceptedRisks": false }); + assert!(s.app.call("desktop.grantWhole", unaccepted).await.is_err()); + let accepted = json!({ "botId": s.bot["id"], "level": "see", "acceptedRisks": true }); + let given = s + .app + .call("desktop.grantWhole", accepted) + .await + .expect("given"); + assert_eq!(given["grants"][0]["scope"], "desktop"); + assert!(given["grants"][0]["acceptedRisksAt"].is_i64()); + let changed = s.app.notification("bot.desktop").await; + assert_eq!(changed["grants"][0]["scope"], "desktop"); + + // Every window with its title; one is read without asking. + let listed = s + .mcp + .tool_text("desktop_windows", json!({})) + .await + .expect("windows"); + assert!(listed.contains(TITLE), "{listed}"); + let look = json!({ "window": window.id, "why": "To read the visit" }); + let text = s.mcp.tool_text("desktop_look", look).await.expect("read"); + assert!(text.contains(r#"button "Save""#), "{text}"); + + // And the whole screen as one picture. + let picture = s + .mcp + .tool_result("desktop_screenshot", json!({ "why": "To see the screen" })) + .await; + assert_eq!(picture["content"][0]["type"], "image", "{picture}"); + assert!( + picture["content"][1]["text"] + .as_str() + .is_some_and(|text| text.contains("whole screen")), + "{picture}" + ); +} diff --git a/docs/spec.md b/docs/spec.md index 3471e603..4c28d336 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1656,7 +1656,7 @@ Tabela `questions`: `id, crew_id, bot_id, chat_item_id, text, options (JSON), st ## 24. Desktop -Status: **D1 a D5 implementados** (24.12). É o item 10 da seção 18. Decisões e riscos em `docs/adr/0002-desktop-use.md`. +Status: **D1 a D6 implementados** (24.12). É o item 10 da seção 18. Decisões e riscos em `docs/adr/0002-desktop-use.md`. ### 24.1 O que é @@ -1672,7 +1672,7 @@ Cada permissão é de um bot. Ela tem um **alcance**, um **nível** e duas opç - **Alcance**, escolhido pelo dono: - **Um app**: as janelas de um programa, identificado pelo executável (caminho completo e nome do produto da versão do arquivo, por exemplo `C:\Program Files\Microsoft Office\root\Office16\EXCEL.EXE`, "Microsoft Excel"). Um executável com o mesmo nome em outra pasta é outro app. - - **O desktop inteiro**: todas as janelas de todos os apps, menos as que nunca são liberadas (24.3). + - **O desktop inteiro**: todas as janelas de todos os apps, menos as que nunca são liberadas (24.3). Uma permissão só (`scope: desktop`, sem executável), no nível que o dono escolhe, e que ele pode baixar de Ver e usar para Ver; as opções (mouse e teclado, sem você na frente) ficam. Com ela, o bot vê o título de toda janela, lê e fotografa qualquer uma sem perguntar e, no nível Ver e usar, age em qualquer uma; no nível Ver, usar um app ainda pede por ele no chat. - **Nível**, separado (o dono libera um, depois o outro): - **Ver**: o bot lista as janelas do alcance, lê o que há nelas (a árvore de acessibilidade, 24.5) e tira fotos delas. - **Mexer**: além de ver, clica, digita, escolhe e rola por acessibilidade (24.5). Pede "Ver" antes. @@ -1702,7 +1702,7 @@ Os programas são reconhecidos pelo nome do executável, sem diferença de maiú - **Janelas**: as de nível superior, visíveis ou minimizadas, com dono (não ferramentas nem janelas sem título). De um app fora do alcance, o bot vê só o nome do app, nunca o título da janela: um título diz o assunto de um e-mail ou o nome de um arquivo. - **Foto de uma janela**: `PrintWindow` com `PW_RENDERFULLCONTENT`, que pega a janela mesmo atrás de outras, só a janela, sem o que estiver por cima dela, recortada na moldura que o DWM diz (`DWMWA_EXTENDED_FRAME_BOUNDS`), sem a borda invisível de redimensionar que o Windows deixa em volta. A medida é em pixels reais (o thread fica com DPI por monitor enquanto fotografa) e a foto é reduzida pela escala da tela da janela (`GetDpiForWindow`). Uma janela minimizada não tem o que fotografar: a tool diz isso. -- **Foto do desktop inteiro** (só com esse alcance): a tela toda, com as janelas que nunca são liberadas (24.3) cobertas de preto. +- **Foto do desktop inteiro** (só com esse alcance): a tela toda, com as janelas que nunca são liberadas (24.3) cobertas de preto. Todos os monitores (a tela virtual, `SM_XVIRTUALSCREEN`...), copiados da tela com `BitBlt` (`SRCCOPY | CAPTUREBLT`), como o dono vê. As janelas cobertas são todas as mostradas na tela de um processo que nunca é liberado, inclusive as sem título e as de ferramenta (uma janelinha de um gerenciador de senhas também conta), cobertas pela moldura do DWM, mesmo com outra janela por cima (o preto sobra, nunca falta). O contorno e o aviso do próprio daemon ficam fora de qualquer foto (`WDA_EXCLUDEFROMCAPTURE`). Reduzida como a de uma janela, pela escala do sistema (`GetDpiForSystem`), até 1600 × 1200. Com a tela bloqueada, a cópia falha e a tool diz isso. O painel do bot (24.9) segue a última janela, não a tela inteira. - **Tamanho**: a foto vai ao bot no tamanho da janela em pixels lógicos (sem a escala do Windows), até 1600 × 1200, reduzida mantendo a proporção. JPEG, qualidade 70, como `browser_screenshot`. ### 24.5 Acessibilidade (UI Automation) @@ -1730,7 +1730,7 @@ No servidor `botloft` (11), como as do navegador. |---|---|---| | `desktop_windows` | | as janelas abertas: as do alcance com título e `window` (id), as outras só pelo nome do app | | `desktop_look` | `window`, `why`, `from?` | lê a janela (24.5) | -| `desktop_screenshot` | `window`, `why` | a foto da janela; a do desktop inteiro vem com esse alcance (D6) | +| `desktop_screenshot` | `window?`, `why` | a foto da janela; sem `window`, a da tela inteira, só com o desktop inteiro (sem ele, a tool diz que só o dono o dá); com o dono fora, só se essa permissão tiver "Sem você na frente", e o aviso na volta diz "Desktop" | | `desktop_click` | `ref`, `why?` | clica no controle (24.5) | | `desktop_type` | `ref`, `text`, `submit?`, `why?` | troca o texto do campo; com `submit`, aperta Enter depois, com o teclado de verdade (só onde ele está ligado) | | `desktop_select` | `ref`, `option`, `why?` | escolhe uma opção pelo texto | @@ -1778,6 +1778,7 @@ No servidor `botloft` (11), como as do navegador. ### 24.10 Permissões no app - Nos detalhes do bot ("Sobre "), a seção **Seu desktop** lista as permissões: o app (ou "O desktop inteiro"), o caminho do executável, Pode ver ou Pode ver e usar, e as opções. Cada uma com um X que tira a permissão; o bot pede de novo na próxima vez. Sem nenhuma, a seção diz que o bot pede no chat na primeira vez que precisar. +- **Dar o desktop inteiro**: um botão na seção, que abre a tela de riscos abaixo com a escolha do nível (Ver, ou Ver e usar). Gravado, ele aparece na lista como "O desktop inteiro", com o X e as opções de qualquer permissão. O app chama `desktop.grantWhole {botId, level, acceptedRisks}`, que devolve as permissões do bot (`BotDesktop`) e manda `bot.desktop`; sem `acceptedRisks: true` é `conflict`. - **Dar o desktop inteiro** e **ligar "Sem você na frente"** abrem uma tela de riscos, em palavras simples, antes de gravar: - o bot vê tudo o que estiver aberto no alcance, inclusive dados pessoais, e-mails, conversas e dados de clientes; - ele pode errar: apagar, enviar ou mudar algo no app como se fosse você;