Skip to content

Commit 4020e7a

Browse files
httpsphlhttpsphl
authored
Merge pull request #180 from httpsphl/feat/desktop-real-app
feat(desktop): the owner turns on the real mouse and keyboard, and sees on screen when a bot has them
2 parents 317d01f + 98262a3 commit 4020e7a

37 files changed

Lines changed: 1441 additions & 46 deletions

‎app/src/features/desktop/DesktopGrants.tsx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import type { Bot, DesktopGrant } from "../../lib/protocol.gen";
88
import { useApi } from "../../store/context";
99
import { Button } from "../../ui/Button";
1010
import { attempt } from "../../ui/toast";
11+
import { RealInput } from "./RealInput";
1112

1213
/** The bot's grants, read when shown and kept current by `bot.desktop`. */
1314
function useDesktopGrants(botId: Bot["id"]): DesktopGrant[] | null {
@@ -60,6 +61,7 @@ export function DesktopGrants({ bot }: { bot: Bot }) {
6061
<span className="block text-ink-soft text-xs">
6162
{grant.level === "act" ? words.act : words.see}
6263
</span>
64+
{grant.level === "act" && <RealInput bot={bot} grant={grant} app={app} />}
6365
{grant.appPath && (
6466
<span
6567
className="block break-all font-mono text-muted text-xs"

‎app/src/features/desktop/DesktopPanel.tsx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ export function actionText(action: DesktopAction | null, t: Words): string {
3434
return t.select(action.option ?? "");
3535
case "scroll":
3636
return target ? t.scroll(target) : t.scrollSomething;
37+
case "press":
38+
return t.press(action.option ?? "");
3739
}
3840
}
3941

‎app/src/features/desktop/RealInput.tsx‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
// The switch of a desktop grant for the owner's real mouse and keyboard
2+
// (spec 24.7, 24.10): off from the start; turning it on first says, in a
3+
// dialog, what changes and how to stop it; turning it off is at once.
4+
5+
import { Check } from "lucide-react";
6+
import { useState } from "react";
7+
import { useT } from "../../i18n";
8+
import type { Bot, DesktopGrant } from "../../lib/protocol.gen";
9+
import { useApi } from "../../store/context";
10+
import { Button } from "../../ui/Button";
11+
import { Dialog } from "../../ui/Dialog";
12+
import { Switch } from "../../ui/Switch";
13+
import { attempt } from "../../ui/toast";
14+
15+
export function RealInput({ bot, grant, app }: { bot: Bot; grant: DesktopGrant; app: string }) {
16+
const words = useT().desktop.grants;
17+
const api = useApi();
18+
const [asking, setAsking] = useState(false);
19+
const set = (realInput: boolean) =>
20+
attempt(words.realFailed, () =>
21+
api.call("desktop.setOptions", { grantId: grant.id, realInput }),
22+
);
23+
return (
24+
<>
25+
<span className="mt-1 flex items-center gap-2 text-xs">
26+
<Switch
27+
checked={grant.realInput}
28+
label={`${words.real}: ${app}`}
29+
onChange={(on) => (on ? setAsking(true) : void set(false))}
30+
/>
31+
<span className="text-ink-soft">{words.real}</span>
32+
</span>
33+
{asking && (
34+
<Dialog
35+
title={words.realTitle(bot.name, app)}
36+
onClose={() => setAsking(false)}
37+
footer={
38+
<>
39+
<Button variant="ghost" onClick={() => setAsking(false)}>
40+
{words.cancel}
41+
</Button>
42+
<Button
43+
variant="primary"
44+
icon={Check}
45+
onClick={() => {
46+
setAsking(false);
47+
void set(true);
48+
}}
49+
>
50+
{words.realConfirm}
51+
</Button>
52+
</>
53+
}
54+
>
55+
<ul className="flex list-disc flex-col gap-1.5 pl-5 text-sm">
56+
{words.realPoints(bot.name).map((point) => (
57+
<li key={point}>{point}</li>
58+
))}
59+
</ul>
60+
</Dialog>
61+
)}
62+
</>
63+
);
64+
}

‎app/src/features/desktop/realInput.test.tsx‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
import { act, cleanup, fireEvent, screen, waitFor, within } from "@testing-library/react";
2+
import { afterEach, describe, expect, test } from "vitest";
3+
import { setLocaleChoice } from "../../i18n";
4+
import { FakeBotloft } from "../../lib/fake";
5+
import { crewOpened, openBot, renderApp } from "../../test/app";
6+
7+
afterEach(() => {
8+
act(() => setLocaleChoice("en"));
9+
cleanup();
10+
});
11+
12+
const NOTEPAD = "C:\\Windows\\notepad.exe";
13+
14+
async function openScout() {
15+
const fake = new FakeBotloft();
16+
const ops = fake.addCrew("Ops");
17+
const scout = fake.addBot(ops.id, "Scout", "Finds sources");
18+
fake.setBotState(scout.id, "idle");
19+
renderApp(fake);
20+
await crewOpened("Ops");
21+
openBot("Scout");
22+
await screen.findByRole("list", { name: "Messages" });
23+
return { fake, scout };
24+
}
25+
26+
const details = () => screen.getByRole("complementary", { name: "About Scout" });
27+
28+
describe("the real mouse and keyboard", () => {
29+
test("are off, and turning them on says first what changes", async () => {
30+
const { fake, scout } = await openScout();
31+
act(() => {
32+
fake.desktop.grant(scout.id, NOTEPAD, "Notepad", "act");
33+
});
34+
fireEvent.click(screen.getByRole("button", { name: /^Show details/ }));
35+
const toggle = await within(details()).findByRole("switch", {
36+
name: "Real mouse and keyboard: Notepad",
37+
});
38+
expect(toggle.getAttribute("aria-checked")).toBe("false");
39+
40+
fireEvent.click(toggle);
41+
const dialog = screen.getByRole("dialog", {
42+
name: "Let Scout use your real mouse and keyboard in Notepad?",
43+
});
44+
expect(within(dialog).getByText(/Move the mouse or press a key and it stops/)).toBeDefined();
45+
expect(within(dialog).getByText(/Ctrl\+Alt\+Esc/)).toBeDefined();
46+
fireEvent.click(within(dialog).getByRole("button", { name: "Cancel" }));
47+
expect(fake.desktop.grants[0]?.realInput).toBe(false);
48+
49+
fireEvent.click(toggle);
50+
fireEvent.click(screen.getByRole("button", { name: "Turn on" }));
51+
await waitFor(() => expect(fake.desktop.grants[0]?.realInput).toBe(true));
52+
expect(toggle.getAttribute("aria-checked")).toBe("true");
53+
54+
// Off at once, without asking.
55+
fireEvent.click(toggle);
56+
await waitFor(() => expect(fake.desktop.grants[0]?.realInput).toBe(false));
57+
});
58+
59+
test("are not offered where the bot may only see", async () => {
60+
const { fake, scout } = await openScout();
61+
act(() => {
62+
fake.desktop.grant(scout.id, NOTEPAD, "Notepad");
63+
});
64+
fireEvent.click(screen.getByRole("button", { name: /^Show details/ }));
65+
await within(details()).findByText("Can see");
66+
expect(within(details()).queryByRole("switch", { name: /Real mouse/ })).toBeNull();
67+
});
68+
69+
test("the daemon hears the owner's language when it changes, for its notice", async () => {
70+
const { fake } = await openScout();
71+
act(() => setLocaleChoice("pt-BR"));
72+
await waitFor(() => expect(fake.locale).toBe("pt-BR"));
73+
});
74+
});

‎app/src/features/desktop/useLocaleToDaemon.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
// The owner's language, told to the daemon when it changes, for the notice
2+
// it shows on screen while a bot uses the real mouse and keyboard (spec
3+
// 24.7). The language at connection goes with `session.hello`.
4+
5+
import { useEffect, useRef } from "react";
6+
import { useLocale } from "../../i18n";
7+
import { useApi } from "../../store/context";
8+
9+
export function useLocaleToDaemon(): void {
10+
const api = useApi();
11+
const { locale } = useLocale();
12+
const told = useRef(locale);
13+
useEffect(() => {
14+
if (told.current === locale) {
15+
return;
16+
}
17+
told.current = locale;
18+
// An older daemon does not know the method: nothing to tell.
19+
api.call("session.setLocale", { locale }).catch(() => {});
20+
}, [api, locale]);
21+
}

‎app/src/i18n/en/desktop.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ export const desktop = {
4141
select: (option: string) => `Chose "${option}"`,
4242
scroll: (target: string) => `Scrolled "${target}"`,
4343
scrollSomething: "Scrolled",
44+
press: (keys: string) => `Pressed ${keys}`,
4445
showInPanel: "Watch on the desktop panel",
4546
close: "Close",
4647
},
@@ -53,5 +54,17 @@ export const desktop = {
5354
whole: "The whole desktop",
5455
remove: (app: string) => `Take back ${app}`,
5556
removeFailed: "Could not take it back",
57+
real: "Real mouse and keyboard",
58+
realFailed: "Could not change it",
59+
realTitle: (bot: string, app: string) =>
60+
`Let ${bot} use your real mouse and keyboard in ${app}?`,
61+
realPoints: (bot: string) => [
62+
`When accessibility cannot do something in this app, ${bot} moves your cursor and types as if it were you.`,
63+
`While it acts, your mouse and keyboard are its, and a notice on your screen says so.`,
64+
`Move the mouse or press a key and it stops at once. Ctrl+Alt+Esc stops every bot on your desktop.`,
65+
`It never types in password fields and never presses Windows' own keys, like the Windows key or Alt+Tab.`,
66+
],
67+
realConfirm: "Turn on",
68+
cancel: "Cancel",
5669
},
5770
};

‎app/src/i18n/es/desktop.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ export const desktop: Messages["desktop"] = {
4242
select: (option) => `Eligió "${option}"`,
4343
scroll: (target) => `Desplazó "${target}"`,
4444
scrollSomething: "Desplazó",
45+
press: (keys) => `Presionó ${keys}`,
4546
showInPanel: "Ver en el panel del escritorio",
4647
close: "Cerrar",
4748
},
@@ -54,5 +55,16 @@ export const desktop: Messages["desktop"] = {
5455
whole: "Todo el escritorio",
5556
remove: (app) => `Quitar ${app}`,
5657
removeFailed: "No se pudo quitar",
58+
real: "Mouse y teclado de verdad",
59+
realFailed: "No se pudo cambiar",
60+
realTitle: (bot, app) => `¿Dejar que ${bot} use tu mouse y teclado de verdad en ${app}?`,
61+
realPoints: (bot) => [
62+
`Cuando la accesibilidad no alcanza algo en esa app, ${bot} mueve tu cursor y escribe como si fueras tú.`,
63+
`Mientras actúa, el mouse y el teclado son suyos, y un aviso en tu pantalla lo dice.`,
64+
`Mueve el mouse o presiona una tecla y se detiene al instante. Ctrl+Alt+Esc detiene a todos los bots en tu escritorio.`,
65+
`Nunca escribe en campos de contraseña ni presiona teclas propias de Windows, como la tecla Windows o Alt+Tab.`,
66+
],
67+
realConfirm: "Activar",
68+
cancel: "Cancelar",
5769
},
5870
};

‎app/src/i18n/pt-BR/desktop.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ export const desktop: Messages["desktop"] = {
4141
select: (option) => `Escolheu "${option}"`,
4242
scroll: (target) => `Rolou "${target}"`,
4343
scrollSomething: "Rolou",
44+
press: (keys) => `Apertou ${keys}`,
4445
showInPanel: "Ver no painel do desktop",
4546
close: "Fechar",
4647
},
@@ -53,5 +54,16 @@ export const desktop: Messages["desktop"] = {
5354
whole: "O desktop inteiro",
5455
remove: (app) => `Tirar ${app}`,
5556
removeFailed: "Não foi possível tirar",
57+
real: "Mouse e teclado de verdade",
58+
realFailed: "Não foi possível mudar",
59+
realTitle: (bot, app) => `Deixar ${bot} usar seu mouse e teclado de verdade no ${app}?`,
60+
realPoints: (bot) => [
61+
`Quando a acessibilidade não alcança algo nesse app, ${bot} move o seu cursor e digita como se fosse você.`,
62+
`Enquanto ele age, o mouse e o teclado são dele, e um aviso na sua tela diz isso.`,
63+
`Mexa o mouse ou aperte uma tecla e ele para na hora. Ctrl+Alt+Esc para todos os bots no seu desktop.`,
64+
`Ele nunca digita em campos de senha e nunca aperta teclas do próprio Windows, como a tecla Windows ou Alt+Tab.`,
65+
],
66+
realConfirm: "Ligar",
67+
cancel: "Cancelar",
5668
},
5769
};

‎app/src/lib/fake.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,8 @@ export class FakeBotloft implements BotloftApi {
7575
readonly chat = new FakeChat(this);
7676
readonly allow = new FakeAllow(this);
7777
readonly desktop = new FakeDesktop(this);
78+
/** The language the app last told, as the daemon keeps it (spec 24.7). */
79+
locale: string | null = null;
7880
readonly conversation = new FakeConversation(this);
7981
readonly routines = new FakeRoutines(this);
8082
readonly files = new FakeFiles(this);
@@ -239,6 +241,10 @@ export class FakeBotloft implements BotloftApi {
239241
daemonVersion: this.system.daemonVersion,
240242
protocol: PROTOCOL_VERSION,
241243
}),
244+
"session.setLocale": ({ locale }) => {
245+
this.locale = locale;
246+
return null;
247+
},
242248
"system.status": () => this.system,
243249
"system.refresh": () => {
244250
this.refreshes += 1;

‎app/src/lib/fakeDesktop.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,17 @@ export class FakeDesktop {
104104
this.changed(grant.botId);
105105
return { botId: grant.botId, grants: this.of(grant.botId) };
106106
},
107+
"desktop.setOptions": ({ grantId, realInput }) => {
108+
const grant = this.grants.find((each) => each.id === grantId);
109+
if (!grant) {
110+
throw notFound(`desktop grant ${grantId}`);
111+
}
112+
if (realInput !== undefined) {
113+
grant.realInput = realInput;
114+
}
115+
this.changed(grant.botId);
116+
return { botId: grant.botId, grants: this.of(grant.botId) };
117+
},
107118
"desktop.watch": ({ botId }) => {
108119
this.fake.bot(botId);
109120
this.watching = botId;

0 commit comments

Comments
 (0)