Skip to content

Commit 7f5d79a

Browse files
authored
Merge pull request #176 from httpsphl/feat/desktop-panel
feat(app): the desktop panel, live, with Stop
2 parents f5e898c + 978bd05 commit 7f5d79a

16 files changed

Lines changed: 418 additions & 13 deletions

File tree

‎app/src/dev/seedDesktop.ts‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,45 @@
11
// Dev only: the preview's desktop (spec 24). Analyst asks to see a
2-
// spreadsheet in Excel, and Scout may already see Notepad.
2+
// spreadsheet in Excel, and Scout may already see Notepad, where it just
3+
// clicked Save.
34

45
import type { FakeBotloft } from "../lib/fake";
56
import type { BotId } from "../lib/protocol.gen";
67

78
const EXCEL = "C:\\Program Files\\Microsoft Office\\root\\Office16\\EXCEL.EXE";
9+
const NOTES = { id: 42, title: "Shopping list - Notepad", app: "Notepad" };
10+
11+
/** A Notepad window, drawn: a title bar, a menu and a few lines. */
12+
function drawNotepad(): string {
13+
const canvas = document.createElement("canvas");
14+
canvas.width = 760;
15+
canvas.height = 480;
16+
const g = canvas.getContext("2d");
17+
if (!g) {
18+
return "";
19+
}
20+
g.fillStyle = "#ffffff";
21+
g.fillRect(0, 0, 760, 480);
22+
g.fillStyle = "#f3f3f3";
23+
g.fillRect(0, 0, 760, 72);
24+
g.fillStyle = "#1f1f1f";
25+
g.font = "14px system-ui, sans-serif";
26+
g.fillText(NOTES.title, 16, 24);
27+
g.fillText("File Edit View", 16, 58);
28+
g.font = "16px Consolas, monospace";
29+
["Flour, 2 kg", "Eggs, 12", "Butter, 500 g", "Sugar, 1 kg"].forEach((line, index) => {
30+
g.fillText(line, 20, 112 + index * 26);
31+
});
32+
return canvas.toDataURL("image/jpeg", 0.8).split(",")[1] ?? "";
33+
}
834

935
export function seedDesktop(fake: FakeBotloft, bots: { scout: BotId; analyst: BotId }) {
1036
fake.desktop.grant(bots.scout, "C:\\Windows\\notepad.exe", "Notepad");
37+
fake.desktop.use(bots.scout, NOTES, { kind: "click", target: "Save", option: null });
38+
fake.desktop.paint(bots.scout, drawNotepad(), 760, 480);
39+
fake.chat.tool(bots.scout, "mcp__botloft__desktop_click", {
40+
summary: "To save the list",
41+
status: "done",
42+
});
1143
fake.chat.ask(
1244
bots.analyst,
1345
"mcp__botloft__desktop",

‎app/src/features/bots/BotView.tsx‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ import { useStable } from "../../ui/useStable";
1111
import { BrowserPanel } from "../browser/BrowserPanel";
1212
import { ShowBrowser } from "../browser/showBrowser";
1313
import { ChatView } from "../chat/ChatView";
14+
import { DesktopPanel } from "../desktop/DesktopPanel";
15+
import { ShowDesktop } from "../desktop/showDesktop";
1416
import { FilesPanel } from "../files/FilesPanel";
1517
import { ShowFile } from "../files/showFile";
1618
import { useBotFiles } from "../files/useBotFiles";
@@ -134,14 +136,22 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) {
134136
}
135137
setSide("terminal");
136138
});
139+
const showDesktop = useStable(() => {
140+
if (filesOpen) {
141+
seen();
142+
}
143+
setSide("desktop");
144+
});
137145
// The dock of the bot's computer: browser, terminal and files.
138146
const dock = {
139147
open: side,
140-
pick: (place: "browser" | "terminal" | "files") => {
148+
pick: (place: "browser" | "terminal" | "files" | "desktop") => {
141149
if (place === "browser") {
142150
showBrowser();
143151
} else if (place === "terminal") {
144152
showTerminal();
153+
} else if (place === "desktop") {
154+
showDesktop();
145155
} else if (!filesOpen) {
146156
toggleFiles();
147157
}
@@ -184,7 +194,9 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) {
184194
<ShowBrowser.Provider value={showBrowser}>
185195
<ShowScreen.Provider value={showScreen}>
186196
<ShowTerminal.Provider value={showTerminal}>
187-
<ChatView bot={bot} stopped={bot.paused || crew.paused} />
197+
<ShowDesktop.Provider value={showDesktop}>
198+
<ChatView bot={bot} stopped={bot.paused || crew.paused} />
199+
</ShowDesktop.Provider>
188200
</ShowTerminal.Provider>
189201
</ShowScreen.Provider>
190202
</ShowBrowser.Provider>
@@ -221,6 +233,7 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) {
221233
/>
222234
)}
223235
{beside === "terminal" && <TerminalPanel bot={bot} onClose={() => setSide(null)} />}
236+
{beside === "desktop" && <DesktopPanel bot={bot} onClose={() => setSide(null)} />}
224237
</PanelClosing.Provider>
225238
</PanelRestored.Provider>
226239
</Dock.Provider>

‎app/src/features/chat/ToolLines.tsx‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import { useT } from "../../i18n";
3030
import type { ChatItem, ToolItem } from "../../lib/protocol.gen";
3131
import { useArrival } from "../../ui/motion";
3232
import { isBrowserTool, ShowBrowser } from "../browser/showBrowser";
33+
import { isDesktopTool, ShowDesktop } from "../desktop/showDesktop";
3334
import { ShowFile } from "../files/showFile";
3435
import { isScreenFile, ShowScreen } from "../screens/showScreen";
3536
import { ShowTerminal } from "../terminal/showTerminal";
@@ -106,6 +107,7 @@ function ToolLine({ tool, createdAt }: { tool: ToolItem; createdAt: number }) {
106107
const showBrowser = useContext(ShowBrowser);
107108
const showScreen = useContext(ShowScreen);
108109
const showTerminal = useContext(ShowTerminal);
110+
const showDesktop = useContext(ShowDesktop);
109111
const arrival = useArrival(createdAt);
110112
const title = toolTitle(tool.name, t.tools);
111113
// A command reads in the bot's own words, when it gave them; the command
@@ -173,6 +175,17 @@ function ToolLine({ tool, createdAt }: { tool: ToolItem; createdAt: number }) {
173175
<SquareTerminal aria-hidden size={14} />
174176
</button>
175177
)}
178+
{showDesktop && isDesktopTool(tool.name) && (
179+
<button
180+
type="button"
181+
title={t.desktop.panel.showInPanel}
182+
aria-label={`${t.desktop.panel.showInPanel}: ${title}`}
183+
onClick={() => showDesktop()}
184+
className="grid size-6 shrink-0 place-items-center rounded-lg text-muted transition-colors hover:bg-sunken hover:text-ink"
185+
>
186+
<Monitor aria-hidden size={14} />
187+
</button>
188+
)}
176189
{showBrowser && isBrowserTool(tool.name) && (
177190
<button
178191
type="button"
Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
// A bot's use of the owner's desktop beside its chat (spec 24.9): the
2+
// window it read or acted in last, live, what it did, and Stop. Stopped,
3+
// the panel says so and offers to let it go on.
4+
5+
import { Monitor, Play, Square, X } from "lucide-react";
6+
import type { CSSProperties } from "react";
7+
import { useT } from "../../i18n";
8+
import type { Bot, DesktopAction } from "../../lib/protocol.gen";
9+
import { useApi } from "../../store/context";
10+
import { Badge } from "../../ui/Badge";
11+
import { Button } from "../../ui/Button";
12+
import { Callout } from "../../ui/Callout";
13+
import { EmptyState } from "../../ui/EmptyState";
14+
import { SidePanel } from "../../ui/SidePanel";
15+
import { attempt } from "../../ui/toast";
16+
import { BotAvatar } from "../bots/BotAvatar";
17+
import { ComputerFooter } from "../terminal/ComputerDock";
18+
import { useDesktopView } from "./useDesktopView";
19+
20+
type Words = ReturnType<typeof useT>["desktop"]["panel"];
21+
22+
/** What the bot did, in the owner's language. */
23+
export function actionText(action: DesktopAction | null, t: Words): string {
24+
if (!action) {
25+
return t.read;
26+
}
27+
const target = action.target.trim();
28+
switch (action.kind) {
29+
case "click":
30+
return target ? t.click(target) : t.clickSomething;
31+
case "type":
32+
return target ? t.type(target) : t.typeSomething;
33+
case "select":
34+
return t.select(action.option ?? "");
35+
case "scroll":
36+
return target ? t.scroll(target) : t.scrollSomething;
37+
}
38+
}
39+
40+
export function DesktopPanel({ bot, onClose }: { bot: Bot; onClose(): void }) {
41+
const t = useT().desktop.panel;
42+
const api = useApi();
43+
const { state, frame } = useDesktopView(bot.id);
44+
const stopped = state?.stopped === true;
45+
const window = state?.window ?? null;
46+
const stop = () => attempt(t.stopFailed, () => api.call("desktop.stop", { botId: bot.id }));
47+
const resume = () => attempt(t.resumeFailed, () => api.call("desktop.resume", { botId: bot.id }));
48+
49+
return (
50+
<SidePanel label={t.label(bot.name)} name="computer" defaultWidth={600}>
51+
<header className="flex h-11 shrink-0 items-center justify-between border-line border-b pr-1.5 pl-4">
52+
<h2 className="flex items-center gap-2 font-semibold text-sm">
53+
{t.heading}
54+
{stopped ? (
55+
<Badge tone="warn">{t.stoppedBadge}</Badge>
56+
) : (
57+
frame &&
58+
window && (
59+
<Badge tone="danger">
60+
<span aria-hidden className="live-dot" />
61+
{t.live}
62+
</Badge>
63+
)
64+
)}
65+
</h2>
66+
<div className="flex items-center gap-0.5">
67+
{state && !stopped && (
68+
<Button variant="ghost" size="sm" icon={Square} onClick={() => void stop()}>
69+
{t.stop}
70+
</Button>
71+
)}
72+
<Button variant="ghost" size="sm" icon={X} label={t.close} onClick={onClose} />
73+
</div>
74+
</header>
75+
<div className="flex min-h-0 flex-1 flex-col gap-3 overflow-y-auto p-3">
76+
{stopped && (
77+
<Callout tone="warn" title={t.stoppedTitle(bot.name)}>
78+
<p>{t.stoppedBody(bot.name)}</p>
79+
<div className="mt-2.5">
80+
<Button variant="primary" size="sm" icon={Play} onClick={() => void resume()}>
81+
{t.resume}
82+
</Button>
83+
</div>
84+
</Callout>
85+
)}
86+
{state && !window && (
87+
<EmptyState
88+
color={bot.color}
89+
icon={Monitor}
90+
title={t.emptyTitle(bot.name)}
91+
body={t.emptyBody}
92+
/>
93+
)}
94+
{window && (
95+
<>
96+
<p className="flex min-w-0 items-baseline gap-2 text-sm">
97+
<span className="shrink-0 font-medium">{window.app}</span>
98+
<span className="truncate text-muted" data-selectable>
99+
{window.title}
100+
</span>
101+
</p>
102+
<div
103+
className="bot-stage grid min-h-40 flex-1 place-items-center p-2.5"
104+
style={{ "--bot": bot.color } as CSSProperties}
105+
>
106+
{frame ? (
107+
<img
108+
src={`data:image/jpeg;base64,${frame.data}`}
109+
alt={window.title}
110+
draggable={false}
111+
className="max-h-full max-w-full select-none rounded-lg object-contain shadow-lift"
112+
/>
113+
) : (
114+
<p className="text-sm text-white/80">{t.waiting}</p>
115+
)}
116+
</div>
117+
<p aria-live="polite" className="flex items-center gap-2 text-ink-soft text-sm">
118+
<BotAvatar color={bot.color} size={18} mood={stopped ? "sleeping" : "working"} />
119+
<span className="truncate">{actionText(state?.action ?? null, t)}</span>
120+
</p>
121+
</>
122+
)}
123+
<p className="mt-auto text-muted text-xs">{t.shortcut}</p>
124+
</div>
125+
<ComputerFooter />
126+
</SidePanel>
127+
);
128+
}
Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
import { act, cleanup, fireEvent, screen, waitFor, within } from "@testing-library/react";
2+
import { afterEach, describe, expect, test } from "vitest";
3+
import { FakeBotloft } from "../../lib/fake";
4+
import { crewOpened, openBot, renderApp } from "../../test/app";
5+
6+
afterEach(cleanup);
7+
8+
async function openScout() {
9+
const fake = new FakeBotloft();
10+
const ops = fake.addCrew("Ops");
11+
const scout = fake.addBot(ops.id, "Scout", "Finds sources");
12+
fake.setBotState(scout.id, "idle");
13+
fake.chat.tool(scout.id, "mcp__botloft__desktop_look", {
14+
summary: "To read the list",
15+
status: "done",
16+
});
17+
renderApp(fake);
18+
await crewOpened("Ops");
19+
openBot("Scout");
20+
await screen.findByRole("list", { name: "Messages" });
21+
return { fake, scout };
22+
}
23+
24+
const panel = () => screen.getByRole("complementary", { name: "Scout's desktop" });
25+
const NOTES = { id: 42, title: "Shopping list - Notepad", app: "Notepad" };
26+
27+
describe("the desktop panel", () => {
28+
test("opens from a desktop tool in the chat and shows the window the bot uses, live", async () => {
29+
const { fake, scout } = await openScout();
30+
fireEvent.click(
31+
screen.getByRole("button", { name: "Watch on the desktop panel: Read a window" }),
32+
);
33+
expect(await within(panel()).findByText("Scout has not used your desktop yet")).toBeDefined();
34+
expect(fake.desktop.watching).toBe(scout.id);
35+
36+
act(() => {
37+
fake.desktop.use(scout.id, NOTES, { kind: "click", target: "Save", option: null });
38+
});
39+
expect(await within(panel()).findByText("Shopping list - Notepad")).toBeDefined();
40+
expect(within(panel()).getByText('Clicked "Save"')).toBeDefined();
41+
expect(within(panel()).getByText("Waiting for the picture…")).toBeDefined();
42+
43+
act(() => {
44+
fake.desktop.paint(scout.id, "AAAA");
45+
});
46+
const picture = await within(panel()).findByRole("img", { name: "Shopping list - Notepad" });
47+
expect(picture.getAttribute("src")).toBe("data:image/jpeg;base64,AAAA");
48+
expect(within(panel()).getByText("Live")).toBeDefined();
49+
50+
// Closed, nobody watches.
51+
fireEvent.click(within(panel()).getByRole("button", { name: "Close" }));
52+
await waitFor(() => expect(fake.desktop.watching).toBeNull());
53+
});
54+
55+
test("stops the bot on the desktop and lets it go on", async () => {
56+
const { fake, scout } = await openScout();
57+
fireEvent.click(
58+
screen.getByRole("button", { name: "Watch on the desktop panel: Read a window" }),
59+
);
60+
act(() => {
61+
fake.desktop.use(scout.id, NOTES);
62+
});
63+
expect(await within(panel()).findByText("Read the window")).toBeDefined();
64+
fireEvent.click(within(panel()).getByRole("button", { name: "Stop" }));
65+
expect(await within(panel()).findByText("You stopped Scout on your desktop")).toBeDefined();
66+
expect(fake.desktop.state(scout.id).stopped).toBe(true);
67+
fireEvent.click(within(panel()).getByRole("button", { name: "Let it go on" }));
68+
await waitFor(() => expect(fake.desktop.state(scout.id).stopped).toBe(false));
69+
expect(within(panel()).queryByText("You stopped Scout on your desktop")).toBeNull();
70+
});
71+
});
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
// How a desktop tool line in the chat opens the bot's desktop panel (spec
2+
// 24.9). Absent where there is no panel, so no button shows.
3+
4+
import { createContext } from "react";
5+
6+
export const ShowDesktop = createContext<(() => void) | null>(null);
7+
8+
/** `mcp__botloft__desktop_look` and the other desktop tools. */
9+
export function isDesktopTool(name: string): boolean {
10+
return name.startsWith("mcp__botloft__desktop_");
11+
}
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
// What a bot does on the owner's desktop, live (spec 24.9): its state and
2+
// the picture of the window it is using, while its panel is open.
3+
4+
import { useEffect, useState } from "react";
5+
import type { BotId, DesktopFrame, DesktopState } from "../../lib/protocol.gen";
6+
import { useApi } from "../../store/context";
7+
8+
export interface DesktopView {
9+
/** `null` until the daemon answers. */
10+
state: DesktopState | null;
11+
frame: DesktopFrame | null;
12+
}
13+
14+
export function useDesktopView(botId: BotId): DesktopView {
15+
const api = useApi();
16+
const [view, setView] = useState<DesktopView>({ state: null, frame: null });
17+
useEffect(() => {
18+
let alive = true;
19+
setView({ state: null, frame: null });
20+
const stop = api.subscribe((event) => {
21+
if (event.name === "desktop.changed" && event.params.botId === botId) {
22+
setView((now) => ({ ...now, state: event.params }));
23+
}
24+
if (event.name === "desktop.frame" && event.params.botId === botId) {
25+
setView((now) => ({ ...now, frame: event.params }));
26+
}
27+
});
28+
api.call("desktop.watch", { botId }).then(
29+
// What arrived meanwhile is newer than the answer.
30+
(watched) =>
31+
alive &&
32+
setView((now) => ({
33+
state: now.state ?? watched.state,
34+
frame: now.frame ?? watched.frame,
35+
})),
36+
// An older daemon has no desktop: nothing to show.
37+
() =>
38+
alive &&
39+
setView({
40+
state: { botId, window: null, action: null, at: null, stopped: false },
41+
frame: null,
42+
}),
43+
);
44+
return () => {
45+
alive = false;
46+
stop();
47+
api.call("desktop.unwatch").catch(() => {});
48+
};
49+
}, [api, botId]);
50+
return view;
51+
}

0 commit comments

Comments
 (0)